class AbstractAction(object):
signalActionBegin = QtCore.Signal(str)
signalActionError = QtCore.Signal(str)
signalActionEnd = QtCore.Signal(DataFrameModel)
def __init__(self, model=None):
pass
class NewProjectDialog(QtGui.QDialog, Ui_NewProjectDialog):
signalProjectNew = QtCore.Signal(list) #[dirname, settings.ini]
def __init__(self, base_dirname=None, parent=None):
QtGui.QDialog.__init__(self, parent=parent)
self._base_dirname = base_dirname
self.setupUi(self)
self.namePushButton.clicked.connect(self.set_dirname)
self.settingsFilePushButton.clicked.connect(self.set_config_ini)
self.buttonBox.accepted.connect(self.create_project)
self.buttonBox.rejected.connect(self.close)
self.namePushButton.hide()
self.nameLineEdit.setText("Folder Name:")
def set_dirname(self):
dirname = QtGui.QFileDialog.get(self)
self.nameLineEdit.setText(dirname)
def set_base_dirname(self, dirname):
assert not os.path.isfile(dirname)
self._base_dirname = dirname
def set_config_ini(self):
ini_file = QtGui.QFileDialog.getOpenFileName()
self.settingsFileLineEdit.setText(ini_file[0])
def create_project(self):
proj_dirname = self.nameLineEdit.text()
if os.path.isdir(proj_dirname):
proj_dirname = os.path.basename(proj_dirname)
if self._base_dirname is not None:
proj_dirname = os.path.join(self._base_dirname, proj_dirname)
else:
raise AttributeError(
"NewProjectDialog.set_base_dirname and try again. Invalid directory: '{}'"
.format(self._base_dirname))
proj_ini = self.settingsFileLineEdit.text()
ini_correct = os.path.join(proj_dirname, os.path.basename(proj_ini))
if not os.path.exists(proj_dirname):
os.mkdir(proj_dirname)
if not os.path.exists(ini_correct):
shutil.copy2(proj_ini, ini_correct)
self.signalProjectNew.emit([proj_dirname, ini_correct])
self.close()
class DirectoryViewDialog(QtGui.QDialog, Ui_DirectoryViewDialog):
signalDownloadReady = QtCore.Signal()
def __init__(self, source_view=None, parent=None):
QtGui.QDialog.__init__(self, parent=parent)
self._source_view = source_view
self.setupUi(self)
def configure(self):
self.btnDelete.clicked.connect(self.delete)
self.btnDownload.clicked.connect(self.download)
self.btnRefresh.clicked.connect(self.refresh)
self.btnUpload.clicked.connect(self.upload)
self.btnAddFolder.clicked.connect(self.add_folder)
self.buttonBox.accepted.connect(self.accept)
self.buttonBox.rejected.connect(self.reject)
def upload(self):
pass
def download(self):
pass
def delete(self):
pass
def refresh(self):
pass
def add_folder(self):
pass
@property
def source_view(self):
return self._source_view
def set_source_view(self, tree_view):
self._source_view = tree_view
class DataFrameModelExportDialog(QtGui.QDialog, Ui_ExportFileDialog):
signalExported = QtCore.Signal(str, str) # original_path, new_path
def __init__(self,
df_manager: DataFrameModelManager,
filename: str = None,
allow_multi_source=True,
**kwargs):
self.df_manager = df_manager
if allow_multi_source:
self.df_manager.signalNewModelRead.connect(
self.append_source_filename)
if filename is None:
if df_manager.last_path_updated is not None:
self._default_path = df_manager.last_path_updated
else:
self._default_path = df_manager.last_path_read
else:
self._default_path = filename
assert filename in df_manager.file_paths, "Invalid filename {} not in DataFrameModelManager paths: {}".format(
filename, df_manager.file_paths)
QtGui.QDialog.__init__(self, **kwargs)
self.setupUi(self)
self.configure()
def configure(self):
self.btnBrowseDestination.clicked.connect(self.browse_destination)
self.btnBrowseSource.clicked.connect(self.browse_source)
self.btnOverwrite.clicked.connect(
self.set_destination_path_from_source)
configure_combo_box(self.comboBoxSource, self.df_manager.file_paths,
self._default_path)
configure_combo_box(self.comboBoxSeparator, list(SEPARATORS.keys()),
'Comma')
configure_combo_box(self.comboBoxEncoding, list(ENCODINGS.keys()),
'ISO-8859-1')
self.buttonBox.accepted.connect(self.export)
if self._default_path is not None:
self.lineEditDestination.setText(
path_incremented(self._default_path, overwrite=False))
def browse_destination(self):
source = self.comboBoxSource.currentText()
if os.path.isfile(source):
kwargs = dict(dir=os.path.dirname(source))
else:
kwargs = dict()
filename = QtGui.QFileDialog.getSaveFileName(self, **kwargs)[0]
if os.path.isfile(filename):
self.lineEditDestination.setText(filename)
def set_separator(self, sep):
try:
SEPARATORS[sep]
except KeyError:
mapper = {v: k for k, v in SEPARATORS.items()}
try:
sep = mapper[sep]
except KeyError:
self.radioBtnOtherSeparator.setChecked(True)
return self.lineEditOtherSeparator.setText(sep)
self.comboBoxSeparator.setCurrentIndex(
self.comboBoxSeparator.findText(sep))
def set_encoding(self, en):
try:
ENCODINGS[en]
except KeyError as e:
mapper = {v: k for k, v in ENCODINGS.items()}
try:
en = mapper[en]
except KeyError as e:
raise KeyError("Invalid Encoding {}".format(e))
self.comboBoxEncoding.setCurrentIndex(
self.comboBoxEncoding.findText(en))
def set_destination_path_from_source(self):
self.lineEditDestination.setText(self.comboBoxSource.currentText())
def browse_source(self):
source = self.comboBoxSource.currentText()
if os.path.isfile(source):
kwargs = dict(dir=os.path.dirname(source))
else:
kwargs = dict()
filename = QtGui.QFileDialog.getOpenFileName(self, **kwargs)[0]
if filename not in self.df_manager.file_paths:
self.df_manager.read_file(filename)
idx = self.comboBoxSource.findText(filename)
self.comboBoxSource.setCurrentIndex(idx)
@QtCore.Slot(str)
def append_source_filename(self, filename: str):
self.comboBoxSource.addItem(filename)
def export(self):
source_path = self.comboBoxSource.currentText()
file_path = self.lineEditDestination.text()
encoding = self.comboBoxEncoding.currentText()
if self.radioBtnOtherSeparator.isChecked():
sep = self.lineEditOtherSeparator.text()
else:
sep = self.comboBoxSeparator.currentText()
kwargs = dict(index=False)
kwargs['encoding'] = ENCODINGS[encoding]
kwargs['sep'] = SEPARATORS[sep]
self.df_manager.save_file(source_path,
save_as=file_path,
keep_orig=True,
**kwargs)
self.signalExported.emit(source_path, file_path)
self.hide()
#box = get_ok_msg_box(self.parent, "Exported {}!".format(file_path))
#box.show()
logging.info("Exported {} to {}".format(source_path, file_path))
class ProjectMainWindow(QtGui.QMainWindow, Ui_ProjectWindow):
"""
The ProjectMainWindow displays a project that the user wants to work on.
The project lives in a directory within the ZeexApp.ROOT_DIRECTORY.
Each project gets a DataFrameModelManager object which controls the
DataFrameModel of each file opened.
The main window shows all files in this directory and provides
very convenient features to work with files containing rows/columns.
Project's settings are stored in a .ini file
in the root project directory.
"""
signalModelChanged = QtCore.Signal(str)
signalModelOpened = QtCore.Signal(str)
signalModelDestroyed = QtCore.Signal(str)
def __init__(self, settings_ini: (str, SettingsINI), parent=None):
"""
:param settings_ini: (str, SettingsINI)
can be a settings_ini file path or configured SettingsINI object.
"""
self.df_manager = DataFrameModelManager()
QtGui.QMainWindow.__init__(self, parent=parent)
self.setupUi(self)
self.icons = Icons()
self.dialog_settings = SettingsDialog(settings=settings_ini)
self.dialog_merge_purge = MergePurgeDialog(self.df_manager)
self.dialog_export = DataFrameModelExportDialog(self.df_manager,
parent=self)
self.dialog_import = DataFrameModelImportDialog(self.df_manager,
parent=self)
self.dialog_new_folder = DirectoryPathCreateDialog(self.treeView,
parent=self)
self.dialog_cloud = None
self.key_delete = QtGui.QShortcut(self)
self.key_enter = QtGui.QShortcut(self)
self.key_zip = QtGui.QShortcut(self)
self.key_rename = QtGui.QShortcut(self)
self.connect_window_title()
self.connect_actions()
self.connect_treeview()
self.connect_icons()
self.connect_settings_dialog()
self.connect_import_dialog()
self.connect_export_dialog()
self.connect_cloud_dialog()
self.current_model = None
@property
def project_directory(self):
"""
The project's root directory stores everything for the project.
:return: (str)
ProjectMainWindow.treeView.QFileSystemModel.rootPath
"""
return self.treeView.model().rootPath()
@property
def log_directory(self):
"""
The project's log directory stores output logs.
:return: (str)
ProjectMainWindow.project_directory/log
"""
return os.path.join(self.project_directory, 'log')
@QtCore.Slot(SettingsINI, str)
def sync_settings(self, config: SettingsINI = None, file_path=None):
"""
Anytime settings are saved this method gets triggered.
Sets defaults for various views.
:param config:
:param file_path:
:return:
"""
self.connect_export_dialog()
self.connect_import_dialog()
def connect_window_title(self):
"""
Sets the ProjectMainWindow.windowTitle to "Project - dirname - dirpath"
:return: None
"""
root_dir = self.dialog_settings.rootDirectoryLineEdit.text()
base_name = os.path.basename(root_dir)
self.dialog_settings.setWindowTitle("{} - Settings".format(base_name))
self.setWindowTitle("Project: {} - {}".format(
base_name, root_dir.replace(base_name, "")))
def connect_actions(self):
"""
Connects all project actions.
:return: None
"""
self.actionPreferences.triggered.connect(self.dialog_settings.show)
self.actionAddFolder.triggered.connect(self.dialog_new_folder.show)
self.actionNew.triggered.connect(self.dialog_import.show)
self.actionOpen.triggered.connect(self.open_tableview_window)
self.actionSave.triggered.connect(self.dialog_export.show)
self.actionRemove.triggered.connect(self.remove_tree_selected_path)
self.actionRename.triggered.connect(self.open_rename_path_dialog)
self.actionMergePurge.triggered.connect(self.open_merge_purge_dialog)
self.actionUnzip.triggered.connect(self.handle_compression)
self.actionZip.triggered.connect(self.handle_compression)
self.key_delete.setKey('del')
self.key_enter.setKey('return')
self.key_zip.setKey(QtGui.QKeySequence(self.tr('Ctrl+Z')))
self.key_rename.setKey(QtGui.QKeySequence(self.tr('Ctrl+R')))
self.key_delete.activated.connect(self.remove_tree_selected_path)
self.key_enter.activated.connect(self.open_tableview_window)
self.key_zip.activated.connect(self.handle_compression)
self.key_rename.activated.connect(self.open_rename_path_dialog)
def connect_icons(self):
"""
Sets all the menu/window icons.
:return: None
"""
self.setWindowIcon(self.icons['folder'])
self.actionNew.setIcon(self.icons['add'])
self.actionAddFolder.setIcon(self.icons['folder'])
self.actionOpen.setIcon(self.icons['spreadsheet'])
self.actionPreferences.setIcon(self.icons['settings'])
self.actionRemove.setIcon(self.icons['delete'])
self.actionSave.setIcon(self.icons['save'])
self.dialog_settings.setWindowIcon(self.icons['settings'])
self.actionMergePurge.setIcon(self.icons['merge'])
self.actionRename.setIcon(self.icons['rename'])
self.dialog_merge_purge.setWindowIcon(self.icons['merge'])
self.actionZip.setIcon(self.icons['archive'])
self.dialog_new_folder.setWindowIcon(self.icons['folder'])
self.actionUnzip.setIcon(self.icons['unzip'])
def connect_treeview(self):
"""
Uses the ProjectMainWindow.dialog_settings.rootDirectoryLineEdit
to get the root directory name of the project. It then
connects the filetree using a QFileSystemModel.
:return: None
"""
rootdir = self.dialog_settings.rootDirectoryLineEdit.text()
model = FileTreeModel(root_dir=rootdir)
self.treeView.setModel(model)
self.treeView.setRootIndex(model.index(rootdir))
self.treeView.setColumnWidth(0, 400)
self.treeView.setSelectionMode(self.treeView.ExtendedSelection)
def connect_settings_dialog(self):
"""
Re-purposes the ProjectMainWindow.dialog_settings
dialog to fit the scope of a project rather than
the application as a whole.
:return: None
"""
#Adjust the box to remove irrelevant items.
self.dialog_settings.cloudProviderComboBox.hide()
self.dialog_settings.cloudProviderLabel.hide()
self.dialog_settings.btnLogDirectory.hide()
self.dialog_settings.btnRootDirectory.hide()
self.dialog_settings.themeComboBox.hide()
self.dialog_settings.themeLabel.hide()
# Override the log/root directory options
self.dialog_settings.logDirectoryLineEdit.setText(self.log_directory)
self.dialog_settings.logDirectoryLineEdit.setReadOnly(True)
self.dialog_settings.rootDirectoryLineEdit.setText(
self.project_directory)
self.dialog_settings.rootDirectoryLineEdit.setReadOnly(True)
self.dialog_settings.btnSetDefault.setVisible(False)
self.dialog_settings.signalSettingsSaved.connect(self.sync_settings)
self.dialog_new_folder.base_dirname = self.dialog_settings.rootDirectoryLineEdit.text(
)
def connect_cloud_dialog(self):
try:
self.dialog_cloud = DropBoxViewDialog(self.treeView, self)
self.actionViewCloud.triggered.connect(self.dialog_cloud.show)
self.actionViewCloud.setIcon(self.icons['cloud'])
except Exception as e:
logging.error("Error connecting to cloud: {}".format(e))
self.actionViewCloud.setVisible(False)
def connect_export_dialog(self):
"""
Sets defaults of the DataFrameModelExport Dialog.
:return: None
"""
self.dialog_export.signalExported.connect(self._flush_export)
self.dialog_export.setWindowIcon(self.icons['export_generic'])
sep = self.dialog_settings.separatorComboBox.currentText()
enc = self.dialog_settings.encodingComboBox.currentText()
self.dialog_export.set_encoding(enc)
self.dialog_export.set_separator(sep)
def connect_import_dialog(self):
"""
Sets defaults of the DataFrameModelImport Dialog.
:return: None
"""
self.dialog_import.signalImported.connect(self.import_file)
self.dialog_import.setWindowIcon(self.icons['add'])
sep = self.dialog_settings.separatorComboBox.currentText()
enc = self.dialog_settings.encodingComboBox.currentText()
self.dialog_import.set_encoding(enc)
self.dialog_import.set_separator(sep)
def open_tableview_window(self, model: DataFrameModel = None):
"""
Opens a FileTableWindow for the filename selected in the
ProjectMainWindow.treeView.
:param model: The qtpandas.models.DataFrameModel to edit.
:return: None
"""
if model is None:
# Maybe it's selected on the tree?
model = self.get_tree_selected_model()
if model is None:
# No, not sure why this was called..
#box = get_ok_msg_box(self, "No model available to open.")
#box.show()
pass
self.df_manager.get_fileview_window(model.filePath).show()
self.add_recent_file_menu_entry(model.filePath, model)
def open_merge_purge_dialog(self, model: DataFrameModel = None):
if model is None:
model = self.get_tree_selected_model()
current_model = self.dialog_merge_purge.source_model
if current_model is None or current_model.filePath is not model.filePath:
self.dialog_merge_purge.set_source_model(model=model,
configure=True)
self.dialog_merge_purge.show()
def get_tree_selected_model(self,
raise_on_error=True) -> (DataFrameModel, None):
"""
Returns a DataFrameModel based on the filepath selected
in the ProjectMainWindow.treeView.
:return: qtpandas.DataFrameModel
"""
# Check if file is selected in tree view
selected = self.treeView.selectedIndexes()
if selected:
idx = selected[0]
file_path = self.treeView.model().filePath(idx)
return self.df_manager.read_file(file_path)
return None
def get_tree_selected_path(self):
selected = self.treeView.selectedIndexes()
if selected:
idx = selected[0]
return self.treeView.model().filePath(idx)
return None
def remove_tree_selected_path(self):
#TODO: need to emit a signal here.
idxes = self.treeView.selectedIndexes()
if idxes:
file_model = self.treeView.model()
for idx in idxes:
if not file_model.isDir(idx):
file_model.remove(idx)
else:
file_model.rmdir(idx)
def open_tableview_current(self, model: DataFrameModel = None):
"""
Opens a tableview window for the current_model or the model kwarg.
:param model: DataFrameModel
:return: None
"""
if model is None:
model = self.current_model
else:
self.set_current_df_model(model)
assert isinstance(model, DataFrameModel), "No current DataFrame model."
return self.open_tableview_window(model)
def set_current_df_model(self, model: DataFrameModel):
"""
Sets the current dataframe model. for the project.
:param model: DataFrameModel
:return: None
"""
self.df_manager.set_model(model, model._filePath)
self.current_model = self.df_manager.get_model(model._filePath)
@QtCore.Slot('DataFrameModel', str)
def import_file(self, filepath, open=True):
if isinstance(filepath, DataFrameModel):
model = filepath
filepath = model.filePath
self.df_manager.set_model(model, filepath)
model = self.df_manager.get_model(filepath)
name = os.path.basename(model.filePath)
dirname = self.dialog_settings.rootDirectoryLineEdit.text()
assert os.path.isdir(
dirname), "Root Directory is not a directory: {}".format(dirname)
if os.path.dirname(filepath) != dirname:
newpath = os.path.join(dirname, name)
self.df_manager.save_file(filepath,
save_as=newpath,
keep_orig=False)
filepath = newpath
if open is True:
model = self.df_manager.get_model(filepath)
self.open_tableview_window(model)
def add_recent_file_menu_entry(self, name, model):
action = QtGui.QAction(name, self.menuRecent_Files)
action.triggered.connect(partial(self.open_tableview_window, model))
actions = self.menuRecent_Files.actions()
if actions:
self.menuRecent_Files.insertAction(actions[0], action)
else:
self.menuRecent_Files.addAction(action)
def maybe_save_copy(self, df, filepath, max_size=20000, **kwargs):
"""
Saves a copy of the file to the project folder if
its smaller than max_size
its larger than 0 rows
:param df: DataFrame: to save
:param filepath: str: the filepath of the DataFrame
:param max_size: int: max size of DataFrame
:param kwargs: DataFrame.to_csv(**kwargs)
:return: None
"""
if df.index.size <= max_size and not df.empty:
kwargs['index'] = kwargs.get('index', False)
df.to_csv(filepath, **kwargs)
def _flush_export(self, orig_path, new_path):
if orig_path != new_path:
self.add_recent_file_menu_entry(
new_path, self.df_manager.get_model(new_path))
def open_rename_path_dialog(self):
current_path = self.get_tree_selected_path()
dialog = FilePathRenameDialog(current_path, parent=self)
dialog.show()
def handle_compression(self, fpath=None, **kwargs):
if fpath is None:
fpath = self.get_tree_selected_path()
assert fpath is not None, "No selected path!"
if not fpath.lower().endswith('.zip'):
return ostools.zipfile_compress(fpath, **kwargs)
else:
return ostools.zipfile_unzip(
file_path=fpath,
dir=self.dialog_settings.rootDirectoryLineEdit.text())
class DataFrameModelImportDialog(QtGui.QDialog, Ui_ImportFileDialog):
signalImported = QtCore.Signal(str) # file path
def __init__(self,
df_manager: DataFrameModelManager,
file_path=None,
dir='',
**kwargs):
QtGui.QDialog.__init__(self, **kwargs)
self.df_manager = df_manager
self.file_path = file_path
self.dir = dir
if file_path is not None and dir == '':
self.dir = os.path.dirname(file_path)
self.setupUi(self)
self.configure()
def configure(self):
self.checkBoxHasHeaders.setChecked(True)
if self.file_path is not None:
self.set_file_path(self.file_path)
self.btnBrowseFilePath.clicked.connect(self.browse_file_path)
self.buttonBox.accepted.connect(self.execute)
configure_combo_box(self.comboBoxSeparator, list(SEPARATORS.keys()),
'Comma')
configure_combo_box(self.comboBoxEncoding, list(ENCODINGS.keys()),
'ISO-8859-1')
def set_file_path(self, file_path=None):
if file_path is None:
if self.file_path is None:
file_path = self.lineEditFilePath.text()
else:
file_path = self.file_path
if not os.path.isfile(file_path):
last = self.df_manager.last_path_updated
if last is not None:
kwargs = dict(dir=os.path.dirname(last))
else:
kwargs = dict(dir=self.dir)
file_path = QtGui.QFileDialog.getOpenFileName(**kwargs)[0]
self.lineEditFilePath.setText(str(file_path))
def browse_file_path(self):
last = self.df_manager.last_path_updated
if last is not None and self.dir == '':
kwargs = dict(dir=os.path.dirname(last))
else:
kwargs = dict(dir=self.dir)
file_path = QtGui.QFileDialog.getOpenFileName(**kwargs)[0]
self.set_file_path(file_path)
def process_dataframe(self,
df,
trim_spaces=False,
remove_linebreaks=False,
parse_dates=False):
if trim_spaces and not remove_linebreaks:
for c in df.columns:
if str(df[c].dtype) == 'object':
df.loc[:, c] = pandatools.series_strip(df.loc[:, c])
elif remove_linebreaks:
df = pandatools.dataframe_remove_linebreaks(df, copy=False)
if parse_dates:
df = pandatools.dataframe_to_datetime(df)
return df
def set_separator(self, sep):
try:
SEPARATORS[sep]
except KeyError:
mapper = {v: k for k, v in SEPARATORS.items()}
try:
sep = mapper[sep]
except KeyError:
self.radioBtnOtherSeparator.setChecked(True)
return self.lineEditOtherSeparator.setText(sep)
self.comboBoxSeparator.setCurrentIndex(
self.comboBoxSeparator.findText(sep))
def set_encoding(self, en):
try:
ENCODINGS[en]
except KeyError as e:
mapper = {v: k for k, v in ENCODINGS.items()}
try:
en = mapper[en]
except KeyError as e:
raise KeyError("Invalid Encoding {}".format(e))
self.comboBoxEncoding.setCurrentIndex(
self.comboBoxEncoding.findText(en))
def execute(self):
file_path = self.lineEditFilePath.text()
file_megabytes = os.path.getsize(file_path) / 1000 / 1000
remove_linebreaks = self.checkBoxScrubLinebreaks.isChecked()
trim_spaces = self.checkBoxTrimSpaces.isChecked()
has_headers = self.checkBoxHasHeaders.isChecked()
parse_dates = self.checkBoxParseDates.isChecked()
encoding = self.comboBoxEncoding.currentText()
kwargs = dict()
if self.radioBtnOtherSeparator.isChecked():
sep = self.lineEditOtherSeparator.text()
kwargs['sep'] = sep
else:
sep = self.comboBoxSeparator.currentText()
kwargs['sep'] = SEPARATORS.get(sep, ',')
if not has_headers:
kwargs['header'] = 0
kwargs['first_codec'] = ENCODINGS.get(encoding, 'utf8')
if file_megabytes > 1000:
kwargs['chunksize'] = 50 * 1000 * 1000
df_reader = pandatools.superReadFile(file_path, **kwargs)
if kwargs.get('chunksize', 0) > 0:
# Process the dataframe in chunks
df = pd.DataFrame()
for chunk in df_reader:
chunk = self.process_dataframe(
chunk,
trim_spaces=trim_spaces,
remove_linebreaks=remove_linebreaks,
parse_dates=parse_dates)
df = df.append(chunk)
else:
df = self.process_dataframe(df_reader,
trim_spaces=trim_spaces,
remove_linebreaks=remove_linebreaks,
parse_dates=parse_dates)
dfm = DataFrameModel(dataFrame=df, filePath=file_path)
self.df_manager.set_model(df_model=dfm, file_path=file_path)
self.signalImported.emit(file_path)
class SplitFileDialog(QtGui.QDialog, Ui_FileSplitDialog):
signalFileSplit = QtCore.Signal(list) # [source_path, [p1, p2, p3, p4]]
def __init__(self, source_model, parent=None):
QtGui.QDialog.__init__(self, parent=parent)
self.df_model = source_model
self.setupUi(self)
self.configure_model()
cols = source_model.dataFrame().columns.tolist()
self.handler_split_on = PushGridHandler(left_button=self.btnSplitOnPushLeft,
left_view=self.listViewSplitOnLeft,
right_button=self.btnSplitOnPushRight,
right_view=self.listViewSplitOnRight,
left_model=cols)
self.handler_use_cols = PushGridHandler(left_button=self.btnUseColsPushLeft,
left_view=self.listViewUseColsLeft,
right_button=self.btnUseColsPushRight,
right_view=self.listViewUseColsRight,
left_model=cols)
self.configure()
def configure(self):
self.btnExportTemplate.clicked.connect(self.export_settings)
self.btnImportTemplate.clicked.connect(self.import_settings)
self.df_model.dataChanged.connect(partial(self.configure_model, reset=True))
self.buttonBox.accepted.connect(self.execute)
self.buttonBox.rejected.connect(self.close)
def configure_model(self, reset=False):
p = self.df_model.filePath
self.lineEditSourcePath.setText(p)
base, ext = os.path.splitext(p)
filename = os.path.basename(base).lower()
dirname = os.path.basename(os.path.dirname(base))
self.lineEditExportPath.setText(os.path.join(os.path.dirname(p), "{}_split".format(filename)))
self.setWindowTitle("Split: {} - Project - {}".format(filename, dirname))
if reset is True:
cols = self.df_model.dataFrame().columns.tolist()
self.handler_split_on.set_model_from_list(cols)
self.handler_use_cols.set_model_from_list(cols)
def get_settings(self, dc:DictConfig=None, section:str="SPLIT"):
if dc is None:
dc = DictConfig()
if dc.has_section(section):
dc.remove_section(section)
dictionary = dict(dropna = self.checkBoxDropNulls.isChecked(),
dirname = self.lineEditExportPath.text(),
source_path = self.lineEditSourcePath.text(),
max_rows = self.lineEditMaxRows.text())
dc.update({section: dictionary})
dc.set_safe(section, 'split_on', self.handler_split_on.get_model_list(left=False))
dc.set_safe(section, 'fields', self.handler_use_cols.get_model_list(left=False))
return dc
def set_settings(self, dc:DictConfig, section='SPLIT'):
self.lineEditTemplate.setText(dc.filename)
self.handler_split_on.set_model_from_list(dc.get_safe(section, 'split_on',
fallback=self.handler_split_on.get_model_list(left=False)),
left=False)
self.handler_split_on.drop_duplicates(left_priority=False)
self.handler_use_cols.set_model_from_list(dc.get_safe(section, 'fields',
fallback=self.handler_use_cols.get_model_list(left=False)),
left=False)
self.handler_use_cols.drop_duplicates(left_priority=False)
self.checkBoxDropNulls.setChecked(dc.getboolean(section, 'dropna', fallback=self.checkBoxDropNulls.isChecked()))
self.lineEditSourcePath.setText(dc.get(section, 'source_path', fallback=self.lineEditSourcePath.text()))
self.lineEditExportPath.setText(dc.get(section, 'dirname', fallback=self.lineEditExportPath.text()))
self.lineEditMaxRows.setText(dc.get(section, 'max_rows', fallback=self.lineEditMaxRows.text()))
def import_settings(self):
import_settings(self.set_settings, parent=self)
def export_settings(self):
export_settings(self.get_settings(),parent=self)
def execute(self):
split_on = self.handler_split_on.get_model_list(left=False)
fields = self.handler_use_cols.get_model_list(left=False)
dropna = self.checkBoxDropNulls.isChecked()
dirname = self.lineEditExportPath.text()
source_path = self.lineEditSourcePath.text()
max_rows = self.lineEditMaxRows.text()
try:
max_rows = int(max_rows)
except ValueError:
max_rows = None
df = self.df_model.dataFrame()
exported_paths = dataframe_split_to_files(df,
source_path,
split_on,
fields=fields,
dropna=dropna,
dest_dirname=dirname,
chunksize=max_rows,
index=False)
emission = [source_path, exported_paths]
self.signalFileSplit.emit(emission)
class AlchemyConnectionDialog(QtGui.QDialog, Ui_AlchemyConnectionDialog):
"""
A dialog that allows a user to enter database connection parameters.
Successful connections are registered to the AlchemyConnectionManager
and the connection name is emitted whenever this happens.
"""
signalConnectionAdded = QtCore.Signal(str)
def __init__(self, connection_manager: AlchemyConnectionManager, **kwargs):
QtGui.QDialog.__init__(self, **kwargs)
self.con_manager = connection_manager
self.configure()
def configure(self):
db_types = list(sorted(DBAPI_MAP.keys(), reverse=True))
db_apis = DBAPI_MAP[db_types[0]]
self.setupUi(self)
# Actions
self.btnTestConnection.clicked.connect(self.test_connection)
self.buttonBox.clicked.connect(self.register_connection)
self.comboBoxDatabaseType.currentIndexChanged.connect(
self.sync_options)
self.lineEditConnectionURL.textChanged.connect(self.sync_options)
self.btnClear.clicked.connect(self.reset_line_edit_text)
# default modes & items
self.lineEditPassword.setEchoMode(QtGui.QLineEdit.Password)
self.comboBoxDatabaseType.addItems(db_types)
self.comboBoxDatabaseAPI.addItems(db_apis)
self.sync_options()
def test_connection(self, show_success=True):
try:
c = self.get_connection()
if show_success is True:
box = get_ok_msg_box(self,
"Connected to database!",
title="CONNECTION SUCCESS")
box.show()
return c
except Exception as e:
box = get_ok_msg_box(self, str(e), title="CONNECTION ERROR")
box.show()
raise
def get_connection(self) -> AlchemyConnection:
"""
Builds an AlchemyConnection as described by the user
in the line edits/combo boxes.
:return:
"""
con_name = self.lineEditConnectionName.text()
db_type = self.comboBoxDatabaseType.currentText()
uri = self.lineEditConnectionURL.text()
if db_type == 'sqlite' and uri == '':
uri = self.lineEditHost.text()
a = AlchemyConnection(name=con_name)
if uri is '':
port = self.lineEditPort.text()
host = self.lineEditHost.text()
username = self.lineEditUsername.text()
password = self.lineEditPassword.text()
database = self.lineEditDefaultDatabase.text()
db_api = self.comboBoxDatabaseAPI.currentText()
DATABASE = {
'drivername': "{}+{}".format(db_type, db_api),
'host': host or None,
'port': port or None,
'username': username or None,
'password': password or None,
'database': database or None
}
try:
__import__(db_api)
except ImportError:
logging.error(
"Unable to import DBAPI: {} - you need to pip install it.".
format(db_api))
DATABASE['drivername'] = db_type
uri = URL(**DATABASE)
else:
if db_type == 'sqlite' and not uri.startswith('sqlite'):
uri = uri.replace("\\", "/").replace("//",
"/").replace("/", "\\\\")
uri = URL('sqlite', database=uri)
a.configure_from_url(uri)
return a
def register_connection(self, connection=None):
"""
:param connection: (AlchemyConnection, default None)
A optional pre-compiled AlchemyConnection to register to the connection.
Otherwise one will attempt to generate.
:return: (None)
"""
if not isinstance(connection, AlchemyConnection):
connection = self.test_connection(show_success=False)
self.con_manager.add_connection(connection=connection,
allow_duplicate=True)
self.con_manager.save_settings()
self.signalConnectionAdded.emit(connection.name)
def sync_options(self):
"""
Keeps the available options in sync on the dialog.
Makes sure users don't see irrelevant options.
Example:
- database type = sqlite
- hide irrelevant options
- database type = 'postgresql'
- show hidden options (if any)
- database type = 'mysql' & URL provided
- only show URL
:return: None
"""
db_type = self.comboBoxDatabaseType.currentText()
db_api = self.comboBoxDatabaseAPI.currentText()
if db_type == 'sqlite' or (self.lineEditConnectionURL.text() != ''
and db_type != 'sqlite'):
if db_type != 'sqlite':
self.labelHost.hide()
self.lineEditHost.hide()
else:
self.lineEditConnectionURL.hide()
self.labelConnectionURL.hide()
self.lineEditPort.hide()
self.labelPort.hide()
self.lineEditDefaultDatabase.hide()
self.lineEditPassword.hide()
self.labelPassword.hide()
self.lineEditUsername.hide()
self.labelUsername.hide()
self.labelDefaultDatabase.hide()
self.labelDatabaseAPI.hide()
self.comboBoxDatabaseAPI.hide()
else:
self.lineEditConnectionURL.show()
self.labelConnectionURL.show()
self.lineEditPort.show()
self.labelPort.show()
self.labelHost.show()
self.lineEditHost.show()
self.lineEditPassword.show()
self.labelPassword.show()
self.lineEditUsername.show()
self.labelUsername.show()
self.labelDefaultDatabase.show()
self.lineEditDefaultDatabase.show()
self.labelDatabaseAPI.show()
self.comboBoxDatabaseAPI.show()
configure_combo_box(self.comboBoxDatabaseAPI, DBAPI_MAP[db_type],
db_api)
def reset_line_edit_text(self):
for line in self.findChildren(QtGui.QLineEdit):
line.setText('')
class MapGridDialog(QtGui.QDialog, Ui_MapGrid):
"""
Small dialog that contains two combo
boxes, 4 buttons, and a table view.
The two combo boxes should be filled with
different lists of values that the user can
map together and add to the table view.
When the user select's the OK button, a
signalNewMapping signal is emitted with the dictionary
containing the left combo box values as dict values, and the right
combo box values as dict keys.
"""
signalNewMapping = QtCore.Signal(dict)
def __init__(self, *args, **kwargs):
self._data_source = kwargs.pop('data_source', None)
QtGui.QDialog.__init__(self, *args, **kwargs)
self.setupUi(self)
self.map_model = QtGui.QStandardItemModel(0, 2)
self.configure()
def configure(self):
self.btnAdd.clicked.connect(self.push_combo_boxes)
self.btnDelete.clicked.connect(self.delete_selection)
self.btnBox.accepted.connect(self.emit_selection)
self.btnBox.rejected.connect(self.hide)
self.tableView.setModel(self.map_model)
self.map_model.setHorizontalHeaderLabels(['Left On', 'Right On'])
self.comboBoxLeft.currentIndexChanged.connect(self.sync_right_box)
def load_combo_box(self, model, default=None, left=True):
if not isinstance(
model, (QtCore.QAbstractItemModel, QtGui.QStandardItemModel)):
model = self.list_to_model(model)
if left:
box = self.comboBoxLeft
else:
box = self.comboBoxRight
box.setModel(model)
if default is not None:
idx = box.findText(default)
box.setCurrentIndex(idx)
def push_combo_boxes(self):
left_item = self.comboBoxLeft.currentText()
right_item = self.comboBoxRight.currentText()
row = self.map_model.rowCount()
for col, val in enumerate([left_item, right_item]):
item = QtGui.QStandardItem(val)
item.setEditable(False)
self.map_model.setItem(row, col, item)
def sync_right_box(self, index):
"""
Tries to set the right-hand combo box
by searching for what's currently selected on the left-hand
combo box. Case-insensitive but the string must match otherwise.
:param index: Useless, part of the signal that comes out of the combo box.
:return: None
"""
try:
text = self.comboBoxLeft.currentText()
idx = self.comboBoxRight.findText(text,
flags=QtCore.Qt.MatchFixedString)
self.comboBoxRight.setCurrentIndex(idx)
except:
pass
def delete_selection(self):
for item in self.tableView.selectedIndexes():
self.map_model.takeRow(item.row())
def emit_selection(self):
"""
Emits a signalNewMapping signal containing a
dictionary with keys from the right-hand combo box selections,
and values from the left-hand combo box selections.
:return: None
"""
data = dict()
for row in range(self.map_model.rowCount()):
left_item = self.map_model.item(row, 0)
right_item = self.map_model.item(row, 1)
data.update({right_item.text(): left_item.text()})
self.signalNewMapping.emit(data)
def clear_boxes(self):
"""
Resets the models of both ComboBoxes.
:return:
"""
for b in self.findChildren(QtGui.QComboBox):
b.setModel(False)
@staticmethod
def list_to_model(items):
"""
Creates a QtGui.StandardItemModel filled
with QtGui.QStandardItems from the list of
items.
:param items: (list)
of string-like items to store.
:return: (QtGui.QStandardItemModel)
With all items from the list.
"""
model = QtGui.QStandardItemModel()
for i in items:
model.appendRow(QtGui.QStandardItem(i))
return model
class SettingsDialog(QtGui.QDialog, Ui_settingsDialog):
signalSettingsExported = QtCore.Signal(SettingsINI, str)
signalSettingsImported = QtCore.Signal(SettingsINI, str)
signalSettingsSaved = QtCore.Signal(SettingsINI, str)
_themes_dir = THEMES_DIR
def __init__(self, settings=None, **kwargs):
QtGui.QDialog.__init__(self, **kwargs)
self.setupUi(self)
self.settings_ini = settings
if isinstance(settings, str) or settings is None:
if settings is None:
print(
"Using default settings - got settings of type {}".format(
type(settings)))
self.settings_ini = SettingsINI(filename=settings)
elif isinstance(settings, dict):
self.settings_ini = DictConfig(dictionary=settings)
self.configure_settings(self.settings_ini)
self.connect_buttons()
@property
def _settings_filename(self):
return self.settings_ini._filename
def connect_buttons(self):
self.btnSave.clicked.connect(self.save_settings)
self.btnExport.clicked.connect(self.export_settings)
self.btnImport.clicked.connect(self.import_settings)
self.btnLogDirectory.clicked.connect(self.open_log_directory)
self.btnRootDirectory.clicked.connect(self.open_root_directory)
self.btnSetDefault.clicked.connect(self.set_and_save_defaults)
self.btnReset.clicked.connect(self.reset_settings)
def configure_settings(self, config: SettingsINI):
"""
Configures settings based on a SettingsINI object.
Defaults are contained in here.
Top-level configurations read from.:
GENERAL
INPUT
OUTPUT
DATABASE
FIELDS
"""
if not hasattr(config, "set_safe"):
# We'll assume get_safe exists to.
raise NotImplementedError("Unsupported settings type: {}".format(
type(config)))
c = config
# Get items/set defaults.
ROOT_DIR = c.get('GENERAL',
'root_directory',
fallback=DEFAULT_ROOT_DIR)
LOG_DIR = c.get('GENERAL', 'log_directory', fallback=DEFAULT_LOG_DIR)
LOG_LEVEL = c.get_safe('GENERAL', 'LOG_LEVEL', fallback="Low")
CLOUD_PROVIDER = c.get_safe('GENERAL', 'CLOUD_PROVIDER', fallback="S3")
THEME_NAME = c.get_safe('GENERAL', 'THEME', fallback=DEFAULT_THEME)
HEADER_CASE = c.get_safe('INPUT', 'HEADER_CASE', fallback="Lower")
HEADER_SPACE = c.get_safe('INPUT', 'HEADER_SPACE', fallback="_")
SEPARATOR = c.get_safe('OUTPUT', 'SEPARATOR', fallback=",")
ENCODING = c.get_safe('OUTPUT', 'ENCODING', fallback="UTF_8")
if not os.path.exists(ROOT_DIR):
try:
os.mkdir(ROOT_DIR)
except OSError:
msg = "Defaulting root and log directories because the root directory '{}' was invalid.".format(
ROOT_DIR)
ROOT_DIR = DEFAULT_ROOT_DIR
LOG_DIR = DEFAULT_LOG_DIR
logging.warning(msg)
#Make sure the directories exist.
for fp in [ROOT_DIR, LOG_DIR]:
try:
if not os.path.exists(fp):
os.mkdir(fp)
except OSError as e:
raise OSError(
"Cannot initialize settings directory {} - {}".format(
fp, e))
#LINE EDITS
self.rootDirectoryLineEdit.setText(ROOT_DIR)
self.logDirectoryLineEdit.setText(LOG_DIR)
#COMBO BOXES
configure_combo_box(self.logLevelComboBox, ['Low', 'Medium', 'High'],
LOG_LEVEL)
configure_combo_box(self.cloudProviderComboBox,
['Google Drive', 'S3', 'DropBox'], CLOUD_PROVIDER)
configure_combo_box(self.headerCaseComboBox,
['lower', 'UPPER', 'Proper'], HEADER_CASE)
configure_combo_box(self.headerSpacesComboBox, [' ', '_'],
HEADER_SPACE)
configure_combo_box(self.separatorComboBox, DEFAULT_SEPARATORS,
SEPARATOR)
configure_combo_box(self.encodingComboBox, DEFAULT_CODECS, ENCODING)
configure_combo_box(self.themeComboBox, DEFAULT_THEME_OPTIONS,
THEME_NAME)
self.set_theme(THEME_NAME)
self.cloudProviderLabel.hide()
self.cloudProviderComboBox.hide()
self.save_settings(to_path=None, write=False)
def set_and_save_defaults(self):
self.export_settings(self.settings_ini.default_path, set_self=True)
try:
self.export_settings(self.settings_ini.backup_path, set_self=False)
except Exception as e:
logging.warning(
"SettingsDialog.set_and_save_defaults: Ignored error setting backup settings path: {}"
.format(e))
def save_settings(self, to_path=None, write=False):
self.set_theme()
self.settings_ini.set_safe('GENERAL', 'ROOT_DIRECTORY',
self.rootDirectoryLineEdit.text())
self.settings_ini.set('GENERAL', 'LOG_DIRECTORY',
self.logDirectoryLineEdit.text())
self.settings_ini.set('GENERAL', 'LOG_LEVEL',
self.logLevelComboBox.currentText())
self.settings_ini.set('GENERAL', 'CLOUD_PROVIDER',
self.cloudProviderComboBox.currentText())
self.settings_ini.set('GENERAL', 'THEME',
self.themeComboBox.currentText())
self.settings_ini.set_safe('INPUT', 'HEADER_CASE',
self.headerCaseComboBox.currentText())
self.settings_ini.set('INPUT', 'HEADER_SPACE',
self.headerSpacesComboBox.currentText())
if write or to_path is not None:
if to_path is None:
to_path = self.settings_ini._filename
self.settings_ini._filename = to_path
self.settings_ini.save()
self.signalSettingsSaved.emit(self.settings_ini,
self.settings_ini.filename)
def clear_settings(self):
self.cloudProviderComboBox.clear()
self.encodingComboBox.clear()
self.headerCaseComboBox.clear()
self.headerSpacesComboBox.clear()
self.logLevelComboBox.clear()
self.separatorComboBox.clear()
self.themeComboBox.clear()
def export_settings(self, to=None, set_self=False):
if to is None:
to = QtGui.QFileDialog.getSaveFileName()[0]
self.save_settings(to_path=None, write=False)
self.settings_ini.save_as(to, set_self=set_self)
self.signalSettingsExported.emit(self.settings_ini, to)
def set_theme(self, theme_name=None):
if theme_name is None:
theme_name = self.themeComboBox.currentText()
if isinstance(theme_name, int):
logging.info("No theme selected.")
theme_name = THEME_NAME2
theme_path = normpath(self._themes_dir, theme_name)
if os.path.exists(theme_path):
with open(theme_path, "r") as fh:
app = QtCore.QCoreApplication.instance()
app.setStyleSheet(fh.read())
else:
logging.warning("Couldnt find theme {}!".format(theme_path))
def import_settings(self, filename=None):
if filename is None:
filename = QtGui.QFileDialog.getOpenFileName()
self.settings_ini.save()
self.settings_ini.clear()
self.settings_ini.read(filename)
self.settings_ini._filename = filename
self.clear_settings()
self.configure_settings(self.settings_ini)
def reset_settings(self):
self.settings_ini.clear()
self.settings_ini.read(self.settings_ini.backup_path)
self.settings_ini.save_as(self.settings_ini.default_path,
set_self=True)
self.clear_settings()
self.configure_settings(self.settings_ini)
def open_root_directory(self):
dirname = QtGui.QFileDialog.getExistingDirectory()
self.rootDirectoryLineEdit.setText(dirname)
def open_log_directory(self):
dirname = QtGui.QFileDialog.getExistingDirectory()
self.logDirectoryLineEdit.setText(dirname)
class MergePurgeDialog(QtGui.QDialog, Ui_MergePurgeDialog):
"""
This dialog allows a user to do large updates on a given source DataFrameModel.
- Merging other file(s) with the source based on common keys/fields
- Purging records from the source using other file(s) based on common keys/fields
- Sorting the DataFrame by multiple columns/ascending/descending
- Deduplicating the DataFrame based on common keys/fields
Settings can exported to a config.ini file and re-imported at a later time.
"""
signalMergeFileOpened = QtCore.Signal(str) # file path
signalSFileOpened = QtCore.Signal(str) # file path
signalSourcePathSet = QtCore.Signal(str) #file path
signalExecuted = QtCore.Signal(str, str,
str) # source_path, dest_path, report_path
def __init__(self,
df_manager: DataFrameModelManager,
parent=None,
source_model=None):
"""
:param df_manager: (DataFrameModelManager)
This will be used to handle reading/updating of DataFrameModels used
in the operation.
:param parent: (QMainWindow)
:param source_model: (DataFrameModel)
An optional source DataFrameModel
"""
self.df_manager = df_manager
QtGui.QDialog.__init__(self, parent)
self.setupUi(self)
self.source_model = source_model
self._merge_view_model = FileViewModel()
self._suppress_view_model = FileViewModel()
self._purge_files = {}
self._merge_files = {}
self._field_map_grids = {}
self._field_map_data = {}
self.sortAscHandler = None
self.sortOnHandler = None
self.dedupeOnHandler = None
self.uniqueFieldsHandler = None
self.gatherFieldsHandler = None
self.configure()
if self.source_model is not None:
self.set_source_model(source_model, configure=True)
def configure(self, source_path=None, dest_path=None):
"""
Connects main buttons and actions.
:param source_path: (str, default None)
If this is None there must be a valid path already in the sourcePathLineEdit or an AssertionError raises.
:param dest_path: (str, default None)
Optional custom destination path to be added to the destPathLineEdit.
:return: None
"""
if source_path is None:
source_path = self.sourcePathLineEdit.text()
if os.path.isfile(source_path):
self.set_line_edit_paths(source_path, dest_path=dest_path)
if self.sortAscHandler is None:
self.set_handler_sort_asc()
source_func = partial(self.open_file,
model_signal=self.signalSourcePathSet)
self.signalSourcePathSet.connect(self.set_source_model_from_browse)
self.btnBrowseSourcePath.clicked.connect(source_func)
self.btnBrowseDestPath.clicked.connect(self.set_dest_path_from_browse)
self.signalMergeFileOpened.connect(self.add_merge_file)
merge_file_func = partial(self.open_file,
model_signal=self.signalMergeFileOpened)
self.btnAddMergeFile.clicked.connect(merge_file_func)
self.btnBrowseMergeFile.clicked.connect(merge_file_func)
self.btnDeleteMergeFile.clicked.connect(
partial(self.remove_file, self.mergeFileTable))
self.btnEditMergeFile.clicked.connect(
partial(self.open_edit_file_window, self.mergeFileTable,
self._merge_files))
self.mergeFileTable.setModel(self._merge_view_model)
self.signalSFileOpened.connect(self.add_purge_file)
sfile_func = partial(self.open_file,
model_signal=self.signalSFileOpened)
self.btnEditSFile.clicked.connect(
partial(self.open_edit_file_window, self.sFileTable,
self._purge_files))
self.btnDeleteSFile.clicked.connect(
partial(self.remove_file, self.sFileTable))
self.btnAddSFile.clicked.connect(sfile_func)
self.btnBrowseSFile.clicked.connect(sfile_func)
self.sFileTable.setModel(self._suppress_view_model)
self.btnMapSFields.clicked.connect(
partial(self.open_field_map, self.sFileTable, self._purge_files))
self.btnMapMergeFields.clicked.connect(
partial(self.open_field_map, self.mergeFileTable,
self._merge_files))
self.btnExecute.clicked.connect(self.execute)
self.btnExportTemplate.clicked.connect(self.export_settings)
self.btnImportTemplate.clicked.connect(self.import_settings)
self.btnReset.clicked.connect(self.reset)
def set_source_model_from_browse(self, filepath):
self.set_line_edit_paths(filepath, dest_path=False)
self.set_source_model(configure=True)
def set_dest_path_from_browse(self, filepath=None):
if filepath is None:
try:
dirname = os.path.dirname(self.df_manager.last_path_read)
except:
dirname = ''
filepath = QtGui.QFileDialog.getOpenFileName(self, dir=dirname)[0]
self.destPathLineEdit.setText(filepath)
def set_source_model(self, model=None, configure=True):
"""
Sets the source DataFrameModel for the Dialog.
:param model: (DataFrameModel)
The DataFrameModel to be set.
:param configure:
True re-configures file path line edits and the listviews.
:return:
"""
if not hasattr(model, 'dataFrame'):
if model is None:
model = self.sourcePathLineEdit.text()
if isinstance(model, str) and os.path.exists(model):
model = self.df_manager.read_file(model)
else:
raise Exception(
"model parameter must be a filepath or a qtpandas.models.DataFrameModel"
)
if self.source_model is not None:
models_different = model.filePath != self.source_model.filePath
if models_different:
try:
self.source_model.dataFrameChanged.disconnect(self.sync)
except RuntimeError:
pass
else:
models_different = True
if models_different:
self.source_model = model
self.source_model.dataFrameChanged.connect(self.sync)
if configure:
self.sync()
def sync(self):
df = self.source_model.dataFrame()
cols = df.columns.tolist()
if self.dedupeOnHandler is None or self.uniqueFieldsHandler is None:
self.set_push_grid_handlers()
else:
self.dedupeOnHandler.set_model_from_list(cols)
self.gatherFieldsHandler.set_model_from_list(cols)
self.sortOnHandler.set_model_from_list(cols)
self.uniqueFieldsHandler.set_model_from_list(cols)
self.set_primary_key_combo_box()
self.set_line_edit_paths(source_path=self.source_model.filePath)
def set_line_edit_paths(self, source_path=None, dest_path=None):
"""
Sets the source/destination line edits in the Dialog.
:param source_path: (str, default None)
An optional valid filepath for the source DataFrameModel.
If None, :param dest_path cannot be None.
:param dest_path: (str, default None)
An optional destination path. One will be created automatically
if None is given.
False will prevent the destination path from being set at all.
:return: None
"""
assert any([dest_path,
source_path]), "source_path or dest_path must be set."
if dest_path is None:
dirname = os.path.dirname(source_path)
base, ext = os.path.splitext(os.path.basename(source_path))
dest_path = os.path.join(dirname, base + "_merged" + ext)
if source_path:
self.sourcePathLineEdit.setText(source_path)
if dest_path:
self.destPathLineEdit.setText(dest_path)
def set_push_grid_handlers(self,
column_model=None,
sorton_model=None,
sortasc_model=None,
dedupe_model=None,
gather_model=None,
unique_model=None):
"""
Sets all default push grid handlers for the dialog.
:param column_model: (QStandardItemModel, default None)
:param sorton_model: ((QStandardItemModel,list) default None)
:param sortasc_model: ((QStandardItemModel,list) default None)
:param dedupe_model: ((QStandardItemModel,list) default None)
:return:
"""
if column_model is None:
column_model = self.get_source_columns_model()
self.set_handler_sort_on(column_model=None, default_model=sorton_model)
self.set_handler_sort_asc(default_model=sortasc_model)
self.set_handler_dedupe_on(column_model=None,
default_model=dedupe_model)
self.set_handler_gather_fields(column_model=None,
default_model=gather_model)
self.set_handler_unique_fields(column_model=None,
default_model=unique_model)
def set_handler_sort_on(self, column_model=None, default_model=None):
if column_model is None:
column_model = self.get_source_columns_model()
self.sortOnHandler = PushGridHandler(
left_model=column_model,
left_view=self.sortOnLeftView,
left_button=self.sortOnLeftButton,
left_delete=True,
right_model=default_model,
right_view=self.sortOnRightView,
right_button=self.sortOnRightButton)
def set_handler_sort_asc(self, default_model=None, overwrite=False):
if self.sortAscHandler is None or default_model is not None or overwrite:
sort_asc = QtGui.QStandardItemModel()
sort_asc.appendRow(QtGui.QStandardItem('True'))
sort_asc.appendRow(QtGui.QStandardItem('False'))
self.sortAscHandler = PushGridHandler(
left_model=sort_asc,
left_view=self.sortAscLeftView,
left_button=self.sortAscLeftButton,
left_delete=False,
right_model=default_model,
right_view=self.sortAscRightView,
right_button=self.sortAscRightButton)
def set_handler_dedupe_on(self, column_model=None, default_model=None):
if column_model is None:
column_model = self.get_source_columns_model()
self.dedupeOnHandler = PushGridHandler(
left_model=column_model,
left_view=self.dedupeOnLeftView,
left_button=self.dedupeOnLeftButton,
left_delete=True,
right_model=default_model,
right_view=self.dedupeOnRightView,
right_button=self.dedupeOnRightButton)
def set_handler_gather_fields(self, column_model=None, default_model=None):
if column_model is None:
column_model = self.get_source_columns_model()
self.gatherFieldsHandler = PushGridHandler(
left_model=column_model,
left_view=self.gatherFieldsListViewLeft,
left_button=self.gatherFieldsButtonLeft,
left_delete=True,
right_model=default_model,
right_view=self.gatherFieldsListViewRight,
right_button=self.gatherFieldsButtonRight)
def set_handler_unique_fields(self, column_model=None, default_model=None):
if column_model is None:
column_model = self.get_source_columns_model()
self.uniqueFieldsHandler = PushGridHandler(
left_model=column_model,
left_view=self.uniqueFieldsListViewLeft,
left_button=self.uniqueFieldsPushButtonLeft,
left_delete=True,
right_model=default_model,
right_view=self.uniqueFieldsListViewRight,
right_button=self.uniqueFieldsPushButtonRight)
def get_source_columns_model(self,
raise_on_error=True
) -> QtGui.QStandardItemModel:
"""
Quick way to get a QStandardItemModel form the DataFrameModel's columns.
:param raise_on_error: (bool, default True)
Raises an error if the source_model has not yet been set.
:return: (QtGui.QStandardItemModel)
"""
if self.source_model is None:
if raise_on_error:
raise Exception(
"Cannot get source_columns as source_model is None!")
else:
columns = []
else:
columns = self.source_model.dataFrame().columns.tolist()
return create_standard_item_model(columns)
def open_file(self,
file_names: list = None,
model_signal=None,
allow_multi=True):
"""
Opens a Merge or Purge file (or really any file) and calls the
given model signal after registering the DataFrameModel with the DataFrameModelManager.
:param file_names: (list, default None)
An optional list of filenames to open.
The user must select filenames otherwise.
:param model_signal: (QtCore.Signal)
A signal to be called after successfully reading the DataFrameModel.
:param allow_multi: (bool, default True)
True allows multiple files to be read (and the signal called each time).
False allows only the first file to be read.
:return: None
You can call MergePurgeDialog.df_manager.get_frame(filename) to
retrieve a DataFrameModel.
"""
if file_names is None:
dirname = os.path.dirname(self.sourcePathLineEdit.text())
file_names = QtGui.QFileDialog.getOpenFileNames(parent=self,
dir=dirname)[0]
if isinstance(file_names, str):
file_names = list(file_names)
assert not isinstance(file_names, str) and hasattr(
file_names, "__iter__"), "file_names is not list-like!"
if allow_multi is False:
file_names = list(file_names[0])
for f in file_names:
try:
if not isinstance(f, str) and hasattr(f, '__iter__'):
f = f[0]
if os.path.exists(f):
self.df_manager.read_file(f)
if model_signal is not None:
model_signal.emit(f)
logging.info("Emitted signal: {}".format(f))
except Exception as e:
logging.error(e)
@QtCore.Slot(str)
def add_merge_file(self, file_path):
"""
Adds a merge file to the merge view and
also updates the internal dictionary storing the filepath/model.
:param file_path: (str)
The file path to add.
:return: None
"""
model = self.df_manager.get_model(file_path)
model.enableEditing(True)
self._merge_files.update({file_path: model})
self._merge_view_model.append_df_model(model)
self.mergeFileTable.setColumnWidth(0, 500)
self._merge_view_model.setHorizontalHeaderLabels(['filepath', 'count'])
@QtCore.Slot(str)
def add_purge_file(self, file_path):
"""
Adds a purge file to the purge view and
also updates the internal dictionary storing the filepath/model.
:param file_path: (str)
The file path to add.
:return: None
"""
model = self.df_manager.get_model(file_path)
model.enableEditing(True)
self._purge_files.update({file_path: model})
self._suppress_view_model.append_df_model(model)
self.sFileTable.setColumnWidth(0, 500)
self._suppress_view_model.setHorizontalHeaderLabels(
['filepath', 'count'])
def remove_file(self, view, indexes=None):
"""
Removes selected file(s) from the given view.
:param view: (QListView)
The view to drop the selected indexes on.
:param indexes: (list, default None)
A list of given indexes to drop.
Otherwise relies on selected indexes in the view.
:return: None
"""
if indexes is None:
indexes = [x.row() for x in view.selectedIndexes()]
model = view.model()
for idx in indexes:
model.takeRow(idx)
def open_field_map(self, view, models):
"""
Connects a MapGridDialog to help the user map field names that
are different between the source DataFrameModel and the
selected merge or suppression DataFrameModel.
:param view: (QtGui.QTableView)
The view that has a selected filepath
:param models: (dict)
The dictionary of {file_path:DataFrameModel} where
dataframe columns can be gathered from.
:return: None
"""
idx = view.selectedIndexes()[0]
view_model = view.model()
view_item = view_model.item(idx.row())
view_item_text = view_item.text()
try:
self._field_map_grids[view_item_text].show()
except KeyError:
dfmodel = models[view_item_text]
colmodel = dfmodel._dataFrame.columns.tolist()
if self.source_model is None:
self.set_source_model()
source_colmodel = self.source_model._dataFrame.columns.tolist()
fmap = MapGridDialog(parent=self)
fmap.load_combo_box(source_colmodel, left=True)
fmap.load_combo_box(colmodel, left=False)
fmap.setWindowTitle("Map Fields")
fmap.labelLeft.setText(os.path.basename(
self.source_model.filePath))
fmap.labelRight.setText(os.path.basename(dfmodel.filePath))
fmap.signalNewMapping.connect(
lambda x: self._field_map_data.update({dfmodel.filePath: x}))
self._field_map_grids[view_item_text] = fmap
self._field_map_grids[view_item_text].show()
def get_map_grid(self, file_path):
"""
Accessor to the MergePurgeDialog._field_map_grids dictionary.
Contains map grid dialogs.
:param file_path: (str)
The filepath related to the desired MapGridDialog.
:return: (MapGridDialog, None)
"""
return self._field_map_grids.get(file_path, None)
def open_edit_file_window(self, view, models):
"""
Connects a DataFrameModel selected in the view
to a FileTableWindow where the model can be edited.
:param view: (QtGui.QTableView)
The view that has a selected filepath
:param models: (dict)
The dictionary of {file_path:DataFrameModel}
to supply the FileTableWindow
:return: None
"""
try:
idx = view.selectedIndexes()[0]
except IndexError:
raise IndexError("No file selected to open.")
vmodel = view.model()
vitem = vmodel.item(idx.row())
model = models.get(vitem.text())
fp = model.filePath
wdw = self.df_manager.get_fileview_window(fp)
# Prevent wierdos from doing an endless loop of MergePurge windows.
# That would be pretty funny, though..
wdw.actionMergePurge.setVisible(False)
wdw.show()
def execute(self):
"""
Executes the merge_purge based upon the given settings.
:return: None
"""
if self.source_model is None:
self.set_source_model()
suppressed_results = {}
merged_results = {}
source_path = self.sourcePathLineEdit.text()
dest_path = self.destPathLineEdit.text()
source_df = self.source_model.dataFrame().copy()
source_df.loc[:, 'ORIG_IDXER'] = source_df.index
source_size = source_df.index.size
index_label = self.primaryKeyComboBox.currentText()
sort_on = self.sortOnHandler.get_model_list(left=False)
ascending = self.sortAscHandler.get_model_list(left=False)
dedupe_on = self.dedupeOnHandler.get_model_list(left=False)
gather_fields = self.gatherFieldsHandler.get_model_list(left=False)
overwrite_existing = self.gatherFieldsOverWriteCheckBox.isChecked()
# Make sure ascending/sort_on lists are equal.
while len(sort_on) < len(ascending):
ascending.append(False)
while len(sort_on) > len(ascending):
ascending.pop()
# Get all merge models and merge.
# Absorb all rows and columns
for file_path, merge_model in self._merge_files.items():
pre_size = source_df.index.size
other_df = merge_model.dataFrame()
if gather_fields:
assert index_label in other_df.columns, "DataFrameModel for {} missing column {}".format(
merge_model.filePath, index_label)
source_df = gather_frame_fields(source_df,
other_df,
index_label=index_label,
fields=gather_fields,
copy_frames=True,
append_missing=True,
overwrite=overwrite_existing)
else:
source_df = pd.concat([source_df, other_df])
merged_results.update(
{merge_model.filePath: source_df.index.size - pre_size})
# Get all suppression models and suppress.
for file_path, suppress_model in self._purge_files.items():
map_dict = self._field_map_data.get(file_path, {})
sframe = suppress_model.dataFrame().copy()
sframe.drop(['ORIG_IDXER'], axis=1, inplace=True, errors='ignore')
if map_dict:
# A mapping exists - rename the data and get the key_cols
key_cols = list(map_dict.values())
sframe.rename(columns=map_dict, inplace=True)
else:
# No mapping exists - Try to use the dedupe_on cols as key_cols
key_cols = dedupe_on.copy()
missing = [x for x in key_cols if x not in sframe.columns]
if missing:
raise KeyError(
"Suppression file {} must have a field mapping or \
have the dedupe column labels, it has neither!."
.format(suppress_model.filePath))
sframe = sframe.loc[:, key_cols].drop_duplicates(key_cols)
badframe = pd.merge(source_df,
sframe,
how='inner',
left_on=key_cols,
right_on=key_cols)
source_df = source_df.loc[
~source_df.index.isin(badframe.loc[:,
'ORIG_IDXER'].tolist()), :]
suppressed_results.update(
{suppress_model.filePath: badframe.index.size})
# Sort the data
if sort_on and ascending:
source_df.sort_values(sort_on, ascending=ascending, inplace=True)
# Deduplicate the data.
if dedupe_on:
pre_size = source_df.index.size
source_df.drop_duplicates(dedupe_on, inplace=True)
dedupe_lost = pre_size - source_df.index.size
else:
dedupe_lost = 0
# Export the data - done!
source_df.drop(['ORIG_IDXER'], axis=1, inplace=True, errors='ignore')
source_df.to_csv(dest_path, index=False)
logging.info("Exported: {}".format(dest_path))
merge_string = "\n".join("Gained {} merging {}".format(v, k)
for k, v in merged_results.items())
suppress_string = "\n".join("Lost {} suppressing {}".format(v, k)
for k, v in suppressed_results.items())
report = """
Merge Purge Report
==================
Original Size: {}
Final Size: {}
Source Path: {}
Output Path: {}
Merge:
==================
{}
Purge:
==================
{}
Sort:
==================
SORT BY: {}
SORT ASCENDING: {}
Dedupe:
==================
DEDUPE ON: {}
RECORDS LOST: {}
""".format(source_size, source_df.index.size, source_path, dest_path,
merge_string, suppress_string, sort_on, ascending,
dedupe_on, dedupe_lost)
report_path = os.path.splitext(dest_path)[0] + "_report.txt"
with open(report_path, "w") as fh:
fh.write(report)
self.signalExecuted.emit(source_path, dest_path, report_path)
def get_settings(self,
dc: DictConfig = None,
section="MERGE_PURGE") -> DictConfig:
"""
Gathers the settings out of the Dialog and
returns a DictConfig object with updated settings.
:param dc (DictConfig, default None)
An optional DictConfig object, one is created if none is given.
:param section (str, default 'MERGE_PURGE')
An optional section name to apply settings to.
A pre-existing section with this name would be overwritten.
:return: (DictConfig)
An updated DictConfig object.
"""
if dc is None:
dc = DictConfig()
if dc.has_section(section):
dc.remove_section(section)
dc.add_section(section)
dc.set_safe(section, 'source_path', self.sourcePathLineEdit.text())
dc.set_safe(section, 'dest_path', self.destPathLineEdit.text())
dc.set_safe(section, 'primary_key',
self.primaryKeyComboBox.currentText())
dc.set_safe(section, 'dedupe_on',
self.dedupeOnHandler.get_model_list(left=False))
dc.set_safe(section, 'gather_fields',
self.gatherFieldsHandler.get_model_list(left=False))
dc.set_safe(section, 'gather_fields_overwrite',
self.gatherFieldsOverWriteCheckBox.isChecked())
dc.set_safe(section, 'sort_on',
self.sortOnHandler.get_model_list(left=False))
dc.set_safe(section, 'sort_ascending',
self.sortAscHandler.get_model_list(left=False))
dc.set_safe(section, 'unique_fields',
self.uniqueFieldsHandler.get_model_list(left=False))
dc.set_safe(section, 'field_map_data', self._field_map_data)
dc.set_safe(section, 'merge_files', list(self._merge_files.keys()))
dc.set_safe(section, 'purge_files', list(self._purge_files.keys()))
return dc
def set_settings(self, dc: DictConfig, section="MERGE_PURGE"):
"""
Applies settings from a DictConfig object to the Dialog.
:param dc (DictConfig, default None)
The DictConfig object that contains the settings to be applied.
:param section (str, default 'MERGE_PURGE')
The section name to read settings from.
:return:
"""
source_path = dc.get(section,
'source_path',
fallback=self.sourcePathLineEdit.text())
current_path = self.sourcePathLineEdit.text()
if source_path != current_path:
dfm = self.df_manager.read_file(source_path)
dest = dc.get(section, 'dest_path', fallback=None)
self.set_source_model(dfm, configure=False)
self.set_line_edit_paths(source_path, dest_path=dest)
self.primaryKeyComboBox.clear()
self.set_primary_key_combo_box()
key_id = self.primaryKeyComboBox.findText(
dc.get(section,
'primary_key',
fallback=self.primaryKeyComboBox.currentText()))
dedupe_on = dc.get_safe(section, 'dedupe_on', fallback=None)
sort_on = dc.get_safe(section, 'sort_on', fallback=None)
gather_fields = dc.get_safe(section, 'gather_fields', fallback=None)
unique_fields = dc.get_safe(section, 'unique_fields', fallback=None)
gather_fields_overwrite = dc.getboolean(section,
'gather_fields_overwrite',
fallback=False)
sort_ascending = dc.get_safe(section, 'sort_ascending', fallback=None)
merge_files = dc.get_safe(section, 'merge_files', fallback=[])
purge_files = dc.get_safe(section, 'purge_files', fallback=[])
field_map_data = dc.get_safe(section, 'field_map_data', fallback={})
self.primaryKeyComboBox.setCurrentIndex(key_id)
self.set_push_grid_handlers(column_model=None,
sorton_model=sort_on,
sortasc_model=sort_ascending,
dedupe_model=dedupe_on,
gather_model=gather_fields,
unique_model=unique_fields)
self.gatherFieldsOverWriteCheckBox.setChecked(gather_fields_overwrite)
self._field_map_data.update(field_map_data)
self.open_file(file_names=merge_files,
model_signal=self.signalMergeFileOpened)
self.open_file(file_names=purge_files,
model_signal=self.signalSFileOpened)
def reset(self):
"""
Resets ListViews/CheckBoxes.
The source/dest line edits are left alone
The suppression/merge files are also left alone.
:return: None
"""
self.set_push_grid_handlers()
self.set_handler_sort_asc(overwrite=True)
self.set_primary_key_combo_box(reset=True)
self.gatherFieldsOverWriteCheckBox.setChecked(False)
def set_primary_key_combo_box(self, default=None, reset=False):
"""
Sets the primary key combo box.
:param default: (str, default None)
An optional default field name to select.
:return:
"""
if default is None and reset is False:
current_model = self.primaryKeyComboBox.model()
if current_model:
default = self.primaryKeyComboBox.currentText()
combo_model = create_standard_item_model(
[''] + self.source_model.dataFrame().columns.tolist(),
editable=False,
checkable=True)
self.primaryKeyComboBox.setModel(combo_model)
if default is not None:
self.primaryKeyComboBox.setCurrentIndex(
self.primaryKeyComboBox.findText(default))
def import_settings(self, from_path=None):
"""
Imports settings to the Dialog from a file.
:param from_path: (str, default None)
None makes the user enter a file path.
:return:
"""
if from_path is None:
try:
dirname = os.path.dirname(self.sourcePathLineEdit.text())
except:
dirname = ''
from_path = QtGui.QFileDialog.getOpenFileName(self, dir=dirname)[0]
config = SettingsINI(filename=from_path)
self.set_settings(config)
def export_settings(self, to=None):
"""
Exports settings from the Dialog to a file.
:param to: (str, default None)
None makes the user enter a file path.
:return: None
"""
if to is None:
try:
dirname = os.path.dirname(self.sourcePathLineEdit.text())
except:
dirname = ''
to = QtGui.QFileDialog.getSaveFileName(self, dir=dirname)[0]
config = self.get_settings()
config.save_as(to, set_self=True)