def set_handler_unique_fields(self, column_model=None, default_model=None): if column_model is None: column_model = self.get_source_columns_model() self.uniqueFieldsHandler = PushGridHandler( left_model=column_model, left_view=self.uniqueFieldsListViewLeft, left_button=self.uniqueFieldsPushButtonLeft, left_delete=True, right_model=default_model, right_view=self.uniqueFieldsListViewRight, right_button=self.uniqueFieldsPushButtonRight)
def set_handler_dedupe_on(self, column_model=None, default_model=None): if column_model is None: column_model = self.get_source_columns_model() self.dedupeOnHandler = PushGridHandler( left_model=column_model, left_view=self.dedupeOnLeftView, left_button=self.dedupeOnLeftButton, left_delete=True, right_model=default_model, right_view=self.dedupeOnRightView, right_button=self.dedupeOnRightButton)
def set_handler_sort_asc(self, default_model=None, overwrite=False): if self.sortAscHandler is None or default_model is not None or overwrite: sort_asc = QtGui.QStandardItemModel() sort_asc.appendRow(QtGui.QStandardItem('True')) sort_asc.appendRow(QtGui.QStandardItem('False')) self.sortAscHandler = PushGridHandler( left_model=sort_asc, left_view=self.sortAscLeftView, left_button=self.sortAscLeftButton, left_delete=False, right_model=default_model, right_view=self.sortAscRightView, right_button=self.sortAscRightButton)
def __init__(self, source_model, parent=None): QtGui.QDialog.__init__(self, parent=parent) self.df_model = source_model self.setupUi(self) self.configure_model() cols = source_model.dataFrame().columns.tolist() self.handler_split_on = PushGridHandler(left_button=self.btnSplitOnPushLeft, left_view=self.listViewSplitOnLeft, right_button=self.btnSplitOnPushRight, right_view=self.listViewSplitOnRight, left_model=cols) self.handler_use_cols = PushGridHandler(left_button=self.btnUseColsPushLeft, left_view=self.listViewUseColsLeft, right_button=self.btnUseColsPushRight, right_view=self.listViewUseColsRight, left_model=cols) self.configure()
class SplitFileDialog(QtGui.QDialog, Ui_FileSplitDialog): signalFileSplit = QtCore.Signal(list) # [source_path, [p1, p2, p3, p4]] def __init__(self, source_model, parent=None): QtGui.QDialog.__init__(self, parent=parent) self.df_model = source_model self.setupUi(self) self.configure_model() cols = source_model.dataFrame().columns.tolist() self.handler_split_on = PushGridHandler(left_button=self.btnSplitOnPushLeft, left_view=self.listViewSplitOnLeft, right_button=self.btnSplitOnPushRight, right_view=self.listViewSplitOnRight, left_model=cols) self.handler_use_cols = PushGridHandler(left_button=self.btnUseColsPushLeft, left_view=self.listViewUseColsLeft, right_button=self.btnUseColsPushRight, right_view=self.listViewUseColsRight, left_model=cols) self.configure() def configure(self): self.btnExportTemplate.clicked.connect(self.export_settings) self.btnImportTemplate.clicked.connect(self.import_settings) self.df_model.dataChanged.connect(partial(self.configure_model, reset=True)) self.buttonBox.accepted.connect(self.execute) self.buttonBox.rejected.connect(self.close) def configure_model(self, reset=False): p = self.df_model.filePath self.lineEditSourcePath.setText(p) base, ext = os.path.splitext(p) filename = os.path.basename(base).lower() dirname = os.path.basename(os.path.dirname(base)) self.lineEditExportPath.setText(os.path.join(os.path.dirname(p), "{}_split".format(filename))) self.setWindowTitle("Split: {} - Project - {}".format(filename, dirname)) if reset is True: cols = self.df_model.dataFrame().columns.tolist() self.handler_split_on.set_model_from_list(cols) self.handler_use_cols.set_model_from_list(cols) def get_settings(self, dc:DictConfig=None, section:str="SPLIT"): if dc is None: dc = DictConfig() if dc.has_section(section): dc.remove_section(section) dictionary = dict(dropna = self.checkBoxDropNulls.isChecked(), dirname = self.lineEditExportPath.text(), source_path = self.lineEditSourcePath.text(), max_rows = self.lineEditMaxRows.text()) dc.update({section: dictionary}) dc.set_safe(section, 'split_on', self.handler_split_on.get_model_list(left=False)) dc.set_safe(section, 'fields', self.handler_use_cols.get_model_list(left=False)) return dc def set_settings(self, dc:DictConfig, section='SPLIT'): self.lineEditTemplate.setText(dc.filename) self.handler_split_on.set_model_from_list(dc.get_safe(section, 'split_on', fallback=self.handler_split_on.get_model_list(left=False)), left=False) self.handler_split_on.drop_duplicates(left_priority=False) self.handler_use_cols.set_model_from_list(dc.get_safe(section, 'fields', fallback=self.handler_use_cols.get_model_list(left=False)), left=False) self.handler_use_cols.drop_duplicates(left_priority=False) self.checkBoxDropNulls.setChecked(dc.getboolean(section, 'dropna', fallback=self.checkBoxDropNulls.isChecked())) self.lineEditSourcePath.setText(dc.get(section, 'source_path', fallback=self.lineEditSourcePath.text())) self.lineEditExportPath.setText(dc.get(section, 'dirname', fallback=self.lineEditExportPath.text())) self.lineEditMaxRows.setText(dc.get(section, 'max_rows', fallback=self.lineEditMaxRows.text())) def import_settings(self): import_settings(self.set_settings, parent=self) def export_settings(self): export_settings(self.get_settings(),parent=self) def execute(self): split_on = self.handler_split_on.get_model_list(left=False) fields = self.handler_use_cols.get_model_list(left=False) dropna = self.checkBoxDropNulls.isChecked() dirname = self.lineEditExportPath.text() source_path = self.lineEditSourcePath.text() max_rows = self.lineEditMaxRows.text() try: max_rows = int(max_rows) except ValueError: max_rows = None df = self.df_model.dataFrame() exported_paths = dataframe_split_to_files(df, source_path, split_on, fields=fields, dropna=dropna, dest_dirname=dirname, chunksize=max_rows, index=False) emission = [source_path, exported_paths] self.signalFileSplit.emit(emission)
class MergePurgeDialog(QtGui.QDialog, Ui_MergePurgeDialog): """ This dialog allows a user to do large updates on a given source DataFrameModel. - Merging other file(s) with the source based on common keys/fields - Purging records from the source using other file(s) based on common keys/fields - Sorting the DataFrame by multiple columns/ascending/descending - Deduplicating the DataFrame based on common keys/fields Settings can exported to a config.ini file and re-imported at a later time. """ signalMergeFileOpened = QtCore.Signal(str) # file path signalSFileOpened = QtCore.Signal(str) # file path signalSourcePathSet = QtCore.Signal(str) #file path signalExecuted = QtCore.Signal(str, str, str) # source_path, dest_path, report_path def __init__(self, df_manager: DataFrameModelManager, parent=None, source_model=None): """ :param df_manager: (DataFrameModelManager) This will be used to handle reading/updating of DataFrameModels used in the operation. :param parent: (QMainWindow) :param source_model: (DataFrameModel) An optional source DataFrameModel """ self.df_manager = df_manager QtGui.QDialog.__init__(self, parent) self.setupUi(self) self.source_model = source_model self._merge_view_model = FileViewModel() self._suppress_view_model = FileViewModel() self._purge_files = {} self._merge_files = {} self._field_map_grids = {} self._field_map_data = {} self.sortAscHandler = None self.sortOnHandler = None self.dedupeOnHandler = None self.uniqueFieldsHandler = None self.gatherFieldsHandler = None self.configure() if self.source_model is not None: self.set_source_model(source_model, configure=True) def configure(self, source_path=None, dest_path=None): """ Connects main buttons and actions. :param source_path: (str, default None) If this is None there must be a valid path already in the sourcePathLineEdit or an AssertionError raises. :param dest_path: (str, default None) Optional custom destination path to be added to the destPathLineEdit. :return: None """ if source_path is None: source_path = self.sourcePathLineEdit.text() if os.path.isfile(source_path): self.set_line_edit_paths(source_path, dest_path=dest_path) if self.sortAscHandler is None: self.set_handler_sort_asc() source_func = partial(self.open_file, model_signal=self.signalSourcePathSet) self.signalSourcePathSet.connect(self.set_source_model_from_browse) self.btnBrowseSourcePath.clicked.connect(source_func) self.btnBrowseDestPath.clicked.connect(self.set_dest_path_from_browse) self.signalMergeFileOpened.connect(self.add_merge_file) merge_file_func = partial(self.open_file, model_signal=self.signalMergeFileOpened) self.btnAddMergeFile.clicked.connect(merge_file_func) self.btnBrowseMergeFile.clicked.connect(merge_file_func) self.btnDeleteMergeFile.clicked.connect( partial(self.remove_file, self.mergeFileTable)) self.btnEditMergeFile.clicked.connect( partial(self.open_edit_file_window, self.mergeFileTable, self._merge_files)) self.mergeFileTable.setModel(self._merge_view_model) self.signalSFileOpened.connect(self.add_purge_file) sfile_func = partial(self.open_file, model_signal=self.signalSFileOpened) self.btnEditSFile.clicked.connect( partial(self.open_edit_file_window, self.sFileTable, self._purge_files)) self.btnDeleteSFile.clicked.connect( partial(self.remove_file, self.sFileTable)) self.btnAddSFile.clicked.connect(sfile_func) self.btnBrowseSFile.clicked.connect(sfile_func) self.sFileTable.setModel(self._suppress_view_model) self.btnMapSFields.clicked.connect( partial(self.open_field_map, self.sFileTable, self._purge_files)) self.btnMapMergeFields.clicked.connect( partial(self.open_field_map, self.mergeFileTable, self._merge_files)) self.btnExecute.clicked.connect(self.execute) self.btnExportTemplate.clicked.connect(self.export_settings) self.btnImportTemplate.clicked.connect(self.import_settings) self.btnReset.clicked.connect(self.reset) def set_source_model_from_browse(self, filepath): self.set_line_edit_paths(filepath, dest_path=False) self.set_source_model(configure=True) def set_dest_path_from_browse(self, filepath=None): if filepath is None: try: dirname = os.path.dirname(self.df_manager.last_path_read) except: dirname = '' filepath = QtGui.QFileDialog.getOpenFileName(self, dir=dirname)[0] self.destPathLineEdit.setText(filepath) def set_source_model(self, model=None, configure=True): """ Sets the source DataFrameModel for the Dialog. :param model: (DataFrameModel) The DataFrameModel to be set. :param configure: True re-configures file path line edits and the listviews. :return: """ if not hasattr(model, 'dataFrame'): if model is None: model = self.sourcePathLineEdit.text() if isinstance(model, str) and os.path.exists(model): model = self.df_manager.read_file(model) else: raise Exception( "model parameter must be a filepath or a qtpandas.models.DataFrameModel" ) if self.source_model is not None: models_different = model.filePath != self.source_model.filePath if models_different: try: self.source_model.dataFrameChanged.disconnect(self.sync) except RuntimeError: pass else: models_different = True if models_different: self.source_model = model self.source_model.dataFrameChanged.connect(self.sync) if configure: self.sync() def sync(self): df = self.source_model.dataFrame() cols = df.columns.tolist() if self.dedupeOnHandler is None or self.uniqueFieldsHandler is None: self.set_push_grid_handlers() else: self.dedupeOnHandler.set_model_from_list(cols) self.gatherFieldsHandler.set_model_from_list(cols) self.sortOnHandler.set_model_from_list(cols) self.uniqueFieldsHandler.set_model_from_list(cols) self.set_primary_key_combo_box() self.set_line_edit_paths(source_path=self.source_model.filePath) def set_line_edit_paths(self, source_path=None, dest_path=None): """ Sets the source/destination line edits in the Dialog. :param source_path: (str, default None) An optional valid filepath for the source DataFrameModel. If None, :param dest_path cannot be None. :param dest_path: (str, default None) An optional destination path. One will be created automatically if None is given. False will prevent the destination path from being set at all. :return: None """ assert any([dest_path, source_path]), "source_path or dest_path must be set." if dest_path is None: dirname = os.path.dirname(source_path) base, ext = os.path.splitext(os.path.basename(source_path)) dest_path = os.path.join(dirname, base + "_merged" + ext) if source_path: self.sourcePathLineEdit.setText(source_path) if dest_path: self.destPathLineEdit.setText(dest_path) def set_push_grid_handlers(self, column_model=None, sorton_model=None, sortasc_model=None, dedupe_model=None, gather_model=None, unique_model=None): """ Sets all default push grid handlers for the dialog. :param column_model: (QStandardItemModel, default None) :param sorton_model: ((QStandardItemModel,list) default None) :param sortasc_model: ((QStandardItemModel,list) default None) :param dedupe_model: ((QStandardItemModel,list) default None) :return: """ if column_model is None: column_model = self.get_source_columns_model() self.set_handler_sort_on(column_model=None, default_model=sorton_model) self.set_handler_sort_asc(default_model=sortasc_model) self.set_handler_dedupe_on(column_model=None, default_model=dedupe_model) self.set_handler_gather_fields(column_model=None, default_model=gather_model) self.set_handler_unique_fields(column_model=None, default_model=unique_model) def set_handler_sort_on(self, column_model=None, default_model=None): if column_model is None: column_model = self.get_source_columns_model() self.sortOnHandler = PushGridHandler( left_model=column_model, left_view=self.sortOnLeftView, left_button=self.sortOnLeftButton, left_delete=True, right_model=default_model, right_view=self.sortOnRightView, right_button=self.sortOnRightButton) def set_handler_sort_asc(self, default_model=None, overwrite=False): if self.sortAscHandler is None or default_model is not None or overwrite: sort_asc = QtGui.QStandardItemModel() sort_asc.appendRow(QtGui.QStandardItem('True')) sort_asc.appendRow(QtGui.QStandardItem('False')) self.sortAscHandler = PushGridHandler( left_model=sort_asc, left_view=self.sortAscLeftView, left_button=self.sortAscLeftButton, left_delete=False, right_model=default_model, right_view=self.sortAscRightView, right_button=self.sortAscRightButton) def set_handler_dedupe_on(self, column_model=None, default_model=None): if column_model is None: column_model = self.get_source_columns_model() self.dedupeOnHandler = PushGridHandler( left_model=column_model, left_view=self.dedupeOnLeftView, left_button=self.dedupeOnLeftButton, left_delete=True, right_model=default_model, right_view=self.dedupeOnRightView, right_button=self.dedupeOnRightButton) def set_handler_gather_fields(self, column_model=None, default_model=None): if column_model is None: column_model = self.get_source_columns_model() self.gatherFieldsHandler = PushGridHandler( left_model=column_model, left_view=self.gatherFieldsListViewLeft, left_button=self.gatherFieldsButtonLeft, left_delete=True, right_model=default_model, right_view=self.gatherFieldsListViewRight, right_button=self.gatherFieldsButtonRight) def set_handler_unique_fields(self, column_model=None, default_model=None): if column_model is None: column_model = self.get_source_columns_model() self.uniqueFieldsHandler = PushGridHandler( left_model=column_model, left_view=self.uniqueFieldsListViewLeft, left_button=self.uniqueFieldsPushButtonLeft, left_delete=True, right_model=default_model, right_view=self.uniqueFieldsListViewRight, right_button=self.uniqueFieldsPushButtonRight) def get_source_columns_model(self, raise_on_error=True ) -> QtGui.QStandardItemModel: """ Quick way to get a QStandardItemModel form the DataFrameModel's columns. :param raise_on_error: (bool, default True) Raises an error if the source_model has not yet been set. :return: (QtGui.QStandardItemModel) """ if self.source_model is None: if raise_on_error: raise Exception( "Cannot get source_columns as source_model is None!") else: columns = [] else: columns = self.source_model.dataFrame().columns.tolist() return create_standard_item_model(columns) def open_file(self, file_names: list = None, model_signal=None, allow_multi=True): """ Opens a Merge or Purge file (or really any file) and calls the given model signal after registering the DataFrameModel with the DataFrameModelManager. :param file_names: (list, default None) An optional list of filenames to open. The user must select filenames otherwise. :param model_signal: (QtCore.Signal) A signal to be called after successfully reading the DataFrameModel. :param allow_multi: (bool, default True) True allows multiple files to be read (and the signal called each time). False allows only the first file to be read. :return: None You can call MergePurgeDialog.df_manager.get_frame(filename) to retrieve a DataFrameModel. """ if file_names is None: dirname = os.path.dirname(self.sourcePathLineEdit.text()) file_names = QtGui.QFileDialog.getOpenFileNames(parent=self, dir=dirname)[0] if isinstance(file_names, str): file_names = list(file_names) assert not isinstance(file_names, str) and hasattr( file_names, "__iter__"), "file_names is not list-like!" if allow_multi is False: file_names = list(file_names[0]) for f in file_names: try: if not isinstance(f, str) and hasattr(f, '__iter__'): f = f[0] if os.path.exists(f): self.df_manager.read_file(f) if model_signal is not None: model_signal.emit(f) logging.info("Emitted signal: {}".format(f)) except Exception as e: logging.error(e) @QtCore.Slot(str) def add_merge_file(self, file_path): """ Adds a merge file to the merge view and also updates the internal dictionary storing the filepath/model. :param file_path: (str) The file path to add. :return: None """ model = self.df_manager.get_model(file_path) model.enableEditing(True) self._merge_files.update({file_path: model}) self._merge_view_model.append_df_model(model) self.mergeFileTable.setColumnWidth(0, 500) self._merge_view_model.setHorizontalHeaderLabels(['filepath', 'count']) @QtCore.Slot(str) def add_purge_file(self, file_path): """ Adds a purge file to the purge view and also updates the internal dictionary storing the filepath/model. :param file_path: (str) The file path to add. :return: None """ model = self.df_manager.get_model(file_path) model.enableEditing(True) self._purge_files.update({file_path: model}) self._suppress_view_model.append_df_model(model) self.sFileTable.setColumnWidth(0, 500) self._suppress_view_model.setHorizontalHeaderLabels( ['filepath', 'count']) def remove_file(self, view, indexes=None): """ Removes selected file(s) from the given view. :param view: (QListView) The view to drop the selected indexes on. :param indexes: (list, default None) A list of given indexes to drop. Otherwise relies on selected indexes in the view. :return: None """ if indexes is None: indexes = [x.row() for x in view.selectedIndexes()] model = view.model() for idx in indexes: model.takeRow(idx) def open_field_map(self, view, models): """ Connects a MapGridDialog to help the user map field names that are different between the source DataFrameModel and the selected merge or suppression DataFrameModel. :param view: (QtGui.QTableView) The view that has a selected filepath :param models: (dict) The dictionary of {file_path:DataFrameModel} where dataframe columns can be gathered from. :return: None """ idx = view.selectedIndexes()[0] view_model = view.model() view_item = view_model.item(idx.row()) view_item_text = view_item.text() try: self._field_map_grids[view_item_text].show() except KeyError: dfmodel = models[view_item_text] colmodel = dfmodel._dataFrame.columns.tolist() if self.source_model is None: self.set_source_model() source_colmodel = self.source_model._dataFrame.columns.tolist() fmap = MapGridDialog(parent=self) fmap.load_combo_box(source_colmodel, left=True) fmap.load_combo_box(colmodel, left=False) fmap.setWindowTitle("Map Fields") fmap.labelLeft.setText(os.path.basename( self.source_model.filePath)) fmap.labelRight.setText(os.path.basename(dfmodel.filePath)) fmap.signalNewMapping.connect( lambda x: self._field_map_data.update({dfmodel.filePath: x})) self._field_map_grids[view_item_text] = fmap self._field_map_grids[view_item_text].show() def get_map_grid(self, file_path): """ Accessor to the MergePurgeDialog._field_map_grids dictionary. Contains map grid dialogs. :param file_path: (str) The filepath related to the desired MapGridDialog. :return: (MapGridDialog, None) """ return self._field_map_grids.get(file_path, None) def open_edit_file_window(self, view, models): """ Connects a DataFrameModel selected in the view to a FileTableWindow where the model can be edited. :param view: (QtGui.QTableView) The view that has a selected filepath :param models: (dict) The dictionary of {file_path:DataFrameModel} to supply the FileTableWindow :return: None """ try: idx = view.selectedIndexes()[0] except IndexError: raise IndexError("No file selected to open.") vmodel = view.model() vitem = vmodel.item(idx.row()) model = models.get(vitem.text()) fp = model.filePath wdw = self.df_manager.get_fileview_window(fp) # Prevent wierdos from doing an endless loop of MergePurge windows. # That would be pretty funny, though.. wdw.actionMergePurge.setVisible(False) wdw.show() def execute(self): """ Executes the merge_purge based upon the given settings. :return: None """ if self.source_model is None: self.set_source_model() suppressed_results = {} merged_results = {} source_path = self.sourcePathLineEdit.text() dest_path = self.destPathLineEdit.text() source_df = self.source_model.dataFrame().copy() source_df.loc[:, 'ORIG_IDXER'] = source_df.index source_size = source_df.index.size index_label = self.primaryKeyComboBox.currentText() sort_on = self.sortOnHandler.get_model_list(left=False) ascending = self.sortAscHandler.get_model_list(left=False) dedupe_on = self.dedupeOnHandler.get_model_list(left=False) gather_fields = self.gatherFieldsHandler.get_model_list(left=False) overwrite_existing = self.gatherFieldsOverWriteCheckBox.isChecked() # Make sure ascending/sort_on lists are equal. while len(sort_on) < len(ascending): ascending.append(False) while len(sort_on) > len(ascending): ascending.pop() # Get all merge models and merge. # Absorb all rows and columns for file_path, merge_model in self._merge_files.items(): pre_size = source_df.index.size other_df = merge_model.dataFrame() if gather_fields: assert index_label in other_df.columns, "DataFrameModel for {} missing column {}".format( merge_model.filePath, index_label) source_df = gather_frame_fields(source_df, other_df, index_label=index_label, fields=gather_fields, copy_frames=True, append_missing=True, overwrite=overwrite_existing) else: source_df = pd.concat([source_df, other_df]) merged_results.update( {merge_model.filePath: source_df.index.size - pre_size}) # Get all suppression models and suppress. for file_path, suppress_model in self._purge_files.items(): map_dict = self._field_map_data.get(file_path, {}) sframe = suppress_model.dataFrame().copy() sframe.drop(['ORIG_IDXER'], axis=1, inplace=True, errors='ignore') if map_dict: # A mapping exists - rename the data and get the key_cols key_cols = list(map_dict.values()) sframe.rename(columns=map_dict, inplace=True) else: # No mapping exists - Try to use the dedupe_on cols as key_cols key_cols = dedupe_on.copy() missing = [x for x in key_cols if x not in sframe.columns] if missing: raise KeyError( "Suppression file {} must have a field mapping or \ have the dedupe column labels, it has neither!." .format(suppress_model.filePath)) sframe = sframe.loc[:, key_cols].drop_duplicates(key_cols) badframe = pd.merge(source_df, sframe, how='inner', left_on=key_cols, right_on=key_cols) source_df = source_df.loc[ ~source_df.index.isin(badframe.loc[:, 'ORIG_IDXER'].tolist()), :] suppressed_results.update( {suppress_model.filePath: badframe.index.size}) # Sort the data if sort_on and ascending: source_df.sort_values(sort_on, ascending=ascending, inplace=True) # Deduplicate the data. if dedupe_on: pre_size = source_df.index.size source_df.drop_duplicates(dedupe_on, inplace=True) dedupe_lost = pre_size - source_df.index.size else: dedupe_lost = 0 # Export the data - done! source_df.drop(['ORIG_IDXER'], axis=1, inplace=True, errors='ignore') source_df.to_csv(dest_path, index=False) logging.info("Exported: {}".format(dest_path)) merge_string = "\n".join("Gained {} merging {}".format(v, k) for k, v in merged_results.items()) suppress_string = "\n".join("Lost {} suppressing {}".format(v, k) for k, v in suppressed_results.items()) report = """ Merge Purge Report ================== Original Size: {} Final Size: {} Source Path: {} Output Path: {} Merge: ================== {} Purge: ================== {} Sort: ================== SORT BY: {} SORT ASCENDING: {} Dedupe: ================== DEDUPE ON: {} RECORDS LOST: {} """.format(source_size, source_df.index.size, source_path, dest_path, merge_string, suppress_string, sort_on, ascending, dedupe_on, dedupe_lost) report_path = os.path.splitext(dest_path)[0] + "_report.txt" with open(report_path, "w") as fh: fh.write(report) self.signalExecuted.emit(source_path, dest_path, report_path) def get_settings(self, dc: DictConfig = None, section="MERGE_PURGE") -> DictConfig: """ Gathers the settings out of the Dialog and returns a DictConfig object with updated settings. :param dc (DictConfig, default None) An optional DictConfig object, one is created if none is given. :param section (str, default 'MERGE_PURGE') An optional section name to apply settings to. A pre-existing section with this name would be overwritten. :return: (DictConfig) An updated DictConfig object. """ if dc is None: dc = DictConfig() if dc.has_section(section): dc.remove_section(section) dc.add_section(section) dc.set_safe(section, 'source_path', self.sourcePathLineEdit.text()) dc.set_safe(section, 'dest_path', self.destPathLineEdit.text()) dc.set_safe(section, 'primary_key', self.primaryKeyComboBox.currentText()) dc.set_safe(section, 'dedupe_on', self.dedupeOnHandler.get_model_list(left=False)) dc.set_safe(section, 'gather_fields', self.gatherFieldsHandler.get_model_list(left=False)) dc.set_safe(section, 'gather_fields_overwrite', self.gatherFieldsOverWriteCheckBox.isChecked()) dc.set_safe(section, 'sort_on', self.sortOnHandler.get_model_list(left=False)) dc.set_safe(section, 'sort_ascending', self.sortAscHandler.get_model_list(left=False)) dc.set_safe(section, 'unique_fields', self.uniqueFieldsHandler.get_model_list(left=False)) dc.set_safe(section, 'field_map_data', self._field_map_data) dc.set_safe(section, 'merge_files', list(self._merge_files.keys())) dc.set_safe(section, 'purge_files', list(self._purge_files.keys())) return dc def set_settings(self, dc: DictConfig, section="MERGE_PURGE"): """ Applies settings from a DictConfig object to the Dialog. :param dc (DictConfig, default None) The DictConfig object that contains the settings to be applied. :param section (str, default 'MERGE_PURGE') The section name to read settings from. :return: """ source_path = dc.get(section, 'source_path', fallback=self.sourcePathLineEdit.text()) current_path = self.sourcePathLineEdit.text() if source_path != current_path: dfm = self.df_manager.read_file(source_path) dest = dc.get(section, 'dest_path', fallback=None) self.set_source_model(dfm, configure=False) self.set_line_edit_paths(source_path, dest_path=dest) self.primaryKeyComboBox.clear() self.set_primary_key_combo_box() key_id = self.primaryKeyComboBox.findText( dc.get(section, 'primary_key', fallback=self.primaryKeyComboBox.currentText())) dedupe_on = dc.get_safe(section, 'dedupe_on', fallback=None) sort_on = dc.get_safe(section, 'sort_on', fallback=None) gather_fields = dc.get_safe(section, 'gather_fields', fallback=None) unique_fields = dc.get_safe(section, 'unique_fields', fallback=None) gather_fields_overwrite = dc.getboolean(section, 'gather_fields_overwrite', fallback=False) sort_ascending = dc.get_safe(section, 'sort_ascending', fallback=None) merge_files = dc.get_safe(section, 'merge_files', fallback=[]) purge_files = dc.get_safe(section, 'purge_files', fallback=[]) field_map_data = dc.get_safe(section, 'field_map_data', fallback={}) self.primaryKeyComboBox.setCurrentIndex(key_id) self.set_push_grid_handlers(column_model=None, sorton_model=sort_on, sortasc_model=sort_ascending, dedupe_model=dedupe_on, gather_model=gather_fields, unique_model=unique_fields) self.gatherFieldsOverWriteCheckBox.setChecked(gather_fields_overwrite) self._field_map_data.update(field_map_data) self.open_file(file_names=merge_files, model_signal=self.signalMergeFileOpened) self.open_file(file_names=purge_files, model_signal=self.signalSFileOpened) def reset(self): """ Resets ListViews/CheckBoxes. The source/dest line edits are left alone The suppression/merge files are also left alone. :return: None """ self.set_push_grid_handlers() self.set_handler_sort_asc(overwrite=True) self.set_primary_key_combo_box(reset=True) self.gatherFieldsOverWriteCheckBox.setChecked(False) def set_primary_key_combo_box(self, default=None, reset=False): """ Sets the primary key combo box. :param default: (str, default None) An optional default field name to select. :return: """ if default is None and reset is False: current_model = self.primaryKeyComboBox.model() if current_model: default = self.primaryKeyComboBox.currentText() combo_model = create_standard_item_model( [''] + self.source_model.dataFrame().columns.tolist(), editable=False, checkable=True) self.primaryKeyComboBox.setModel(combo_model) if default is not None: self.primaryKeyComboBox.setCurrentIndex( self.primaryKeyComboBox.findText(default)) def import_settings(self, from_path=None): """ Imports settings to the Dialog from a file. :param from_path: (str, default None) None makes the user enter a file path. :return: """ if from_path is None: try: dirname = os.path.dirname(self.sourcePathLineEdit.text()) except: dirname = '' from_path = QtGui.QFileDialog.getOpenFileName(self, dir=dirname)[0] config = SettingsINI(filename=from_path) self.set_settings(config) def export_settings(self, to=None): """ Exports settings from the Dialog to a file. :param to: (str, default None) None makes the user enter a file path. :return: None """ if to is None: try: dirname = os.path.dirname(self.sourcePathLineEdit.text()) except: dirname = '' to = QtGui.QFileDialog.getSaveFileName(self, dir=dirname)[0] config = self.get_settings() config.save_as(to, set_self=True)