def set_settings(self, dc: DictConfig, section="MERGE_PURGE"):
"""
Applies settings from a DictConfig object to the Dialog.
:param dc (DictConfig, default None)
The DictConfig object that contains the settings to be applied.
:param section (str, default 'MERGE_PURGE')
The section name to read settings from.
:return:
"""
source_path = dc.get(section,
'source_path',
fallback=self.sourcePathLineEdit.text())
current_path = self.sourcePathLineEdit.text()
if source_path != current_path:
dfm = self.df_manager.read_file(source_path)
dest = dc.get(section, 'dest_path', fallback=None)
self.set_source_model(dfm, configure=False)
self.set_line_edit_paths(source_path, dest_path=dest)
self.primaryKeyComboBox.clear()
self.set_primary_key_combo_box()
key_id = self.primaryKeyComboBox.findText(
dc.get(section,
'primary_key',
fallback=self.primaryKeyComboBox.currentText()))
dedupe_on = dc.get_safe(section, 'dedupe_on', fallback=None)
sort_on = dc.get_safe(section, 'sort_on', fallback=None)
gather_fields = dc.get_safe(section, 'gather_fields', fallback=None)
unique_fields = dc.get_safe(section, 'unique_fields', fallback=None)
gather_fields_overwrite = dc.getboolean(section,
'gather_fields_overwrite',
fallback=False)
sort_ascending = dc.get_safe(section, 'sort_ascending', fallback=None)
merge_files = dc.get_safe(section, 'merge_files', fallback=[])
purge_files = dc.get_safe(section, 'purge_files', fallback=[])
field_map_data = dc.get_safe(section, 'field_map_data', fallback={})
self.primaryKeyComboBox.setCurrentIndex(key_id)
self.set_push_grid_handlers(column_model=None,
sorton_model=sort_on,
sortasc_model=sort_ascending,
dedupe_model=dedupe_on,
gather_model=gather_fields,
unique_model=unique_fields)
self.gatherFieldsOverWriteCheckBox.setChecked(gather_fields_overwrite)
self._field_map_data.update(field_map_data)
self.open_file(file_names=merge_files,
model_signal=self.signalMergeFileOpened)
self.open_file(file_names=purge_files,
model_signal=self.signalSFileOpened)
def get_settings(self, dc:DictConfig=None, section:str="SPLIT"):
if dc is None:
dc = DictConfig()
if dc.has_section(section):
dc.remove_section(section)
dictionary = dict(dropna = self.checkBoxDropNulls.isChecked(),
dirname = self.lineEditExportPath.text(),
source_path = self.lineEditSourcePath.text(),
max_rows = self.lineEditMaxRows.text())
dc.update({section: dictionary})
dc.set_safe(section, 'split_on', self.handler_split_on.get_model_list(left=False))
dc.set_safe(section, 'fields', self.handler_use_cols.get_model_list(left=False))
return dc
def __init__(self, settings=None, **kwargs):
QtGui.QDialog.__init__(self, **kwargs)
self.setupUi(self)
self.settings_ini = settings
if isinstance(settings, str) or settings is None:
if settings is None:
print(
"Using default settings - got settings of type {}".format(
type(settings)))
self.settings_ini = SettingsINI(filename=settings)
elif isinstance(settings, dict):
self.settings_ini = DictConfig(dictionary=settings)
self.configure_settings(self.settings_ini)
self.connect_buttons()
def export_settings(self, to=None):
"""
Outputs Dialog's settings to a text-like file object.
These settings can be re-imported later.
:param to: (str, default None)
The filepath to output the settings to.
None opens a QFileDialog requiring the user to select a filename to
save to.
:return: None
"""
if to is None:
try:
dirname = os.path.dirname(self.df_model.filePath)
except:
dirname = ''
to = QtGui.QFileDialog.getSaveFileName(self, dir=dirname)[0]
if to != '' and to is not None:
settings = self.get_settings()
ini = DictConfig(dictionary=dict(NORMALIZE=settings), filename=to)
ini.save()
def set_settings(self, dc:DictConfig, section='SPLIT'):
self.lineEditTemplate.setText(dc.filename)
self.handler_split_on.set_model_from_list(dc.get_safe(section, 'split_on',
fallback=self.handler_split_on.get_model_list(left=False)),
left=False)
self.handler_split_on.drop_duplicates(left_priority=False)
self.handler_use_cols.set_model_from_list(dc.get_safe(section, 'fields',
fallback=self.handler_use_cols.get_model_list(left=False)),
left=False)
self.handler_use_cols.drop_duplicates(left_priority=False)
self.checkBoxDropNulls.setChecked(dc.getboolean(section, 'dropna', fallback=self.checkBoxDropNulls.isChecked()))
self.lineEditSourcePath.setText(dc.get(section, 'source_path', fallback=self.lineEditSourcePath.text()))
self.lineEditExportPath.setText(dc.get(section, 'dirname', fallback=self.lineEditExportPath.text()))
self.lineEditMaxRows.setText(dc.get(section, 'max_rows', fallback=self.lineEditMaxRows.text()))
def get_default_config(dictionary:dict = None) -> DictConfig:
"""
Returns a DictConfig object that contains data from
zeex.configs.databases.ini if the path exists.
:param dictionary: (dict, default None)
An optional dictionary of compiled settings to load.
:return: (DictConfig)
With compiled settings.
"""
filename = os.path.join(get_default_config_directory(), "databases.ini")
c = DictConfig(filename=filename)
if os.path.exists(filename):
c.read(filename)
if dictionary:
c.read_dict(dictionary)
return c
def add_connections_from_settings(self, settings: DictConfig=None, sections=None,
raise_on_error=False) -> list:
"""
Adds database connections from a DictConfig (ConfigParser) object.
List of configuration options available from SQLAlchemy:
(http://docs.sqlalchemy.org/en/latest/core/engines.html)
========================================================
drivername – the name of the database backend.
This name will correspond to a module in sqlalchemy/databases
or a third party plug-in.
username – The user name.
password – database password.
host – The name of the host.
port¶ – The port number.
database – The database name.
query – A dictionary of options to be passed to the
dialect and/or the DBAPI upon connect.
========================================================
:param settings: (DictConfig, default None)
None uses the last saved zeex.configs.databases.ini file
example configuration setup:
[connection name1]
database=C:/Users/User/database1.db
drivername=sqlite+pysqlite
The above example could be accessed like:
alchemy_connection = AlchemyConnectionManager.connection('connection name1')
:param sections: (list, default None)
An optional list of connection names to connect to.
None uses all available sections in the :param settings object
:param raise_on_error: (bool, default False)
True raises errors when a connection fails to add
False prints that the connection failed to add on the console.
:return: (list)
Of connection names that were successfully added
"""
if settings is None:
settings = self.config
if sections is None:
sections = settings.sections()
else:
sections = [s for s in settings.sections() if s in sections]
added = []
if sections:
for s in sections:
options = settings.options(s)
if 'database' in options and s not in self.connections:
# should be a valid section
try:
info = {o: settings.get(s, o) for o in options}
username = info.get('username')
if username is not None:
info['password'] = settings.get_password(s, username=username)
url = sqlalchemy.engine.url.URL(info.pop('drivername'), **info)
con = AlchemyConnection(name=s)
con.configure_from_url(url)
self.add_connection(connection=con)
added.append(s)
except Exception as e:
if raise_on_error:
raise
else:
logging.error("Error adding connection: {}, {}".format(s, e))
return added
class SettingsDialog(QtGui.QDialog, Ui_settingsDialog):
signalSettingsExported = QtCore.Signal(SettingsINI, str)
signalSettingsImported = QtCore.Signal(SettingsINI, str)
signalSettingsSaved = QtCore.Signal(SettingsINI, str)
_themes_dir = THEMES_DIR
def __init__(self, settings=None, **kwargs):
QtGui.QDialog.__init__(self, **kwargs)
self.setupUi(self)
self.settings_ini = settings
if isinstance(settings, str) or settings is None:
if settings is None:
print(
"Using default settings - got settings of type {}".format(
type(settings)))
self.settings_ini = SettingsINI(filename=settings)
elif isinstance(settings, dict):
self.settings_ini = DictConfig(dictionary=settings)
self.configure_settings(self.settings_ini)
self.connect_buttons()
@property
def _settings_filename(self):
return self.settings_ini._filename
def connect_buttons(self):
self.btnSave.clicked.connect(self.save_settings)
self.btnExport.clicked.connect(self.export_settings)
self.btnImport.clicked.connect(self.import_settings)
self.btnLogDirectory.clicked.connect(self.open_log_directory)
self.btnRootDirectory.clicked.connect(self.open_root_directory)
self.btnSetDefault.clicked.connect(self.set_and_save_defaults)
self.btnReset.clicked.connect(self.reset_settings)
def configure_settings(self, config: SettingsINI):
"""
Configures settings based on a SettingsINI object.
Defaults are contained in here.
Top-level configurations read from.:
GENERAL
INPUT
OUTPUT
DATABASE
FIELDS
"""
if not hasattr(config, "set_safe"):
# We'll assume get_safe exists to.
raise NotImplementedError("Unsupported settings type: {}".format(
type(config)))
c = config
# Get items/set defaults.
ROOT_DIR = c.get('GENERAL',
'root_directory',
fallback=DEFAULT_ROOT_DIR)
LOG_DIR = c.get('GENERAL', 'log_directory', fallback=DEFAULT_LOG_DIR)
LOG_LEVEL = c.get_safe('GENERAL', 'LOG_LEVEL', fallback="Low")
CLOUD_PROVIDER = c.get_safe('GENERAL', 'CLOUD_PROVIDER', fallback="S3")
THEME_NAME = c.get_safe('GENERAL', 'THEME', fallback=DEFAULT_THEME)
HEADER_CASE = c.get_safe('INPUT', 'HEADER_CASE', fallback="Lower")
HEADER_SPACE = c.get_safe('INPUT', 'HEADER_SPACE', fallback="_")
SEPARATOR = c.get_safe('OUTPUT', 'SEPARATOR', fallback=",")
ENCODING = c.get_safe('OUTPUT', 'ENCODING', fallback="UTF_8")
if not os.path.exists(ROOT_DIR):
try:
os.mkdir(ROOT_DIR)
except OSError:
msg = "Defaulting root and log directories because the root directory '{}' was invalid.".format(
ROOT_DIR)
ROOT_DIR = DEFAULT_ROOT_DIR
LOG_DIR = DEFAULT_LOG_DIR
logging.warning(msg)
#Make sure the directories exist.
for fp in [ROOT_DIR, LOG_DIR]:
try:
if not os.path.exists(fp):
os.mkdir(fp)
except OSError as e:
raise OSError(
"Cannot initialize settings directory {} - {}".format(
fp, e))
#LINE EDITS
self.rootDirectoryLineEdit.setText(ROOT_DIR)
self.logDirectoryLineEdit.setText(LOG_DIR)
#COMBO BOXES
configure_combo_box(self.logLevelComboBox, ['Low', 'Medium', 'High'],
LOG_LEVEL)
configure_combo_box(self.cloudProviderComboBox,
['Google Drive', 'S3', 'DropBox'], CLOUD_PROVIDER)
configure_combo_box(self.headerCaseComboBox,
['lower', 'UPPER', 'Proper'], HEADER_CASE)
configure_combo_box(self.headerSpacesComboBox, [' ', '_'],
HEADER_SPACE)
configure_combo_box(self.separatorComboBox, DEFAULT_SEPARATORS,
SEPARATOR)
configure_combo_box(self.encodingComboBox, DEFAULT_CODECS, ENCODING)
configure_combo_box(self.themeComboBox, DEFAULT_THEME_OPTIONS,
THEME_NAME)
self.set_theme(THEME_NAME)
self.cloudProviderLabel.hide()
self.cloudProviderComboBox.hide()
self.save_settings(to_path=None, write=False)
def set_and_save_defaults(self):
self.export_settings(self.settings_ini.default_path, set_self=True)
try:
self.export_settings(self.settings_ini.backup_path, set_self=False)
except Exception as e:
logging.warning(
"SettingsDialog.set_and_save_defaults: Ignored error setting backup settings path: {}"
.format(e))
def save_settings(self, to_path=None, write=False):
self.set_theme()
self.settings_ini.set_safe('GENERAL', 'ROOT_DIRECTORY',
self.rootDirectoryLineEdit.text())
self.settings_ini.set('GENERAL', 'LOG_DIRECTORY',
self.logDirectoryLineEdit.text())
self.settings_ini.set('GENERAL', 'LOG_LEVEL',
self.logLevelComboBox.currentText())
self.settings_ini.set('GENERAL', 'CLOUD_PROVIDER',
self.cloudProviderComboBox.currentText())
self.settings_ini.set('GENERAL', 'THEME',
self.themeComboBox.currentText())
self.settings_ini.set_safe('INPUT', 'HEADER_CASE',
self.headerCaseComboBox.currentText())
self.settings_ini.set('INPUT', 'HEADER_SPACE',
self.headerSpacesComboBox.currentText())
if write or to_path is not None:
if to_path is None:
to_path = self.settings_ini._filename
self.settings_ini._filename = to_path
self.settings_ini.save()
self.signalSettingsSaved.emit(self.settings_ini,
self.settings_ini.filename)
def clear_settings(self):
self.cloudProviderComboBox.clear()
self.encodingComboBox.clear()
self.headerCaseComboBox.clear()
self.headerSpacesComboBox.clear()
self.logLevelComboBox.clear()
self.separatorComboBox.clear()
self.themeComboBox.clear()
def export_settings(self, to=None, set_self=False):
if to is None:
to = QtGui.QFileDialog.getSaveFileName()[0]
self.save_settings(to_path=None, write=False)
self.settings_ini.save_as(to, set_self=set_self)
self.signalSettingsExported.emit(self.settings_ini, to)
def set_theme(self, theme_name=None):
if theme_name is None:
theme_name = self.themeComboBox.currentText()
if isinstance(theme_name, int):
logging.info("No theme selected.")
theme_name = THEME_NAME2
theme_path = normpath(self._themes_dir, theme_name)
if os.path.exists(theme_path):
with open(theme_path, "r") as fh:
app = QtCore.QCoreApplication.instance()
app.setStyleSheet(fh.read())
else:
logging.warning("Couldnt find theme {}!".format(theme_path))
def import_settings(self, filename=None):
if filename is None:
filename = QtGui.QFileDialog.getOpenFileName()
self.settings_ini.save()
self.settings_ini.clear()
self.settings_ini.read(filename)
self.settings_ini._filename = filename
self.clear_settings()
self.configure_settings(self.settings_ini)
def reset_settings(self):
self.settings_ini.clear()
self.settings_ini.read(self.settings_ini.backup_path)
self.settings_ini.save_as(self.settings_ini.default_path,
set_self=True)
self.clear_settings()
self.configure_settings(self.settings_ini)
def open_root_directory(self):
dirname = QtGui.QFileDialog.getExistingDirectory()
self.rootDirectoryLineEdit.setText(dirname)
def open_log_directory(self):
dirname = QtGui.QFileDialog.getExistingDirectory()
self.logDirectoryLineEdit.setText(dirname)
def get_settings(self,
dc: DictConfig = None,
section="MERGE_PURGE") -> DictConfig:
"""
Gathers the settings out of the Dialog and
returns a DictConfig object with updated settings.
:param dc (DictConfig, default None)
An optional DictConfig object, one is created if none is given.
:param section (str, default 'MERGE_PURGE')
An optional section name to apply settings to.
A pre-existing section with this name would be overwritten.
:return: (DictConfig)
An updated DictConfig object.
"""
if dc is None:
dc = DictConfig()
if dc.has_section(section):
dc.remove_section(section)
dc.add_section(section)
dc.set_safe(section, 'source_path', self.sourcePathLineEdit.text())
dc.set_safe(section, 'dest_path', self.destPathLineEdit.text())
dc.set_safe(section, 'primary_key',
self.primaryKeyComboBox.currentText())
dc.set_safe(section, 'dedupe_on',
self.dedupeOnHandler.get_model_list(left=False))
dc.set_safe(section, 'gather_fields',
self.gatherFieldsHandler.get_model_list(left=False))
dc.set_safe(section, 'gather_fields_overwrite',
self.gatherFieldsOverWriteCheckBox.isChecked())
dc.set_safe(section, 'sort_on',
self.sortOnHandler.get_model_list(left=False))
dc.set_safe(section, 'sort_ascending',
self.sortAscHandler.get_model_list(left=False))
dc.set_safe(section, 'unique_fields',
self.uniqueFieldsHandler.get_model_list(left=False))
dc.set_safe(section, 'field_map_data', self._field_map_data)
dc.set_safe(section, 'merge_files', list(self._merge_files.keys()))
dc.set_safe(section, 'purge_files', list(self._purge_files.keys()))
return dc