def __get_selector(self):
if self.paras.selector_file:
try:
selector = Selector(location=self.paras.selector_file)
self.last_directory = os.path.dirname(self.paras.selector_file)
LOG.debug("Loaded selector from %s" % self.paras.selector_file)
except Exception as err:
self.warn(
"Unable to read selector file '%s'" %
self.paras.selector_file, err)
selector = Selector()
else:
selector = Selector()
#Difficult keeping track of a selector that is not saved:
if selector.location is None:
selector_dir = os.path.join(Configurations.rspub_config_dir(),
"selectors")
os.makedirs(selector_dir, exist_ok=True)
location = os.path.join(selector_dir, "default_selector.csv")
if os.path.exists(location):
selector = Selector(location=location)
else:
selector.write(location)
self.paras.selector_file = location
return selector
def on_btn_run_clicked(self):
if self.paras.select_mode == SelectMode.simple:
selector = Selector()
if self.paras.simple_select_file:
selector.include(self.paras.simple_select_file)
else:
selector = self.ctrl.selector
self.paras.is_saving_sitemaps = not self.chk_trial_run.isChecked()
self.pte_events1.setPlainText("")
self.pte_events2.setPlainText("")
self.pte_events3.setPlainText("")
self.lbl_processing.setVisible(True)
self.lbl_processing_file.setVisible(True)
self.btn_close.setEnabled(False)
self.btn_run.setEnabled(False)
self.chk_trial_run.setEnabled(False)
self.btn_stop.setEnabled(True)
self.btn_stop.setStyleSheet(Style.alarm())
self.executor_thread = ExecutorThread(self.paras, selector, self)
self.executor_thread.signal_exception.connect(self.on_signal_exception)
self.executor_thread.ask_confirmation.connect(self.on_ask_confirmation)
self.executor_thread.signal_main_event.connect(
self.on_signal_main_event)
self.executor_thread.signal_minor_event.connect(
self.on_signal_minor_event)
self.executor_thread.signal_next_file.connect(self.on_signal_next_file)
self.executor_thread.signal_end_processing.connect(
self.on_signal_end_processing)
self.executor_thread.finished.connect(self.on_executor_thread_finished)
self.executor_thread.start()
self.update()
def test_list(self):
selector = Selector()
selector.include(os.path.join(test_dir(), "collection1"))
selector.include(os.path.join(test_dir(), "collection2"))
selector.exclude(os.path.join(test_dir(), "collection2/folder2"))
selector.exclude(os.path.join(test_dir(), "collection2/.DS_Store"))
for filename in selector.list_includes():
print(filename)
print(len([x for x in selector.list_includes()]))
def __init__(self):
global SELECTOR
SuperCmd.__init__(self)
if PARAS.selector_file:
try:
SELECTOR = Selector(PARAS.selector_file)
print("Loaded Selector from", SELECTOR.abs_location())
except Exception as err:
print("\nSelector error: {0}".format(err))
if SELECTOR is None:
SELECTOR = Selector()
def open_selector(self, filename):
try:
self.selector = Selector(filename)
LOG.debug("Opened selector %s" % self.selector.abs_location())
self.last_directory = os.path.dirname(self.selector.abs_location())
self.paras.selector_file = filename
self.paras.save_configuration()
self.switch_configuration.emit(self.paras.configuration_name())
self.switch_selector.emit(self.selector.abs_location())
except Exception as err:
self.warn(
"Unable to open selector file '%s'" %
self.selector.abs_location(), err)
def do_load_selector(self, path):
"""
load_selector::
load_selector [path] - Load Selector from location [path]
---------------------------------------------------------
If the current Selector has unsaved changes, you will be
prompted to save or discard.
"""
global SELECTOR
if path and self.check_exit():
try:
SELECTOR = Selector(path)
print("Loaded Selector from", SELECTOR.abs_location())
except Exception as err:
print("\nSelector error: {0}".format(err))
def test_discard_self(self):
selector = Selector()
selector.include("collection1")
selector.include("collection2")
selector.discard_include(selector)
self.assertEquals(0, len(selector))
def do_run(self, line):
"""
run::
run rspub with the current configuration.
"""
# SELECTOR -> yes -> SELECTOR.location -> yes -> associated -> yes -> [runs]
# no| no| no|
# P.selector > y > [run] [runs] P.selector -> yes -> ask ->yes-> [runs]
# no| no| no|
# [abort] [runs] [abort]
# ----------------------------------------------------------------------------------------
global PARAS
global SELECTOR
run = False # rs.execute()
runs = False # rs.execute(SELECTOR)
abort = False
if PARAS.select_mode == SelectMode.simple and PARAS.simple_select_file:
SELECTOR = Selector()
SELECTOR.include(PARAS.simple_select_file)
if SELECTOR is None:
if PARAS.selector_file is None:
abort = "No selector and configuration not associated with selector. Run aborted."
else:
run = True
else:
if SELECTOR.location is None \
or SELECTOR.abs_location() == PARAS.selector_file \
or PARAS.selector_file is None:
runs = True
elif self.__confirm__(
"Associate current configuration with selector?"):
runs = True
else:
abort = "Not associating current configuration with selector. Run aborted."
if abort:
print(abort)
elif run or runs:
try:
rs = ResourceSync(**PARAS.__dict__)
rs.register(self)
rs.execute(
SELECTOR) # == rs.execute() if SELECTOR is None for [run]
PARAS = RsParameters(
**rs.__dict__) # catch up with updated paras
except Exception as err:
traceback.print_exc()
print("\nUncompleted run: {0}".format(err))
else: # we should not end here!
location = None
if SELECTOR:
location = SELECTOR.abs_location()
print("Missed a path in tree: ", SELECTOR, location,
PARAS.selector_file)
def on_btn_run_clicked(self):
super(ExecuteWidget, self).on_btn_run_clicked()
if self.paras.select_mode == SelectMode.simple:
selector = Selector()
if self.paras.simple_select_file:
selector.include(self.paras.simple_select_file)
else:
selector = self.ctrl.selector
self.paras.is_saving_sitemaps = not self.chk_trial_run.isChecked()
self.executor_thread = ExecutorThread(self.paras, selector, self)
self.executor_thread.signal_exception.connect(self.on_signal_exception)
self.executor_thread.ask_confirmation.connect(self.on_ask_confirmation)
self.executor_thread.signal_main_event.connect(
self.on_signal_main_event)
self.executor_thread.signal_minor_event.connect(
self.on_signal_minor_event)
self.executor_thread.signal_next_file.connect(self.on_signal_next_file)
self.executor_thread.signal_end_processing.connect(
self.on_signal_end_processing)
self.executor_thread.finished.connect(self.on_executor_thread_finished)
self.executor_thread.start()
self.update()
def test_discard(self):
selector = Selector()
selector.include(os.path.join(test_dir(), "collection1"))
selector.include(os.path.join(test_dir(), "collection2"))
selector.exclude(os.path.join(test_dir(), "collection2/folder2"))
selector.exclude(os.path.join(test_dir(), "collection2/.DS_Store"))
# selector2 = Selector()
# selector2.exclude(os.path.join(test_dir(), "collection2/folder1"))
# selector2.include(os.path.join(test_dir(), "directory_1"))
# selector2.exclude(os.path.join(test_dir(), "collection1"))
#
# selector.include(selector2)
# selector.exclude(selector2)
# selector.discard_exclude(selector2)
for filename in selector:
print(filename)
def test_with_selector(self):
metadata = os.path.join("tmp", "rs", "md_select")
selector = Selector()
os.makedirs(os.path.join(resource_dir(), "tmp", "rs", "test_data"),
exist_ok=True)
selector.location = os.path.join(resource_dir(), "tmp", "rs",
"test_data", "selector1.txt")
selector.include(test_resource())
selector.exclude(os.path.join(test_resource(), "collection2"))
rs = ResourceSync(resource_dir=resource_dir(), metadata_dir=metadata)
rs.register(EventLogger(logging_level=logging.INFO))
rs.execute(selector)
# once executed we can just call rs.execute() because selector associated with paras
rs.execute()
def test_iter(self):
selector = Selector()
selector.include("foo", "bee")
selector.include(["foo", "boz", ["kaz", "koz"]])
selector.include("baz")
selector2 = Selector()
selector2.include("2foo", "2bar", ["2kaz", "2koz", {"2beez"}])
selector.include(selector2)
expected_length = 0
selector_length = 0 #len(selector)
self.assertEquals(expected_length, selector_length)
names = []
for filename in selector:
names.append(filename)
self.assertEquals(expected_length, len(names))
def test_filter_base_path(self):
paths = {"abc", "abc/def", "abc/def/ghi", "foo/bar", "abc/def"}
base_paths = Selector.filter_base_paths(paths)
self.assertEqual(2, len(base_paths))
self.assertTrue("abc" in base_paths)
self.assertTrue("foo/bar" in base_paths)
def test_read_write(self):
selector = Selector()
selector.include("collection1")
selector.exclude("collection2")
selector.write(os.path.join(test_dir(), "test_selector.txt"))
selector2 = Selector(os.path.join(test_dir(), "test_selector.txt"))
self.assertEqual(selector2.get_included_entries().pop(), "collection1")
self.assertEqual(selector2.get_excluded_entries().pop(), "collection2")
def test_read_write(self):
selector = Selector()
selector.read_includes(os.path.join(test_dir(), "includes.txt"))
selector.read_excludes(os.path.join(test_dir(), "excludes.txt"))
print("\nselector 1")
for filename in selector:
print(filename)
selector.write_includes(os.path.join(test_dir(), "includes_w.txt"))
selector.write_excludes(os.path.join(test_dir(), "excludes_w.txt"))
selector.write(os.path.join(test_dir(), "selector_1.txt"))
selector2 = Selector(os.path.join(test_dir(), "selector_1.txt"))
#
print("\nselector 2")
for filename in selector2:
print(filename)
selector2.clear_excludes()
selector2.write(os.path.join(test_dir(), "test_data",
"selector_2.txt"))
def execute(self, filenames: iter = None, start_new=False):
"""
:samp:`Publish ResourceSync documents under conditions of current {parameters}`
Call appropriate executor and publish sitemap documents on the resources found in `filenames`.
If no file/files 'resourcelist_*.xml' are found in metadata directory will always dispatch to
strategy (new) ``resourcelist``.
If ``parameter`` :func:`~rspub.core.rs_paras.RsParameters.is_saving_sitemaps` is ``False`` will do
a dry run: no existing sitemaps will be changed and no new sitemaps will be written to disk.
:param filenames: filenames and/or directories to scan
:param start_new: erase metadata directory and create new resourcelists
"""
# always start fresh publication with resourcelist
resourcelist_files = sorted(
glob(self.abs_metadata_path("resourcelist_*.xml")))
start_new = start_new or len(resourcelist_files) == 0
# do we have filenames or look for a saved Selector?
if filenames is None and self.selector_file:
try:
filenames = Selector(self.selector_file)
LOG.info("Loaded selector from '%s'" % self.selector_file)
except Exception as err:
LOG.warning("Unable to load selector: {0}".format(err))
if filenames is None:
raise RuntimeError("Unable to execute: no filenames.")
paras = RsParameters(**self.__dict__)
executor = None
if self.strategy == Strategy.resourcelist or start_new:
executor = ResourceListExecutor(paras)
elif self.strategy == Strategy.new_changelist:
executor = NewChangeListExecutor(paras)
elif self.strategy == Strategy.inc_changelist:
executor = IncrementalChangeListExecutor(paras)
if executor:
executor.register(*self.observers)
sitemap_data_iter = executor.execute(filenames)
else:
raise NotImplementedError("Strategy not implemented: %s" %
self.strategy)
# associate current parameters with a selector
if isinstance(filenames, Selector):
if filenames.location:
try:
filenames.write()
self.selector_file = filenames.abs_location()
LOG.info(
"Associated parameters '%s' with selector at '%s'" %
(self.configuration_name(), self.selector_file))
except Exception as err:
LOG.warning("Unable to save selector: {0}".format(err))
# set a timestamp, save paths to newly created sitemaps
if self.is_saving_sitemaps:
self.last_execution = executor.date_start_processing
self.last_strategy = self.strategy
sitemaps = [sitemap.path for sitemap in sitemap_data_iter]
self.last_sitemaps = sitemaps
self.save_configuration(True)
def __get_simple_selector(self):
selector = Selector()
if self.paras.simple_select_file:
selector.include(self.paras.simple_select_file)
return selector
class Ctrl(QObject):
switch_language = pyqtSignal(str)
switch_configuration = pyqtSignal(str)
switch_selector = pyqtSignal(str)
switch_tab = pyqtSignal(int, int)
def __init__(self, application_home, locale_dir):
QObject.__init__(self)
self.application_home = application_home
self.locale_dir = locale_dir
self.config = GuiConf()
self.last_directory = os.path.expanduser("~")
try:
self.paras = RsParameters(
config_name=self.config.last_configuration_name())
except:
self.paras = RsParameters()
self.selector = self.__get_selector()
def report_tab_switch(self, from_index, to_index):
self.switch_tab.emit(from_index, to_index)
def locales(self):
return [
os.path.basename(x)
for x in glob.glob(os.path.join(self.locale_dir, "*-*"))
if os.path.isdir(x)
]
def system_language(self):
loc = None
try:
loc = locale.getdefaultlocale()
except:
pass
return DEFAULT_LOCALE if loc is None or len(loc) < 2 else loc[0]
def current_language(self):
return GuiConf().language(fallback=self.system_language())
def set_language(self, code):
gettext.translation(LOCALE_DOMAIN,
localedir=self.locale_dir,
languages=[code],
fallback=True).install()
self.config.set_language(code)
self.config.persist()
self.switch_language.emit(code)
def iso_lang(self):
data = {}
iso_path = os.path.join(self.application_home, "i18n", "iso-lang.json")
try:
with open(iso_path, "r", encoding="utf-8") as iso_file:
data = json.load(iso_file)
except Exception as err:
self.warn(_("Could not read iso-lang.json"), err)
return data
def load_configuration(self, name):
try:
self.paras = RsParameters(config_name=name)
self.selector = self.__get_selector()
self.switch_configuration.emit(name)
self.switch_selector.emit(self.selector.abs_location())
LOG.debug("Loaded configuration: '%s'" % name)
except ValueError as err:
self.error("Unable to load configuration %s" % name, err)
def save_configuration_as(self, name):
try:
if self.paras.selector_file:
selector_dir = os.path.dirname(self.paras.selector_file)
new_name = os.path.join(selector_dir, name + "-selector.csv")
self.selector.write(new_name)
self.paras.selector_file = new_name
LOG.debug("Saved selector '%s' as '%s'" %
(self.paras.selector_file, new_name))
self.paras.save_configuration_as(name)
LOG.debug("Saved configuration as '%s'" % name)
self.switch_configuration.emit(name)
self.switch_selector.emit(self.selector.abs_location())
except ValueError as err:
self.error("Unable to save configuration '%s'" % name, err)
def reset_configuration(self):
try:
self.paras.reset()
self.selector = self.__get_selector()
LOG.debug("Configuration was reset")
self.switch_configuration.emit(self.paras.configuration_name())
self.switch_selector.emit(self.selector.abs_location())
except Exception as err:
self.error("Unable to reset parameters.", err)
def update_configuration(self, paras):
self.paras = paras
self.paras.save_configuration()
LOG.debug("Configuration updated")
self.switch_configuration.emit(self.paras.configuration_name())
def __get_selector(self):
if self.paras.selector_file:
try:
selector = Selector(location=self.paras.selector_file)
self.last_directory = os.path.dirname(self.paras.selector_file)
LOG.debug("Loaded selector from %s" % self.paras.selector_file)
except Exception as err:
self.warn(
"Unable to read selector file '%s'" %
self.paras.selector_file, err)
selector = Selector()
else:
selector = Selector()
#Difficult keeping track of a selector that is not saved:
if selector.location is None:
selector_dir = os.path.join(Configurations.rspub_config_dir(),
"selectors")
os.makedirs(selector_dir, exist_ok=True)
location = os.path.join(selector_dir, "default_selector.csv")
if os.path.exists(location):
selector = Selector(location=location)
else:
selector.write(location)
self.paras.selector_file = location
return selector
def save_selector(self):
if self.selector.location:
try:
self.selector.write()
LOG.debug("Saved selector as %s" %
self.selector.abs_location())
except Exception as err:
self.warn(
"Unable to save selector file '%s'" %
self.selector.abs_location(), err)
def save_selector_as(self, filename):
try:
self.selector.write(filename)
LOG.debug("Saved selector as %s" % self.selector.abs_location())
self.last_directory = os.path.dirname(self.selector.abs_location())
self.paras.selector_file = filename
self.paras.save_configuration()
self.switch_configuration.emit(self.paras.configuration_name())
self.switch_selector.emit(self.selector.abs_location())
except Exception as err:
self.warn(
"Unable to save selector file as '%s'" %
self.selector.abs_location(), err)
def open_selector(self, filename):
try:
self.selector = Selector(filename)
LOG.debug("Opened selector %s" % self.selector.abs_location())
self.last_directory = os.path.dirname(self.selector.abs_location())
self.paras.selector_file = filename
self.paras.save_configuration()
self.switch_configuration.emit(self.paras.configuration_name())
self.switch_selector.emit(self.selector.abs_location())
except Exception as err:
self.warn(
"Unable to open selector file '%s'" %
self.selector.abs_location(), err)
def load_selector_includes(self, filename):
try:
self.selector.read_includes(filename)
LOG.debug("Loaded includes %s" % filename)
self.last_directory = os.path.dirname(filename)
self.switch_selector.emit(self.selector.abs_location())
except Exception as err:
self.warn("Unable to load includes from %s." % filename, err)
def load_selector_excludes(self, filename):
try:
self.selector.read_excludes(filename)
LOG.debug("Loaded excludes %s" % filename)
self.last_directory = os.path.dirname(filename)
self.switch_selector.emit(self.selector.abs_location())
except Exception as err:
self.warn("Unable to load excludes from %s." % filename, err)
@staticmethod
def error(msg, cause=None):
LOG.exception(msg)
Ctrl.__msg(QMessageBox.Critical, msg, cause)
@staticmethod
def warn(msg, cause=None):
LOG.warning(msg, exc_info=True)
Ctrl.__msg(QMessageBox.Warning, msg, cause)
@staticmethod
def __msg(icon, text, cause=None):
msg_box = QMessageBox()
msg_box.setWindowTitle(_("MPT"))
if cause:
msg_box.setText(_("%s\nCaused by:\n\n%s") % (text, cause))
else:
msg_box.setText(text)
msg_box.setIcon(icon)
msg_box.exec()