def retrieve_element(self, element, config) -> LocalElement:
if element.id == "skip":
raise ElementShouldSkipError("test")
elif element.id == "retry3" and self.retryCount < 3:
self.retryCount += 1
raise ElementShouldRetryError("test")
elif element.id == "retryN":
raise ElementShouldRetryError("test")
else:
return None
def analyse_element(self, element, config):
if element.id == "skip":
raise ElementShouldSkipError("test")
elif element.id == "retry3" and self.retryCount < 3:
self.retryCount += 1
raise ElementShouldRetryError("test")
elif element.id == "retryN":
raise ElementShouldRetryError("test")
else:
pass
def __attempt_retrieve(self, attempts, element_index):
try:
new_element = self.retrieve_element(element_index, self.config)
if new_element is None:
return
success = self.disk.write_element(self.name, new_element)
if not success:
raise ElementShouldRetryError("Unsuccessful storage")
except ElementShouldSkipError as e:
self.error_logger(str(e), element_index)
except ElementShouldRetryError as e:
self.error_logger(str(e), element_index)
if attempts > 1:
return self.__attempt_retrieve(attempts - 1, element_index)
else:
self.error_logger(
"failed after maximum retries - skipping element",
element_index)
# TODO: flag to turn this off during development should be passed during run
except Exception as e:
if self.is_dev():
raise e
else:
self.error_logger(
"unknown exception raised - skipping element",
element_index)
def __attempt_analyse(self, attempts, element):
try:
new_element = self.analyse_element(element, self.config)
if new_element is None:
return
success = self.disk.write_element(self.get_dest_q(), new_element)
if not success:
raise ElementShouldRetryError("Unsuccessful storage")
except ElementShouldSkipError as e:
self.error_logger(str(e), element)
except ElementShouldRetryError as e:
self.error_logger(str(e), element)
if attempts > 1:
return self.__attempt_analyse(attempts - 1, element)
else:
self.error_logger(
"failed after maximum retries - skipping element", element)
self.errored = True
except Exception as e:
if self.is_dev():
raise e
else:
self.error_logger(f"{str(e)}: skipping element", element)
print(traceback.format_exc())
def __post_analyse(self):
# TODO: is there a way to only do this work if overridden?
analysed_els = self.disk.read_elements([self.get_dest_q()])
outel = self.post_analyse(analysed_els)
if outel is None:
return
successes = []
for q in self.config["elements_in"]:
selname, _ = super(type(self.disk), self.disk).read_query(q)
success = self.disk.write_element(f"{selname}/{self.name}", outel)
successes.append(success)
if not all(successes):
raise ElementShouldRetryError(
"Some instances of the final element produced via 'post_analyse' failed to save."
)
def __post_analyse(self):
# TODO: is there a way to only do this work if overridden?
analysed_els = self.disk.read_elements([self.get_dest_q()])
outel = self.post_analyse(analysed_els)
if outel is None:
return
successes = []
# NOTE: this is duplicated code from `get_selector` in this same class,
# as it supports `elements_in` as a list. We perhaps need to just
# enforce `elements_in` as a single query, rather than a list of
# queries.
for q in self.config["elements_in"]:
selname, _ = super(type(self.disk), self.disk).read_query(q)
success = self.disk.write_element(f"{selname}/{self.name}", outel)
successes.append(success)
if not all(successes):
raise ElementShouldRetryError(
"Some instances of the final element produced via 'post_analyse' failed to save."
)