def remove_importer_entries(limit: datetime):
"""
Remove the entries that were persisted by the simulate_importer method
:param limit: the date and time limit after which entries should be
deleted
"""
ImporterStatuses.remove_where(limit)
def test_request_for_retry_success(self):
"""
Test whether the correct response is returned when a importer
retry succeeds
"""
importer = ImporterStatuses.find_by_api_id(2)
current_timestamp = importer.timestamp
access_token_header = self.generate_access_token()
response = self.testing_client.post('/importer_retry',
data=dict(api_id=importer.api_id),
headers=access_token_header)
self.assertEqual(response.status_code, 200)
self.assertIn(b"success", response.data)
updated_importer = ImporterStatuses.find_by_api_id(2)
self.assertGreater(updated_importer.timestamp, current_timestamp)
def status_has_changed(self, status: ImporterStatus):
"""
Receive the status of an importer and persist it to the Importer
Status table
:param status: Status object identifying the
"""
logger.info(status)
importer_status = ImporterStatuses.find_by_name(status.name)
if importer_status:
if status.state == "success":
importer_status.state = "success"
else:
importer_status.state = "failure"
importer_status.reason = status.reason
importer_status.trace = status.stack_trace
importer_status.timestamp = datetime.now()
importer_status.commit()
if status.state == "failure" and not importer_status.retrying:
importer_status.retrying = True
importer_status.save()
importer_status.commit()
try:
Scheduler.retry_importer(status.name)
logger.info("Exponential retry procedure was "
"successful for class {}".format(status.name))
except RetryingException:
logger.info("Exponential retry procedure was "
"unsuccessful for class {}".format(
status.name))
importer_status.retrying = False
importer_status.save()
importer_status.commit()
def simulate_importers() -> int:
"""
Persist a simulated importer status for each of the entries in
the api table
:return: the number of importer statuses added
"""
states = ["pending", "success", "failure"]
apis = API.get_all()
number_of_importers = 0
for api in apis:
importer_state = numpy.random.choice(states, 1, True,
[0.1, 0.6, 0.3])[0]
if importer_state == "failure":
reason = "Exceeded daily limit"
else:
reason = ""
new_importer = ImporterStatuses(api.id, api.api_class,
importer_state, reason, "",
datetime.now())
new_importer.save()
new_importer.commit()
number_of_importers += 1
return number_of_importers
def status_has_changed(self, status: ImporterStatus):
"""
Receive the status of an importer and persist it to the Importer
Status table
:param status: Status object defining the state of an importer
"""
logger.info(status)
importer_status = ImporterStatuses.find_by_name(status.name)
if importer_status:
if status.state == "success":
importer_status.state = "success"
else:
importer_status.state = "failure"
importer_status.reason = status.reason
importer_status.trace = status.stack_trace
importer_status.timestamp = datetime.datetime.now()
importer_status.commit()
def retry_importer(importer_class_name: str):
"""
Retry importer if it has failed at intervals that correspond to a
exponential back-off procedure
:param importer_class_name: name of class that implements the importer
:raise RetryingException: raise if importer retry was unsuccessful
"""
logger.info("Importer retry for {}".format(importer_class_name))
retry_class = Api_Class.get_by_api_class(importer_class_name)
if retry_class:
api_name = retry_class.name
class_name = retry_class.api_class
Scheduler.fetch_data(class_name, api_name)
result_entry = ImporterStatuses.find_by_name(importer_class_name)
if result_entry:
if result_entry.state != "success":
raise RetryingException
def run():
""" Schedule main_task to execute once a day """
apis = Scheduler.get_apis()
for api in apis:
if not ImporterStatuses.find_by_api_id(api.id):
class_name = api.api_class.split('.')[2]
new_entry = ImporterStatuses(api.id, class_name, 'pending', '',
'', False, datetime.now())
new_entry.save()
new_entry.commit()
all_attribute = Attributes.get_all()
for attribute in all_attribute:
if not AttributeRange.get_by_attr_id(attribute.id):
attr_min = Attributes.attribute_min(attribute.table_name)
attr_max = Attributes.attribute_max(attribute.table_name)
try:
new_range = AttributeRange(attribute.id, attr_min.s_id,
attr_min.value,
attr_min.timestamp,
attr_max.s_id, attr_max.value,
attr_max.timestamp,
datetime.now())
except AttributeError:
new_range = AttributeRange(attribute.id, None, None,
None, None, None, None,
datetime.now())
new_range.save()
new_range.commit()
sched.add_job(Scheduler.main_task,
'interval',
start_date=datetime.now() + timedelta(seconds=5),
days=1,
name='Primary_Scheduler',
replace_existing=True,
id='Primary_Scheduler',
jobstore='sqlalchemy')
try:
# This is here to simulate application activity (which keeps
# the main thread alive).
while True:
time.sleep(2)
except (KeyboardInterrupt, SystemExit):
sched.shutdown()
def get(self) -> (str, int):
"""
GET request endpoint. Retrieve all entries in the importer status
table.
:return: a dictionary containing importer status entries if
successful or a dictionary containing an error message if unsuccessful
"""
if get_jwt_claims()["admin"]:
statuses = ImporterStatuses.get_all()
if statuses:
status_list = []
for status in statuses:
status_list.append(status.json())
return status_list, 200
else:
return [], 200
else:
return {"message": "Not Authorised"}, 403