async def reevaluate_all_async():
data = get_db_controller().get_all()
for record in data:
identifier = record.get("identifier")
get_db_controller().update_status(identifier,
status="PENDING_REEVALUATION")
await asyncio.gather(*[reevaluate(d) for d in data])
def _save_to_db(data: iter):
records = get_db_controller().get_all_loaded_run_identifiers()
identifiers = list(map(lambda x: x["identifier"], records))
for identifier, params, scores in data:
if (identifier in identifiers):
get_db_controller().update_user_score(identifier, params, scores)
print(f"saved: {identifier} {params}")
else:
print(f"not found in database: {identifier} {params}")
def fetch_data_by_identifier(identifier: dict):
"""Loads resource from database from database, or if not found,
initiates to fetching it from external api in separate thread"""
data = get_db_controller().get_one(identifier)
if data == None:
get_db_controller().save(identifier, status="LOADING")
asyncUtils.run_async_in_thread(_load_process_and_save, identifier)
return None
return data
async def reevaluate_all_async():
data = get_db_controller().get_all()
downloaded_data = [
x for x in data
if x["status"] in ["REEVALUATING", "FINISHED", "PENDING_REEVALUATION"]
]
for record in downloaded_data:
identifier = record.get("identifier")
get_db_controller().update_status(identifier,
status="PENDING_REEVALUATION")
await asyncio.gather(*[reevaluate(d) for d in downloaded_data])
def _reevaluate_all():
try:
data = get_db_controller().get_all()
for record in data:
identifier = record.get("identifier")
get_db_controller().update_status(identifier,
status="REEVALUATING")
data = dataEvaluation.process(record["data"])
print(f":: Successfully reevaluated data for: {identifier}")
get_db_controller().update(identifier,
data=data,
status="FINISHED")
except Exception as exception:
print(f"Unknown error occurred while fetching: {exception}")
get_db_controller().update_status(identifier,
status="ERROR",
other={"exception": str(exception)})
raise exception
async def _load_process_and_save(identifier):
"""Loads, processes and saves data from external api"""
try:
data = await async_fetch_all_data(identifier)
print(f":: Successfully fetched data for: {identifier}")
data = dataEvaluation.process(data)
print(f":: Successfully processed data for: {identifier}")
get_db_controller().update(identifier, data=data, status="FINISHED")
except FetchError as fetchError:
print(f"Error occurred while fetching: {fetchError}")
get_db_controller().update(identifier,
status="ERROR",
other={"exception": str(exception)})
except Exception as exception:
print(f"Unknown error occurred while fetching: {exception}")
get_db_controller().update(identifier,
status="ERROR",
other={"exception": str(exception)})
raise exception
def get_not_evaluated_network_scores(limit=20, page=1):
skip_count = (page - 1) * limit
return get_db_controller().get_not_evaluated_network_scores(
limit, skip=skip_count)
def get_one_record(identifier: dict):
return get_db_controller().get_one(identifier)
def get_contamination_scores(limit=20, page=1):
skip_count = (page - 1) * limit
return get_db_controller().get_contamination(limit, skip=skip_count)
def get_scores_data():
return get_db_controller().get_all_user_scores()
def get_fetched_data():
return get_db_controller().get_all()
def get_not_evaluated_network_scores(limit=20):
return get_db_controller().get_not_evaluated_network_scores(limit)
def delete(identifier: dict):
return get_db_controller().delete(identifier)
async def reevaluate(record):
identifier = record.get("identifier")
get_db_controller().update_status(identifier, status="REEVALUATING")
data = dataEvaluation.process(record["data"])
print(f":: Successfully reevaluated data for: {identifier}")
get_db_controller().update(identifier, data=data, status="FINISHED")
def _load_from_database():
return get_db_controller().get_all_user_scores()
def update_user_score(identifier: dict, params: dict, layers: list):
return get_db_controller().update_user_score(identifier, params, layers)
def mark_as_skipped(identifier: dict, params: dict):
return get_db_controller().skip_user_score(identifier, params)
def visualize(identifier: dict, params: dict):
record = get_db_controller().get_single_record_matrix(identifier, params)
return dataEvaluation.visualize_saliency(record)
def get_matrix_from_DB(identifier: dict, params: dict = {}):
return get_db_controller().get_single_record_matrix(identifier, params)
def get_network_scores(limit=20):
return get_db_controller().get_all_network_scores(limit)
