async def reevaluate_all_async(): data = get_db_controller().get_all() for record in data: identifier = record.get("identifier") get_db_controller().update_status(identifier, status="PENDING_REEVALUATION") await asyncio.gather(*[reevaluate(d) for d in data])
def _save_to_db(data: iter): records = get_db_controller().get_all_loaded_run_identifiers() identifiers = list(map(lambda x: x["identifier"], records)) for identifier, params, scores in data: if (identifier in identifiers): get_db_controller().update_user_score(identifier, params, scores) print(f"saved: {identifier} {params}") else: print(f"not found in database: {identifier} {params}")
def fetch_data_by_identifier(identifier: dict): """Loads resource from database from database, or if not found, initiates to fetching it from external api in separate thread""" data = get_db_controller().get_one(identifier) if data == None: get_db_controller().save(identifier, status="LOADING") asyncUtils.run_async_in_thread(_load_process_and_save, identifier) return None return data
async def reevaluate_all_async(): data = get_db_controller().get_all() downloaded_data = [ x for x in data if x["status"] in ["REEVALUATING", "FINISHED", "PENDING_REEVALUATION"] ] for record in downloaded_data: identifier = record.get("identifier") get_db_controller().update_status(identifier, status="PENDING_REEVALUATION") await asyncio.gather(*[reevaluate(d) for d in downloaded_data])
def _reevaluate_all(): try: data = get_db_controller().get_all() for record in data: identifier = record.get("identifier") get_db_controller().update_status(identifier, status="REEVALUATING") data = dataEvaluation.process(record["data"]) print(f":: Successfully reevaluated data for: {identifier}") get_db_controller().update(identifier, data=data, status="FINISHED") except Exception as exception: print(f"Unknown error occurred while fetching: {exception}") get_db_controller().update_status(identifier, status="ERROR", other={"exception": str(exception)}) raise exception
async def _load_process_and_save(identifier): """Loads, processes and saves data from external api""" try: data = await async_fetch_all_data(identifier) print(f":: Successfully fetched data for: {identifier}") data = dataEvaluation.process(data) print(f":: Successfully processed data for: {identifier}") get_db_controller().update(identifier, data=data, status="FINISHED") except FetchError as fetchError: print(f"Error occurred while fetching: {fetchError}") get_db_controller().update(identifier, status="ERROR", other={"exception": str(exception)}) except Exception as exception: print(f"Unknown error occurred while fetching: {exception}") get_db_controller().update(identifier, status="ERROR", other={"exception": str(exception)}) raise exception
def get_not_evaluated_network_scores(limit=20, page=1): skip_count = (page - 1) * limit return get_db_controller().get_not_evaluated_network_scores( limit, skip=skip_count)
def get_one_record(identifier: dict): return get_db_controller().get_one(identifier)
def get_contamination_scores(limit=20, page=1): skip_count = (page - 1) * limit return get_db_controller().get_contamination(limit, skip=skip_count)
def get_scores_data(): return get_db_controller().get_all_user_scores()
def get_fetched_data(): return get_db_controller().get_all()
def get_not_evaluated_network_scores(limit=20): return get_db_controller().get_not_evaluated_network_scores(limit)
def delete(identifier: dict): return get_db_controller().delete(identifier)
async def reevaluate(record): identifier = record.get("identifier") get_db_controller().update_status(identifier, status="REEVALUATING") data = dataEvaluation.process(record["data"]) print(f":: Successfully reevaluated data for: {identifier}") get_db_controller().update(identifier, data=data, status="FINISHED")
def _load_from_database(): return get_db_controller().get_all_user_scores()
def update_user_score(identifier: dict, params: dict, layers: list): return get_db_controller().update_user_score(identifier, params, layers)
def mark_as_skipped(identifier: dict, params: dict): return get_db_controller().skip_user_score(identifier, params)
def visualize(identifier: dict, params: dict): record = get_db_controller().get_single_record_matrix(identifier, params) return dataEvaluation.visualize_saliency(record)
def get_matrix_from_DB(identifier: dict, params: dict = {}): return get_db_controller().get_single_record_matrix(identifier, params)
def get_network_scores(limit=20): return get_db_controller().get_all_network_scores(limit)