def test_answer_get_registered_queries(): db = _get_test_db() qm = QueryManager(db=db, model_managers=[test_mm]) # Put all types of queries in db, answer together qm.db.put_queries(test_email, 1, query_object, ['test'], subscribe=True) qm.db.put_queries(test_email, 1, dyn_query, ['test'], subscribe=True) qm.db.put_queries(test_email, 1, open_query, ['test'], subscribe=True) qm.db.put_queries(test_email, 1, interv_query, ['test'], subscribe=True) qm.answer_registered_queries('test') # Retrieve results for path query results = qm.get_registered_queries(test_email, 'path_property') qh = query_object.get_hash_with_model('test') assert qh in results assert len(results) == 1 assert results[qh]['model'] == 'test' assert results[qh]['query'] == simple_query assert isinstance(results[qh]['date'], str) assert results[qh]['pysb'] == ['Pass', [test_response['3801854542']]], \ (results[qh]['pysb'], test_response['3801854542']) assert results[qh]['pybel'] == ['Pass', [test_response['3801854542']]] assert results[qh]['signed_graph'][0] == 'Pass' assert results[qh]['unsigned_graph'][0] == 'Pass' # Retrieve results for dynamic query results = qm.get_registered_queries(test_email, 'dynamic_property') qh = dyn_query.get_hash_with_model('test') assert qh in results assert results[qh]['model'] == 'test' assert results[qh]['query'] == 'Active MAP2K1 is eventually high.' assert isinstance(results[qh]['date'], str) assert results[qh]['result'] == [ 'Pass', 'Satisfaction rate is 100% after 2 simulations.' ] assert isinstance(results[qh]['image'], str) # Retrieve results for open query results = qm.get_registered_queries(test_email, 'open_search_query') qh = open_query.get_hash_with_model('test') assert qh in results assert results[qh]['model'] == 'test' assert results[qh]['query'] == 'What does BRAF activate?' assert isinstance(results[qh]['date'], str) for mc_type in ['pysb', 'pybel', 'signed_graph', 'unsigned_graph']: assert results[qh][mc_type][0] == 'Pass' assert isinstance(results[qh][mc_type][1], list) assert test_response['3801854542']['path'] in [ res['path'] for res in results[qh][mc_type][1] ] # Retrieve results for intervention query results = qm.get_registered_queries(test_email, 'simple_intervention_property') qh = interv_query.get_hash_with_model('test') assert qh in results, results assert results[qh]['model'] == 'test' assert results[qh]['query'] == 'BRAF increases active MAP2K1.' assert isinstance(results[qh]['date'], str) assert results[qh]['result'] == [ 'Pass', 'Yes, the amount of target entity increased.' ] assert isinstance(results[qh]['image'], str)
def test_answer_get_registered_queries(): db = _get_test_db() qm = QueryManager(db=db, model_managers=[test_mm]) qm.db.put_queries('*****@*****.**', 1, query_object, ['test'], subscribe=True) qm.answer_registered_queries('test') results = qm.get_registered_queries('*****@*****.**') assert len(results) == 1 assert results[0]['model'] == 'test' assert results[0]['query'] == simple_query assert isinstance(results[0]['response'], str) assert 'BRAF activates MAP2K1.' in results[0]['response'], \ results[0]['response'] assert isinstance(results[0]['date'], str)
def run_model_tests_from_s3(model_name, upload_mm=True, upload_results=True, upload_stats=True, registered_queries=True, db=None): """Run a given set of tests on a given model, both loaded from S3. After loading both the model and the set of tests, model/test overlap is determined using a ScopeTestConnector and tests are run. Parameters ---------- model_name : str Name of EmmaaModel to load from S3. upload_mm : Optional[bool] Whether to upload a model manager instance to S3 as a pickle file. Default: True upload_results : Optional[bool] Whether to upload test results to S3 in JSON format. Can be set to False when running tests. Default: True upload_stats : Optional[bool] Whether to upload latest statistics about model and a test. Default: True registered_queries : Optional[bool] If True, registered queries are fetched from the database and executed, the results are then saved to the database. Default: True db : Optional[emmaa.db.manager.EmmaaDatabaseManager] If given over-rides the default primary database. Returns ------- emmaa.model_tests.ModelManager Instance of ModelManager containing the model data, list of applied tests and the test results. emmaa.analyze_test_results.StatsGenerator Instance of StatsGenerator containing statistics about model and test. """ model = EmmaaModel.load_from_s3(model_name) test_corpus = model.test_config.get('test_corpus', 'large_corpus_tests.pkl') tests = load_tests_from_s3(test_corpus) mm = ModelManager(model) if upload_mm: save_model_manager_to_s3(model_name, mm) tm = TestManager([mm], tests) tm.make_tests(ScopeTestConnector()) tm.run_tests() results_json_dict = mm.results_to_json() results_json_str = json.dumps(results_json_dict, indent=1) # Optionally upload test results to S3 if upload_results: client = get_s3_client(unsigned=False) date_str = make_date_str() result_key = f'results/{model_name}/results_{date_str}.json' logger.info(f'Uploading test results to {result_key}') client.put_object(Bucket='emmaa', Key=result_key, Body=results_json_str.encode('utf8')) tr = TestRound(results_json_dict) sg = StatsGenerator(model_name, latest_round=tr) sg.make_stats() # Optionally upload statistics to S3 if upload_stats: sg.save_to_s3() if registered_queries: qm = QueryManager(db=db, model_managers=[mm]) qm.answer_registered_queries(model_name) return (mm, sg)