def test_answer_get_registered_queries():
db = _get_test_db()
qm = QueryManager(db=db, model_managers=[test_mm])
# Put all types of queries in db, answer together
qm.db.put_queries(test_email, 1, query_object, ['test'], subscribe=True)
qm.db.put_queries(test_email, 1, dyn_query, ['test'], subscribe=True)
qm.db.put_queries(test_email, 1, open_query, ['test'], subscribe=True)
qm.db.put_queries(test_email, 1, interv_query, ['test'], subscribe=True)
qm.answer_registered_queries('test')
# Retrieve results for path query
results = qm.get_registered_queries(test_email, 'path_property')
qh = query_object.get_hash_with_model('test')
assert qh in results
assert len(results) == 1
assert results[qh]['model'] == 'test'
assert results[qh]['query'] == simple_query
assert isinstance(results[qh]['date'], str)
assert results[qh]['pysb'] == ['Pass', [test_response['3801854542']]], \
(results[qh]['pysb'], test_response['3801854542'])
assert results[qh]['pybel'] == ['Pass', [test_response['3801854542']]]
assert results[qh]['signed_graph'][0] == 'Pass'
assert results[qh]['unsigned_graph'][0] == 'Pass'
# Retrieve results for dynamic query
results = qm.get_registered_queries(test_email, 'dynamic_property')
qh = dyn_query.get_hash_with_model('test')
assert qh in results
assert results[qh]['model'] == 'test'
assert results[qh]['query'] == 'Active MAP2K1 is eventually high.'
assert isinstance(results[qh]['date'], str)
assert results[qh]['result'] == [
'Pass', 'Satisfaction rate is 100% after 2 simulations.'
]
assert isinstance(results[qh]['image'], str)
# Retrieve results for open query
results = qm.get_registered_queries(test_email, 'open_search_query')
qh = open_query.get_hash_with_model('test')
assert qh in results
assert results[qh]['model'] == 'test'
assert results[qh]['query'] == 'What does BRAF activate?'
assert isinstance(results[qh]['date'], str)
for mc_type in ['pysb', 'pybel', 'signed_graph', 'unsigned_graph']:
assert results[qh][mc_type][0] == 'Pass'
assert isinstance(results[qh][mc_type][1], list)
assert test_response['3801854542']['path'] in [
res['path'] for res in results[qh][mc_type][1]
]
# Retrieve results for intervention query
results = qm.get_registered_queries(test_email,
'simple_intervention_property')
qh = interv_query.get_hash_with_model('test')
assert qh in results, results
assert results[qh]['model'] == 'test'
assert results[qh]['query'] == 'BRAF increases active MAP2K1.'
assert isinstance(results[qh]['date'], str)
assert results[qh]['result'] == [
'Pass', 'Yes, the amount of target entity increased.'
]
assert isinstance(results[qh]['image'], str)
def test_answer_get_registered_queries():
db = _get_test_db()
qm = QueryManager(db=db, model_managers=[test_mm])
qm.db.put_queries('*****@*****.**',
1,
query_object, ['test'],
subscribe=True)
qm.answer_registered_queries('test')
results = qm.get_registered_queries('*****@*****.**')
assert len(results) == 1
assert results[0]['model'] == 'test'
assert results[0]['query'] == simple_query
assert isinstance(results[0]['response'], str)
assert 'BRAF activates MAP2K1.' in results[0]['response'], \
results[0]['response']
assert isinstance(results[0]['date'], str)
def run_model_tests_from_s3(model_name,
upload_mm=True,
upload_results=True,
upload_stats=True,
registered_queries=True,
db=None):
"""Run a given set of tests on a given model, both loaded from S3.
After loading both the model and the set of tests, model/test overlap
is determined using a ScopeTestConnector and tests are run.
Parameters
----------
model_name : str
Name of EmmaaModel to load from S3.
upload_mm : Optional[bool]
Whether to upload a model manager instance to S3 as a pickle file.
Default: True
upload_results : Optional[bool]
Whether to upload test results to S3 in JSON format. Can be set
to False when running tests. Default: True
upload_stats : Optional[bool]
Whether to upload latest statistics about model and a test.
Default: True
registered_queries : Optional[bool]
If True, registered queries are fetched from the database and
executed, the results are then saved to the database. Default: True
db : Optional[emmaa.db.manager.EmmaaDatabaseManager]
If given over-rides the default primary database.
Returns
-------
emmaa.model_tests.ModelManager
Instance of ModelManager containing the model data, list of applied
tests and the test results.
emmaa.analyze_test_results.StatsGenerator
Instance of StatsGenerator containing statistics about model and test.
"""
model = EmmaaModel.load_from_s3(model_name)
test_corpus = model.test_config.get('test_corpus',
'large_corpus_tests.pkl')
tests = load_tests_from_s3(test_corpus)
mm = ModelManager(model)
if upload_mm:
save_model_manager_to_s3(model_name, mm)
tm = TestManager([mm], tests)
tm.make_tests(ScopeTestConnector())
tm.run_tests()
results_json_dict = mm.results_to_json()
results_json_str = json.dumps(results_json_dict, indent=1)
# Optionally upload test results to S3
if upload_results:
client = get_s3_client(unsigned=False)
date_str = make_date_str()
result_key = f'results/{model_name}/results_{date_str}.json'
logger.info(f'Uploading test results to {result_key}')
client.put_object(Bucket='emmaa',
Key=result_key,
Body=results_json_str.encode('utf8'))
tr = TestRound(results_json_dict)
sg = StatsGenerator(model_name, latest_round=tr)
sg.make_stats()
# Optionally upload statistics to S3
if upload_stats:
sg.save_to_s3()
if registered_queries:
qm = QueryManager(db=db, model_managers=[mm])
qm.answer_registered_queries(model_name)
return (mm, sg)
