def test_report_one_query():
db = _get_test_db()
qm = QueryManager(db=db, model_managers=[test_mm])
# Using results from db
qm.db.put_queries('*****@*****.**',
1,
query_object, ['test'],
subscribe=True)
qm.db.put_results('test', [(query_object, 'pysb', test_response),
(query_object, 'pysb', query_not_appl)])
str_msg = qm.get_report_per_query('test', query_object)[0]
assert str_msg
assert 'A new result to query' in str_msg
assert 'Query is not applicable for this model' in str_msg
assert 'BRAF activates MAP2K1.' in str_msg
# String report given two responses explicitly
str_msg = qm.make_str_report_one_query('test', query_object, 'pysb',
test_response, query_not_appl)
assert str_msg
assert 'A new result to query' in str_msg, str_msg
assert 'Query is not applicable for this model' in str_msg
assert 'BRAF activates MAP2K1.' in str_msg
# Html report given two responses explicitly
html_msg = qm.make_html_one_query_report('test', query_object, 'pysb',
test_response, query_not_appl)
assert html_msg
assert 'A new result to query' in html_msg
assert 'Query is not applicable for this model' in html_msg
assert processed_link in html_msg
def test_answer_immediate_query():
db = _get_test_db()
qm = QueryManager(db=db, model_managers=[test_mm])
results = qm.answer_immediate_query('*****@*****.**',
1,
query_object, ['test'],
subscribe=False)
assert len(results) == 1
assert results[0]['model'] == 'test'
assert results[0]['query'] == simple_query
assert isinstance(results[0]['response'], str)
assert 'BRAF activates MAP2K1.' in results[0]['response'], \
results[0]['response']
assert isinstance(results[0]['date'], str)
def test_user_query_delta():
db = _get_test_db()
qm = QueryManager(db=db, model_managers=[test_mm])
# Using results from db
qm.db.put_queries(test_email, 1, query_object, ['test'],
subscribe=True)
qm.db.put_results('test', [(query_object, 'pysb', test_response)])
qm.db.put_results('test', [(query_object, 'pysb', query_not_appl)])
str_rep, html_rep = qm.get_user_query_delta(user_email=test_email)
assert str_rep, print(str_rep)
assert html_rep, print(html_rep)
assert '</html>' in html_rep
assert '</body>' in html_rep
assert 'pysb' in html_rep
assert 'unsubscribe' in html_rep
def test_answer_get_registered_queries():
db = _get_test_db()
qm = QueryManager(db=db, model_managers=[test_mm])
# Put all types of queries in db, answer together
qm.db.put_queries(test_email, 1, query_object, ['test'], subscribe=True)
qm.db.put_queries(test_email, 1, dyn_query, ['test'], subscribe=True)
qm.db.put_queries(test_email, 1, open_query, ['test'], subscribe=True)
qm.db.put_queries(test_email, 1, interv_query, ['test'], subscribe=True)
qm.answer_registered_queries('test')
# Retrieve results for path query
results = qm.get_registered_queries(test_email, 'path_property')
qh = query_object.get_hash_with_model('test')
assert qh in results
assert len(results) == 1
assert results[qh]['model'] == 'test'
assert results[qh]['query'] == simple_query
assert isinstance(results[qh]['date'], str)
assert results[qh]['pysb'] == ['Pass', [test_response['3801854542']]], \
(results[qh]['pysb'], test_response['3801854542'])
assert results[qh]['pybel'] == ['Pass', [test_response['3801854542']]]
assert results[qh]['signed_graph'][0] == 'Pass'
assert results[qh]['unsigned_graph'][0] == 'Pass'
# Retrieve results for dynamic query
results = qm.get_registered_queries(test_email, 'dynamic_property')
qh = dyn_query.get_hash_with_model('test')
assert qh in results
assert results[qh]['model'] == 'test'
assert results[qh]['query'] == 'Active MAP2K1 is eventually high.'
assert isinstance(results[qh]['date'], str)
assert results[qh]['result'] == [
'Pass', 'Satisfaction rate is 100% after 2 simulations.'
]
assert isinstance(results[qh]['image'], str)
# Retrieve results for open query
results = qm.get_registered_queries(test_email, 'open_search_query')
qh = open_query.get_hash_with_model('test')
assert qh in results
assert results[qh]['model'] == 'test'
assert results[qh]['query'] == 'What does BRAF activate?'
assert isinstance(results[qh]['date'], str)
for mc_type in ['pysb', 'pybel', 'signed_graph', 'unsigned_graph']:
assert results[qh][mc_type][0] == 'Pass'
assert isinstance(results[qh][mc_type][1], list)
assert test_response['3801854542']['path'] in [
res['path'] for res in results[qh][mc_type][1]
]
# Retrieve results for intervention query
results = qm.get_registered_queries(test_email,
'simple_intervention_property')
qh = interv_query.get_hash_with_model('test')
assert qh in results, results
assert results[qh]['model'] == 'test'
assert results[qh]['query'] == 'BRAF increases active MAP2K1.'
assert isinstance(results[qh]['date'], str)
assert results[qh]['result'] == [
'Pass', 'Yes, the amount of target entity increased.'
]
assert isinstance(results[qh]['image'], str)
def test_answer_get_registered_queries():
db = _get_test_db()
qm = QueryManager(db=db, model_managers=[test_mm])
qm.db.put_queries('*****@*****.**',
1,
query_object, ['test'],
subscribe=True)
qm.answer_registered_queries('test')
results = qm.get_registered_queries('*****@*****.**')
assert len(results) == 1
assert results[0]['model'] == 'test'
assert results[0]['query'] == simple_query
assert isinstance(results[0]['response'], str)
assert 'BRAF activates MAP2K1.' in results[0]['response'], \
results[0]['response']
assert isinstance(results[0]['date'], str)
def test_immediate_dynamic():
db = _get_test_db()
qm = QueryManager(db=db, model_managers=[test_mm])
query_hashes = qm.answer_immediate_query(
test_email, 1, dyn_query, ['test'], subscribe=False)[
'dynamic_property']
assert query_hashes == [-27775603206605897], query_hashes
results = qm.retrieve_results_from_hashes(query_hashes, 'dynamic_property')
assert len(results) == 1, results
assert query_hashes[0] in results
result_values = results[query_hashes[0]]
assert result_values['model'] == 'test'
assert result_values['query'] == 'Active MAP2K1 is eventually high.'
assert isinstance(result_values['date'], str)
assert result_values['result'] == [
'Pass', 'Satisfaction rate is 100% after 2 simulations.']
assert isinstance(result_values['image'], str)
def test_immediate_intervention():
db = _get_test_db()
qm = QueryManager(db=db, model_managers=[test_mm])
query_hashes = qm.answer_immediate_query(
test_email, 1, interv_query, ['test'],
subscribe=False)['simple_intervention_property']
assert query_hashes == [14834228758745381], query_hashes
results = qm.retrieve_results_from_hashes(query_hashes,
'simple_intervention_property')
assert len(results) == 1, results
assert query_hashes[0] in results
result_values = results[query_hashes[0]]
assert result_values['model'] == 'test'
assert result_values['query'] == 'BRAF increases active MAP2K1.'
assert isinstance(result_values['date'], str)
assert result_values['result'] == [
'Pass', 'Yes, the amount of target entity increased.'
]
assert isinstance(result_values['image'], str)
def test_answer_immediate_query():
db = _get_test_db()
qm = QueryManager(db=db, model_managers=[test_mm])
query_hashes = qm.answer_immediate_query(
test_email, 1, query_object, ['test'], subscribe=False)[
'path_property']
assert query_hashes == [35683418474694258], query_hashes
results = qm.retrieve_results_from_hashes(query_hashes)
assert len(results) == 1
assert query_hashes[0] in results
result_values = results[query_hashes[0]]
assert result_values['model'] == 'test'
assert result_values['query'] == simple_query
assert isinstance(result_values['date'], str)
assert result_values['pysb'] == ['Pass', [test_response['3801854542']]], \
result_values['pysb']
assert result_values['pybel'] == ['Pass', [test_response['3801854542']]]
assert result_values['signed_graph'][0] == 'Pass'
assert result_values['unsigned_graph'][0] == 'Pass'
def test_immediate_open():
db = _get_test_db()
qm = QueryManager(db=db, model_managers=[test_mm])
query_hashes = qm.answer_immediate_query(
test_email, 1, open_query, ['test'], subscribe=False)[
'open_search_query']
assert query_hashes == [-13552944417558866], query_hashes
results = qm.retrieve_results_from_hashes(query_hashes)
assert len(results) == 1
assert query_hashes[0] in results
result_values = results[query_hashes[0]]
assert result_values['model'] == 'test'
assert result_values['query'] == 'What does BRAF activate?'
assert isinstance(result_values['date'], str)
for mc_type in ['pysb', 'pybel', 'signed_graph', 'unsigned_graph']:
assert result_values[mc_type][0] == 'Pass'
assert isinstance(result_values[mc_type][1], list)
assert test_response['3801854542']['path'] in [
res['path'] for res in result_values[mc_type][1]]
def test_report_files():
db = _get_test_db()
qm = QueryManager(db=db, model_managers=[test_mm])
qm.db.put_queries('*****@*****.**',
1,
query_object, ['test'],
subscribe=True)
qm.db.put_results('test', [(query_object, 'pysb', query_not_appl)])
results = qm.db.get_results('*****@*****.**', latest_order=1)
qm.make_str_report_per_user(results, filename='test_query_delta.txt')
report_file = join(dirname(abspath(__file__)), 'test_query_delta.txt')
with open(report_file, 'r') as f:
msg = f.read()
assert msg
assert 'This is the first result to query' in msg, msg
assert 'Query is not applicable for this model' in msg
qm.db.put_results('test', [(query_object, 'pysb', test_response)])
results = qm.db.get_results('*****@*****.**', latest_order=1)
qm.make_str_report_per_user(results, filename='new_test_query_delta.txt')
new_report_file = join(dirname(abspath(__file__)),
'new_test_query_delta.txt')
with open(new_report_file, 'r') as f:
msg = f.read()
assert msg
assert 'A new result to query' in msg
assert 'BRAF activates MAP2K1.' in msg
app.register_blueprint(path_temps)
app.config['DEBUG'] = True
app.config['SECRET_KEY'] = os.environ.get('EMMAA_SERVICE_SESSION_KEY', '')
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
DEVMODE = int(os.environ.get('DEVMODE', 0))
GLOBAL_PRELOAD = int(os.environ.get('GLOBAL_PRELOAD', 0))
TITLE = 'emmaa title'
ALL_MODEL_TYPES = ['pysb', 'pybel', 'signed_graph', 'unsigned_graph']
LINKAGE_SYMBOLS = {'LEFT TACK': '\u22a3', 'RIGHTWARDS ARROW': '\u2192'}
link_list = [('/home', 'EMMAA Dashboard'), ('/query?tab=static', 'Queries')]
pass_fail_msg = 'Click to see detailed results for this test'
stmt_db_link_msg = 'Click to see the evidence for this statement'
SC, jwt = config_auth(app)
qm = QueryManager()
def _sort_pass_fail(row):
def _translator(status):
if status.lower() == 'pass':
return 0
elif status.lower() == 'fail':
return 1
elif status.lower() == 'n_a':
return 2
else:
raise ValueError(f'Status {status} not handled in sorting test '
f'table')
# First sort on count of passing tests per row, then model type from
def run_model_tests_from_s3(model_name,
upload_mm=True,
upload_results=True,
upload_stats=True,
registered_queries=True,
db=None):
"""Run a given set of tests on a given model, both loaded from S3.
After loading both the model and the set of tests, model/test overlap
is determined using a ScopeTestConnector and tests are run.
Parameters
----------
model_name : str
Name of EmmaaModel to load from S3.
upload_mm : Optional[bool]
Whether to upload a model manager instance to S3 as a pickle file.
Default: True
upload_results : Optional[bool]
Whether to upload test results to S3 in JSON format. Can be set
to False when running tests. Default: True
upload_stats : Optional[bool]
Whether to upload latest statistics about model and a test.
Default: True
registered_queries : Optional[bool]
If True, registered queries are fetched from the database and
executed, the results are then saved to the database. Default: True
db : Optional[emmaa.db.manager.EmmaaDatabaseManager]
If given over-rides the default primary database.
Returns
-------
emmaa.model_tests.ModelManager
Instance of ModelManager containing the model data, list of applied
tests and the test results.
emmaa.analyze_test_results.StatsGenerator
Instance of StatsGenerator containing statistics about model and test.
"""
model = EmmaaModel.load_from_s3(model_name)
test_corpus = model.test_config.get('test_corpus',
'large_corpus_tests.pkl')
tests = load_tests_from_s3(test_corpus)
mm = ModelManager(model)
if upload_mm:
save_model_manager_to_s3(model_name, mm)
tm = TestManager([mm], tests)
tm.make_tests(ScopeTestConnector())
tm.run_tests()
results_json_dict = mm.results_to_json()
results_json_str = json.dumps(results_json_dict, indent=1)
# Optionally upload test results to S3
if upload_results:
client = get_s3_client(unsigned=False)
date_str = make_date_str()
result_key = f'results/{model_name}/results_{date_str}.json'
logger.info(f'Uploading test results to {result_key}')
client.put_object(Bucket='emmaa',
Key=result_key,
Body=results_json_str.encode('utf8'))
tr = TestRound(results_json_dict)
sg = StatsGenerator(model_name, latest_round=tr)
sg.make_stats()
# Optionally upload statistics to S3
if upload_stats:
sg.save_to_s3()
if registered_queries:
qm = QueryManager(db=db, model_managers=[mm])
qm.answer_registered_queries(model_name)
return (mm, sg)
