def update_with_cord19(self):
"""Update model with new CORD19 dataset statements."""
current_stmts = self.get_indra_stmts()
drug_stmts = load_pickle_from_s3('indra-covid19', 'drug_stmts.pkl')
gordon_stmts = load_pickle_from_s3('indra-covid19',
'gordon_ndex_stmts.pkl')
virhostnet_stmts = load_pickle_from_s3('indra-covid19',
'virhostnet_stmts.pkl')
ctd_stmts = load_pickle_from_s3('indra-covid19', 'ctd_stmts.pkl')
logger.info(f'Loaded {len(current_stmts)} current model statements, '
f'{len(drug_stmts)} drug statements, {len(gordon_stmts)} '
f'Gordon statements, {len(virhostnet_stmts)} '
f'VirHostNet statements, {len(ctd_stmts)} CTD statements.')
other_stmts = drug_stmts + gordon_stmts + virhostnet_stmts + ctd_stmts
new_stmts = make_model_stmts(current_stmts, other_stmts)
self.stmts = to_emmaa_stmts(new_stmts, datetime.datetime.now(), [])
def update_from_files(self, files_config):
"""Add custom statements from files.
Relevant part of reading config should look similar to:
{"other_files": [
{
"bucket": "indra-covid19",
"filename": "ctd_stmts.pkl",
"metadata": {"internal": true, "curated": true}
}
]
}
"""
new_estmts = []
for file_dict in files_config:
bucket = file_dict['bucket']
fname = file_dict['filename']
metadata = file_dict['metadata']
file_stmts = load_pickle_from_s3(bucket, fname)
logger.info(f'Loaded {len(file_stmts)} statements from {fname}.')
file_estmts = to_emmaa_stmts(file_stmts, datetime.datetime.now(),
[], metadata)
new_estmts += file_estmts
return new_estmts
def load_model_manager_from_s3(model_name=None,
key=None,
bucket=EMMAA_BUCKET_NAME):
# First try find the file from specified key
if key:
try:
model_manager = load_pickle_from_s3(bucket, key)
if not model_manager.model.assembled_stmts:
stmts, _ = get_assembled_statements(model_manager.model.name,
strip_out_date(
model_manager.date_str,
'date'),
bucket=bucket)
model_manager.model.assembled_stmts = stmts
return model_manager
except Exception as e:
logger.info('Could not load the model manager directly')
logger.info(e)
if not model_name:
model_name = key.split('/')[1]
date = strip_out_date(key, 'date')
logger.info('Trying to load model manager from statements')
try:
model_manager = ModelManager.load_from_statements(
model_name, date=date, bucket=bucket)
return model_manager
except Exception as e:
logger.info('Could not load the model manager from '
'statements')
logger.info(e)
return None
# Now try find the latest key for given model
if model_name:
# Versioned
key = find_latest_s3_file(bucket,
f'results/{model_name}/model_manager_',
'.pkl')
if key is None:
# Non-versioned
key = f'results/{model_name}/latest_model_manager.pkl'
return load_model_manager_from_s3(model_name=model_name,
key=key,
bucket=bucket)
# Could not find either from key or from model name.
logger.info('Could not find the model manager.')
return None
def load_stmts_from_s3(model_name, bucket=EMMAA_BUCKET_NAME):
"""Return the list of EMMAA Statements constituting the latest model.
Parameters
----------
model_name : str
The name of the model whose config should be loaded.
Returns
-------
stmts : list of emmaa.statements.EmmaaStatement
The list of EMMAA Statements in the latest model version.
"""
base_key = f'models/{model_name}'
latest_model_key = find_latest_s3_file(bucket, f'{base_key}/model_',
extension='.pkl')
logger.info(f'Loading model state from {latest_model_key}')
stmts = load_pickle_from_s3(bucket, latest_model_key)
return stmts, latest_model_key
def update_with_cord19(self):
"""Update model with new CORD19 dataset statements."""
# Using local import to avoid dependency
from covid_19.emmaa_update import make_model_stmts
current_stmts = self.get_indra_stmts()
default_filenames = [
'drug_stmts_v2.pkl', 'gordon_ndex_stmts.pkl',
'virhostnet_stmts.pkl', 'ctd_stmts.pkl']
if isinstance(self.reading_config['cord19_update'], dict):
fnames = self.reading_config['cord19_update'].get(
'filenames', default_filenames)
else: # if it's a boolean
fnames = default_filenames
other_stmts = []
for fname in fnames:
file_stmts = load_pickle_from_s3('indra-covid19', fname)
logger.info(f'Loaded {len(file_stmts)} statements from {fname}.')
other_stmts += file_stmts
new_stmts, paper_ids = make_model_stmts(current_stmts, other_stmts)
self.stmts = to_emmaa_stmts(new_stmts, datetime.datetime.now(), [])
self.add_paper_ids(paper_ids, 'TRID')
def load_tests_from_s3(test_name, bucket=EMMAA_BUCKET_NAME):
"""Load Emmaa Tests with the given name from S3.
Parameters
----------
test_name : str
Looks for a test file in the emmaa bucket on S3 with key
'tests/{test_name}'.
Return
------
list of EmmaaTest
List of EmmaaTest objects loaded from S3.
"""
prefix = f'tests/{test_name}'
try:
test_key = find_latest_s3_file(bucket, prefix, '.pkl')
except ValueError:
test_key = f'tests/{test_name}.pkl'
logger.info(f'Loading tests from {test_key}')
tests = load_pickle_from_s3(bucket, test_key)
return tests, test_key
def load_model_manager_from_s3(model_name=None,
key=None,
bucket=EMMAA_BUCKET_NAME):
# First try find the file from specified key
if key:
try:
model_manager = load_pickle_from_s3(bucket, key)
return model_manager
except Exception as e:
logger.info('Could not load the model manager')
logger.info(e)
# Now try find the latest key for given model
if model_name:
# Versioned
key = find_latest_s3_file(bucket,
f'results/{model_name}/model_manager_',
'.pkl')
if key is None:
# Non-versioned
key = f'results/{model_name}/latest_model_manager.pkl'
return load_model_manager_from_s3(key=key, bucket=bucket)
# Could not find either from key or from model name.
logger.info('Could not find the model manager.')
return None