def trial_request(trial_id):
# determine mode.
mode = request.args.get('delete')
if mode == '1':
delete = True
else:
delete = False
# set headers.
headers = {
'Content-Type': 'application/json',
'Authorization': 'Basic ' + base64.b64encode(f'{API_TOKEN}'.encode('utf-8')).decode('utf-8'),
}
# create the request.
qstr = "where=%s" % json.dumps(({"protocol_no": trial_id}))
# get the trial list from db.
url = '%s/trial?%s' % (API_ADDRESS, qstr)
r = requests.get(url, headers=headers)
output = r.json()
# sanity check item.
if len(output['_items']) == 0:
return "No trial found with id: %s, try uploading from the file" % trial_id
trial = r.json()['_items'][0]
mongo_id = trial['_id']
etag = trial['_etag']
# add the defaults
headers['If-Match'] = etag
# strip meta
for key in list(trial.keys()):
if key[0] == "_":
del trial[key]
# return the file if we aren't looking to delete it.
if not delete:
response = _yamlize(trial, trial_id)
else:
# issue delete.
get_db().trial.remove({"protocol_no": trial_id})
remove_trial_from_elasticsearch_by_es_id(trial_id)
if r.status_code < 300:
return redirect("/curate")
else:
return r.text
return response
def test_extract_cancer_types(self):
m = MatchEngine(get_db())
match_tree = match_tree_example
g = m.create_match_tree(match_tree)
pmt = ParseMatchTree(g)
cancer_type_dict = pmt.extract_cancer_types()
assert sorted(cancer_type_dict['diagnoses']) == sorted(
['Ocular Melanoma']), cancer_type_dict['diagnoses']
assert sorted(cancer_type_dict['cancer_types_expanded']) == sorted([
'Ocular Melanoma', 'Uveal Melanoma', 'Conjunctival Melanoma'
]), cancer_type_dict['cancer_types_expanded']
assert sorted(cancer_type_dict['excluded_cancer_types']
) == [], cancer_type_dict['excluded_cancer_types']
assert cancer_type_dict['primary_cancer_types'] == [
'Eye'
], cancer_type_dict['primary_cancer_types']
m = MatchEngine(get_db())
match_tree = match_tree_example2
g = m.create_match_tree(match_tree)
pmt = ParseMatchTree(g)
cancer_type_dict = pmt.extract_cancer_types()
assert sorted(cancer_type_dict['diagnoses']) == sorted(
['_SOLID_']), cancer_type_dict['diagnoses']
assert 'Acute Lymphoid Leukemia' not in cancer_type_dict[
'cancer_types_expanded'], cancer_type_dict['cancer_types_expanded']
assert sorted(cancer_type_dict['excluded_cancer_types']
) == [], cancer_type_dict['excluded_cancer_types']
assert cancer_type_dict['primary_cancer_types'] == [
'All Solid Tumors'
], cancer_type_dict['primary_cancer_types']
m = MatchEngine(get_db())
match_tree = match_tree_example3
g = m.create_match_tree(match_tree)
pmt = ParseMatchTree(g)
cancer_type_dict = pmt.extract_cancer_types()
assert sorted(cancer_type_dict['diagnoses']) == sorted(
['_LIQUID_']), cancer_type_dict['diagnoses']
assert 'Acute Lymphoid Leukemia' in cancer_type_dict[
'cancer_types_expanded'], cancer_type_dict['cancer_types_expanded']
assert sorted(cancer_type_dict['excluded_cancer_types']
) == [], cancer_type_dict['excluded_cancer_types']
assert cancer_type_dict['primary_cancer_types'] == [
'All Liquid Tumors'
], cancer_type_dict['primary_cancer_types']
def _validate_normalized(self, normalized, field, value):
''' use normalization dictionary to control values
return validation error if it isn't a true value
in the dictionary'''
# load the mapping
db = get_db()
normalize_table = db['normalize'].find_one()
if not normalize_table:
return
for key, val in value.iteritems():
if (isinstance(val, str) or isinstance(val, unicode)) and val[0] == "!":
val = val[1:]
if key == 'oncotree_primary_diagnosis':
if val not in normalize_table['values']['oncotree_primary_diagnosis'].values():
self._error(field, "%s is not a valid value for oncotree_primary_diagnosis" % val)
elif key == 'hugo_symbol':
if 'hugo_symbol' in normalize_table['values'] and val not in normalize_table['values']['hugo_symbol']:
self._error(422, "%s is not a valid hugo symbol" % val)
elif isinstance(val, dict):
self._validate_normalized(normalized, field, val)
elif isinstance(val, list):
for subitem in val:
if isinstance(subitem, dict):
self._validate_normalized(normalized, field, subitem)
def after_request(response):
# test for response
is_response, item_id = parse_response(request.url)
# only redirect if response
if is_response:
# do the redirect.
db = get_db()
item = db['response'].find_one({"_id": ObjectId(item_id)})
# if it exists return the redirect.
if item is not None:
return make_response(redirect(item['return_url']))
# remove these headers
response.headers.add('Last-Modified', datetime.now())
response.headers.add('Expires', '-1')
# dont use these headers because IE11 doesn't like them with fonts.
if response.content_type != 'application/json':
response.headers.add(
'Cache-Control',
'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
)
response.headers.add('Pragma', 'no-cache')
return response
def get_public_stats(resp):
"""Returns the number of clinical trials and the number of patients to the UI"""
db = database.get_db()
resp['_items'].append({
'num_clinical_trials': len(list(db['trial'].distinct('protocol_no'))),
'num_patients': len(list(db['clinical'].distinct("MRN")))
})
def email_matchminer(patient, status):
"""Emails the matchminer service email account to keep track of physician responses to the email blast"""
cur_stamp = datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y")
html = '''<html><head></head><body>%s</body></html>''' % emails.EMAIL_BLAST_RESPONSE_BODY.format(
patient['ORD_PHYSICIAN_NAME'],
patient['FIRST_NAME'],
patient['LAST_NAME'],
patient['MRN'],
status,
cur_stamp
)
# put email object in db
db = database.get_db()
email_item = {
'email_from': settings.EMAIL_AUTHOR_PROTECTED,
'email_to': settings.EMAIL_AUTHOR_PROTECTED,
'subject': 'Email Blast Response',
'body': html,
'cc': [],
'sent': False,
'num_failures': 0,
'errors': []
}
db['email'].insert(email_item)
def email_filter_owner(user, patient, filter_name, variant, status):
if variant:
alteration = get_alterations(variant)
else:
alteration = 'unknown'
cur_stamp = datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y")
html = _user_email_filter_owner(user, patient, alteration, status, cur_stamp)
subject = '%s : Patient %s' % (filter_name, status)
author = settings.EMAIL_AUTHOR_PROTECTED
# put email object in db
db = database.get_db()
email_item = {
'email_from': author,
'email_to': user['email'],
'subject': subject,
'body': html,
'cc': settings.EMAIL_TRIAL_CC_LIST,
'sent': False,
'num_failures': 0,
'errors': []
}
db['email'].insert(email_item)
def _get_new_filters_match_counts(team_id, filters_with_new_matches, run_id):
"""
Aggregate new filter match counts by team
:param team_id: Team with associated filters
:param filters_with_new_matches: List of filter IDs associated with single team
:return:
"""
db = database.get_db()
new_filter_match_counts = {}
for filter_id in filters_with_new_matches:
matches_query = {
"TEAM_ID": team_id,
"FILTER_ID": filter_id,
"is_disabled": False,
"_me_id": run_id
}
num_matches = len(list(db.match.find(matches_query, {"_id": 1})))
if num_matches < 1:
continue
else:
filter_ = list(db.filter.find({'_id': filter_id}))[0]
new_filter_match_counts[filter_id] = {
"num_matches": num_matches,
"description": filter_['description'],
"label": filter_['label'],
"protocol_id": filter_['protocol_id']
}
return new_filter_match_counts
def get_public_stats(resp):
"""Returns the number of clinical trials and the number of patients to the UI"""
db = database.get_db()
resp['_items'].append({
'num_clinical_trials': len(list(db['trial'].distinct('protocol_no'))),
'num_patients': len(list(db['clinical'].distinct("MRN")))
})
def setUp(self):
# prepare the app
settings_file = os.path.join(SETTINGS_DIR, 'settings.py')
self.app = eve.Eve(settings=settings_file,
url_converters=None,
auth=TokenAuth,
validator=ConsentValidatorEve)
# load normalization.
with open(BSON_FILE) as fin:
mappings = list(bson.decode_iter(fin.read()))
# add normalization.
self.db = get_db()
for mapping in mappings:
self.db['normalize'].drop()
self.db['normalize'].insert(mapping)
# create the validator.
resource_def = self.app.config['DOMAIN']['trial']
schema = resource_def['schema']
#print self.app.config['SOURCES']
with self.app.app_context():
self.v = self.app.validator(schema=schema, resource='trial')
def setUp(self):
# prepare the app
settings_file = os.path.join(SETTINGS_DIR, 'settings.py')
self.app = eve.Eve(settings=settings_file,
url_converters = None,
auth = TokenAuth,
validator = ConsentValidatorEve)
# load normalization.
with open(BSON_FILE) as fin:
mappings = list(bson.decode_iter(fin.read()))
# add normalization.
self.db = get_db()
for mapping in mappings:
self.db['normalize'].drop()
self.db['normalize'].insert(mapping)
# create the validator.
resource_def = self.app.config['DOMAIN']['trial']
schema = resource_def['schema']
#print self.app.config['SOURCES']
with self.app.app_context():
self.v = self.app.validator(schema=schema, resource='trial')
def build_redirect_url_epic(user, trial_match):
"""
When redirecting to a patient for integration with EPIC, set appropriate tokens, headers, and cookies
:param user:
:param trial_match:
:return:
"""
db = database.get_db()
# Set token. Must match token set in cookie
token = str(uuid.uuid4())
db['user'].update_one({'_id': user['_id']}, {
'$set': {'token': token, 'last_auth': datetime.datetime.now()}
})
# Build redirect URL
patient_id = str(trial_match["_id"])
url = FRONT_END_ADDRESS + 'dashboard/patients/' + patient_id + '?epic=true'
redirect_to_patient = redirect(url)
logging.info('[EPIC] redirect to URL: ' + url)
# Build response headers
response = app.make_response(redirect_to_patient)
response.headers.add('Authorization', 'Basic' + base64.b64encode(f'{token}:'.encode('utf-8')).decode())
response.headers.add('Last-Modified', datetime.datetime.now())
response.headers.add('Cache-Control', 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0')
response.headers.add('Pragma', 'no-cache')
response.headers.add('Content-Type', 'application/json')
response.headers.add('Location', url)
# Set cookies
response.set_cookie('user_id', value=str(user['_id']), expires=0)
response.set_cookie('team_id', value=str(user['teams'][0]), expires=0)
response.set_cookie('token', value=token, expires=0)
return response
def email_filter_owner(user, patient, filter_name, variant, status):
if variant:
alteration = get_alterations(variant)
else:
alteration = 'unknown'
cur_stamp = datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y")
html = _user_email_filter_owner(user, patient, alteration, status,
cur_stamp)
subject = '%s : Patient %s' % (filter_name, status)
author = settings.EMAIL_AUTHOR_PROTECTED
# put email object in db
db = database.get_db()
email_item = {
'email_from': author,
'email_to': user['email'],
'subject': subject,
'body': html,
'cc': settings.EMAIL_TRIAL_CC_LIST,
'sent': False,
'num_failures': 0,
'errors': []
}
db['email'].insert(email_item)
def email_matchminer(patient, status):
"""Emails the matchminer service email account to keep track of physician responses to the email blast"""
cur_stamp = datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y")
html = '''<html><head></head><body>%s</body></html>''' % emails.EMAIL_BLAST_RESPONSE_BODY.format(
patient['ORD_PHYSICIAN_NAME'],
patient['FIRST_NAME'],
patient['LAST_NAME'],
patient['MRN'],
status,
cur_stamp
)
# put email object in db
db = database.get_db()
email_item = {
'email_from': settings.EMAIL_AUTHOR_PROTECTED,
'email_to': settings.EMAIL_AUTHOR_PROTECTED,
'subject': 'Email Blast Response',
'body': html,
'cc': [],
'sent': False,
'num_failures': 0,
'errors': []
}
db['email'].insert(email_item)
def start_filter_run(silent=False, datapush_id=None):
"""
Wrapper function which calls rerun filters.
Creates a record in active_process collection to make sure multiple filter
matching runs are not created simultaneously.
:param silent: Whether to send emails or not
:param datapush_id: ID to append to output matches if relevant
:return:
"""
db = database.get_db()
db.active_processes.insert({"filters_running": True})
filters = list(
db.filter.find({
"temporary": False,
"status": {
"$in": [0, 1]
}
}))
transform_filter_to_CTML(filters, save=True)
_, run_id = rerun_filters(datapush_id=datapush_id)
db.active_processes.drop()
if not silent:
email_matches(run_id)
return run_id
def maintain_filters():
"""
recalculate text descriptions
:return:
"""
logging.info("maintain filters")
# get database connection.
db = database.get_db()
# find all filters status.
filters = list(db['filter'].find())
for filter in filters:
# skip inactive.
if 'temporary' not in filter:
continue
if filter['temporary'] == True:
continue
# get the text string.
try:
c, g, txt = miner.prepare_criteria(filter)
gen_txt, clin_txt = txt
description = []
except KeyError:
continue
# setup clincal portion.
cancer, age, gender = clin_txt
c_test = cancer == ""
g_test = gender == ""
a_test = age == ""
# handle cancer sentance.
if not c_test:
description = "%s in %s" % (gen_txt, cancer)
else:
description = gen_txt
# handle the rest.
if not g_test and a_test:
description = "%s, Gender: %s" % (description, gender)
elif not g_test and not a_test:
description = "%s, Gender: %s, Age %s" % (description, gender, age)
elif g_test and not a_test:
description = "%s, Age %s" % (description, age)
# hack fix
if description == []:
description = ''
# update record with this.
result = db['filter'].update_one({"_id": filter["_id"]}, {"$set": {"description": description}})
def assess_vital_status(update, original):
"""If a patient's VITAL STATUS has been changed to deceased, remove their matches from the database"""
db = database.get_db()
if 'VITAL_STATUS' in update and update[
'VITAL_STATUS'] == 'deceased' and original[
'VITAL_STATUS'] == 'alive':
db['match'].remove({'CLINICAL_ID': original['_id']})
def maintain_matches():
"""
this function does 2 tasks on matches: 1st it determines if any matches belong to a deleted filter and
also deletes them. 2nd it adds the MRN to existing matches.
:return:
"""
logging.info("maintain matches")
# get database connection.
db = database.get_db()
# find all filters status.
filters = list(db['filter'].find())
filters_status = {}
for filter in filters:
filters_status[filter['_id']] = filter['status']
# build a patient lu.
patient_lu = {}
# loop over each match and delete if its stale.
to_delete = list()
for match in db['match'].find():
# check if the filter is deleted.
delete = False
if match['FILTER_ID'] not in filters_status:
delete = True
elif filters_status[match['FILTER_ID']] != 1:
delete = True
if delete:
db['match'].remove({"_id": match['_id']})
to_delete.append(match['_id'])
# update it if not deleted.
if not delete:
# get patient info.
if match['CLINICAL_ID'] not in patient_lu:
patient_lu[match['CLINICAL_ID']] = db['clinical'].find_one(
{"_id": match['CLINICAL_ID']})
# get MRN.
patient_mrn = patient_lu[match['CLINICAL_ID']]['MRN']
# update the record.
db['match'].update_one({"_id": match['_id']},
{"$set": {
"PATIENT_MRN": patient_mrn
}})
# return the count.
return to_delete
def test_extract_hr_status(self):
m = MatchEngine(get_db())
match_tree = trial['treatment_list']['step'][0]['match'][5]
g = m.create_match_tree(match_tree)
pmt = ParseMatchTree(g)
hr = pmt.extract_hr_status()
assert sorted(hr) == sorted(['HER2 Negative', 'ER Negative', 'PR Positive'])
def setUp(self):
# set up fake db
self.db = get_db()
self.db['response'].drop()
self.db['clinical'].insert_many([self.fake_patient, self.fake_patient2])
self.db['match'].insert_many([self.match_contacted, self.match_flagged])
self.db['filter'].insert_one(self.fake_filter)
self.db['user'].insert_one(self.fake_filter_owner)
def update_response(item):
if 'allow_update' not in item or not item['allow_update']:
abort(501)
bin_key = settings.match_status_mapping
db = database.get_db()
# add time_clicked and ip_address to response
db['response'].update_one({'_id': item['_id']}, {
'$set': {
'time_clicked':
formatdate(time.mktime(datetime.datetime.now().timetuple()),
localtime=False,
usegmt=True),
'ip_address':
_get_ip()
}
})
# Get match from match table
match_db = db['match']
match = match_db.find_one({'_id': item['match_id']})
patient = db['clinical'].find_one({'_id': match['CLINICAL_ID']})
# get clinician response
status = item['match_status']
# get new status
new_status = bin_key[status]
# Update match status
result = match_db.update_one({"_id": item["match_id"]}, {
"$set": {
"MATCH_STATUS": new_status,
"_updated": datetime.datetime.utcnow().replace(microsecond=0)
}
})
if status == 'Eligible' or status == 'Deferred':
# get the filter owner
user_db = db['user']
filter_owner = user_db.find_one({'_id': item['notification_id']})
# get filter
filter_name = match['FILTER_NAME']
# get the list of matched variant
variant = []
genomic_db = db['genomic']
for v in match['VARIANTS']:
variant.extend(list(genomic_db.find({'_id': v})))
email_filter_owner(filter_owner, patient, filter_name, variant, status)
elif status == 'Not Eligible' or status == 'Deceased':
email_matchminer(patient, status)
def test_extract_hr_status(self):
m = MatchEngine(get_db())
match_tree = trial['treatment_list']['step'][0]['match'][5]
g = m.create_match_tree(match_tree)
pmt = ParseMatchTree(g)
hr = pmt.extract_hr_status()
assert sorted(hr) == sorted(['HER2 Negative', 'ER Negative', 'PR Positive'])
def setUp(self):
# set up fake db
self.db = get_db()
self.db['response'].drop()
self.db['clinical'].insert_many([self.fake_patient, self.fake_patient2])
self.db['match'].insert_many([self.match_contacted, self.match_flagged])
self.db['filter'].insert_one(self.fake_filter)
self.db['user'].insert_one(self.fake_filter_owner)
def test_extract_cancer_types(self):
m = MatchEngine(get_db())
match_tree = match_tree_example
g = m.create_match_tree(match_tree)
pmt = ParseMatchTree(g)
cancer_type_dict = pmt.extract_cancer_types()
assert sorted(cancer_type_dict['diagnoses']) == sorted([
'Ocular Melanoma'
]), cancer_type_dict['diagnoses']
assert sorted(cancer_type_dict['cancer_types_expanded']) == sorted([
'Ocular Melanoma',
'Uveal Melanoma',
'Conjunctival Melanoma'
]), cancer_type_dict['cancer_types_expanded']
assert sorted(cancer_type_dict['excluded_cancer_types']) == [], cancer_type_dict['excluded_cancer_types']
assert cancer_type_dict['primary_cancer_types'] == ['Eye'], cancer_type_dict['primary_cancer_types']
m = MatchEngine(get_db())
match_tree = match_tree_example2
g = m.create_match_tree(match_tree)
pmt = ParseMatchTree(g)
cancer_type_dict = pmt.extract_cancer_types()
assert sorted(cancer_type_dict['diagnoses']) == sorted(['_SOLID_']), cancer_type_dict['diagnoses']
assert 'Acute Lymphoid Leukemia' not in cancer_type_dict['cancer_types_expanded'], cancer_type_dict['cancer_types_expanded']
assert sorted(cancer_type_dict['excluded_cancer_types']) == [], cancer_type_dict['excluded_cancer_types']
assert cancer_type_dict['primary_cancer_types'] == ['All Solid Tumors'], cancer_type_dict['primary_cancer_types']
m = MatchEngine(get_db())
match_tree = match_tree_example3
g = m.create_match_tree(match_tree)
pmt = ParseMatchTree(g)
cancer_type_dict = pmt.extract_cancer_types()
assert sorted(cancer_type_dict['diagnoses']) == sorted(['_LIQUID_']), cancer_type_dict['diagnoses']
assert 'Acute Lymphoid Leukemia' in cancer_type_dict['cancer_types_expanded'], cancer_type_dict[
'cancer_types_expanded']
assert sorted(cancer_type_dict['excluded_cancer_types']) == [], cancer_type_dict['excluded_cancer_types']
assert cancer_type_dict['primary_cancer_types'] == ['All Liquid Tumors'], cancer_type_dict['primary_cancer_types']
def test_extract_signatures(self):
m = MatchEngine(get_db())
match_tree = trial['treatment_list']['step'][0]['match'][3]
g = m.create_match_tree(match_tree)
pmt = ParseMatchTree(g)
s = pmt.extract_signatures()
assert 'MMR-D' in s[0]
assert 'MSI-H' in s[1]
def test_extract_signatures(self):
m = MatchEngine(get_db())
match_tree = trial['treatment_list']['step'][0]['match'][3]
g = m.create_match_tree(match_tree)
pmt = ParseMatchTree(g)
s = pmt.extract_signatures()
assert 'MMR-D' in s[0]
assert 'MSI-H' in s[1]
def assess_vital_status(update, original):
"""If a patient's VITAL STATUS has been changed to deceased, remove their matches from the database"""
db = database.get_db()
if 'VITAL_STATUS' in update and update[
'VITAL_STATUS'] == 'deceased' and original[
'VITAL_STATUS'] == 'alive':
update = {'is_disabled': True, "_updated": datetime.datetime.now()}
db['match'].update_many({'CLINICAL_ID': original['_id']},
{'$set': update})
def update_response(item):
if 'allow_update' not in item or not item['allow_update']:
abort(501)
bin_key = settings.match_status_mapping
db = database.get_db()
# add time_clicked and ip_address to response
db['response'].update_one(
{'_id': item['_id']},
{'$set': {
'time_clicked': formatdate(time.mktime(datetime.datetime.now().timetuple())),
'ip_address': _get_ip()
}}
)
# Get match from match table
match_db = db['match']
match = match_db.find_one({'_id': item['match_id']})
patient = db['clinical'].find_one({'_id': match['CLINICAL_ID']})
# get clinician response
status = item['match_status']
# get new status
new_status = bin_key[status]
# Update match status
result = match_db.update_one(
{"_id": item["match_id"]},
{"$set": {
"MATCH_STATUS": new_status,
"_updated": datetime.datetime.utcnow().replace(microsecond=0)
}}
)
if status == 'Eligible' or status == 'Deferred':
# get the filter owner
user_db = db['user']
filter_owner = user_db.find_one({'_id': item['notification_id']})
# get filter
filter_name = match['FILTER_NAME']
# get the list of matched variant
variant = []
genomic_db = db['genomic']
for v in match['VARIANTS']:
variant.extend(list(genomic_db.find({'_id': v})))
email_filter_owner(filter_owner, patient, filter_name, variant, status)
elif status == 'Not Eligible' or status == 'Deceased':
email_matchminer(patient, status)
def run_matchengine():
"""
Computes matches between all trials in the database and the given subset list of patient MRNs
:param mrns: List of patient MRNs
:return: database collection of trial matches
"""
db = database.get_db()
me = MatchEngine(db)
me.find_trial_matches()
def run_matchengine():
"""
Computes matches between all trials in the database and the given subset list of patient MRNs
:param mrns: List of patient MRNs
:return: database collection of trial matches
"""
db = database.get_db()
me = MatchEngine(db)
me.find_trial_matches()
def is_engine_running():
db = database.get_db()
running_processes = list(db.active_processes.find())
is_running = True if len(running_processes) > 0 else False
logging.info(f"/api/is_matchengine_running {str(is_running)}")
resp = Response(response=json.dumps({"is_running": is_running}),
status=200,
mimetype="application/json")
return resp
def entry_replace(resource, item, original):
# get all records from db
db = get_db()
records = db['normalize'].find_one({"key": resource})
if records is None:
return
mapping = records['values']
# check all keys.
NormalizeHook._recursive_check(item, mapping)
def entry_replace(resource, item, original):
# get all records from db
db = get_db()
records = db['normalize'].find_one({"key": resource})
if records is None:
return
mapping = records['values']
# check all keys.
NormalizeHook._recursive_check(item, mapping)
def maintain_matches():
"""
this function does 2 tasks on matches: 1st it determines if any matches belong to a deleted filter and
also deletes them. 2nd it adds the MRN to existing matches.
:return:
"""
logging.info("maintain matches")
# get database connection.
db = database.get_db()
# find all filters status.
filters = list(db['filter'].find())
filters_status = {}
for filter in filters:
filters_status[filter['_id']] = filter['status']
# build a patient lu.
patient_lu = {}
# loop over each match and delete if its stale.
to_delete = list()
for match in db['match'].find():
# check if the filter is deleted.
delete = False
if match['FILTER_ID'] not in filters_status:
delete = True
elif filters_status[match['FILTER_ID']] != 1:
delete = True
if delete:
db['match'].remove({"_id": match['_id']})
to_delete.append(match['_id'])
# update it if not deleted.
if not delete:
# get patient info.
if match['CLINICAL_ID'] not in patient_lu:
patient_lu[match['CLINICAL_ID']] = db['clinical'].find_one({"_id": match['CLINICAL_ID']})
# get MRN.
patient_mrn = patient_lu[match['CLINICAL_ID']]['MRN']
# update the record.
db['match'].update_one({"_id": match['_id']}, {"$set": {"PATIENT_MRN": patient_mrn}})
# return the count.
return to_delete
def entry_insert(resource, items):
# get all records from db
db = get_db()
records = db['normalize'].find_one({"key": resource})
if records is None:
return
mapping = records['values']
# check all keys.
for item in items:
NormalizeHook._recursive_check(item, mapping)
def entry_insert(resource, items):
# get all records from db
db = get_db()
records = db['normalize'].find_one({"key": resource})
if records is None:
return
mapping = records['values']
# check all keys.
for item in items:
NormalizeHook._recursive_check(item, mapping)
def bootstrap_restore(args):
# restore the important collections.
data_dir = settings.DATA_DIR_PROD
if data_dir == "":
data_dir = settings.DATA_DIR
restore_collections(data_dir, settings)
# fetch the database.
db = get_db()
db['clinical'].update_many({"VITAL_STATUS": "dead"}, {"$set": {"VITAL_STATUS": "deceased"}})
def reannotate_trials():
db = database.get_db()
trials = list(db['trial'].find())
# modify trials to be inserted in bulk later
events.trial_insert(trials)
# re-insert.
for trial in trials:
db['trial'].delete_one({'_id': trial['_id']})
db['trial'].insert_one(trial)
logging.info("DONE")
def test_extract_variants(self):
m = MatchEngine(get_db())
match_tree = trial['treatment_list']['step'][0]['match'][0]
g = m.create_match_tree(match_tree)
pmt = ParseMatchTree(g)
v = pmt.extract_variants()
assert 'BRAF V600E' in v['variants']
assert 'BRAF V600K' in v['variants']
assert 'KRAS any' in v['variants']
assert 'EGFR wt' in v['wts']
assert len(v['variants']) == 3
assert len(v['wts']) == 1
match_tree = trial['treatment_list']['step'][0]['match'][1]
g = m.create_match_tree(match_tree)
pmt = ParseMatchTree(g)
v = pmt.extract_variants()
assert 'PTEN CNV' in v['cnvs']
assert 'BRCA1 SV' in v['svs']
assert 'BRAF V600' in v['exclusions']
assert len(v['variants']) == 0
assert len(v['cnvs']) == 1
assert len(v['svs']) == 1
assert len(v['wts']) == 0
assert len(v['exclusions']) == 1
match_tree = trial['treatment_list']['step'][0]['match'][2]
g = m.create_match_tree(match_tree)
pmt = ParseMatchTree(g)
v = pmt.extract_variants()
assert 'BRAF V600E' not in v['variants']
assert len(v['variants']) == 0
assert len(v['wts']) == 0
assert 'BRAF V600E' in v['exclusions']
match_tree = trial['treatment_list']['step'][0]['match'][4]
g = m.create_match_tree(match_tree)
pmt = ParseMatchTree(g)
v = pmt.extract_variants()
assert 'BRAF V600K' in v['variants']
assert 'EGFR any' in v['variants']
assert len(v['variants']) == 2
assert 'PTEN CNV' in v['cnvs']
assert len(v['cnvs']) == 1
assert 'KRAS' in v['exclusions']
assert 'NRAS' in v['exclusions']
assert len(v['exclusions']) == 2
assert 'NTRK1 wt' in v['wts']
assert len(v['wts']) == 1
def trial_insert(items):
# get database connection.
db = database.get_db()
# loop over each item.
for item in items:
# log this.
logging.info("trial inserted")
# build tree.
me = MatchEngine(db)
status, trial_tree = me.create_trial_tree(item, no_validate=True)
# look at every node.
genomic = {}
clinical = {}
other = {}
for n in trial_tree.nodes():
# get parent.
if 'node_id' not in trial_tree.node[n]:
continue
node_id = trial_tree.node[n]['node_id']
# look for multi-level nodes (right now its only match).
if 'match_tree' in trial_tree.node[n]:
# compress categories.
mt = trial_tree.node[n]['match_tree']
for x in mt:
if mt.node[x]['type'] == 'genomic':
insert_data_genomic(genomic, mt.node[x]['value'],
node_id)
if mt.node[x]['type'] == 'clinical':
insert_data_clinical(clinical, mt.node[x]['value'],
node_id)
# add the other nodes.
insert_data_other(trial_tree, node_id, n, other)
# create _summary, _suggest, and _elasticsearch fields
summary = Summary(clinical, genomic, other, trial_tree)
item['_summary'] = summary.create_summary(item)
autocomplete = Autocomplete(item)
item['_suggest'], item['_elasticsearch'], item['_summary']['primary_tumor_types'] = \
autocomplete.add_autocomplete()
return items
def test_extract_variants(self):
m = MatchEngine(get_db())
match_tree = trial['treatment_list']['step'][0]['match'][0]
g = m.create_match_tree(match_tree)
pmt = ParseMatchTree(g)
v = pmt.extract_variants()
assert 'BRAF V600E' in v['variants']
assert 'BRAF V600K' in v['variants']
assert 'KRAS any' in v['variants']
assert 'EGFR wt' in v['wts']
assert len(v['variants']) == 3
assert len(v['wts']) == 1
match_tree = trial['treatment_list']['step'][0]['match'][1]
g = m.create_match_tree(match_tree)
pmt = ParseMatchTree(g)
v = pmt.extract_variants()
assert 'PTEN CNV' in v['cnvs']
assert 'BRCA1 SV' in v['svs']
assert 'BRAF V600' in v['exclusions']
assert len(v['variants']) == 0
assert len(v['cnvs']) == 1
assert len(v['svs']) == 1
assert len(v['wts']) == 0
assert len(v['exclusions']) == 1
match_tree = trial['treatment_list']['step'][0]['match'][2]
g = m.create_match_tree(match_tree)
pmt = ParseMatchTree(g)
v = pmt.extract_variants()
assert 'BRAF V600E' not in v['variants']
assert len(v['variants']) == 0
assert len(v['wts']) == 0
assert 'BRAF V600E' in v['exclusions']
match_tree = trial['treatment_list']['step'][0]['match'][4]
g = m.create_match_tree(match_tree)
pmt = ParseMatchTree(g)
v = pmt.extract_variants()
assert 'BRAF V600K' in v['variants']
assert 'EGFR any' in v['variants']
assert len(v['variants']) == 2
assert 'PTEN CNV' in v['cnvs']
assert len(v['cnvs']) == 1
assert 'KRAS' in v['exclusions']
assert 'NRAS' in v['exclusions']
assert len(v['exclusions']) == 2
assert 'NTRK1 wt' in v['wts']
assert len(v['wts']) == 1
def find_filter_matches(items):
db = database.get_db()
for item in items:
do_update = False if item['temporary'] else True
num_matches, run_id = rerun_filters(filters=[item['_id']],
do_update=do_update,
datapush_id=None)
enrollments = get_enrollment(num_matches[item['_id']])
item['num_samples'] = len(num_matches[item['_id']])
item['enrollment'] = enrollments
# don't persist temporary filters
if item['status'] == 2 and item['temporary'] == True:
db.filter.remove({"_id": item['_id']})
def setUp(self):
self.db = get_db()
self.db.status.drop()
self.now = '2017-0{0}-01 05:00:00'
self.db.status.insert_many([{
'updated_genomic': 0,
'new_genomic': 10,
'updated_clinical': 0,
'new_clinical': 5,
'last_update': dt.now(),
'data_push_id': self.now.format(m)
} for m in [1, 2, 3, 4, 5, 6]])
def setUp(self, settings_file=None, url_converters=None):
super(TestResponse, self).setUp(settings_file=None, url_converters=None)
self.user_token = None
# add to database
self.db = get_db()
self.db.response.drop()
self.db['clinical'].insert_one(self.fake_patient)
self.db['match'].insert_one(self.match_flagged)
self.db['filter'].insert_one(self.fake_filter)
self.db['user'].insert_one(self.fake_filter_owner)
self.db['response'].insert_many(demo_resps)
self.response_ids = [resp['_id'] for resp in demo_resps]
self.email_ids = [email['_id'] for email in list(self.db['email'].find())]
def setUp(self):
self.db = get_db()
self.db.status.drop()
self.now = '2017-0{0}-01 05:00:00'
self.db.status.insert_many([{
'updated_genomic': 0,
'new_genomic': 10,
'updated_clinical': 0,
'new_clinical': 5,
'last_update': dt.now(),
'data_push_id': self.now.format(m)
} for m in [1, 2, 3, 4, 5, 6]])
def trial_insert(items):
# get database connection.
db = database.get_db()
# loop over each item.
for item in items:
# log this.
logging.info("trial inserted")
# build tree.
me = MatchEngine(db)
status, trial_tree = me.create_trial_tree(item, no_validate=True)
# look at every node.
genomic = {}
clinical = {}
other = {}
for n in trial_tree.nodes():
# get parent.
if 'node_id' not in trial_tree.node[n]:
continue
node_id = trial_tree.node[n]['node_id']
# look for multi-level nodes (right now its only match).
if 'match_tree' in trial_tree.node[n]:
# compress categories.
mt = trial_tree.node[n]['match_tree']
for x in mt:
if mt.node[x]['type'] == 'genomic':
insert_data_genomic(genomic, mt.node[x]['value'], node_id)
if mt.node[x]['type'] == 'clinical':
insert_data_clinical(clinical, mt.node[x]['value'], node_id)
# add the other nodes.
insert_data_other(trial_tree, node_id, n, other)
# create _summary, _suggest, and _elasticsearch fields
summary = Summary(clinical, genomic, other, trial_tree)
item['_summary'] = summary.create_summary(item)
autocomplete = Autocomplete(item)
item['_suggest'], item['_elasticsearch'], item['_summary']['primary_tumor_types'] = \
autocomplete.add_autocomplete()
return items
def test_trial_summary(self):
on_trial['_genomic'] = {'hugo_symbol': [{'value': 'TEST'}]}
on_trial['_clinical'] = {'disease_status': [{'value': ['TEST']}]}
other = {'management_group': [{'value': 'TEST'}]}
# create trial tree
db = get_db()
me = MatchEngine(db)
status, trial_tree = me.create_trial_tree(on_trial, no_validate=True)
# validate summary
summary = Summary(on_trial['_clinical'],
on_trial['_genomic'],
other,
trial_tree)
item = summary.create_summary(on_trial)
status_fields = ['drugs', 'genes', 'tumor_types', 'sponsor',
'phase_summary', 'accrual_goal', 'investigator', 'age_summary', 'protocol_number',
'disease_status', 'nct_number', 'disease_center', 'short_title',
'hormone_receptor_status']
for field in status_fields:
assert field in item, self._debug(item, field)
if field not in ['dfci_investigator', 'hormone_receptor_status']:
assert item[field], '%s| %s' % (field, item)
# remove all fields and validate that the summary will not error
del on_trial['age']
del on_trial['phase']
del on_trial['nct_id']
del on_trial['protocol_no']
del on_trial['principal_investigator']
del on_trial['cancer_center_accrual_goal_upper']
del on_trial['site_list']
del on_trial['sponsor_list']
del on_trial['drug_list']
del on_trial['staff_list']
status, trial_tree = me.create_trial_tree(on_trial, no_validate=True)
summary = Summary(on_trial['_clinical'],
on_trial['_genomic'],
other,
trial_tree)
item = summary.create_summary(on_trial)
for field in status_fields:
assert field in item, self._debug(item, field)
def test_trial_summary(self):
on_trial['_genomic'] = {'hugo_symbol': [{'value': 'TEST'}]}
on_trial['_clinical'] = {'disease_status': [{'value': ['TEST']}]}
other = {'management_group': [{'value': 'TEST'}]}
# create trial tree
db = get_db()
me = MatchEngine(db)
status, trial_tree = me.create_trial_tree(on_trial, no_validate=True)
# validate summary
summary = Summary(on_trial['_clinical'],
on_trial['_genomic'],
other,
trial_tree)
item = summary.create_summary(on_trial)
status_fields = ['drugs', 'genes', 'tumor_types', 'sponsor',
'phase_summary', 'accrual_goal', 'investigator', 'age_summary', 'protocol_number',
'disease_status', 'nct_number', 'disease_center', 'short_title',
'hormone_receptor_status']
for field in status_fields:
assert field in item, self._debug(item, field)
if field not in ['dfci_investigator', 'hormone_receptor_status']:
assert item[field], '%s| %s' % (field, item)
# remove all fields and validate that the summary will not error
del on_trial['age']
del on_trial['phase']
del on_trial['nct_id']
del on_trial['protocol_no']
del on_trial['principal_investigator']
del on_trial['cancer_center_accrual_goal_upper']
del on_trial['site_list']
del on_trial['sponsor_list']
del on_trial['drug_list']
del on_trial['staff_list']
status, trial_tree = me.create_trial_tree(on_trial, no_validate=True)
summary = Summary(on_trial['_clinical'],
on_trial['_genomic'],
other,
trial_tree)
item = summary.create_summary(on_trial)
for field in status_fields:
assert field in item, self._debug(item, field)
def setUp(self, settings_file=None, url_converters=None):
super(TestResponse, self).setUp(settings_file=None, url_converters=None)
self.user_token = None
# add to database
self.db = get_db()
self.db.response.drop()
self.db['clinical'].insert_one(self.fake_patient)
self.db['match'].insert_one(self.match_flagged)
self.db['filter'].insert_one(self.fake_filter)
self.db['user'].insert_one(self.fake_filter_owner)
self.db['response'].insert_many(demo_resps)
self.response_ids = [resp['_id'] for resp in demo_resps]
self.email_ids = [email['_id'] for email in list(self.db['email'].find())]
def setUp(self):
self.db = get_db()
self.db.negative_genomic.drop()
self.item = {
'clinical_id': ObjectId(),
'sample_id': 'XXXX',
'true_hugo_symbol': 'PTEN',
'true_transcript_exon': 1,
'true_codon': None,
'coverage': 100000.,
'coverage_type': 'PN',
'panel': 'None',
'roi_type': 'G'
}
def test_trial_normalization(self):
db = get_db()
db['normalize'].insert(MAPPING)
# define test json
with open(os.path.join(YAML_DIR, "00-001.yml")) as fin:
test_json = yaml.load(fin)
# do the mapping.
UtilHooks.NormalizeHook.entry_insert("trial", [test_json])
trial_insert([test_json])
db['normalize'].drop()
def setUp(self):
self.db = get_db()
self.db.negative_genomic.drop()
self.item = {
'clinical_id': ObjectId(),
'sample_id': 'XXXX',
'true_hugo_symbol': 'PTEN',
'true_transcript_exon': 1,
'true_codon': None,
'coverage': 100000.,
'coverage_type': 'PN',
'panel': 'None',
'roi_type': 'G'
}
def reannotate_trials():
# connect to the database.
db = database.get_db()
# get all trials.
trials = list(db['trial'].find())
# call hooks.
events.trial_insert(trials)
# re-insert.
for trial in trials:
db['trial'].delete_one({'_id': trial['_id']})
db['trial'].insert_one(trial)
def maintain_users():
"""
ensure all users have a role
:return:
"""
logging.info("maintain users")
# get database connection.
db = database.get_db()
# find all filters status.
users = list(db['user'].find())
# check each user.
for user in users:
if 'roles' not in user:
db['user'].update_one({"_id": user['_id']}, {"$set": {"roles": ["user"]}})
def test_extract_genes(self):
m = MatchEngine(get_db())
match_tree = trial['treatment_list']['step'][0]['match'][0]
g = m.create_match_tree(match_tree)
pmt = ParseMatchTree(g)
genes = pmt.extract_genes()
assert 'BRAF' in genes, genes
assert 'KRAS' in genes, genes
assert 'EGFR' not in genes
assert 'test' not in genes, genes
assert len(genes) == 2, genes
match_tree = trial['treatment_list']['step'][0]['match'][2]
g = m.create_match_tree(match_tree)
pmt = ParseMatchTree(g)
genes = pmt.extract_genes()
assert 'BRAF' not in genes, genes
def test_ms_status(self):
on_trial['_clinical'] = {}
on_trial['_genomic'] = {'mmr_status': [{'value': 'MMR-Proficient'}]}
db = get_db()
me = MatchEngine(db)
status, trial_tree = me.create_trial_tree(on_trial, no_validate=True)
summary = Summary(on_trial['_clinical'],
on_trial['_genomic'],
{},
trial_tree)
item = summary.create_summary(on_trial)
assert 'mmr_status' in item, self._debug(item, 'mmr_status')
on_trial['_genomic'] = {'ms_status': [{'value': 'MSI-H'}]}
status, trial_tree = me.create_trial_tree(on_trial, no_validate=True)
summary = Summary(on_trial['_clinical'],
on_trial['_genomic'],
{},
trial_tree)
item = summary.create_summary(on_trial)
assert 'ms_status' in item, self._debug(item, 'ms_status')
def mapping_oncotree(self):
# insert the mapping
db = get_db()
mapping = {
"key": "trial",
"values": {
"oncotree_primary_diagnosis": {"cns/brain": "CNS/Brain"}
}
}
db['normalize'].insert(mapping)
# set data example
data = {'match': [{'or': [{'or': [{'clinical': {'oncotree_primary_diagnosis': 'Breast', 'age_numerical': '>=18'}}, {
'clinical': {'oncotree_primary_diagnosis': 'Renal Cell Carcinoma', 'age_numerical': '>=18'}}, {
'clinical': {'oncotree_primary_diagnosis': 'Pleural Mesothelioma',
'age_numerical': '>=18'}}, {
'clinical': {'oncotree_primary_diagnosis': 'Peritoneal Mesotheliom',
'age_numerical': '>=18'}}, {
'clinical': {'oncotree_primary_diagnosis': 'Gastrointestinal Stromal Tumor',
'age_numerical': '>=18'}}]}, {'and': [{'or': [
{'genomic': {'wildcard_protein_change': 'p.G12', 'hugo_symbol': 'KRAS'}},
{'genomic': {'wildcard_protein_change': 'p.G13', 'hugo_symbol': 'KRAS'}},
{'genomic': {'wildcard_protein_change': 'p.Q61', 'hugo_symbol': 'KRAS'}}]}, {'clinical': {
'oncotree_primary_diagnosis': 'Lung Adenocarcinoma', 'age_numerical': '>=18'}}]}, {'and': [{'or': [
{'genomic': {'variant_category': 'Mutation', 'hugo_symbol': 'IDH1'}},
{'genomic': {'variant_category': 'Mutation', 'hugo_symbol': 'IDH2'}}]}, {'and': [
{'clinical': {'oncotree_primary_diagnosis': '_SOLID_', 'age_numerical': '>=18'}},
{'clinical': {'oncotree_primary_diagnosis': '!CNS/Brain', 'age_numerical': '>=18'}}]}]}, {
'and': [{'genomic': {'hugo_symbol': 'MYC', 'cnv_call': 'High Amplification'}}, {
'clinical': {'oncotree_primary_diagnosis': '_SOLID_', 'age_numerical': '>=18'}}]}]}]}
# do the mapping.
UtilHooks.NormalizeHook.entry_insert("trial", [data])
# assert it is updated.
assert data['match'][0]['or'][2]['and'][1]['and'][1]['clinical']['oncotree_primary_diagnosis'] == '!CNS/Brain'
def setUp(self, settings_file=None, url_converters=None):
# call parent
super(TestNegGenomic, self).setUp(settings_file=None, url_converters=None)
# switch to service account.
self.user_token = self.service_token
self.db = get_db()
self.db.negative_genomic.drop()
self.clinical_id = ObjectId()
self.db.clinical.insert_one({'_id': self.clinical_id})
self.item = {
'clinical_id': str(self.clinical_id),
'sample_id': 'XXXX',
'true_hugo_symbol': 'PTEN',
'true_transcript_exon': 1,
'true_codon': None,
'coverage': 100000.,
'coverage_type': 'PN',
'panel': 'none',
'roi_type': 'G'
}
def email_user(items):
# skip unless production.
if settings.WELCOME_EMAIL != "YES":
logging.debug("welcome email skipped")
return
# loop over each user.
for user in items:
# do not email users not approved by Susan
if user['user_name'] == '':
continue
# generate email.
recipient_email = user['email']
# create the message.
cur_date = datetime.date.today().strftime("%B %d, %Y")
cur_stamp = datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y")
# generate text
html = _user_email_text(user, cur_date, cur_stamp)
db = database.get_db()
email_item = {
'email_from': settings.EMAIL_AUTHOR_PROTECTED,
'email_to': recipient_email,
'subject': 'Welcome to MatchMiner - %s' % cur_date,
'body': html,
'cc': [],
'sent': False,
'num_failures': 0,
'errors': []
}
db['email'].insert(email_item)
def email_matches():
# get the database links.
match_db = database.get_collection("match")
user_db = database.get_collection('user')
filter_db = database.get_collection('filter')
logging.info("starting email search")
# get distinct list of team ids
teams = match_db.find().distinct("TEAM_ID")
# loop over each team.
message_list = []
for teamid in teams:
# get the counts.
num_filters, num_matches = _email_counts(teamid, match_db, filter_db)
# skip if no updates.
if num_matches < 1:
continue
# get users in this team
team_members = list(user_db.find({'teams': {'$elemMatch': {'$in': [teamid]}}}))
for user in team_members:
# skip if silenced.
if 'silent' in user and user['silent']:
continue
# simplify.
recipient_email = user['email']
match_str = "matches"
if num_matches == 1:
match_str = "match"
# create the message.
cur_date = datetime.date.today().strftime("%B %d, %Y")
cur_stamp = datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y")
# generate text
html = _email_text(user, num_matches, match_str, num_filters, cur_date, cur_stamp)
db = database.get_db()
email_item = {
'email_from': settings.EMAIL_AUTHOR_PROTECTED,
'email_to': recipient_email,
'subject': 'New MatchMiner Hits - %s' % cur_date,
'body': html,
'cc': [],
'sent': False,
'num_failures': 0,
'errors': []
}
db['email'].insert(email_item)
message_list.append(html)
# return the message lists
return message_list