def create_gene_note_handler(request, gene_id):
request_json = json.loads(request.body)
create_model_from_json(GeneNote, {'note': request_json.get('note'), 'gene_id': gene_id}, request.user)
return create_json_response({'genesById': {gene_id: {
'notes': _get_gene_notes(gene_id, request.user)
}}})
def create_variant_note_handler(request, variant_guids):
request_json = json.loads(request.body)
save_as_gene_note = request_json.get('saveAsGeneNote')
family_guid = request_json.pop('familyGuid')
family = Family.objects.get(guid=family_guid)
check_project_permissions(family.project, request.user)
all_variant_guids = variant_guids.split(',')
saved_variants = SavedVariant.objects.filter(guid__in=all_variant_guids)
if len(saved_variants) != len(all_variant_guids):
error = 'Unable to find the following variant(s): {}'.format(', '.join(
[
guid for guid in all_variant_guids
if guid not in {sv.guid
for sv in saved_variants}
]))
return create_json_response({'error': error}, status=400, reason=error)
# update saved_variants
note = _create_variant_note(saved_variants, request_json, request.user)
note_json = get_json_for_variant_note(note, add_variant_guids=False)
note_json['variantGuids'] = all_variant_guids
response = {
'savedVariantsByGuid': {
saved_variant.guid: {
'noteGuids':
[n.guid for n in saved_variant.variantnote_set.all()]
}
for saved_variant in saved_variants
},
'variantNotesByGuid': {
note.guid: note_json
},
}
if save_as_gene_note:
main_transcript_id = saved_variants[
0].selected_main_transcript_id or saved_variants[
0].saved_variant_json['mainTranscriptId']
gene_id = next(
(gene_id for gene_id, transcripts in saved_variants[0].
saved_variant_json['transcripts'].items() if any(
t['transcriptId'] == main_transcript_id
for t in transcripts)), None) if main_transcript_id else None
create_model_from_json(GeneNote, {
'note': request_json.get('note'),
'gene_id': gene_id
}, request.user)
response['genesById'] = {
gene_id: {
'notes':
get_json_for_gene_notes_by_gene_id([gene_id],
request.user)[gene_id],
}
}
return create_json_response(response)
def get_mme_matches(patient_data, origin_request_host=None, user=None, originating_submission=None):
hpo_terms_by_id, genes_by_id, gene_symbols_to_ids = get_mme_genes_phenotypes_for_results([patient_data])
genomic_features = _get_patient_genomic_features(patient_data)
feature_ids = [
feature['id'] for feature in (_get_patient_features(patient_data) or []) if
feature.get('observed', 'yes') == 'yes' and feature['id'] in hpo_terms_by_id
]
if genomic_features:
for feature in genomic_features:
feature['gene_ids'] = get_gene_ids_for_feature(feature, gene_symbols_to_ids)
get_submission_kwargs = {
'query_ids': list(genes_by_id.keys()),
'filter_key': 'genomic_features',
'id_filter_func': lambda gene_id: {'gene': {'id': gene_id}},
}
else:
get_submission_kwargs = {
'query_ids': feature_ids,
'filter_key': 'features',
'id_filter_func': lambda feature_id: {'id': feature_id, 'observed': 'yes'},
}
query_patient_id = patient_data['patient']['id']
scored_matches = _get_matched_submissions(
query_patient_id,
get_match_genotype_score=lambda match: _get_genotype_score(genomic_features, match) if genomic_features else 0,
get_match_phenotype_score=lambda match: _get_phenotype_score(feature_ids, match) if feature_ids else 0,
**get_submission_kwargs
)
incoming_query = create_model_from_json(MatchmakerIncomingQuery, {
'institution': patient_data['patient']['contact'].get('institution') or origin_request_host,
'patient_id': query_patient_id if scored_matches else None,
}, user)
if not scored_matches:
return [], incoming_query
prefetch_related_objects(list(scored_matches.keys()), 'matchmakerresult_set')
for match_submission in scored_matches.keys():
if not match_submission.matchmakerresult_set.filter(result_data__patient__id=query_patient_id):
create_model_from_json( MatchmakerResult, {
'submission': match_submission,
'originating_submission': originating_submission,
'originating_query': incoming_query,
'result_data': patient_data,
'last_modified_by': user,
}, user)
return [get_submission_json_for_external_match(match_submission, score=score)
for match_submission, score in scored_matches.items()], incoming_query
def create_locus_list_handler(request):
request_json = json.loads(request.body)
if not request_json.get('name'):
return create_json_response({},
status=400,
reason='"Name" is required')
genes_by_id, intervals, invalid_items = parse_locus_list_items(
request_json)
if invalid_items and not request_json.get('ignoreInvalidItems'):
return create_json_response({'invalidLocusListItems': invalid_items},
status=400,
reason=INVALID_ITEMS_ERROR)
try:
locus_list = create_model_from_json(
LocusList, {
'name': request_json['name'],
'description': request_json.get('description') or '',
'is_public': request_json.get('isPublic') or False,
}, request.user)
except IntegrityError:
return create_json_response({'error': 'This list already exists'},
status=400)
_update_locus_list_items(locus_list, genes_by_id, intervals, request_json,
request.user)
return create_json_response({
'locusListsByGuid': {
locus_list.guid: get_json_for_locus_list(locus_list, request.user)
},
'genesById': genes_by_id,
})
def _update_tags(saved_variants,
tags_json,
user,
tag_key='tags',
model_cls=VariantTag,
get_tag_create_data=_get_tag_type_create_data):
tags = tags_json.get(tag_key, [])
updated_models = []
for tag in tags:
if tag.get('tagGuid'):
model = model_cls.objects.get(guid=tag.get('tagGuid'))
update_model_from_json(model,
tag,
user=user,
allow_unknown_keys=True)
else:
create_data = get_tag_create_data(tag,
saved_variants=saved_variants)
create_data.update({
'metadata': tag.get('metadata'),
'search_hash': tags_json.get('searchHash'),
})
model = create_model_from_json(model_cls, create_data, user)
model.saved_variants.set(saved_variants)
updated_models.append(model)
return updated_models
def update_family_analysed_by(request, family_guid):
"""Updates the specified field in the Family model.
Args:
family_guid (string): GUID of the family.
field_name (string): Family model field name to update
"""
family = Family.objects.get(guid=family_guid)
# analysed_by can be edited by anyone with access to the project
check_project_permissions(family.project, request.user, can_edit=False)
create_model_from_json(FamilyAnalysedBy, {'family': family}, request.user)
return create_json_response(
{family.guid: _get_json_for_family(family, request.user)})
def _get_or_create_results_model(search_hash, search_context, user):
results_model = VariantSearchResults.objects.filter(search_hash=search_hash).first()
if not results_model:
if not search_context:
raise Exception('Invalid search hash: {}'.format(search_hash))
project_families = search_context.get('projectFamilies')
if project_families:
all_families = set()
for project_family in project_families:
all_families.update(project_family['familyGuids'])
families = Family.objects.filter(guid__in=all_families)
elif _is_all_project_family_search(search_context):
omit_projects = [p.guid for p in ProjectCategory.objects.get(name='Demo').projects.only('guid').all()]
project_guids = [project_guid for project_guid in get_project_guids_user_can_view(user) if project_guid not in omit_projects]
families = Family.objects.filter(project__guid__in=project_guids)
elif search_context.get('projectGuids'):
families = Family.objects.filter(project__guid__in=search_context['projectGuids'])
else:
raise Exception('Invalid search: no projects/ families specified')
search_dict = search_context.get('search', {})
search_model = VariantSearch.objects.filter(search=search_dict).filter(
Q(created_by=user) | Q(name__isnull=False)).first()
if not search_model:
search_model = create_model_from_json(VariantSearch, {'search': search_dict}, user)
# If a search_context request and results request are dispatched at the same time, its possible the other
# request already created the model
results_model, _ = get_or_create_model_from_json(
VariantSearchResults, {'search_hash': search_hash, 'variant_search': search_model},
update_json=None, user=user)
results_model.families.set(families)
return results_model
def update_mme_submission(request, submission_guid=None):
"""
Create or update the submission for the given individual.
"""
submission_json = json.loads(request.body)
phenotypes = submission_json.pop('phenotypes', [])
gene_variants = submission_json.pop('geneVariants', [])
if not phenotypes and not gene_variants:
return create_json_response({}, status=400, reason='Genotypes or phentoypes are required')
genomic_features = []
for gene_variant in gene_variants:
if not gene_variant.get('geneId'):
return create_json_response({}, status=400, reason='Gene id is required for genomic features')
feature = {'gene': {'id': gene_variant['geneId']}}
if 'numAlt' in gene_variant and gene_variant['numAlt'] > 0:
feature['zygosity'] = gene_variant['numAlt']
if gene_variant.get('pos'):
genome_version = gene_variant['genomeVersion']
feature['variant'] = {
'referenceName': gene_variant['chrom'],
'start': gene_variant['pos'],
'assembly': GENOME_VERSION_LOOKUP.get(genome_version, genome_version),
}
if gene_variant.get('alt'):
feature['variant'].update({
'alternateBases': gene_variant['alt'],
'referenceBases': gene_variant['ref'],
})
elif gene_variant.get('end'):
feature['variant']['end'] = gene_variant['end']
genomic_features.append(feature)
submission_json.update({
'features': phenotypes,
'genomicFeatures': genomic_features,
'deletedDate': None,
'deletedBy': None,
})
if submission_guid:
submission = MatchmakerSubmission.objects.get(guid=submission_guid)
check_mme_permissions(submission, request.user)
else:
individual_guid = submission_json.get('individualGuid')
if not individual_guid:
return create_json_response({}, status=400, reason='Individual is required for a new submission')
individual = Individual.objects.get(guid=individual_guid)
check_project_permissions(individual.family.project, request.user)
submission = create_model_from_json(MatchmakerSubmission, {
'individual': individual,
'submission_id': individual.guid,
'label': individual.individual_id,
}, request.user)
update_model_from_json(submission, submission_json, user=request.user, allow_unknown_keys=True)
# search for new matches
return _search_matches(submission, request.user)
def _create_variant_note(saved_variants, note_json, user):
note = create_model_from_json(VariantNote, {
'note': note_json.get('note'),
'submit_to_clinvar': note_json.get('submitToClinvar') or False,
'search_hash': note_json.get('searchHash'),
}, user)
note.saved_variants.set(saved_variants)
return note
def _update_project_categories(project, user, category_guids):
"""Updates the stored categories for the given project.
Args:
project (project): Django Project model
user (User): Django User model
category_guids (set): set of category GUIDs to apply to the given project
"""
category_guids = set(category_guids)
project_categories_by_guid = {
} # keep track of new and removed categories so client can be updated.
# remove ProjectCategory => Project mappings for categories the user wants to remove from this project
current_categories_in_db = set()
for project_category in project.projectcategory_set.all():
if project_category.guid not in category_guids:
if project_category.name == ANALYST_PROJECT_CATEGORY:
continue
project_category.projects.remove(project)
if project_category.projects.count() == 0:
project_category.delete_model(user, user_can_delete=True)
project_categories_by_guid[project_category.guid] = None
else:
# also record the project_category guids for which there's already a ProjectCategory
# object mapped to this project and doesn't need to be added or removed
current_categories_in_db.add(project_category.guid)
# add ProjectCategory => Project mappings for categories that already exist in the system, and that the user now wants to add to this project also
project_categories_to_create = set(category_guids) # copy the set
for project_category in ProjectCategory.objects.filter(
guid__in=category_guids):
if project_category.guid not in current_categories_in_db:
project_category.projects.add(project)
project_categories_to_create.remove(project_category.guid)
# create ProjectCategory objects for new categories, and add ProjectCategory => Project mappings for them to this project
for category_name in project_categories_to_create:
project_category = create_model_from_json(ProjectCategory,
{'name': category_name},
user)
project_category.projects.add(project)
project_categories_by_guid[
project_category.guid] = project_category.json()
return project_categories_by_guid
def _create_new_tags(saved_variants, tags_json, user):
tags = tags_json.get('tags', [])
new_tags = [tag for tag in tags if not tag.get('tagGuid')]
new_tag_models = []
for tag in new_tags:
variant_tag_type = VariantTagType.objects.get(
Q(name=tag['name']),
Q(project=saved_variants[0].family.project) | Q(project__isnull=True)
)
tag_model = create_model_from_json(VariantTag, {
'variant_tag_type': variant_tag_type,
'search_hash': tags_json.get('searchHash'),
}, user)
tag_model.saved_variants.set(saved_variants)
new_tag_models.append(tag_model)
return new_tag_models
def update_variant_functional_data_handler(request, variant_guids):
request_json = json.loads(request.body)
family_guid = request_json.pop('familyGuid')
family = Family.objects.get(guid=family_guid)
check_project_permissions(family.project, request.user)
all_variant_guids = set(variant_guids.split(','))
saved_variants = SavedVariant.objects.filter(guid__in=all_variant_guids)
if len(saved_variants) != len(all_variant_guids):
error = 'Unable to find the following variant(s): {}'.format(
', '.join([guid for guid in all_variant_guids if guid not in {sv.guid for sv in saved_variants}]))
return create_json_response({'error': error}, status=400, reason=error)
updated_functional_data = request_json.get('functionalData', [])
deleted_functional_guids = _delete_removed_tags(
saved_variants, all_variant_guids, updated_functional_data, request.user, tag_type='functionaldata')
updated_functional_models = []
for tag in updated_functional_data:
if tag.get('tagGuid'):
functional_data = VariantFunctionalData.objects.get(guid=tag.get('tagGuid'))
update_model_from_json(functional_data, tag, user=request.user, allow_unknown_keys=True)
else:
functional_data = create_model_from_json(VariantFunctionalData, {
'functional_data_tag': tag.get('name'),
'metadata': tag.get('metadata'),
'search_hash': request_json.get('searchHash'),
}, request.user)
functional_data.saved_variants.set(saved_variants)
updated_functional_models.append(functional_data)
functional_updates = {tag['tagGuid']: tag for tag in get_json_for_variant_functional_data_tags(updated_functional_models)}
functional_updates.update({guid: None for guid in deleted_functional_guids})
return create_json_response({
'savedVariantsByGuid': {saved_variant.guid: {
'functionalDataGuids': [t.guid for t in saved_variant.variantfunctionaldata_set.all()],
} for saved_variant in saved_variants},
'variantFunctionalDataByGuid': functional_updates,
})
def create_project_handler(request):
"""Create a new project.
HTTP POST
Request body - should contain json params:
name: Project name
description: Project description
Response body - will be json with the following structure, representing the ,created project:
{
'projectsByGuid': { <projectGuid1> : { ... <project key-value pairs> ... } }
}
"""
request_json = json.loads(request.body)
missing_fields = [
field for field in ['name', 'genomeVersion']
if not request_json.get(field)
]
if missing_fields:
error = 'Field(s) "{}" are required'.format(', '.join(missing_fields))
return create_json_response({'error': error}, status=400, reason=error)
project_args = {
'name': request_json['name'],
'genome_version': request_json['genomeVersion'],
'description': request_json.get('description', ''),
}
project = create_model_from_json(Project, project_args, user=request.user)
if ANALYST_PROJECT_CATEGORY:
ProjectCategory.objects.get(
name=ANALYST_PROJECT_CATEGORY).projects.add(project)
return create_json_response({
'projectsByGuid': {
project.guid: _get_json_for_project(project, request.user)
},
})
def create_project_from_workspace(request, namespace, name):
"""
Create a project when a cooperator requests to load data from an AnVIL workspace.
:param request: Django request object
:param namespace: The namespace (or the billing account) of the workspace
:param name: The name of the workspace. It also be used as the project name
:return the projectsByGuid with the new project json
"""
# Validate that the current user has logged in through google and has sufficient permissions
workspace_meta = check_workspace_perm(request.user, CAN_EDIT, namespace, name, can_share=True, meta_fields=['workspace.bucketName'])
projects = Project.objects.filter(workspace_namespace=namespace, workspace_name=name)
if projects:
error = 'Project "{}" for workspace "{}/{}" exists.'.format(projects.first().name, namespace, name)
return create_json_response({'error': error}, status=400, reason=error)
# Validate all the user inputs from the post body
request_json = json.loads(request.body)
missing_fields = [field for field in ['genomeVersion', 'uploadedFileId', 'dataPath'] if not request_json.get(field)]
if missing_fields:
error = 'Field(s) "{}" are required'.format(', '.join(missing_fields))
return create_json_response({'error': error}, status=400, reason=error)
if not request_json.get('agreeSeqrAccess'):
error = 'Must agree to grant seqr access to the data in the associated workspace.'
return create_json_response({'error': error}, status=400, reason=error)
# Add the seqr service account to the corresponding AnVIL workspace
added_account_to_workspace = add_service_account(request.user, namespace, name)
if added_account_to_workspace:
_wait_for_service_account_access(request.user,namespace, name)
# Validate the data path
bucket_name = workspace_meta['workspace']['bucketName']
data_path = 'gs://{bucket}/{path}'.format(bucket=bucket_name.rstrip('/'), path=request_json['dataPath'].lstrip('/'))
if not does_file_exist(data_path):
error = 'Data file or path {} is not found.'.format(request_json['dataPath'])
return create_json_response({'error': error}, status=400, reason=error)
# Parse families/individuals in the uploaded pedigree file
json_records = load_uploaded_file(request_json['uploadedFileId'])
pedigree_records, errors, ped_warnings = parse_pedigree_table(json_records, 'uploaded pedigree file', user=request.user)
errors += ped_warnings
if errors:
return create_json_response({'errors': errors}, status=400)
# Create a new Project in seqr
project_args = {
'name': name,
'genome_version': request_json['genomeVersion'],
'description': request_json.get('description', ''),
'workspace_namespace': namespace,
'workspace_name': name,
}
project = create_model_from_json(Project, project_args, user=request.user)
# add families and individuals according to the uploaded individual records
_, updated_individuals = add_or_update_individuals_and_families(
project, individual_records=pedigree_records, user=request.user
)
# Send an email to all seqr data managers
try:
_send_load_data_email(project, updated_individuals, data_path, request.user)
except Exception as ee:
message = 'Exception while sending email to user {}. {}'.format(request.user, str(ee))
logger.error(message)
return create_json_response({'projectGuid': project.guid})
def add_or_update_individuals_and_families(project, individual_records, user):
"""
Add or update individual and family records in the given project.
Args:
project (object): Django ORM model for the project to add families to
individual_records (list): A list of JSON records representing individuals. See
the return value of pedigree_info_utils#convert_fam_file_rows_to_json(..)
Return:
2-tuple: updated_families, updated_individuals containing Django ORM models
"""
updated_families = set()
updated_individuals = set()
parent_updates = []
family_ids = {_get_record_family_id(record) for record in individual_records}
families_by_id = {f.family_id: f for f in Family.objects.filter(project=project, family_id__in=family_ids)}
missing_family_ids = family_ids - set(families_by_id.keys())
for family_id in missing_family_ids:
family = create_model_from_json(Family, {'project': project, 'family_id': family_id}, user)
families_by_id[family_id] = family
updated_families.add(family)
individual_models = Individual.objects.filter(family__project=project).prefetch_related(
'family', 'mother', 'father')
has_individual_guid = any(record.get('individualGuid') for record in individual_records)
if has_individual_guid:
individual_lookup = {
i.guid: i for i in individual_models.filter(
guid__in=[record['individualGuid'] for record in individual_records])
}
else:
individual_lookup = defaultdict(dict)
for i in individual_models.filter(
individual_id__in=[_get_record_individual_id(record) for record in individual_records]):
individual_lookup[i.individual_id][i.family] = i
for record in individual_records:
family_id = _get_record_family_id(record)
family = families_by_id.get(family_id)
if has_individual_guid:
individual = individual_lookup[record.pop('individualGuid')]
else:
# uploaded files do not have unique guid's so fall back to a combination of family and individualId
individual_id = _get_record_individual_id(record)
individual = individual_lookup[individual_id].get(family)
if not individual:
individual = create_model_from_json(
Individual, {'family': family, 'individual_id': individual_id, 'case_review_status': 'I'}, user)
record['family'] = family
record.pop('familyId', None)
if individual.family != family:
family = individual.family
updated_families.add(family)
previous_id = record.pop(JsonConstants.PREVIOUS_INDIVIDUAL_ID_COLUMN, None)
if previous_id:
updated_individuals.update(individual.maternal_children.all())
updated_individuals.update(individual.paternal_children.all())
record['displayName'] = ''
# Update the parent ids last, so if they are referencing updated individuals they will check for the correct ID
if record.get('maternalId') or record.get('paternalId'):
parent_updates.append({
'individual': individual,
'maternalId': record.pop('maternalId', None),
'paternalId': record.pop('paternalId', None),
})
family_notes = record.pop(JsonConstants.FAMILY_NOTES_COLUMN, None)
if family_notes:
update_family_from_json(family, {'analysis_notes': family_notes}, user)
updated_families.add(family)
is_updated = update_individual_from_json(individual, record, user=user, allow_unknown_keys=True)
if is_updated:
updated_individuals.add(individual)
updated_families.add(family)
for update in parent_updates:
individual = update.pop('individual')
is_updated = update_individual_from_json(individual, update, user=user)
if is_updated:
updated_individuals.add(individual)
updated_families.add(individual.family)
# update pedigree images
update_pedigree_images(updated_families, user, project_guid=project.guid)
return list(updated_families), list(updated_individuals)
def _search_matches(submission, user):
patient_data = get_submission_json_for_external_match(submission)
nodes_to_query = [node for node in MME_NODES.values() if node.get('url')]
if not nodes_to_query:
message = 'No external MME nodes are configured'
logger.error(message)
return create_json_response({'error': message}, status=400, reason=message)
external_results = _search_external_matches(nodes_to_query, patient_data)
local_results, incoming_query = get_mme_matches(patient_data, user=user, originating_submission=submission)
results = local_results + external_results
initial_saved_results = {
result.result_data['patient']['id']: result for result in MatchmakerResult.objects.filter(submission=submission)
}
local_result_submissions = {
s.submission_id: s for s in MatchmakerSubmission.objects.filter(matchmakerresult__originating_submission=submission)
}
new_results = []
saved_results = {}
for result in results:
result_patient_id = result['patient']['id']
saved_result = initial_saved_results.get(result_patient_id)
if not saved_result:
saved_result = create_model_from_json(MatchmakerResult, {
'submission': submission,
'originating_submission': local_result_submissions.get(result_patient_id),
'originating_query': incoming_query,
'result_data': result,
'last_modified_by': user,
}, user)
new_results.append(result)
else:
update_model_from_json(saved_result, {'result_data': result}, user)
saved_results[result['patient']['id']] = saved_result
if new_results:
try:
_generate_notification_for_seqr_match(submission, new_results)
except Exception as e:
logger.error('Unable to create notification for new MME match: {}'.format(str(e)))
logger.info('Found {} matches for {} ({} new)'.format(len(results), submission.submission_id, len(new_results)))
removed_patients = set(initial_saved_results.keys()) - set(saved_results.keys())
removed_count = 0
for patient_id in removed_patients:
saved_result = initial_saved_results[patient_id]
if saved_result.we_contacted or saved_result.host_contacted or saved_result.comments:
if not saved_result.match_removed:
update_model_from_json(saved_result, {'match_removed': True}, user)
removed_count += 1
saved_results[patient_id] = saved_result
else:
saved_result.delete_model(user, user_can_delete=True)
removed_count += 1
if removed_count:
logger.info('Removed {} old matches for {}'.format(removed_count, submission.submission_id))
return _parse_mme_results(submission, list(saved_results.values()), user)
