def fetch_igv_track(request, project_guid, igv_track_path):
get_project_and_check_permissions(project_guid, request.user)
logger.info("Proxy Request: %s %s" % (request.method, igv_track_path))
is_cram = igv_track_path.split('?')[0].endswith('.cram')
if is_cram:
absolute_path = "/alignments?reference=igvjs/static/data/public/Homo_sapiens_assembly38.fasta&file=igvjs/static/data/readviz-mounts/{0}&options={1}®ion={2}".format(
igv_track_path, request.GET.get('options', ''),
request.GET.get('region', ''))
request_kwargs = {
'host': settings.READ_VIZ_CRAM_PATH,
'scheme': 'http',
'filter_request_headers': True,
'stream': True
}
else:
absolute_path = os.path.join(settings.READ_VIZ_BAM_PATH,
igv_track_path)
request_kwargs = {
'auth_tuple': ('xbrowse-bams', 'xbrowse-bams'),
'verify': False
}
return proxy_request(request, absolute_path, **request_kwargs)
def fetch_igv_track(request, project_guid, igv_track_path):
get_project_and_check_permissions(project_guid, request.user)
logger.info("Proxy Request: %s %s" % (request.method, igv_track_path))
return proxy_to_igv(igv_track_path, request.GET, request)
def fetch_igv_track(request, project_guid, igv_track_path):
get_project_and_check_permissions(project_guid, request.user)
logger.info("Proxy Request: %s %s" % (request.method, igv_track_path))
return proxy_to_igv(igv_track_path, request.GET, request)
def fetch_igv_track(request, project_guid, igv_track_path):
get_project_and_check_permissions(project_guid, request.user)
if igv_track_path.endswith(
'.bam.bai') and not does_file_exist(igv_track_path):
igv_track_path = igv_track_path.replace('.bam.bai', '.bai')
return _stream_file(request, igv_track_path)
def saved_variant_data(request, project_guid, variant_guid=None):
project = get_project_and_check_permissions(project_guid, request.user)
family_guids = request.GET['families'].split(',') if request.GET.get(
'families') else None
if family_guids:
variant_query = SavedVariant.objects.filter(
family__guid__in=family_guids)
else:
variant_query = SavedVariant.objects.filter(family__project=project)
if variant_guid:
variant_query = variant_query.filter(guid=variant_guid)
if variant_query.count() < 1:
return create_json_response(
{},
status=404,
reason='Variant {} not found'.format(variant_guid))
saved_variants = get_json_for_saved_variants(variant_query,
add_tags=True,
add_details=True)
variants = {
variant['variantGuid']: variant
for variant in saved_variants if variant['notes'] or variant['tags']
}
genes = _saved_variant_genes(variants.values())
_add_locus_lists([project], variants.values(), genes)
return create_json_response({
'savedVariantsByGuid': variants,
'genesById': genes,
})
def project_page_data(request, project_guid):
"""Returns a JSON object containing information used by the project page:
::
json_response = {
'project': {..},
'familiesByGuid': {..},
'individualsByGuid': {..},
'samplesByGuid': {..},
}
Args:
project_guid (string): GUID of the Project to retrieve data for.
"""
project = get_project_and_check_permissions(project_guid, request.user)
update_project_from_json(project, {'last_accessed_date': timezone.now()})
response = _get_project_child_entities(project, request.user)
project_json = _get_json_for_project(project, request.user)
project_json['collaborators'] = get_json_for_project_collaborator_list(project)
project_json['locusListGuids'] = response['locusListsByGuid'].keys()
project_json['detailsLoaded'] = True
project_json.update(_get_json_for_variant_tag_types(project))
gene_ids = set()
for tag in project_json['discoveryTags']:
gene_ids.update(tag.get('transcripts', {}).keys())
response.update({
'projectsByGuid': {project_guid: project_json},
'genesById': get_genes(gene_ids),
})
return create_json_response(response)
def create_project_collaborator(request, project_guid):
project = get_project_and_check_permissions(project_guid,
request.user,
can_edit=True)
try:
user = _create_user(request)
except CreateUserException as e:
if e.existing_user:
return _update_existing_user(e.existing_user, project,
json.loads(request.body))
else:
return create_json_response({'error': str(e)},
status=e.status_code,
reason=str(e))
project.can_view_group.user_set.add(user)
return create_json_response({
'projectsByGuid': {
project_guid: {
'collaborators':
get_json_for_project_collaborator_list(project)
}
}
})
def save_hpo_table_handler(request, project_guid, upload_file_id):
"""
Handler for 'save' requests to apply HPO terms tables previously uploaded through receive_hpo_table_handler
"""
project = get_project_and_check_permissions(project_guid, request.user)
json_records, _ = load_uploaded_file(upload_file_id)
individual_guids = [record[INDIVIDUAL_GUID_COLUMN] for record in json_records]
individuals_by_guid = {
i.guid: i for i in Individual.objects.filter(family__project=project, guid__in=individual_guids)
}
for record in json_records:
individual = individuals_by_guid[record[INDIVIDUAL_GUID_COLUMN]]
individual.features = [{'id': feature} for feature in record[HPO_TERMS_PRESENT_COLUMN]]
individual.absent_features = [{'id': feature} for feature in record[HPO_TERMS_ABSENT_COLUMN]]
individual.save()
return create_json_response({
'individualsByGuid': {
individual['individualGuid']: individual for individual in _get_json_for_individuals(
individuals_by_guid.values(), user=request.user, add_hpo_details=True,
)},
})
def export_case_review_individuals_handler(request, project_guid):
"""Export case review Individuals table.
Args:
project_guid (string): GUID of the project for which to export case review individual data
"""
format = request.GET.get('file_format', 'tsv')
project = get_project_and_check_permissions(project_guid, request.user)
individuals = Individual.objects.filter(
family__project=project,
case_review_status__regex="[\w].*").order_by('family__family_id',
'affected')
filename_prefix = "%s_case_review_individuals" % _slugify(project.name)
return export_individuals(
filename_prefix,
individuals,
format,
include_case_review_status=True,
include_case_review_discussion=True,
include_hpo_terms_present=True,
include_hpo_terms_absent=True,
)
def save_individuals_table_handler(request, project_guid, upload_file_id):
"""Handler for 'save' requests to apply Individual tables previously uploaded through receive_individuals_table(..)
Args:
request (object): Django request object
project_guid (string): project GUID
uploadedFileId (string): a token sent to the client by receive_individuals_table(..)
"""
project = get_project_and_check_permissions(project_guid, request.user)
json_records = load_uploaded_file(upload_file_id)
updated_families, updated_individuals = add_or_update_individuals_and_families(
project, individual_records=json_records, user=request.user
)
# edit individuals
individuals = _get_json_for_individuals(updated_individuals, request.user, add_sample_guids_field=True)
individuals_by_guid = {individual['individualGuid']: individual for individual in individuals}
families = _get_json_for_families(updated_families, request.user, add_individual_guids_field=True)
families_by_guid = {family['familyGuid']: family for family in families}
updated_families_and_individuals_by_guid = {
'individualsByGuid': individuals_by_guid,
'familiesByGuid': families_by_guid,
}
return create_json_response(updated_families_and_individuals_by_guid)
def save_individuals_metadata_table_handler(request, project_guid,
upload_file_id):
"""
Handler for 'save' requests to apply HPO terms tables previously uploaded through receive_individuals_metadata_handler
"""
project = get_project_and_check_permissions(project_guid, request.user)
json_records, _ = load_uploaded_file(upload_file_id)
individual_guids = [record[INDIVIDUAL_GUID_COL] for record in json_records]
individuals_by_guid = {
i.guid: i
for i in Individual.objects.filter(family__project=project,
guid__in=individual_guids)
}
for record in json_records:
individual = individuals_by_guid[record[INDIVIDUAL_GUID_COL]]
update_model_from_json(individual, {
k: record[k]
for k in INDIVIDUAL_METADATA_FIELDS.keys() if k in record
},
user=request.user)
return create_json_response({
'individualsByGuid': {
individual['individualGuid']: individual
for individual in _get_json_for_individuals(
list(individuals_by_guid.values()),
user=request.user,
add_hpo_details=True,
)
},
})
def receive_hpo_table_handler(request, project_guid):
"""Handler for bulk update of hpo terms. This handler parses the records, but doesn't save them in the database.
Instead, it saves them to a temporary file and sends a 'uploadedFileId' representing this file back to the client.
Args:
request (object): Django request object
project_guid (string): project GUID
"""
project = get_project_and_check_permissions(project_guid, request.user)
def process_records(json_records, filename=''):
records, errors, warnings = _process_hpo_records(json_records, filename, project)
if errors:
raise ErrorsWarningsException(errors, warnings)
return records, warnings
try:
uploaded_file_id, _, (json_records, warnings) = save_uploaded_file(request, process_records=process_records)
except ErrorsWarningsException as e:
return create_json_response({'errors': e.errors, 'warnings': e.warnings}, status=400, reason=e.errors)
except Exception as e:
return create_json_response({'errors': [e.message or str(e)], 'warnings': []}, status=400, reason=e.message or str(e))
response = {
'uploadedFileId': uploaded_file_id,
'errors': [],
'warnings': warnings,
'info': ['{} individuals will be updated'.format(len(json_records))],
}
return create_json_response(response)
def export_project_individuals_handler(request, project_guid):
"""Export project Individuals table.
Args:
project_guid (string): GUID of the project for which to export individual data
"""
format = request.GET.get('file_format', 'tsv')
include_phenotypes = bool(request.GET.get('include_phenotypes'))
project = get_project_and_check_permissions(project_guid, request.user)
# get all individuals in this project
individuals = Individual.objects.filter(family__project=project).order_by(
'family__family_id', 'affected')
filename_prefix = "%s_individuals" % _slugify(project.name)
return export_individuals(
filename_prefix,
individuals,
format,
include_hpo_terms_present=include_phenotypes,
include_hpo_terms_absent=include_phenotypes,
)
def saved_variant_data(request, project_guid, variant_guids=None):
project = get_project_and_check_permissions(project_guid, request.user)
family_guids = request.GET['families'].split(',') if request.GET.get('families') else None
variant_guids = variant_guids.split(',') if variant_guids else None
if family_guids:
variant_query = SavedVariant.objects.filter(family__guid__in=family_guids)
else:
get_note_only = bool(request.GET.get('includeNoteVariants'))
variant_query = SavedVariant.objects.filter(family__project=project, varianttag__isnull=get_note_only).distinct()
if variant_guids:
variant_query = variant_query.filter(guid__in=variant_guids)
if variant_query.count() < 1:
return create_json_response({}, status=404, reason='Variant {} not found'.format(', '.join(variant_guids)))
response = get_json_for_saved_variants_with_tags(variant_query, add_details=True)
discovery_tags = None
if user_is_analyst(request.user):
discovery_tags, discovery_response = get_json_for_discovery_tags(response['savedVariantsByGuid'].values())
response.update(discovery_response)
variants = list(response['savedVariantsByGuid'].values())
genes = saved_variant_genes(variants)
response['locusListsByGuid'] = _add_locus_lists([project], genes)
if discovery_tags:
_add_discovery_tags(variants, discovery_tags)
response['genesById'] = genes
return create_json_response(response)
def saved_variant_data(request, project_guid, variant_guid=None):
project = get_project_and_check_permissions(project_guid, request.user)
variants = {}
variant_query = SavedVariant.objects.filter(project=project)\
.select_related('family')\
.only('xpos_start', 'ref', 'alt', 'saved_variant_json', 'family__guid', 'guid')\
.prefetch_related('varianttag_set', 'varianttag_set__created_by', 'varianttag_set__variant_tag_type',
'variantfunctionaldata_set', 'variantfunctionaldata_set__created_by', 'variantnote_set',
'variantnote_set__created_by')
if request.GET.get('families'):
variant_query = variant_query.filter(
family__guid__in=request.GET.get('families').split(','))
if variant_guid:
variant_query = variant_query.filter(guid=variant_guid)
if variant_query.count() < 1:
return create_json_response(
{},
status=404,
reason='Variant {} not found'.format(variant_guid))
for saved_variant in variant_query:
variant = get_json_for_saved_variant(saved_variant, add_tags=True)
if variant['tags'] or variant['notes']:
variant_json = json.loads(saved_variant.saved_variant_json or '{}')
variant.update(variant_details(variant_json, project,
request.user))
variants[variant['variantId']] = variant
genes = _saved_variant_genes(variants)
_add_locus_lists(project, variants, genes)
return create_json_response({
'savedVariants': variants,
'genesById': genes,
})
def saved_variant_data(request, project_guid, variant_guids=None):
project = get_project_and_check_permissions(project_guid, request.user)
family_guids = request.GET['families'].split(',') if request.GET.get('families') else None
variant_guids = variant_guids.split(',') if variant_guids else None
if family_guids:
variant_query = SavedVariant.objects.filter(family__guid__in=family_guids)
else:
variant_query = SavedVariant.objects.filter(family__project=project)
if variant_guids:
variant_query = variant_query.filter(guid__in=variant_guids)
if variant_query.count() < 1:
return create_json_response({}, status=404, reason='Variant {} not found'.format(', '.join(variant_guids)))
discovery_tags_query = None
if user_is_analyst(request.user):
discovery_tags_query = Q()
for variant in variant_query:
discovery_tags_query |= Q(Q(variant_id=variant.variant_id) & ~Q(family_id=variant.family_id))
discovery_tags_query &= Q(family__project__projectcategory__name=ANALYST_PROJECT_CATEGORY)
response = get_json_for_saved_variants_with_tags(variant_query, add_details=True, discovery_tags_query=discovery_tags_query)
variants = list(response['savedVariantsByGuid'].values())
genes = saved_variant_genes(variants)
response['locusListsByGuid'] = _add_locus_lists([project], genes)
discovery_tags = response.pop('discoveryTags', None)
if discovery_tags:
_add_discovery_tags(variants, discovery_tags)
response['genesById'] = genes
return create_json_response(response)
def receive_igv_table_handler(request, project_guid):
project = get_project_and_check_permissions(project_guid,
request.user,
can_edit=True)
info = []
def _process_alignment_records(rows, **kwargs):
invalid_row = next((row for row in rows if len(row) != 2), None)
if invalid_row:
raise ValueError("Must contain 2 columns: " +
', '.join(invalid_row))
return {row[0]: row[1] for row in rows}
try:
uploaded_file_id, filename, individual_dataset_mapping = save_uploaded_file(
request, process_records=_process_alignment_records)
matched_individuals = Individual.objects.filter(
family__project=project,
individual_id__in=individual_dataset_mapping.keys())
unmatched_individuals = set(individual_dataset_mapping.keys()) - {
i.individual_id
for i in matched_individuals
}
if len(unmatched_individuals) > 0:
raise Exception(
'The following Individual IDs do not exist: {}'.format(
", ".join(unmatched_individuals)))
info.append('Parsed {} rows from {}'.format(
len(individual_dataset_mapping), filename))
existing_samples = IgvSample.objects.select_related(
'individual').filter(individual__in=matched_individuals)
unchanged_individual_ids = {
s.individual.individual_id
for s in existing_samples if individual_dataset_mapping[
s.individual.individual_id] == s.file_path
}
if unchanged_individual_ids:
info.append('No change detected for {} individuals'.format(
len(unchanged_individual_ids)))
updates_by_individual_guid = {
i.guid: individual_dataset_mapping[i.individual_id]
for i in matched_individuals
if i.individual_id not in unchanged_individual_ids
}
except Exception as e:
traceback.print_exc()
return create_json_response({'errors': [str(e)]}, status=400)
response = {
'updatesByIndividualGuid': updates_by_individual_guid,
'uploadedFileId': uploaded_file_id,
'errors': [],
'info': info,
}
return create_json_response(response)
def update_project_collaborator(request, project_guid, username):
project = get_project_and_check_permissions(project_guid,
request.user,
can_edit=True)
user = User.objects.get(username=username)
request_json = json.loads(request.body)
return _update_existing_user(user, project, request_json)
def delete_individuals_handler(request, project_guid):
"""Delete one or more Individual records.
Args:
request (object): Django HTTP Request object.
project_guid (string): GUID of project that contains these individuals.
Request:
body should be a json dictionary that contains a 'recordIdsToDelete' list of individual
GUIDs to delete - for example:
{
'form': {
'recordIdsToDelete': [
<individualGuid1>,
<individualGuid2>,
...
}
}
}
Response:
json dictionary with the deleted GUIDs mapped to None:
{
<individualGuid1> : None,
<individualGuid2> : None,
...
}
"""
# validate request
project = get_project_and_check_permissions(project_guid, request.user, CAN_EDIT)
request_json = json.loads(request.body)
individuals_list = request_json.get('individuals')
if individuals_list is None:
return create_json_response(
{}, status=400, reason="Invalid request: 'individuals' not in request_json")
logger.info("delete_individuals_handler %s", request_json)
individual_guids_to_delete = [ind['individualGuid'] for ind in individuals_list]
# delete the individuals
families_with_deleted_individuals = delete_individuals(project, individual_guids_to_delete)
deleted_individuals_by_guid = {
individual_guid: None for individual_guid in individual_guids_to_delete
}
families_by_guid = {
family.guid: _get_json_for_family(family, request.user, add_individual_guids_field=True) for family in families_with_deleted_individuals
} # families whose list of individuals may have changed
# send response
return create_json_response({
'individualsByGuid': deleted_individuals_by_guid,
'familiesByGuid': families_by_guid,
})
def update_saved_variant_json(request, project_guid):
project = get_project_and_check_permissions(project_guid,
request.user,
permission_level=CAN_EDIT)
updated_saved_variant_guids = update_project_saved_variant_json(project)
return create_json_response(
{variant_guid: None
for variant_guid in updated_saved_variant_guids})
def receive_individuals_table_handler(request, project_guid):
"""Handler for the initial upload of an Excel or .tsv table of individuals. This handler
parses the records, but doesn't save them in the database. Instead, it saves them to
a temporary file and sends a 'uploadedFileId' representing this file back to the client. If/when the
client then wants to 'apply' this table, it can send the uploadedFileId to the
save_individuals_table(..) handler to actually save the data in the database.
Args:
request (object): Django request object
project_guid (string): project GUID
"""
project = get_project_and_check_permissions(project_guid, request.user)
def parse_file(filename, stream):
pedigree_records, errors, warnings = parse_pedigree_table(filename, stream, user=request.user, project=project)
if errors:
raise ErrorsWarningsException(errors, warnings)
return pedigree_records
try:
uploaded_file_id, filename, json_records = save_uploaded_file(request, parse_file)
except ErrorsWarningsException as e:
return create_json_response({'errors': e.errors, 'warnings': e.warnings}, status=400, reason=e.errors)
except Exception as e:
return create_json_response({'errors': [e.message or str(e)], 'warnings': []}, status=400, reason=e.message or str(e))
# send back some stats
num_families = len(set(r['familyId'] for r in json_records))
num_individuals = len(set(r['individualId'] for r in json_records))
num_families_to_create = len([
family_id for family_id in set(r['familyId'] for r in json_records)
if not Family.objects.filter(family_id=family_id, project=project)])
num_individuals_to_create = len(set(
r['individualId'] for r in json_records
if not Individual.objects.filter(
individual_id=r['individualId'],
family__family_id=r['familyId'],
family__project=project)))
info = [
"{num_families} families, {num_individuals} individuals parsed from {filename}".format(
num_families=num_families, num_individuals=num_individuals, filename=filename
),
"%d new families, %d new individuals will be added to the project" % (num_families_to_create, num_individuals_to_create),
"%d existing individuals will be updated" % (num_individuals - num_individuals_to_create),
]
response = {
'uploadedFileId': uploaded_file_id,
'errors': [],
'warnings': [],
'info': info,
}
logger.info(response)
return create_json_response(response)
def update_saved_variant_json(request, project_guid):
project = get_project_and_check_permissions(project_guid,
request.user,
can_edit=True)
reset_cached_search_results(project)
updated_saved_variant_guids = update_project_saved_variant_json(project)
return create_json_response(
{variant_guid: None
for variant_guid in updated_saved_variant_guids})
def delete_project_collaborator(request, project_guid, username):
project = get_project_and_check_permissions(project_guid, request.user, permission_level=CAN_EDIT)
user = User.objects.get(username=username)
project.can_view_group.user_set.remove(user)
project.can_edit_group.user_set.remove(user)
return create_json_response({
'projectsByGuid': {project_guid: {'collaborators': get_json_for_project_collaborator_list(project)}}
})
def update_saved_variant_json(request, project_guid):
project = get_project_and_check_permissions(project_guid, request.user, can_edit=True)
reset_cached_search_results(project)
try:
updated_saved_variant_guids = update_project_saved_variant_json(project, user=request.user)
except Exception as e:
logger.error('Unable to reset saved variant json for {}: {}'.format(project_guid, e))
updated_saved_variant_guids = []
return create_json_response({variant_guid: None for variant_guid in updated_saved_variant_guids})
def delete_families_handler(request, project_guid):
"""Edit or delete one or more Individual records.
Args:
project_guid (string): GUID of project that contains these individuals.
"""
project = get_project_and_check_permissions(project_guid, request.user,
CAN_EDIT)
request_json = json.loads(request.body)
if 'form' not in request_json:
return create_json_response(
{},
status=400,
reason="Invalid request: 'form' not in request_json")
logger.info("delete_families_handler %s", request_json)
form_data = request_json['form']
family_guids_to_delete = form_data.get('recordIdsToDelete')
if family_guids_to_delete is None:
return create_json_response({},
status=400,
reason="'recordIdsToDelete' not specified")
# delete individuals 1st
individual_guids_to_delete = [
i.guid for i in Individual.objects.filter(
family__project=project, family__guid__in=family_guids_to_delete)
]
delete_individuals(project, individual_guids_to_delete)
# delete families
for family in Family.objects.filter(project=project,
guid__in=family_guids_to_delete):
base_family = BaseFamily.objects.get(
project__project_id=project.deprecated_project_id,
family_id=family.family_id)
base_family.delete()
family.delete()
# send response
return create_json_response({
'individualsByGuid': {
individual_guid: None
for individual_guid in individual_guids_to_delete
},
'familiesByGuid':
{family_guid: None
for family_guid in family_guids_to_delete},
})
def delete_analysis_group_handler(request, project_guid, analysis_group_guid):
project = get_project_and_check_permissions(project_guid,
request.user,
can_edit=True)
AnalysisGroup.objects.get(guid=analysis_group_guid,
project=project).delete()
return create_json_response(
{'analysisGroupsByGuid': {
analysis_group_guid: None
}})
def project_page_data(request, project_guid):
"""Returns a JSON object containing information used by the project page:
::
json_response = {
'project': {..},
'familiesByGuid': {..},
'individualsByGuid': {..},
'samplesByGuid': {..},
}
Args:
project_guid (string): GUID of the Project to retrieve data for.
"""
project = get_project_and_check_permissions(project_guid, request.user)
families_by_guid, individuals_by_guid, samples_by_guid, analysis_groups_by_guid, locus_lists_by_guid = get_project_child_entities(
project, request.user)
project_json = _get_json_for_project(project, request.user)
project_json['collaborators'] = _get_json_for_collaborator_list(project)
project_json.update(
_get_json_for_variant_tag_types(project, request.user,
individuals_by_guid))
project_json['locusListGuids'] = locus_lists_by_guid.keys()
# gene search will be deprecated once the new database is online.
project_json['hasGeneSearch'] = _has_gene_search(project)
# TODO once all project data is reloaded get rid of this
sorted_es_samples = sorted([
sample
for sample in samples_by_guid.values() if sample['elasticsearchIndex']
],
key=lambda sample: sample['loadedDate'],
reverse=True)
project_json[
'hasNewSearch'] = sorted_es_samples and is_nested_genotype_index(
sorted_es_samples[0]['elasticsearchIndex'])
project_json['detailsLoaded'] = True
return create_json_response({
'projectsByGuid': {
project_guid: project_json
},
'familiesByGuid': families_by_guid,
'individualsByGuid': individuals_by_guid,
'samplesByGuid': samples_by_guid,
'locusListsByGuid': locus_lists_by_guid,
'analysisGroupsByGuid': analysis_groups_by_guid,
'matchmakerSubmissions': {
project.guid: _project_matchmaker_submissions(project)
},
})
def project_page_data(request, project_guid):
"""Returns a JSON object containing information used by the project page:
::
json_response = {
'user': {..},
'familiesByGuid': {..},
'individualsByGuid': {..},
'samplesByGuid': {..},
'datasetsByGuid': {..},
}
Args:
project_guid (string): GUID of the Project to retrieve data for.
"""
project = get_project_and_check_permissions(project_guid, request.user)
cursor = connection.cursor()
families_by_guid, individuals_by_guid = _retrieve_families_and_individuals(
cursor, project.guid)
samples_by_guid, datasets_by_guid = _retrieve_samples(
cursor, project.guid, individuals_by_guid)
cursor.close()
project_json = _get_json_for_project(project, request.user)
project_json['collaborators'] = _get_json_for_collaborator_list(project)
project_json['locusLists'] = _get_json_for_locus_lists(project)
project_json['variantTagTypes'] = _get_json_for_variant_tag_types(project)
#project_json['referencePopulations'] = _get_json_for_reference_populations(project)
# gene search will be deprecated once the new database is online.
project_json['hasGeneSearch'] = _has_gene_search(project)
user_json = _get_json_for_user(request.user)
user_json[
'hasEditPermissions'] = request.user.is_staff or request.user.has_perm(
CAN_EDIT, project)
user_json['hasViewPermissions'] = user_json[
'hasEditPermissions'] or request.user.has_perm(CAN_VIEW, project)
json_response = {
'user': user_json,
'project': project_json,
'familiesByGuid': families_by_guid,
'individualsByGuid': individuals_by_guid,
'samplesByGuid': samples_by_guid,
'datasetsByGuid': datasets_by_guid,
}
return create_json_response(json_response)
def delete_analysis_group_handler(request, project_guid, analysis_group_guid):
project = get_project_and_check_permissions(project_guid,
request.user,
permission_level=CAN_EDIT)
analysis_group = AnalysisGroup.objects.get(guid=analysis_group_guid,
project=project)
delete_seqr_model(analysis_group)
return create_json_response(
{'analysisGroupsByGuid': {
analysis_group_guid: None
}})
def project_page_data(request, project_guid):
"""Returns a JSON object containing information used by the project page:
::
json_response = {
'project': {..},
'familiesByGuid': {..},
'individualsByGuid': {..},
'samplesByGuid': {..},
}
Args:
project_guid (string): GUID of the Project to retrieve data for.
"""
project = get_project_and_check_permissions(project_guid, request.user)
cursor = connection.cursor()
families_by_guid = _retrieve_families(cursor, project.guid, request.user)
individuals_by_guid = _retrieve_individuals(project.guid, request.user)
for individual_guid, individual in individuals_by_guid.items():
families_by_guid[individual['familyGuid']]['individualGuids'].add(
individual_guid)
samples_by_guid = _retrieve_samples(cursor, project.guid,
individuals_by_guid)
cursor.close()
project_json = _get_json_for_project(project, request.user)
project_json['collaborators'] = _get_json_for_collaborator_list(project)
project_json.update(_get_json_for_variant_tag_types(project))
locus_lists = get_sorted_project_locus_lists(project, request.user)
project_json['locusListGuids'] = [
locus_list['locusListGuid'] for locus_list in locus_lists
]
# gene search will be deprecated once the new database is online.
project_json['hasGeneSearch'] = _has_gene_search(project)
project_json['detailsLoaded'] = True
json_response = {
'project': project_json,
'familiesByGuid': families_by_guid,
'individualsByGuid': individuals_by_guid,
'samplesByGuid': samples_by_guid,
'locusListsByGuid': {
locus_list['locusListGuid']: locus_list
for locus_list in locus_lists
}
}
return create_json_response(json_response)
def edit_families_handler(request, project_guid):
"""Edit or one or more Family records.
Args:
project_guid (string): GUID of project that contains these individuals.
"""
request_json = json.loads(request.body)
if request_json.get('uploadedFileId'):
modified_families = load_uploaded_file(
request_json.get('uploadedFileId'))
else:
modified_families = request_json.get('families')
if modified_families is None:
return create_json_response({},
status=400,
reason="'families' not specified")
project = get_project_and_check_permissions(project_guid,
request.user,
can_edit=True)
updated_families = []
for fields in modified_families:
if fields.get('familyGuid'):
family = Family.objects.get(project=project,
guid=fields['familyGuid'])
elif fields.get(PREVIOUS_FAMILY_ID_FIELD):
family = Family.objects.get(
project=project, family_id=fields[PREVIOUS_FAMILY_ID_FIELD])
else:
family, _ = Family.objects.get_or_create(
project=project, family_id=fields[FAMILY_ID_FIELD])
update_family_from_json(family,
fields,
user=request.user,
allow_unknown_keys=True)
updated_families.append(family)
updated_families_by_guid = {
'familiesByGuid': {
family.guid: _get_json_for_family(family,
request.user,
add_individual_guids_field=True)
for family in updated_families
}
}
return create_json_response(updated_families_by_guid)
def delete_project_locus_lists(request, project_guid):
project = get_project_and_check_permissions(project_guid, request.user, CAN_EDIT)
xbrowse_project = find_matching_xbrowse_model(project)
request_json = json.loads(request.body)
locus_lists = LocusList.objects.filter(guid__in=request_json['locusListGuids'])
for locus_list in locus_lists:
remove_perm(user_or_group=project.can_view_group, perm=CAN_VIEW, obj=locus_list)
xbrowse_gene_list = find_matching_xbrowse_model(locus_list)
if xbrowse_project and xbrowse_gene_list:
BaseProjectGeneList.objects.filter(project=xbrowse_project, gene_list=xbrowse_gene_list).delete()
return create_json_response({
'locusListGuids': [locus_list['locusListGuid'] for locus_list in get_sorted_project_locus_lists(project, request.user)],
})
def delete_project_handler(request, project_guid):
"""Delete project - request handler.
Args:
project_guid (string): GUID of the project to delete
"""
project = get_project_and_check_permissions(project_guid, request.user, permission_level=IS_OWNER)
delete_project(project)
return create_json_response({
'projectsByGuid': {
project.guid: None
},
})
def update_analysis_group_handler(request, project_guid, analysis_group_guid=None):
project = get_project_and_check_permissions(project_guid, request.user, permission_level=CAN_EDIT)
request_json = json.loads(request.body)
missing_fields = [field for field in REQUIRED_FIELDS.keys() if not request_json.get(field)]
if missing_fields:
return create_json_response(
{}, status=400, reason='Missing required field(s): {missing_field_names}'.format(
missing_field_names=', '.join([REQUIRED_FIELDS[field] for field in missing_fields])
))
families = Family.objects.filter(guid__in=request_json['familyGuids']).only('guid')
if len(families) != len(request_json['familyGuids']):
return create_json_response(
{}, status=400, reason='The following families do not exist: {missing_families}'.format(
missing_families=', '.join(set(request_json['familyGuids']) - set([family.guid for family in families]))
))
if analysis_group_guid:
analysis_group = AnalysisGroup.objects.get(guid=analysis_group_guid, project=project)
update_model_from_json(analysis_group, request_json, allow_unknown_keys=True)
else:
try:
analysis_group = create_seqr_model(
AnalysisGroup,
project=project,
name=request_json['name'],
description=request_json.get('description'),
created_by=request.user,
)
except IntegrityError:
return create_json_response(
{}, status=400, reason='An analysis group named "{name}" already exists for project "{project}"'.format(
name=request_json['name'], project=project.name
))
analysis_group.families.set(families)
base_family_group = find_matching_xbrowse_model(analysis_group)
if base_family_group:
base_family_group.families.set(BaseFamily.objects.filter(seqr_family__in=families))
return create_json_response({
'analysisGroupsByGuid': {
analysis_group.guid: get_json_for_analysis_group(analysis_group, project_guid=project_guid)
},
})
def delete_families_handler(request, project_guid):
"""Edit or delete one or more Individual records.
Args:
project_guid (string): GUID of project that contains these individuals.
"""
project = get_project_and_check_permissions(project_guid, request.user, CAN_EDIT)
request_json = json.loads(request.body)
logger.info("delete_families_handler %s", request_json)
families_to_delete = request_json.get('families')
if families_to_delete is None:
return create_json_response(
{}, status=400, reason="'recordIdsToDelete' not specified")
family_guids_to_delete = [f['familyGuid'] for f in families_to_delete]
# delete individuals 1st
individual_guids_to_delete = [i.guid for i in Individual.objects.filter(
family__project=project, family__guid__in=family_guids_to_delete)]
delete_individuals(project, individual_guids_to_delete)
# delete families
for family in Family.objects.filter(project=project, guid__in=family_guids_to_delete):
base_family = BaseFamily.objects.get(
project__project_id=project.deprecated_project_id,
family_id=family.family_id)
base_family.delete()
family.delete()
# send response
return create_json_response({
'individualsByGuid': {
individual_guid: None for individual_guid in individual_guids_to_delete
},
'familiesByGuid': {
family_guid: None for family_guid in family_guids_to_delete
},
})
def edit_families_handler(request, project_guid):
"""Edit or one or more Family records.
Args:
project_guid (string): GUID of project that contains these individuals.
"""
request_json = json.loads(request.body)
if request_json.get('uploadedFileId'):
modified_families = load_uploaded_file(request_json.get('uploadedFileId'))
else:
modified_families = request_json.get('families')
if modified_families is None:
return create_json_response(
{}, status=400, reason="'families' not specified")
project = get_project_and_check_permissions(project_guid, request.user, CAN_EDIT)
updated_families = []
for fields in modified_families:
if fields.get('familyGuid'):
family = Family.objects.get(project=project, guid=fields['familyGuid'])
elif fields.get(PREVIOUS_FAMILY_ID_FIELD):
family = Family.objects.get(project=project, family_id=fields[PREVIOUS_FAMILY_ID_FIELD])
else:
family, _ = get_or_create_seqr_model(Family, project=project, family_id=fields[FAMILY_ID_FIELD])
update_family_from_json(family, fields, user=request.user, allow_unknown_keys=True)
updated_families.append(family)
updated_families_by_guid = {
'familiesByGuid': {
family.guid: _get_json_for_family(family, request.user, add_individual_guids_field=True) for family in updated_families
}
}
return create_json_response(updated_families_by_guid)
def saved_variant_data(request, project_guid, variant_guid=None):
project = get_project_and_check_permissions(project_guid, request.user)
family_guids = request.GET['families'].split(',') if request.GET.get('families') else None
variant_query = SavedVariant.objects.filter(project=project)
if family_guids:
variant_query = variant_query.filter(family__guid__in=family_guids)
if variant_guid:
variant_query = variant_query.filter(guid=variant_guid)
if variant_query.count() < 1:
return create_json_response({}, status=404, reason='Variant {} not found'.format(variant_guid))
individual_guids_by_id = {i.individual_id: i.guid for i in Individual.objects.filter(family__project=project)}
saved_variants = get_json_for_saved_variants(variant_query, add_tags=True, add_details=True, project=project,
user=request.user, individual_guids_by_id=individual_guids_by_id)
variants = {variant['variantGuid']: variant for variant in saved_variants if variant['notes'] or variant['tags']}
genes = _saved_variant_genes(variants.values())
_add_locus_lists([project], variants.values(), genes)
return create_json_response({
'savedVariantsByGuid': variants,
'genesById': genes,
})
def delete_analysis_group_handler(request, project_guid, analysis_group_guid):
project = get_project_and_check_permissions(project_guid, request.user, permission_level=CAN_EDIT)
analysis_group = AnalysisGroup.objects.get(guid=analysis_group_guid, project=project)
delete_seqr_model(analysis_group)
return create_json_response({'analysisGroupsByGuid': {analysis_group_guid: None}})
def add_variants_dataset_handler(request, project_guid):
"""Create or update samples for the given variant dataset
Args:
request: Django request object
project_guid (string): GUID of the project that should be updated
HTTP POST
Request body - should contain the following json structure:
{
'elasticsearchIndex': <String> (required)
'ignoreExtraSamplesInCallset': <Boolean>
'ignoreMissingFamilyMembers': <Boolean>
'mappingFilePath': <String>
}
Response body - will contain the following structure:
"""
project = get_project_and_check_permissions(project_guid, request.user, permission_level=CAN_EDIT)
request_json = json.loads(request.body)
try:
if 'elasticsearchIndex' not in request_json:
raise ValueError('"elasticsearchIndex" is required')
elasticsearch_index = request_json['elasticsearchIndex'].strip()
sample_ids, index_metadata = get_elasticsearch_index_samples(elasticsearch_index)
validate_index_metadata(index_metadata, project, elasticsearch_index)
sample_type = index_metadata['sampleType']
dataset_path = index_metadata['sourceFilePath']
sample_id_to_individual_id_mapping = load_mapping_file(
request_json['mappingFilePath']) if request_json.get('mappingFilePath') else {}
matched_sample_id_to_sample_record = match_sample_ids_to_sample_records(
project=project,
sample_ids=sample_ids,
sample_type=sample_type,
dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS,
elasticsearch_index=elasticsearch_index,
sample_id_to_individual_id_mapping=sample_id_to_individual_id_mapping,
)
unmatched_samples = set(sample_ids) - set(matched_sample_id_to_sample_record.keys())
if request_json.get('ignoreExtraSamplesInCallset'):
if len(matched_sample_id_to_sample_record) == 0:
raise Exception(
"None of the individuals or samples in the project matched the {} expected sample id(s)".format(
len(sample_ids)
))
elif len(unmatched_samples) > 0:
raise Exception(
'Matches not found for ES sample ids: {}. Uploading a mapping file for these samples, or select the "Ignore extra samples in callset" checkbox to ignore.'.format(
", ".join(unmatched_samples)))
if not request_json.get('ignoreMissingFamilyMembers'):
included_family_individuals = defaultdict(set)
for sample in matched_sample_id_to_sample_record.values():
included_family_individuals[sample.individual.family].add(sample.individual.individual_id)
missing_family_individuals = []
for family, individual_ids in included_family_individuals.items():
missing_indivs = family.individual_set.filter(
sample__sample_status=Sample.SAMPLE_STATUS_LOADED,
sample__dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS
).exclude(individual_id__in=individual_ids)
if missing_indivs:
missing_family_individuals.append(
'{} ({})'.format(family.family_id, ', '.join([i.individual_id for i in missing_indivs]))
)
if missing_family_individuals:
raise Exception(
'The following families are included in the callset but are missing some family members: {}. This can lead to errors in variant search. If you still want to upload this callset, select the "Ignore missing family members" checkbox.'.format(
', '.join(missing_family_individuals)
))
_update_samples(
matched_sample_id_to_sample_record, elasticsearch_index=elasticsearch_index, dataset_path=dataset_path
)
except Exception as e:
traceback.print_exc()
return create_json_response({'errors': [e.message or str(e)]}, status=400)
if not matched_sample_id_to_sample_record:
return create_json_response({'samplesByGuid': {}})
update_project_from_json(project, {'has_new_search': True})
reset_cached_search_results(project)
_deprecated_update_vcfffiles(
project, sample_type, elasticsearch_index, dataset_path, matched_sample_id_to_sample_record
)
return create_json_response(_get_samples_json(matched_sample_id_to_sample_record, project_guid))
def receive_families_table_handler(request, project_guid):
"""Handler for the initial upload of an Excel or .tsv table of families. This handler
parses the records, but doesn't save them in the database. Instead, it saves them to
a temporary file and sends a 'uploadedFileId' representing this file back to the client.
Args:
request (object): Django request object
project_guid (string): project GUID
"""
project = get_project_and_check_permissions(project_guid, request.user)
def _process_records(records, filename=''):
column_map = {}
for i, field in enumerate(records[0]):
key = field.lower()
if 'family' in key:
if 'prev' in key:
column_map[PREVIOUS_FAMILY_ID_FIELD] = i
else:
column_map[FAMILY_ID_FIELD] = i
elif 'display' in key:
column_map['displayName'] = i
elif 'description' in key:
column_map['description'] = i
elif 'phenotype' in key:
column_map['codedPhenotype'] = i
if FAMILY_ID_FIELD not in column_map:
raise ValueError('Invalid header, missing family id column')
return [{column: row[index] if isinstance(index, int) else next((row[i] for i in index if row[i]), None)
for column, index in column_map.items()} for row in records[1:]]
try:
uploaded_file_id, filename, json_records = save_uploaded_file(request, process_records=_process_records)
except Exception as e:
return create_json_response({'errors': [e.message or str(e)], 'warnings': []}, status=400, reason=e.message or str(e))
prev_fam_ids = {r[PREVIOUS_FAMILY_ID_FIELD] for r in json_records if r.get(PREVIOUS_FAMILY_ID_FIELD)}
existing_prev_fam_ids = {f.family_id for f in Family.objects.filter(family_id__in=prev_fam_ids, project=project).only('family_id')}
if len(prev_fam_ids) != len(existing_prev_fam_ids):
missing_prev_ids = [family_id for family_id in prev_fam_ids if family_id not in existing_prev_fam_ids]
return create_json_response(
{'errors': [
'Could not find families with the following previous IDs: {}'.format(', '.join(missing_prev_ids))
], 'warnings': []},
status=400, reason='Invalid input')
fam_ids = {r[FAMILY_ID_FIELD] for r in json_records if not r.get(PREVIOUS_FAMILY_ID_FIELD)}
num_families_to_update = len(prev_fam_ids) + Family.objects.filter(family_id__in=fam_ids, project=project).count()
num_families = len(json_records)
num_families_to_create = num_families - num_families_to_update
info = [
"{num_families} families parsed from {filename}".format(num_families=num_families, filename=filename),
"{} new families will be added, {} existing families will be updated".format(num_families_to_create, num_families_to_update),
]
return create_json_response({
'uploadedFileId': uploaded_file_id,
'errors': [],
'warnings': [],
'info': info,
})
def edit_individuals_handler(request, project_guid):
"""Modify one or more Individual records.
Args:
request (object): Django HTTP Request object.
project_guid (string): GUID of project that contains these individuals.
Request:
body should be a json dictionary that contains a 'individuals' list that includes the individuals to update,
represented by dictionaries of their guid and fields to update -
for example:
{
'individuals': [
{ 'individualGuid': <individualGuid1>, 'paternalId': <paternalId>, 'affected': 'A' },
{ 'individualGuid': <individualGuid1>, 'sex': 'U' },
...
[
}
Response:
json dictionary representing the updated individual(s) like:
{
<individualGuid1> : { individualId: xxx, sex: xxx, affected: xxx, ...},
<individualGuid2> : { individualId: xxx, sex: xxx, affected: xxx, ...},
...
}
"""
project = get_project_and_check_permissions(project_guid, request.user, CAN_EDIT)
request_json = json.loads(request.body)
modified_individuals_list = request_json.get('individuals')
if modified_individuals_list is None:
return create_json_response(
{}, status=400, reason="'individuals' not specified")
update_individuals = {ind['individualGuid']: ind for ind in modified_individuals_list}
update_individual_models = {ind.guid: ind for ind in Individual.objects.filter(guid__in=update_individuals.keys())}
for modified_ind in modified_individuals_list:
model = update_individual_models[modified_ind['individualGuid']]
if modified_ind[JsonConstants.INDIVIDUAL_ID_COLUMN] != model.individual_id:
modified_ind[JsonConstants.PREVIOUS_INDIVIDUAL_ID_COLUMN] = model.individual_id
modified_family_ids = {ind.get('familyId') or ind['family']['familyId'] for ind in modified_individuals_list}
modified_family_ids.update({ind.family.family_id for ind in update_individual_models.values()})
related_individuals = Individual.objects.filter(
family__family_id__in=modified_family_ids, family__project=project).exclude(guid__in=update_individuals.keys())
related_individuals_json = _get_json_for_individuals(related_individuals, project_guid=project_guid, family_fields=['family_id'])
individuals_list = modified_individuals_list + related_individuals_json
# TODO more validation?
errors, warnings = validate_fam_file_records(individuals_list, fail_on_warnings=True)
if errors:
return create_json_response({'errors': errors, 'warnings': warnings}, status=400, reason='Invalid updates')
try:
updated_families, updated_individuals = add_or_update_individuals_and_families(
project, modified_individuals_list, user=request.user
)
except Exception as e:
return create_json_response({'errors': [e.message]}, status=400, reason='Invalid updates')
individuals_by_guid = {
individual.guid: _get_json_for_individual(individual, request.user) for individual in updated_individuals
}
families_by_guid = {
family.guid: _get_json_for_family(family, request.user, add_individual_guids_field=True)
for family in updated_families
}
return create_json_response({
'individualsByGuid': individuals_by_guid,
'familiesByGuid': families_by_guid,
})
def receive_hpo_table_handler(request, project_guid):
"""Handler for bulk update of hpo terms. This handler parses the records, but doesn't save them in the database.
Instead, it saves them to a temporary file and sends a 'uploadedFileId' representing this file back to the client.
Args:
request (object): Django request object
project_guid (string): project GUID
"""
project = get_project_and_check_permissions(project_guid, request.user)
try:
uploaded_file_id, _, json_records = save_uploaded_file(request, process_records=_process_hpo_records)
except Exception as e:
return create_json_response({'errors': [e.message or str(e)], 'warnings': []}, status=400, reason=e.message or str(e))
updates_by_individual_guid = {}
missing_individuals = []
unchanged_individuals = []
all_hpo_terms = set()
for record in json_records:
family_id = record.get(FAMILY_ID_COLUMN, None)
individual_id = record.get(INDIVIDUAL_ID_COLUMN)
individual_q = Individual.objects.filter(
individual_id__in=[individual_id, '{}_{}'.format(family_id, individual_id)],
family__project=project,
)
if family_id:
individual_q = individual_q.filter(family__family_id=family_id)
individual = individual_q.first()
if individual:
features = record.get(FEATURES_COLUMN) or []
if individual.phenotips_data and features and \
_feature_set(features) == _feature_set(json.loads(individual.phenotips_data).get('features', [])):
unchanged_individuals.append(individual_id)
else:
all_hpo_terms.update([feature['id'] for feature in features])
updates_by_individual_guid[individual.guid] = features
else:
missing_individuals.append(individual_id)
if not updates_by_individual_guid:
return create_json_response({
'errors': ['Unable to find individuals to update for any of the {total} parsed individuals.{missing}{unchanged}'.format(
total=len(missing_individuals) + len(unchanged_individuals),
missing=' No matching ids found for {} individuals'.format(len(missing_individuals)) if missing_individuals else '',
unchanged=' No changes detected for {} individuals'.format(len(unchanged_individuals)) if unchanged_individuals else '',
)],
'warnings': []
}, status=400, reason='Unable to find any matching individuals')
hpo_terms = {hpo.hpo_id: hpo for hpo in HumanPhenotypeOntology.objects.filter(hpo_id__in=all_hpo_terms)}
invalid_hpo_terms = set()
for features in updates_by_individual_guid.values():
for feature in features:
hpo_data = hpo_terms.get(feature['id'])
if hpo_data:
feature['category'] = hpo_data.category_id
feature['label'] = hpo_data.name
else:
invalid_hpo_terms.add(feature['id'])
if invalid_hpo_terms:
return create_json_response({
'errors': [
"The following HPO terms were not found in seqr's HPO data: {}".format(', '.join(invalid_hpo_terms))
],
'warnings': []
}, status=400, reason='Invalid HPO terms')
info = ['{} individuals will be updated'.format(len(updates_by_individual_guid))]
warnings = []
if missing_individuals:
warnings.append(
'Unable to find matching ids for {} individuals. The following entries will not be updated: {}'.format(
len(missing_individuals), ', '.join(missing_individuals)
))
if unchanged_individuals:
warnings.append(
'No changes detected for {} individuals. The following entries will not be updated: {}'.format(
len(unchanged_individuals), ', '.join(unchanged_individuals)
))
response = {
'updatesByIndividualGuid': updates_by_individual_guid,
'uploadedFileId': uploaded_file_id,
'errors': [],
'warnings': warnings,
'info': info,
}
return create_json_response(response)
def receive_individuals_table_handler(request, project_guid):
"""Handler for the initial upload of an Excel or .tsv table of individuals. This handler
parses the records, but doesn't save them in the database. Instead, it saves them to
a temporary file and sends a 'uploadedFileId' representing this file back to the client. If/when the
client then wants to 'apply' this table, it can send the uploadedFileId to the
save_individuals_table(..) handler to actually save the data in the database.
Args:
request (object): Django request object
project_guid (string): project GUID
"""
project = get_project_and_check_permissions(project_guid, request.user)
def process_records(json_records, filename='ped_file'):
pedigree_records, errors, warnings = parse_pedigree_table(json_records, filename, user=request.user, project=project)
if errors:
raise ErrorsWarningsException(errors, warnings)
return pedigree_records
try:
uploaded_file_id, filename, json_records = save_uploaded_file(request, process_records=process_records)
except ErrorsWarningsException as e:
return create_json_response({'errors': e.errors, 'warnings': e.warnings}, status=400, reason=e.errors)
except Exception as e:
return create_json_response({'errors': [e.message or str(e)], 'warnings': []}, status=400, reason=e.message or str(e))
# send back some stats
individual_ids_by_family = defaultdict(list)
for r in json_records:
if r.get(JsonConstants.PREVIOUS_INDIVIDUAL_ID_COLUMN):
individual_ids_by_family[r[JsonConstants.FAMILY_ID_COLUMN]].append(
(r[JsonConstants.PREVIOUS_INDIVIDUAL_ID_COLUMN], True)
)
else:
individual_ids_by_family[r[JsonConstants.FAMILY_ID_COLUMN]].append(
(r[JsonConstants.INDIVIDUAL_ID_COLUMN], False)
)
num_individuals = sum([len(indiv_ids) for indiv_ids in individual_ids_by_family.values()])
num_existing_individuals = 0
missing_prev_ids = []
for family_id, indiv_ids in individual_ids_by_family.items():
existing_individuals = {i.individual_id for i in Individual.objects.filter(
individual_id__in=[indiv_id for (indiv_id, _) in indiv_ids], family__family_id=family_id, family__project=project
).only('individual_id')}
num_existing_individuals += len(existing_individuals)
missing_prev_ids += [indiv_id for (indiv_id, is_previous) in indiv_ids if is_previous and indiv_id not in existing_individuals]
num_individuals_to_create = num_individuals - num_existing_individuals
if missing_prev_ids:
return create_json_response(
{'errors': [
'Could not find individuals with the following previous IDs: {}'.format(', '.join(missing_prev_ids))
], 'warnings': []},
status=400, reason='Invalid input')
family_ids = set(r[JsonConstants.FAMILY_ID_COLUMN] for r in json_records)
num_families = len(family_ids)
num_existing_families = Family.objects.filter(family_id__in=family_ids, project=project).count()
num_families_to_create = num_families - num_existing_families
info = [
"{num_families} families, {num_individuals} individuals parsed from {filename}".format(
num_families=num_families, num_individuals=num_individuals, filename=filename
),
"{} new families, {} new individuals will be added to the project".format(num_families_to_create, num_individuals_to_create),
"{} existing individuals will be updated".format(num_existing_individuals),
]
response = {
'uploadedFileId': uploaded_file_id,
'errors': [],
'warnings': [],
'info': info,
}
logger.info(response)
return create_json_response(response)
def update_saved_variant_json(request, project_guid):
project = get_project_and_check_permissions(project_guid, request.user, permission_level=CAN_EDIT)
updated_saved_variant_guids = update_project_saved_variant_json(project)
return create_json_response({variant_guid: None for variant_guid in updated_saved_variant_guids})
def add_alignment_dataset_handler(request, project_guid):
"""Create or update samples for the given dataset
Args:
request: Django request object
project_guid (string): GUID of the project that should be updated
HTTP POST
Request body - should contain the following json structure:
{
'sampleType': <"WGS", "WES", or "RNA"> (required)
'datasetType': <"VARIANTS", or "ALIGN"> (required)
'elasticsearchIndex': <String>
'datasetPath': <String>
'datasetName': <String>
'ignoreExtraSamplesInCallset': <Boolean>
'mappingFile': { 'uploadedFileId': <Id for temporary uploaded file> }
}
Response body - will contain the following structure:
"""
project = get_project_and_check_permissions(project_guid, request.user, permission_level=CAN_EDIT)
request_json = json.loads(request.body)
try:
required_fields = ['sampleType', 'mappingFile']
if any(field not in request_json for field in required_fields):
raise ValueError(
"request must contain fields: {}".format(', '.join(required_fields)))
sample_type = request_json['sampleType']
if sample_type not in {choice[0] for choice in Sample.SAMPLE_TYPE_CHOICES}:
raise Exception("Sample type not supported: {}".format(sample_type))
mapping_file_id = request_json['mappingFile']['uploadedFileId']
sample_id_to_individual_id_mapping = {}
sample_dataset_path_mapping = {}
for individual_id, dataset_path in load_uploaded_mapping_file(mapping_file_id).items():
if not (dataset_path.endswith(".bam") or dataset_path.endswith(".cram")):
raise Exception('BAM / CRAM file "{}" must have a .bam or .cram extension'.format(dataset_path))
validate_alignment_dataset_path(dataset_path)
sample_id = dataset_path.split('/')[-1].split('.')[0]
sample_id_to_individual_id_mapping[sample_id] = individual_id
sample_dataset_path_mapping[sample_id] = dataset_path
matched_sample_id_to_sample_record = match_sample_ids_to_sample_records(
project=project,
sample_ids=sample_id_to_individual_id_mapping.keys(),
sample_type=sample_type,
dataset_type=Sample.DATASET_TYPE_READ_ALIGNMENTS,
sample_id_to_individual_id_mapping=sample_id_to_individual_id_mapping,
)
unmatched_samples = set(sample_id_to_individual_id_mapping.keys()) - set(matched_sample_id_to_sample_record.keys())
if len(unmatched_samples) > 0:
raise Exception('The following Individual IDs do not exist: {}'.format(", ".join(unmatched_samples)))
_update_samples(matched_sample_id_to_sample_record, sample_dataset_path_mapping=sample_dataset_path_mapping)
except Exception as e:
traceback.print_exc()
return create_json_response({'errors': [e.message or str(e)]}, status=400)
if not matched_sample_id_to_sample_record:
return create_json_response({'samplesByGuid': {}})
# Deprecated update VCFFile records
for sample in matched_sample_id_to_sample_record.values():
for base_indiv in BaseIndividual.objects.filter(seqr_individual=sample.individual).only('id'):
base_indiv.bam_file_path = sample.dataset_file_path
base_indiv.save()
return create_json_response(_get_samples_json(matched_sample_id_to_sample_record, project_guid))
