def update_model_from_json(model_obj,
json,
user=None,
verbose=False,
allow_unknown_keys=False,
immutable_keys=None):
seqr_update_fields = {}
internal_fields = model_obj._meta.internal_json_fields if hasattr(
model_obj._meta, 'internal_json_fields') else []
for json_key, value in json.items():
orm_key = _to_snake_case(json_key)
if orm_key in (immutable_keys or []):
if allow_unknown_keys:
continue
raise ValueError('Cannot edit field {}'.format(orm_key))
if allow_unknown_keys and not hasattr(model_obj, orm_key):
continue
if getattr(model_obj, orm_key) != value:
if orm_key in internal_fields and not (user and user.is_staff):
raise ValueError(
'User {0} is not authorized to edit the internal field {1}'
.format(user, orm_key))
if verbose:
model_obj_name = getattr(model_obj, 'guid', model_obj.__name__)
logger.info("Setting {0}.{1} to {2}".format(
model_obj_name, orm_key, value))
seqr_update_fields[orm_key] = value
if seqr_update_fields:
update_seqr_model(model_obj, **seqr_update_fields)
def _create_patient_if_missing(project, individual):
"""Create a new PhenoTips patient record with the given patient id.
Args:
project (Model): seqr Project - used to retrieve PhenoTips credentials
individual (Model): seqr Individual
Returns:
True if patient created
Raises:
PhenotipsException: if unable to create patient record
"""
if _phenotips_patient_exists(individual):
return False
url = '/rest/patients'
headers = {"Content-Type": "application/json"}
data = json.dumps({'external_id': individual.guid})
auth_tuple = _get_phenotips_uname_and_pwd_for_project(project.phenotips_user_id)
response_items = _make_api_call('POST', url, auth_tuple=auth_tuple, http_headers=headers, data=data, expected_status_code=201, parse_json_resonse=False)
patient_id = response_items['Location'].split('/')[-1]
logger.info("Created PhenoTips record with patient id {patient_id} and external id {external_id}".format(patient_id=patient_id, external_id=individual.guid))
username_read_only, _ = _get_phenotips_uname_and_pwd_for_project(project.phenotips_user_id, read_only=True)
_add_user_to_patient(username_read_only, patient_id, allow_edit=False)
logger.info("Added PhenoTips user {username} to {patient_id}".format(username=username_read_only, patient_id=patient_id))
update_seqr_model(individual, phenotips_patient_id=patient_id, phenotips_eid=individual.guid)
return True
def update_family_assigned_analyst(request, family_guid):
"""Updates the specified field in the Family model.
Args:
family_guid (string): GUID of the family.
"""
family = Family.objects.get(guid=family_guid)
check_permissions(family.project, request.user, CAN_EDIT)
request_json = json.loads(request.body)
assigned_analyst_username = request_json.get('assigned_analyst_username')
if assigned_analyst_username:
try:
assigned_analyst = User.objects.get(
username=assigned_analyst_username)
except Exception:
return create_json_response({},
status=400,
reason="specified user does not exist")
else:
assigned_analyst = None
update_seqr_model(family, assigned_analyst=assigned_analyst)
return create_json_response(
{family.guid: _get_json_for_family(family, request.user)})
def update_model_from_json(model_obj, json, user=None, verbose=False, allow_unknown_keys=False, immutable_keys=None, conditional_edit_keys=None):
immutable_keys = (immutable_keys or []) + ['created_by', 'created_date', 'last_modified_date', 'id']
seqr_update_fields = {}
internal_fields = model_obj._meta.internal_json_fields if hasattr(model_obj._meta, 'internal_json_fields') else []
for json_key, value in json.items():
orm_key = _to_snake_case(json_key)
if orm_key in immutable_keys:
if allow_unknown_keys:
continue
raise ValueError('Cannot edit field {}'.format(orm_key))
if allow_unknown_keys and not hasattr(model_obj, orm_key):
continue
if getattr(model_obj, orm_key) != value:
if orm_key in internal_fields and not (user and user.is_staff):
raise ValueError('User {0} is not authorized to edit the internal field {1}'.format(user, orm_key))
if conditional_edit_keys and orm_key in conditional_edit_keys:
conditional_edit_keys[orm_key](model_obj)
if verbose:
model_obj_name = getattr(model_obj, 'guid', None) or model_obj.__name__
logger.info("Setting {0}.{1} to {2}".format(model_obj_name, orm_key, value))
seqr_update_fields[orm_key] = value
if seqr_update_fields:
update_seqr_model(model_obj, **seqr_update_fields)
def update_individual_hpo_terms(request, individual_guid):
individual = Individual.objects.get(guid=individual_guid)
project = individual.family.project
check_permissions(project, request.user, CAN_EDIT)
features = json.loads(request.body)
_create_patient_if_missing(project, individual)
patient_json = _get_patient_data(project, individual)
patient_json["features"] = features
patient_json_string = json.dumps(patient_json)
url = _phenotips_patient_url(individual)
auth_tuple = _get_phenotips_uname_and_pwd_for_project(project.phenotips_user_id, read_only=False)
_make_api_call('PUT', url, data=patient_json_string, auth_tuple=auth_tuple, expected_status_code=204)
phenotips_patient_id = patient_json['id']
phenotips_eid = patient_json.get('external_id')
update_seqr_model(
individual,
phenotips_data=json.dumps(patient_json),
phenotips_patient_id=phenotips_patient_id,
phenotips_eid=phenotips_eid)
return create_json_response({
individual.guid: {
'phenotipsData': patient_json,
'phenotipsPatientId': phenotips_patient_id,
'phenotipsEid': phenotips_eid
}
})
def update_family_pedigree_image(request, family_guid):
"""Updates the specified field in the Family model.
Args:
family_guid (string): GUID of the family.
"""
family = Family.objects.get(guid=family_guid)
# check permission
check_permissions(family.project, request.user, CAN_EDIT)
if len(request.FILES) == 0:
pedigree_image = None
elif len(request.FILES) > 1:
return create_json_response({},
status=400,
reason='Received {} files'.format(
len(request.FILES)))
else:
pedigree_image = request.FILES.values()[0]
update_seqr_model(family, pedigree_image=pedigree_image)
return create_json_response(
{family.guid: _get_json_for_family(family, request.user)})
def _update_pedigree_image(family, project_guid=None):
"""Uses HaploPainter to (re)generate the pedigree image for the given family.
Args:
family (object): seqr Family model.
"""
individual_records = _get_parsed_individuals(family, project_guid)
if not individual_records:
return
# run HaploPainter to generate the pedigree image
png_file_path = os.path.join(tempfile.gettempdir(),
"pedigree_image_%s.png" % _random_string(10))
family_id = family.family_id
with tempfile.NamedTemporaryFile('w', suffix=".fam",
delete=True) as fam_file:
# columns: family, individual id, paternal id, maternal id, sex, affected
for i in individual_records.values():
row = [family_id] + [
i[key] for key in [
'individualId', 'paternalId', 'maternalId', 'sex',
'affected'
]
]
fam_file.write("\t".join(row).encode('UTF-8'))
fam_file.write("\n")
fam_file.flush()
fam_file_path = fam_file.name
haplopainter_command = "perl " + os.path.join(
BASE_DIR,
"xbrowse_server/base/management/commands/HaploPainter1.043.pl")
haplopainter_command += " -b -outformat png -pedfile {fam_file_path} -family {family_id} -outfile {png_file_path}".format(
fam_file_path=fam_file_path,
family_id=family_id,
png_file_path=png_file_path)
os.system(haplopainter_command)
if not os.path.isfile(png_file_path):
logger.error("Failed to generated pedigree image for family: %s" %
family_id)
update_seqr_model(family, pedigree_image=None)
return
_save_pedigree_image_file(family, png_file_path)
os.remove(png_file_path)
def _create_patient_if_missing(project, individual):
"""Create a new PhenoTips patient record with the given patient id.
Args:
project (Model): seqr Project - used to retrieve PhenoTips credentials
individual (Model): seqr Individual
Returns:
True if patient created
Raises:
PhenotipsException: if unable to create patient record
"""
if _phenotips_patient_exists(individual):
return False
url = '/rest/patients'
headers = {"Content-Type": "application/json"}
data = json.dumps({'external_id': individual.guid})
auth_tuple = _get_phenotips_uname_and_pwd_for_project(
project.phenotips_user_id)
response_items = _make_api_call('POST',
url,
auth_tuple=auth_tuple,
http_headers=headers,
data=data,
expected_status_code=201,
parse_json_resonse=False)
patient_id = response_items['Location'].split('/')[-1]
logger.info(
"Created PhenoTips record with patient id {patient_id} and external id {external_id}"
.format(patient_id=patient_id, external_id=individual.guid))
username_read_only, _ = _get_phenotips_uname_and_pwd_for_project(
project.phenotips_user_id, read_only=True)
_add_user_to_patient(username_read_only, patient_id, allow_edit=False)
logger.info("Added PhenoTips user {username} to {patient_id}".format(
username=username_read_only, patient_id=patient_id))
update_seqr_model(individual,
phenotips_patient_id=patient_id,
phenotips_eid=individual.guid)
return True
def update_project_handler(request, project_guid):
"""Update project metadata - including one or more of these fields: name, description
Args:
project_guid (string): GUID of the project that should be updated
HTTP POST
Request body - should contain the following json structure:
{
'form' : {
'name': <project name>,
'description': <project description>,
}
}
Response body - will contain the following structure, representing the updated project:
{
'projectsByGuid': {
<projectGuid1> : { ... <project key-value pairs> ... }
}
}
"""
project = Project.objects.get(guid=project_guid)
check_permissions(project, request.user, CAN_EDIT)
request_json = json.loads(request.body)
if 'name' in request_json:
update_seqr_model(project, name=request_json.get('name'))
if 'description' in request_json:
update_seqr_model(project, description=request_json.get('description'))
return create_json_response({
'projectsByGuid': {
project.guid: _get_json_for_project(project, request.user)
},
})
def _update_individual_phenotips_data(individual, patient_json):
"""Process and store the given patient_json in the given Individual model.
Args:
individual (Individual): Django Individual model
patient_json (json): json dict representing the patient record in PhenoTips
"""
# for each HPO term, get the top level HPO category (eg. Musculoskeletal)
for feature in patient_json.get('features', []):
hpo_id = feature['id']
try:
feature['category'] = HumanPhenotypeOntology.objects.get(hpo_id=hpo_id).category_id
except ObjectDoesNotExist:
logger.error("ERROR: PhenoTips HPO id %s not found in seqr HumanPhenotypeOntology table." % hpo_id)
update_seqr_model(
individual,
phenotips_data=json.dumps(patient_json),
phenotips_patient_id=patient_json['id'], # phenotips internal id
phenotips_eid=patient_json.get('external_id')) # phenotips external id
def save_internal_case_review_notes(request, family_guid):
"""Updates the `case_review_notes` field for the given family.
Args:
family_guid (string): GUID of the family.
"""
family = Family.objects.get(guid=family_guid)
request_json = json.loads(request.body)
if "value" not in request_json:
raise ValueError("Request is missing 'value' key: %s" %
(request.body, ))
update_seqr_model(family, internal_case_review_notes=request_json['value'])
return create_json_response({
family.guid:
_get_json_for_family(family,
request.user,
add_individual_guids_field=True)
})
def update_individual_hpo_terms(request, individual_guid):
individual = Individual.objects.get(guid=individual_guid)
project = individual.family.project
check_permissions(project, request.user, CAN_EDIT)
features = json.loads(request.body)
_create_patient_if_missing(project, individual)
patient_json = _get_patient_data(project, individual)
patient_json["features"] = features
patient_json_string = json.dumps(patient_json)
url = _phenotips_patient_url(individual)
auth_tuple = _get_phenotips_uname_and_pwd_for_project(
project.phenotips_user_id, read_only=False)
_make_api_call('PUT',
url,
data=patient_json_string,
auth_tuple=auth_tuple,
expected_status_code=204)
phenotips_patient_id = patient_json['id']
phenotips_eid = patient_json.get('external_id')
update_seqr_model(individual,
phenotips_data=json.dumps(patient_json),
phenotips_patient_id=phenotips_patient_id,
phenotips_eid=phenotips_eid)
return create_json_response({
individual.guid: {
'phenotipsData': patient_json,
'phenotipsPatientId': phenotips_patient_id,
'phenotipsEid': phenotips_eid
}
})
def update_pedigree_image(family, project_guid=None):
"""Uses HaploPainter to (re)generate the pedigree image for the given family.
Args:
family (object): seqr Family model.
"""
individual_records = _get_parsed_individuals(family, project_guid)
if not individual_records:
return
# run HaploPainter to generate the pedigree image
png_file_path = os.path.join(tempfile.gettempdir(), "pedigree_image_%s.png" % _random_string(10))
family_id = family.family_id
with tempfile.NamedTemporaryFile('w', suffix=".fam", delete=True) as fam_file:
# columns: family, individual id, paternal id, maternal id, sex, affected
for i in individual_records.values():
row = [family_id] + [i[key] for key in ['individualId', 'paternalId', 'maternalId', 'sex', 'affected']]
fam_file.write("\t".join(row).encode('UTF-8'))
fam_file.write("\n")
fam_file.flush()
fam_file_path = fam_file.name
haplopainter_command = "perl " + os.path.join(BASE_DIR, "xbrowse_server/base/management/commands/HaploPainter1.043.pl")
haplopainter_command += " -b -outformat png -pedfile %(fam_file_path)s -family %(family_id)s -outfile %(png_file_path)s" % locals()
os.system(haplopainter_command)
if not os.path.isfile(png_file_path):
logger.error("Failed to generated pedigree image for family: %s" % family_id)
update_seqr_model(family, pedigree_image=None)
return
_save_pedigree_image_file(family, png_file_path)
os.remove(png_file_path)
def _update_individual_phenotips_data(individual, patient_json):
"""Process and store the given patient_json in the given Individual model.
Args:
individual (Individual): Django Individual model
patient_json (json): json dict representing the patient record in PhenoTips
"""
# for each HPO term, get the top level HPO category (eg. Musculoskeletal)
for feature in patient_json.get('features', []):
hpo_id = feature['id']
try:
feature['category'] = HumanPhenotypeOntology.objects.get(
hpo_id=hpo_id).category_id
except ObjectDoesNotExist:
logger.error(
"ERROR: PhenoTips HPO id %s not found in seqr HumanPhenotypeOntology table."
% hpo_id)
update_seqr_model(
individual,
phenotips_data=json.dumps(patient_json),
phenotips_patient_id=patient_json['id'], # phenotips internal id
phenotips_eid=patient_json.get('external_id')) # phenotips external id
def _set_phenotips_patient_id_if_missing(project, individual):
if individual.phenotips_patient_id:
return
patient_json = _get_patient_data(project, individual)
update_seqr_model(individual, phenotips_patient_id=patient_json['id'])
def _get_parsed_individuals(family, project_guid=None):
"""Uses HaploPainter to (re)generate the pedigree image for the given family.
Args:
family (object): seqr Family model.
"""
individuals = Individual.objects.filter(family=family)
if len(individuals) < 2:
update_seqr_model(family, pedigree_image=None)
return None
# convert individuals to json
individual_records = {
individual['individualId']: individual for individual in
_get_json_for_individuals(individuals, project_guid=project_guid, family_guid=family.guid)
}
# compute a map of parent ids to list of children
parent_ids_to_children_map = collections.defaultdict(list)
for individual_id, individual_json in individual_records.items():
if not individual_json['paternalId'] and not individual_json['maternalId']:
continue
key = (individual_json['paternalId'], individual_json['maternalId'])
parent_ids_to_children_map[key].append(individual_json)
# generate placeholder individuals as needed, since HaploPainter1.043.pl doesn't support families with only 1 parent
for ((paternal_id, maternal_id), children) in parent_ids_to_children_map.items():
for parent_id_key, parent_id, sex in [
('paternalId', paternal_id, 'M'),
('maternalId', maternal_id, 'F')
]:
if not parent_id or parent_id not in individual_records:
placeholder_parent_id = 'placeholder_%s'% _random_string(10)
placeholder_parent_json = {
'individualId': placeholder_parent_id, # fake indiv id
'paternalId': '',
'maternalId': '',
'sex': sex,
'affected': 'INVISIBLE', # use a special value to tell HaploPainter to draw this individual as '?'
}
for child_json in children:
child_json[parent_id_key] = placeholder_parent_id
individual_records[placeholder_parent_id] = placeholder_parent_json
# convert to FAM file values
SEX_TO_FAM_FILE_VALUE = {"M": "1", "F": "2", "U": "0"}
AFFECTED_STATUS_TO_FAM_FILE_VALUE = {"A": "2", "N": "1", "U": "0", "INVISIBLE": "9"} # HaploPainter1.043.pl has been modified to hide individuals with affected-status='9'
return {
individual_id: {
'individualId': individual_id,
'paternalId': individual_json['paternalId'] or '0',
'maternalId': individual_json['maternalId'] or '0',
'sex': SEX_TO_FAM_FILE_VALUE[individual_json['sex']],
'affected': AFFECTED_STATUS_TO_FAM_FILE_VALUE[individual_json['affected']],
} for individual_id, individual_json in individual_records.items()
}
def handle(self, *args, **options):
from_project = Project.objects.get(guid=options['from_project'])
to_project = Project.objects.get(guid=options['to_project'])
to_base_project = find_matching_xbrowse_model(to_project)
family_ids = options['family_ids']
families = Family.objects.filter(project=from_project,
family_id__in=family_ids)
print('Found {} out of {} families. No match for: {}.'.format(
len(families), len(set(family_ids)),
set(family_ids) - set([f.family_id for f in families])))
for f in families:
print("==> Moving {}".format(f))
for seqr_individual in f.individual_set.all():
base_individual = find_matching_xbrowse_model(seqr_individual)
base_individual.project = to_base_project
base_individual.save()
# Update individuals in phenotips
if _phenotips_patient_exists(seqr_individual):
# make sure phenotips_patient_id is up to date
data_json = _get_patient_data(
from_project,
seqr_individual,
)
seqr_individual.phenotips_patient_id = data_json["id"]
seqr_individual.save()
# update permissions
phenotips_readonly_username, _ = _get_phenotips_uname_and_pwd_for_project(
to_project.phenotips_user_id, read_only=True)
_add_user_to_patient(phenotips_readonly_username,
seqr_individual.phenotips_patient_id,
allow_edit=False)
phenotips_readwrite_username, _ = _get_phenotips_uname_and_pwd_for_project(
to_project.phenotips_user_id, read_only=False)
_add_user_to_patient(phenotips_readwrite_username,
seqr_individual.phenotips_patient_id,
allow_edit=True)
# Update individuals samples/ VCFs
for from_vcf_file in base_individual.vcf_files.all():
to_vcf_file, _ = VCFFile.objects.get_or_create(
project=to_base_project,
elasticsearch_index=from_vcf_file.elasticsearch_index,
file_path=from_vcf_file.file_path,
dataset_type=from_vcf_file.dataset_type,
sample_type=from_vcf_file.sample_type,
loaded_date=from_vcf_file.loaded_date,
)
base_individual.vcf_files.add(to_vcf_file)
base_individual.vcf_files.remove(from_vcf_file)
# Update variant tags/ notes
saved_variants = SavedVariant.objects.filter(family=f)
for variant_tag in VariantTag.objects.filter(
saved_variant__in=saved_variants).select_related(
'variant_tag_type'):
if variant_tag.variant_tag_type.project:
to_tag_type, created = get_or_create_seqr_model(
VariantTagType,
project=to_project,
name=variant_tag.variant_tag_type.name)
if created:
update_seqr_model(
to_tag_type,
category=variant_tag.variant_tag_type.category,
description=variant_tag.variant_tag_type.
description,
color=variant_tag.variant_tag_type.color,
order=variant_tag.variant_tag_type.order,
)
update_seqr_model(variant_tag,
variant_tag_type=to_tag_type)
else:
to_project_tag, created = ProjectTag.objects.get_or_create(
project=to_base_project,
tag=variant_tag.variant_tag_type.name)
if created:
to_project_tag.category = variant_tag.variant_tag_type.category
to_project_tag.title = variant_tag.variant_tag_type.description
to_project_tag.color = variant_tag.variant_tag_type.color
to_project_tag.order = variant_tag.variant_tag_type.order
to_project_tag.save()
variant_tag.project_tag = to_project_tag
variant_tag.save()
base_family = find_matching_xbrowse_model(f)
for note in BaseVariantNote.objects.filter(family=base_family):
note.project = to_base_project
note.save()
# Update families
update_seqr_model(f, project=to_project)
print("Done.")
def _get_parsed_individuals(family, project_guid=None):
"""Uses HaploPainter to (re)generate the pedigree image for the given family.
Args:
family (object): seqr Family model.
"""
individuals = Individual.objects.filter(family=family)
if len(individuals) < 2:
update_seqr_model(family, pedigree_image=None)
return None
# convert individuals to json
individual_records = {
individual['individualId']: individual for individual in
_get_json_for_individuals(individuals, project_guid=project_guid, family_guid=family.guid)
}
# compute a map of parent ids to list of children
parent_ids_to_children_map = collections.defaultdict(list)
for individual_id, individual_json in individual_records.items():
if not individual_json['paternalId'] and not individual_json['maternalId']:
continue
key = (individual_json['paternalId'], individual_json['maternalId'])
parent_ids_to_children_map[key].append(individual_json)
# generate placeholder individuals as needed, since HaploPainter1.043.pl doesn't support families with only 1 parent
for ((paternal_id, maternal_id), children) in parent_ids_to_children_map.items():
for parent_id_key, parent_id, sex in [
('paternalId', paternal_id, 'M'),
('maternalId', maternal_id, 'F')
]:
if not parent_id or parent_id not in individual_records:
placeholder_parent_id = 'placeholder_%s'% _random_string(10)
placeholder_parent_json = {
'individualId': placeholder_parent_id, # fake indiv id
'paternalId': '',
'maternalId': '',
'sex': sex,
'affected': 'INVISIBLE', # use a special value to tell HaploPainter to draw this individual as '?'
}
for child_json in children:
child_json[parent_id_key] = placeholder_parent_id
individual_records[placeholder_parent_id] = placeholder_parent_json
# convert to FAM file values
SEX_TO_FAM_FILE_VALUE = {"M": "1", "F": "2", "U": "0"}
AFFECTED_STATUS_TO_FAM_FILE_VALUE = {"A": "2", "N": "1", "U": "0", "INVISIBLE": "9"} # HaploPainter1.043.pl has been modified to hide individuals with affected-status='9'
return {
individual_id: {
'individualId': individual_id,
'paternalId': individual_json['paternalId'] or '0',
'maternalId': individual_json['maternalId'] or '0',
'sex': SEX_TO_FAM_FILE_VALUE[individual_json['sex']],
'affected': AFFECTED_STATUS_TO_FAM_FILE_VALUE[individual_json['affected']],
} for individual_id, individual_json in individual_records.items()
}
def _set_phenotips_patient_id_if_missing(project, individual):
if individual.phenotips_patient_id:
return
patient_json = _get_patient_data(project, individual)
update_seqr_model(individual, phenotips_patient_id=patient_json['id'])
def handle(self, *args, **options):
from_project = Project.objects.get(guid=options['from_project'])
to_project = Project.objects.get(guid=options['to_project'])
family_ids = options['family_ids']
families = Family.objects.filter(project=from_project,
family_id__in=family_ids)
print('Found {} out of {} families. No match for: {}.'.format(
len(families), len(set(family_ids)),
set(family_ids) - set([f.family_id for f in families])))
for f in families:
print("==> Moving phenotips for {}".format(f))
for seqr_individual in f.individual_set.all():
if phenotips_patient_exists(seqr_individual):
# make sure phenotips_patient_id is up to date
data_json = _get_patient_data(
from_project,
seqr_individual,
)
seqr_individual.phenotips_patient_id = data_json["id"]
seqr_individual.save()
# update permissions
phenotips_readonly_username, _ = get_phenotips_uname_and_pwd_for_project(
to_project.phenotips_user_id, read_only=True)
_add_user_to_patient(phenotips_readonly_username,
seqr_individual.phenotips_patient_id,
allow_edit=False)
phenotips_readwrite_username, _ = get_phenotips_uname_and_pwd_for_project(
to_project.phenotips_user_id, read_only=False)
_add_user_to_patient(phenotips_readwrite_username,
seqr_individual.phenotips_patient_id,
allow_edit=True)
for variant_tag_type in VariantTagType.objects.filter(
project=from_project):
variant_tags = VariantTag.objects.filter(
saved_variant__family__in=families,
variant_tag_type=variant_tag_type)
if variant_tags:
print('Updating "{}" tags'.format(variant_tag_type.name))
to_tag_type, created = get_or_create_seqr_model(
VariantTagType,
project=to_project,
name=variant_tag_type.name)
if created:
update_seqr_model(
to_tag_type,
category=variant_tag_type.category,
description=variant_tag_type.description,
color=variant_tag_type.color,
order=variant_tag_type.order,
)
variant_tags.update(variant_tag_type=to_tag_type)
print("Updating families")
families.update(project=to_project)
print("Done.")
def add_or_update_individuals_and_families(project,
individual_records,
user=None):
"""Add or update individual and family records in the given project.
Args:
project (object): Django ORM model for the project to add families to
individual_records (list): A list of JSON records representing individuals. See
the return value of pedigree_info_utils#convert_fam_file_rows_to_json(..)
Return:
2-tuple: updated_families, updated_individuals containing Django ORM models
"""
families = {}
updated_individuals = []
for i, record in enumerate(individual_records):
# family id will be in different places in the json depending on whether it comes from a flat uploaded file or from the nested individual object
family_id = record.get('familyId') or record.get('family',
{}).get('familyId')
if not family_id:
raise ValueError(
"record #%s doesn't contain a 'familyId' key: %s" %
(i, record))
if 'individualId' not in record and 'individualGuid' not in record:
raise ValueError(
"record #%s doesn't contain an 'individualId' key: %s" %
(i, record))
family, created = get_or_create_seqr_model(Family,
project=project,
family_id=family_id)
if created:
logger.info("Created family: %s", family)
if not family.display_name:
update_seqr_model(family, display_name=family.family_id)
# uploaded files do not have unique guid's so fall back to a combination of family and individualId
if record.get('individualGuid'):
individual_filters = {'guid': record['individualGuid']}
else:
individual_filters = {
'family': family,
'individual_id': record['individualId']
}
individual, created = get_or_create_seqr_model(Individual,
**individual_filters)
record['family'] = family
record.pop('familyId', None)
update_individual_from_json(individual,
record,
allow_unknown_keys=True,
user=user)
updated_individuals.append(individual)
# apply additional json fields which don't directly map to Individual model fields
individual.phenotips_eid = individual.guid # use guid instead of indiv_id to avoid collisions
if created:
# create new PhenoTips patient record
patient_record = create_patient(project, individual)
update_seqr_model(individual,
phenotips_patient_id=patient_record['id'],
case_review_status='I')
logger.info(
"Created PhenoTips record with patient id %s and external id %s"
% (str(individual.phenotips_patient_id),
str(individual.phenotips_eid)))
if record.get(JsonConstants.HPO_TERMS_PRESENT_COLUMN) or record.get(
JsonConstants.FINAL_DIAGNOSIS_OMIM_COLUMN):
# update phenotips hpo ids
logger.info("Setting PhenoTips HPO Terms to: %s" %
(record.get(JsonConstants.HPO_TERMS_PRESENT_COLUMN), ))
set_patient_hpo_terms(
project,
individual,
hpo_terms_present=record.get(
JsonConstants.HPO_TERMS_PRESENT_COLUMN, []),
hpo_terms_absent=record.get(
JsonConstants.HPO_TERMS_ABSENT_COLUMN, []),
final_diagnosis_mim_ids=record.get(
JsonConstants.FINAL_DIAGNOSIS_OMIM_COLUMN, []))
if not individual.display_name:
update_seqr_model(individual,
display_name=individual.individual_id)
families[family.family_id] = family
updated_families = list(families.values())
# update pedigree images
update_pedigree_images(updated_families)
return updated_families, updated_individuals
def update_pedigree_image(family):
"""Uses HaploPainter to (re)generate the pedigree image for the given family.
Args:
family (object): seqr Family model.
"""
family_id = family.family_id
individuals = Individual.objects.filter(family=family)
if len(individuals) < 2:
update_seqr_model(family, pedigree_image=None)
return
# convert individuals to json
individual_records = {}
for i in individuals:
individual_records[i.individual_id] = _get_json_for_individual(i)
individual_records[i.individual_id]['familyId'] = i.family.family_id
# compute a map of parent ids to list of children
parent_ids_to_children_map = collections.defaultdict(list)
for individual_id, individual_json in individual_records.items():
if not individual_json['paternalId'] and not individual_json[
'maternalId']:
continue
key = (individual_json['paternalId'], individual_json['maternalId'])
parent_ids_to_children_map[key].append(individual_json)
# generate placeholder individuals as needed, since HaploPainter1.043.pl doesn't support families with only 1 parent
for ((paternal_id, maternal_id),
children) in parent_ids_to_children_map.items():
for parent_id_key, parent_id, sex in [('paternalId', paternal_id, 'M'),
('maternalId', maternal_id, 'F')
]:
if not parent_id or parent_id not in individual_records:
placeholder_parent_id = 'placeholder_%s' % _random_string(10)
placeholder_parent_json = {
'familyId': family_id,
'individualId': placeholder_parent_id, # fake indiv id
'paternalId': '',
'maternalId': '',
'sex': sex,
'affected':
'INVISIBLE', # use a special value to tell HaploPainter to draw this individual as '?'
}
for child_json in children:
child_json[parent_id_key] = placeholder_parent_id
individual_records[
placeholder_parent_id] = placeholder_parent_json
# convert to FAM file values
SEX_TO_FAM_FILE_VALUE = {"M": "1", "F": "2", "U": "0"}
AFFECTED_STATUS_TO_FAM_FILE_VALUE = {
"A": "2",
"N": "1",
"U": "0",
"INVISIBLE": "9"
} # HaploPainter1.043.pl has been modified to hide individuals with affected-status='9'
for individual_json in individual_records.values():
if not individual_json['paternalId']:
individual_json['paternalId'] = '0'
if not individual_json['maternalId']:
individual_json['maternalId'] = '0'
individual_json['sex'] = SEX_TO_FAM_FILE_VALUE[individual_json['sex']]
individual_json['affected'] = AFFECTED_STATUS_TO_FAM_FILE_VALUE[
individual_json['affected']]
# run HaploPainter to generate the pedigree image
png_file_path = os.path.join(tempfile.gettempdir(),
"pedigree_image_%s.png" % _random_string(10))
with tempfile.NamedTemporaryFile('w', suffix=".fam",
delete=True) as fam_file:
# columns: family, individual id, paternal id, maternal id, sex, affected
for i in individual_records.values():
row = [
i[key] for key in [
'familyId', 'individualId', 'paternalId', 'maternalId',
'sex', 'affected'
]
]
fam_file.write("\t".join(row).encode('UTF-8'))
fam_file.write("\n")
fam_file.flush()
fam_file_path = fam_file.name
haplopainter_command = "/usr/bin/perl " + os.path.join(
BASE_DIR,
"xbrowse_server/base/management/commands/HaploPainter1.043.pl")
haplopainter_command += " -b -outformat png -pedfile %(fam_file_path)s -family %(family_id)s -outfile %(png_file_path)s" % locals(
)
os.system(haplopainter_command)
if not os.path.isfile(png_file_path):
logger.error("Failed to generated pedigree image for family: %s" %
family_id)
update_seqr_model(family, pedigree_image=None)
return
_save_pedigree_image_file(family, png_file_path)
os.remove(png_file_path)
def update_mme_submission(request, individual_guid):
"""
Create or update the submission for the given individual.
"""
individual = Individual.objects.get(guid=individual_guid)
project = individual.family.project
check_permissions(project, request.user)
submission_json = json.loads(request.body)
submission_json.pop('individualGuid', {})
phenotypes = submission_json.pop('phenotypes', [])
gene_variants = submission_json.pop('geneVariants', [])
if not phenotypes and not gene_variants:
return create_json_response({}, status=400, reason='Genotypes or phentoypes are required')
if not submission_json.get('patient', {}).get('id'):
return create_json_response({}, status=400, reason='Patient id is required')
genomic_features = []
for gene_variant in gene_variants:
if not gene_variant.get('geneId'):
return create_json_response({}, status=400, reason='Gene id is required for genomic features')
feature = {'gene': {'id': gene_variant['geneId']}}
if 'numAlt' in gene_variant:
feature['zygosity'] = gene_variant['numAlt'] % 2
if gene_variant.get('pos'):
feature['variant'] = {
'alternateBases': gene_variant['alt'],
'referenceBases': gene_variant['ref'],
'referenceName': gene_variant['chrom'],
'start': gene_variant['pos'],
'assembly': gene_variant['genomeVersion'],
}
genomic_features.append(feature)
submission_json['patient']['genomicFeatures'] = genomic_features
submission_json['patient']['features'] = phenotypes
response = requests.post(url=MME_ADD_INDIVIDUAL_URL, headers=MME_HEADERS, data=json.dumps(submission_json))
if response.status_code not in (200, 409):
try:
response_json = response.json()
except Exception:
response_json = {}
return create_json_response(response_json, status=response.status_code, reason=response.content)
submitted_date = datetime.now()
individual.mme_submitted_data = submission_json
individual.mme_submitted_date = submitted_date
individual.mme_deleted_date = None
individual.mme_deleted_by = None
individual.save()
# update the project contact information if anything new was added
new_contact_names = set(submission_json['patient']['contact']['name'].split(',')) - set(project.mme_primary_data_owner.split(','))
new_contact_urls = set(submission_json['patient']['contact']['href'].replace('mailto:', '').split(',')) - set(project.mme_contact_url.replace('mailto:', '').split(','))
updates = {}
if new_contact_names:
updates['mme_primary_data_owner'] = '{},{}'.format(project.mme_primary_data_owner, ','.join(new_contact_names))
if new_contact_urls:
updates['mme_contact_url'] = '{},{}'.format(project.mme_contact_url, ','.join(new_contact_urls))
if updates:
update_seqr_model(project, **updates)
# search for new matches
return _search_individual_matches(individual, request.user)
def add_dataset_handler(request, project_guid):
"""Create or update samples for the given dataset
Args:
request: Django request object
project_guid (string): GUID of the project that should be updated
HTTP POST
Request body - should contain the following json structure:
{
'sampleType': <"WGS", "WES", or "RNA"> (required)
'datasetType': <"VARIANTS", or "ALIGN"> (required)
'elasticsearchIndex': <String>
'datasetPath': <String>
'datasetName': <String>
'ignoreExtraSamplesInCallset': <Boolean>
'mappingFile': { 'uploadedFileId': <Id for temporary uploaded file> }
}
Response body - will contain the following structure:
"""
logger.info("add_dataset_handler: " + str(request))
project = get_project_and_check_permissions(project_guid,
request.user,
permission_level=CAN_EDIT)
request_json = json.loads(request.body)
logger.info("add_dataset_handler: received %s" % pformat(request_json))
required_fields = ['sampleType', 'datasetType']
if any(field not in request_json for field in required_fields):
raise ValueError("request must contain fields: {}".format(
', '.join(required_fields)))
sample_type = request_json['sampleType']
dataset_type = request_json['datasetType']
elasticsearch_index = request_json.get('elasticsearchIndex')
if elasticsearch_index:
elasticsearch_index = elasticsearch_index.strip()
dataset_path = request_json.get('datasetPath')
if dataset_path:
dataset_path = dataset_path.strip()
dataset_name = request_json.get('datasetName')
if dataset_name:
dataset_name = dataset_name.strip()
ignore_extra_samples_in_callset = request_json.get(
'ignoreExtraSamplesInCallset')
ignore_missing_family_members = request_json.get(
'ignoreMissingFamilyMembers')
mapping_file_id = request_json.get('mappingFile', {}).get('uploadedFileId')
mapping_file_path = request_json.get('mappingFilePath')
try:
updated_samples, created_sample_ids = add_dataset(
project=project,
sample_type=sample_type,
dataset_type=dataset_type,
elasticsearch_index=elasticsearch_index,
dataset_path=dataset_path,
dataset_name=dataset_name,
max_edit_distance=0,
ignore_extra_samples_in_callset=ignore_extra_samples_in_callset,
ignore_missing_family_members=ignore_missing_family_members,
mapping_file_path=mapping_file_path,
mapping_file_id=mapping_file_id,
)
# update VCFFile records
if updated_samples:
if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS:
base_project = BaseProject.objects.get(seqr_project=project)
get_datastore(base_project).bust_project_cache(
base_project.project_id)
clear_project_results_cache(base_project.project_id)
vcf_file = VCFFile.objects.filter(
project=base_project,
dataset_type=dataset_type,
sample_type=sample_type,
elasticsearch_index=elasticsearch_index).order_by(
'-pk').first()
if not vcf_file:
vcf_file = VCFFile.objects.create(
project=base_project,
dataset_type=dataset_type,
sample_type=sample_type,
elasticsearch_index=elasticsearch_index,
)
logger.info("Created vcf file: " + str(vcf_file.__dict__))
vcf_file.file_path = dataset_path or "{}.vcf.gz".format(
elasticsearch_index
) # legacy VCFFile model requires non-empty vcf path
vcf_file.loaded_date = iter(updated_samples).next().loaded_date
vcf_file.save()
for indiv in [s.individual for s in updated_samples]:
for base_indiv in BaseIndividual.objects.filter(
seqr_individual=indiv).only('id'):
base_indiv.vcf_files.add(vcf_file)
elif dataset_type == Sample.DATASET_TYPE_READ_ALIGNMENTS:
for sample in updated_samples:
for base_indiv in BaseIndividual.objects.filter(
seqr_individual=sample.individual).only('id'):
base_indiv.bam_file_path = sample.dataset_file_path
base_indiv.save()
updated_sample_json = get_json_for_samples(updated_samples,
project_guid=project_guid)
response = {
'samplesByGuid': {s['sampleGuid']: s
for s in updated_sample_json}
}
updated_individuals = {
s['individualGuid']
for s in updated_sample_json if s['sampleId'] in created_sample_ids
}
if updated_individuals:
individuals = Individual.objects.filter(
guid__in=updated_individuals).prefetch_related(
'sample_set', 'family').only('guid')
response['individualsByGuid'] = {
ind.guid: {
'sampleGuids':
[s.guid for s in ind.sample_set.only('guid').all()]
}
for ind in individuals
}
for ind in individuals:
family = ind.family
if family.analysis_status == Family.ANALYSIS_STATUS_WAITING_FOR_DATA:
update_seqr_model(family,
analysis_status=Family.
ANALYSIS_STATUS_ANALYSIS_IN_PROGRESS)
return create_json_response(response)
except Exception as e:
traceback.print_exc()
return create_json_response({'errors': [e.message or str(e)]},
status=400)
def update_mme_submission(request, individual_guid):
"""
Create or update the submission for the given individual.
"""
individual = Individual.objects.get(guid=individual_guid)
project = individual.family.project
check_permissions(project, request.user)
submission_json = json.loads(request.body)
submission_json.pop('individualGuid', {})
phenotypes = submission_json.pop('phenotypes', [])
gene_variants = submission_json.pop('geneVariants', [])
if not phenotypes and not gene_variants:
return create_json_response({}, status=400, reason='Genotypes or phentoypes are required')
if not submission_json.get('patient', {}).get('id'):
return create_json_response({}, status=400, reason='Patient id is required')
genomic_features = []
for gene_variant in gene_variants:
if not gene_variant.get('geneId'):
return create_json_response({}, status=400, reason='Gene id is required for genomic features')
feature = {'gene': {'id': gene_variant['geneId']}}
if 'numAlt' in gene_variant:
feature['zygosity'] = gene_variant['numAlt'] % 2
if gene_variant.get('pos'):
feature['variant'] = {
'alternateBases': gene_variant['alt'],
'referenceBases': gene_variant['ref'],
'referenceName': gene_variant['chrom'],
'start': gene_variant['pos'],
'assembly': gene_variant['genomeVersion'],
}
genomic_features.append(feature)
submission_json['patient']['genomicFeatures'] = genomic_features
submission_json['patient']['features'] = phenotypes
response = requests.post(url=MME_ADD_INDIVIDUAL_URL, headers=MME_HEADERS, data=json.dumps(submission_json))
if response.status_code not in (200, 409):
try:
response_json = response.json()
except Exception:
response_json = {}
return create_json_response(response_json, status=response.status_code, reason=response.content)
submitted_date = datetime.now()
individual.mme_submitted_data = submission_json
individual.mme_submitted_date = submitted_date
individual.mme_deleted_date = None
individual.mme_deleted_by = None
individual.save()
# update the project contact information if anything new was added
new_contact_names = set(submission_json['patient']['contact']['name'].split(',')) - set(project.mme_primary_data_owner.split(','))
new_contact_urls = set(submission_json['patient']['contact']['href'].replace('mailto:', '').split(',')) - set(project.mme_contact_url.replace('mailto:', '').split(','))
updates = {}
if new_contact_names:
updates['mme_primary_data_owner'] = '{},{}'.format(project.mme_primary_data_owner, ','.join(new_contact_names))
if new_contact_urls:
updates['mme_contact_url'] = '{},{}'.format(project.mme_contact_url, ','.join(new_contact_urls))
if updates:
update_seqr_model(project, **updates)
# search for new matches
return _search_individual_matches(individual, request.user)
