def update_model_from_json(model_obj, json, user=None, verbose=False, allow_unknown_keys=False, immutable_keys=None): seqr_update_fields = {} internal_fields = model_obj._meta.internal_json_fields if hasattr( model_obj._meta, 'internal_json_fields') else [] for json_key, value in json.items(): orm_key = _to_snake_case(json_key) if orm_key in (immutable_keys or []): if allow_unknown_keys: continue raise ValueError('Cannot edit field {}'.format(orm_key)) if allow_unknown_keys and not hasattr(model_obj, orm_key): continue if getattr(model_obj, orm_key) != value: if orm_key in internal_fields and not (user and user.is_staff): raise ValueError( 'User {0} is not authorized to edit the internal field {1}' .format(user, orm_key)) if verbose: model_obj_name = getattr(model_obj, 'guid', model_obj.__name__) logger.info("Setting {0}.{1} to {2}".format( model_obj_name, orm_key, value)) seqr_update_fields[orm_key] = value if seqr_update_fields: update_seqr_model(model_obj, **seqr_update_fields)
def _create_patient_if_missing(project, individual): """Create a new PhenoTips patient record with the given patient id. Args: project (Model): seqr Project - used to retrieve PhenoTips credentials individual (Model): seqr Individual Returns: True if patient created Raises: PhenotipsException: if unable to create patient record """ if _phenotips_patient_exists(individual): return False url = '/rest/patients' headers = {"Content-Type": "application/json"} data = json.dumps({'external_id': individual.guid}) auth_tuple = _get_phenotips_uname_and_pwd_for_project(project.phenotips_user_id) response_items = _make_api_call('POST', url, auth_tuple=auth_tuple, http_headers=headers, data=data, expected_status_code=201, parse_json_resonse=False) patient_id = response_items['Location'].split('/')[-1] logger.info("Created PhenoTips record with patient id {patient_id} and external id {external_id}".format(patient_id=patient_id, external_id=individual.guid)) username_read_only, _ = _get_phenotips_uname_and_pwd_for_project(project.phenotips_user_id, read_only=True) _add_user_to_patient(username_read_only, patient_id, allow_edit=False) logger.info("Added PhenoTips user {username} to {patient_id}".format(username=username_read_only, patient_id=patient_id)) update_seqr_model(individual, phenotips_patient_id=patient_id, phenotips_eid=individual.guid) return True
def update_family_assigned_analyst(request, family_guid): """Updates the specified field in the Family model. Args: family_guid (string): GUID of the family. """ family = Family.objects.get(guid=family_guid) check_permissions(family.project, request.user, CAN_EDIT) request_json = json.loads(request.body) assigned_analyst_username = request_json.get('assigned_analyst_username') if assigned_analyst_username: try: assigned_analyst = User.objects.get( username=assigned_analyst_username) except Exception: return create_json_response({}, status=400, reason="specified user does not exist") else: assigned_analyst = None update_seqr_model(family, assigned_analyst=assigned_analyst) return create_json_response( {family.guid: _get_json_for_family(family, request.user)})
def update_model_from_json(model_obj, json, user=None, verbose=False, allow_unknown_keys=False, immutable_keys=None, conditional_edit_keys=None): immutable_keys = (immutable_keys or []) + ['created_by', 'created_date', 'last_modified_date', 'id'] seqr_update_fields = {} internal_fields = model_obj._meta.internal_json_fields if hasattr(model_obj._meta, 'internal_json_fields') else [] for json_key, value in json.items(): orm_key = _to_snake_case(json_key) if orm_key in immutable_keys: if allow_unknown_keys: continue raise ValueError('Cannot edit field {}'.format(orm_key)) if allow_unknown_keys and not hasattr(model_obj, orm_key): continue if getattr(model_obj, orm_key) != value: if orm_key in internal_fields and not (user and user.is_staff): raise ValueError('User {0} is not authorized to edit the internal field {1}'.format(user, orm_key)) if conditional_edit_keys and orm_key in conditional_edit_keys: conditional_edit_keys[orm_key](model_obj) if verbose: model_obj_name = getattr(model_obj, 'guid', None) or model_obj.__name__ logger.info("Setting {0}.{1} to {2}".format(model_obj_name, orm_key, value)) seqr_update_fields[orm_key] = value if seqr_update_fields: update_seqr_model(model_obj, **seqr_update_fields)
def update_individual_hpo_terms(request, individual_guid): individual = Individual.objects.get(guid=individual_guid) project = individual.family.project check_permissions(project, request.user, CAN_EDIT) features = json.loads(request.body) _create_patient_if_missing(project, individual) patient_json = _get_patient_data(project, individual) patient_json["features"] = features patient_json_string = json.dumps(patient_json) url = _phenotips_patient_url(individual) auth_tuple = _get_phenotips_uname_and_pwd_for_project(project.phenotips_user_id, read_only=False) _make_api_call('PUT', url, data=patient_json_string, auth_tuple=auth_tuple, expected_status_code=204) phenotips_patient_id = patient_json['id'] phenotips_eid = patient_json.get('external_id') update_seqr_model( individual, phenotips_data=json.dumps(patient_json), phenotips_patient_id=phenotips_patient_id, phenotips_eid=phenotips_eid) return create_json_response({ individual.guid: { 'phenotipsData': patient_json, 'phenotipsPatientId': phenotips_patient_id, 'phenotipsEid': phenotips_eid } })
def update_family_pedigree_image(request, family_guid): """Updates the specified field in the Family model. Args: family_guid (string): GUID of the family. """ family = Family.objects.get(guid=family_guid) # check permission check_permissions(family.project, request.user, CAN_EDIT) if len(request.FILES) == 0: pedigree_image = None elif len(request.FILES) > 1: return create_json_response({}, status=400, reason='Received {} files'.format( len(request.FILES))) else: pedigree_image = request.FILES.values()[0] update_seqr_model(family, pedigree_image=pedigree_image) return create_json_response( {family.guid: _get_json_for_family(family, request.user)})
def _update_pedigree_image(family, project_guid=None): """Uses HaploPainter to (re)generate the pedigree image for the given family. Args: family (object): seqr Family model. """ individual_records = _get_parsed_individuals(family, project_guid) if not individual_records: return # run HaploPainter to generate the pedigree image png_file_path = os.path.join(tempfile.gettempdir(), "pedigree_image_%s.png" % _random_string(10)) family_id = family.family_id with tempfile.NamedTemporaryFile('w', suffix=".fam", delete=True) as fam_file: # columns: family, individual id, paternal id, maternal id, sex, affected for i in individual_records.values(): row = [family_id] + [ i[key] for key in [ 'individualId', 'paternalId', 'maternalId', 'sex', 'affected' ] ] fam_file.write("\t".join(row).encode('UTF-8')) fam_file.write("\n") fam_file.flush() fam_file_path = fam_file.name haplopainter_command = "perl " + os.path.join( BASE_DIR, "xbrowse_server/base/management/commands/HaploPainter1.043.pl") haplopainter_command += " -b -outformat png -pedfile {fam_file_path} -family {family_id} -outfile {png_file_path}".format( fam_file_path=fam_file_path, family_id=family_id, png_file_path=png_file_path) os.system(haplopainter_command) if not os.path.isfile(png_file_path): logger.error("Failed to generated pedigree image for family: %s" % family_id) update_seqr_model(family, pedigree_image=None) return _save_pedigree_image_file(family, png_file_path) os.remove(png_file_path)
def _create_patient_if_missing(project, individual): """Create a new PhenoTips patient record with the given patient id. Args: project (Model): seqr Project - used to retrieve PhenoTips credentials individual (Model): seqr Individual Returns: True if patient created Raises: PhenotipsException: if unable to create patient record """ if _phenotips_patient_exists(individual): return False url = '/rest/patients' headers = {"Content-Type": "application/json"} data = json.dumps({'external_id': individual.guid}) auth_tuple = _get_phenotips_uname_and_pwd_for_project( project.phenotips_user_id) response_items = _make_api_call('POST', url, auth_tuple=auth_tuple, http_headers=headers, data=data, expected_status_code=201, parse_json_resonse=False) patient_id = response_items['Location'].split('/')[-1] logger.info( "Created PhenoTips record with patient id {patient_id} and external id {external_id}" .format(patient_id=patient_id, external_id=individual.guid)) username_read_only, _ = _get_phenotips_uname_and_pwd_for_project( project.phenotips_user_id, read_only=True) _add_user_to_patient(username_read_only, patient_id, allow_edit=False) logger.info("Added PhenoTips user {username} to {patient_id}".format( username=username_read_only, patient_id=patient_id)) update_seqr_model(individual, phenotips_patient_id=patient_id, phenotips_eid=individual.guid) return True
def update_project_handler(request, project_guid): """Update project metadata - including one or more of these fields: name, description Args: project_guid (string): GUID of the project that should be updated HTTP POST Request body - should contain the following json structure: { 'form' : { 'name': <project name>, 'description': <project description>, } } Response body - will contain the following structure, representing the updated project: { 'projectsByGuid': { <projectGuid1> : { ... <project key-value pairs> ... } } } """ project = Project.objects.get(guid=project_guid) check_permissions(project, request.user, CAN_EDIT) request_json = json.loads(request.body) if 'name' in request_json: update_seqr_model(project, name=request_json.get('name')) if 'description' in request_json: update_seqr_model(project, description=request_json.get('description')) return create_json_response({ 'projectsByGuid': { project.guid: _get_json_for_project(project, request.user) }, })
def _update_individual_phenotips_data(individual, patient_json): """Process and store the given patient_json in the given Individual model. Args: individual (Individual): Django Individual model patient_json (json): json dict representing the patient record in PhenoTips """ # for each HPO term, get the top level HPO category (eg. Musculoskeletal) for feature in patient_json.get('features', []): hpo_id = feature['id'] try: feature['category'] = HumanPhenotypeOntology.objects.get(hpo_id=hpo_id).category_id except ObjectDoesNotExist: logger.error("ERROR: PhenoTips HPO id %s not found in seqr HumanPhenotypeOntology table." % hpo_id) update_seqr_model( individual, phenotips_data=json.dumps(patient_json), phenotips_patient_id=patient_json['id'], # phenotips internal id phenotips_eid=patient_json.get('external_id')) # phenotips external id
def save_internal_case_review_notes(request, family_guid): """Updates the `case_review_notes` field for the given family. Args: family_guid (string): GUID of the family. """ family = Family.objects.get(guid=family_guid) request_json = json.loads(request.body) if "value" not in request_json: raise ValueError("Request is missing 'value' key: %s" % (request.body, )) update_seqr_model(family, internal_case_review_notes=request_json['value']) return create_json_response({ family.guid: _get_json_for_family(family, request.user, add_individual_guids_field=True) })
def update_individual_hpo_terms(request, individual_guid): individual = Individual.objects.get(guid=individual_guid) project = individual.family.project check_permissions(project, request.user, CAN_EDIT) features = json.loads(request.body) _create_patient_if_missing(project, individual) patient_json = _get_patient_data(project, individual) patient_json["features"] = features patient_json_string = json.dumps(patient_json) url = _phenotips_patient_url(individual) auth_tuple = _get_phenotips_uname_and_pwd_for_project( project.phenotips_user_id, read_only=False) _make_api_call('PUT', url, data=patient_json_string, auth_tuple=auth_tuple, expected_status_code=204) phenotips_patient_id = patient_json['id'] phenotips_eid = patient_json.get('external_id') update_seqr_model(individual, phenotips_data=json.dumps(patient_json), phenotips_patient_id=phenotips_patient_id, phenotips_eid=phenotips_eid) return create_json_response({ individual.guid: { 'phenotipsData': patient_json, 'phenotipsPatientId': phenotips_patient_id, 'phenotipsEid': phenotips_eid } })
def update_pedigree_image(family, project_guid=None): """Uses HaploPainter to (re)generate the pedigree image for the given family. Args: family (object): seqr Family model. """ individual_records = _get_parsed_individuals(family, project_guid) if not individual_records: return # run HaploPainter to generate the pedigree image png_file_path = os.path.join(tempfile.gettempdir(), "pedigree_image_%s.png" % _random_string(10)) family_id = family.family_id with tempfile.NamedTemporaryFile('w', suffix=".fam", delete=True) as fam_file: # columns: family, individual id, paternal id, maternal id, sex, affected for i in individual_records.values(): row = [family_id] + [i[key] for key in ['individualId', 'paternalId', 'maternalId', 'sex', 'affected']] fam_file.write("\t".join(row).encode('UTF-8')) fam_file.write("\n") fam_file.flush() fam_file_path = fam_file.name haplopainter_command = "perl " + os.path.join(BASE_DIR, "xbrowse_server/base/management/commands/HaploPainter1.043.pl") haplopainter_command += " -b -outformat png -pedfile %(fam_file_path)s -family %(family_id)s -outfile %(png_file_path)s" % locals() os.system(haplopainter_command) if not os.path.isfile(png_file_path): logger.error("Failed to generated pedigree image for family: %s" % family_id) update_seqr_model(family, pedigree_image=None) return _save_pedigree_image_file(family, png_file_path) os.remove(png_file_path)
def _update_individual_phenotips_data(individual, patient_json): """Process and store the given patient_json in the given Individual model. Args: individual (Individual): Django Individual model patient_json (json): json dict representing the patient record in PhenoTips """ # for each HPO term, get the top level HPO category (eg. Musculoskeletal) for feature in patient_json.get('features', []): hpo_id = feature['id'] try: feature['category'] = HumanPhenotypeOntology.objects.get( hpo_id=hpo_id).category_id except ObjectDoesNotExist: logger.error( "ERROR: PhenoTips HPO id %s not found in seqr HumanPhenotypeOntology table." % hpo_id) update_seqr_model( individual, phenotips_data=json.dumps(patient_json), phenotips_patient_id=patient_json['id'], # phenotips internal id phenotips_eid=patient_json.get('external_id')) # phenotips external id
def _set_phenotips_patient_id_if_missing(project, individual): if individual.phenotips_patient_id: return patient_json = _get_patient_data(project, individual) update_seqr_model(individual, phenotips_patient_id=patient_json['id'])
def _get_parsed_individuals(family, project_guid=None): """Uses HaploPainter to (re)generate the pedigree image for the given family. Args: family (object): seqr Family model. """ individuals = Individual.objects.filter(family=family) if len(individuals) < 2: update_seqr_model(family, pedigree_image=None) return None # convert individuals to json individual_records = { individual['individualId']: individual for individual in _get_json_for_individuals(individuals, project_guid=project_guid, family_guid=family.guid) } # compute a map of parent ids to list of children parent_ids_to_children_map = collections.defaultdict(list) for individual_id, individual_json in individual_records.items(): if not individual_json['paternalId'] and not individual_json['maternalId']: continue key = (individual_json['paternalId'], individual_json['maternalId']) parent_ids_to_children_map[key].append(individual_json) # generate placeholder individuals as needed, since HaploPainter1.043.pl doesn't support families with only 1 parent for ((paternal_id, maternal_id), children) in parent_ids_to_children_map.items(): for parent_id_key, parent_id, sex in [ ('paternalId', paternal_id, 'M'), ('maternalId', maternal_id, 'F') ]: if not parent_id or parent_id not in individual_records: placeholder_parent_id = 'placeholder_%s'% _random_string(10) placeholder_parent_json = { 'individualId': placeholder_parent_id, # fake indiv id 'paternalId': '', 'maternalId': '', 'sex': sex, 'affected': 'INVISIBLE', # use a special value to tell HaploPainter to draw this individual as '?' } for child_json in children: child_json[parent_id_key] = placeholder_parent_id individual_records[placeholder_parent_id] = placeholder_parent_json # convert to FAM file values SEX_TO_FAM_FILE_VALUE = {"M": "1", "F": "2", "U": "0"} AFFECTED_STATUS_TO_FAM_FILE_VALUE = {"A": "2", "N": "1", "U": "0", "INVISIBLE": "9"} # HaploPainter1.043.pl has been modified to hide individuals with affected-status='9' return { individual_id: { 'individualId': individual_id, 'paternalId': individual_json['paternalId'] or '0', 'maternalId': individual_json['maternalId'] or '0', 'sex': SEX_TO_FAM_FILE_VALUE[individual_json['sex']], 'affected': AFFECTED_STATUS_TO_FAM_FILE_VALUE[individual_json['affected']], } for individual_id, individual_json in individual_records.items() }
def handle(self, *args, **options): from_project = Project.objects.get(guid=options['from_project']) to_project = Project.objects.get(guid=options['to_project']) to_base_project = find_matching_xbrowse_model(to_project) family_ids = options['family_ids'] families = Family.objects.filter(project=from_project, family_id__in=family_ids) print('Found {} out of {} families. No match for: {}.'.format( len(families), len(set(family_ids)), set(family_ids) - set([f.family_id for f in families]))) for f in families: print("==> Moving {}".format(f)) for seqr_individual in f.individual_set.all(): base_individual = find_matching_xbrowse_model(seqr_individual) base_individual.project = to_base_project base_individual.save() # Update individuals in phenotips if _phenotips_patient_exists(seqr_individual): # make sure phenotips_patient_id is up to date data_json = _get_patient_data( from_project, seqr_individual, ) seqr_individual.phenotips_patient_id = data_json["id"] seqr_individual.save() # update permissions phenotips_readonly_username, _ = _get_phenotips_uname_and_pwd_for_project( to_project.phenotips_user_id, read_only=True) _add_user_to_patient(phenotips_readonly_username, seqr_individual.phenotips_patient_id, allow_edit=False) phenotips_readwrite_username, _ = _get_phenotips_uname_and_pwd_for_project( to_project.phenotips_user_id, read_only=False) _add_user_to_patient(phenotips_readwrite_username, seqr_individual.phenotips_patient_id, allow_edit=True) # Update individuals samples/ VCFs for from_vcf_file in base_individual.vcf_files.all(): to_vcf_file, _ = VCFFile.objects.get_or_create( project=to_base_project, elasticsearch_index=from_vcf_file.elasticsearch_index, file_path=from_vcf_file.file_path, dataset_type=from_vcf_file.dataset_type, sample_type=from_vcf_file.sample_type, loaded_date=from_vcf_file.loaded_date, ) base_individual.vcf_files.add(to_vcf_file) base_individual.vcf_files.remove(from_vcf_file) # Update variant tags/ notes saved_variants = SavedVariant.objects.filter(family=f) for variant_tag in VariantTag.objects.filter( saved_variant__in=saved_variants).select_related( 'variant_tag_type'): if variant_tag.variant_tag_type.project: to_tag_type, created = get_or_create_seqr_model( VariantTagType, project=to_project, name=variant_tag.variant_tag_type.name) if created: update_seqr_model( to_tag_type, category=variant_tag.variant_tag_type.category, description=variant_tag.variant_tag_type. description, color=variant_tag.variant_tag_type.color, order=variant_tag.variant_tag_type.order, ) update_seqr_model(variant_tag, variant_tag_type=to_tag_type) else: to_project_tag, created = ProjectTag.objects.get_or_create( project=to_base_project, tag=variant_tag.variant_tag_type.name) if created: to_project_tag.category = variant_tag.variant_tag_type.category to_project_tag.title = variant_tag.variant_tag_type.description to_project_tag.color = variant_tag.variant_tag_type.color to_project_tag.order = variant_tag.variant_tag_type.order to_project_tag.save() variant_tag.project_tag = to_project_tag variant_tag.save() base_family = find_matching_xbrowse_model(f) for note in BaseVariantNote.objects.filter(family=base_family): note.project = to_base_project note.save() # Update families update_seqr_model(f, project=to_project) print("Done.")
def handle(self, *args, **options): from_project = Project.objects.get(guid=options['from_project']) to_project = Project.objects.get(guid=options['to_project']) family_ids = options['family_ids'] families = Family.objects.filter(project=from_project, family_id__in=family_ids) print('Found {} out of {} families. No match for: {}.'.format( len(families), len(set(family_ids)), set(family_ids) - set([f.family_id for f in families]))) for f in families: print("==> Moving phenotips for {}".format(f)) for seqr_individual in f.individual_set.all(): if phenotips_patient_exists(seqr_individual): # make sure phenotips_patient_id is up to date data_json = _get_patient_data( from_project, seqr_individual, ) seqr_individual.phenotips_patient_id = data_json["id"] seqr_individual.save() # update permissions phenotips_readonly_username, _ = get_phenotips_uname_and_pwd_for_project( to_project.phenotips_user_id, read_only=True) _add_user_to_patient(phenotips_readonly_username, seqr_individual.phenotips_patient_id, allow_edit=False) phenotips_readwrite_username, _ = get_phenotips_uname_and_pwd_for_project( to_project.phenotips_user_id, read_only=False) _add_user_to_patient(phenotips_readwrite_username, seqr_individual.phenotips_patient_id, allow_edit=True) for variant_tag_type in VariantTagType.objects.filter( project=from_project): variant_tags = VariantTag.objects.filter( saved_variant__family__in=families, variant_tag_type=variant_tag_type) if variant_tags: print('Updating "{}" tags'.format(variant_tag_type.name)) to_tag_type, created = get_or_create_seqr_model( VariantTagType, project=to_project, name=variant_tag_type.name) if created: update_seqr_model( to_tag_type, category=variant_tag_type.category, description=variant_tag_type.description, color=variant_tag_type.color, order=variant_tag_type.order, ) variant_tags.update(variant_tag_type=to_tag_type) print("Updating families") families.update(project=to_project) print("Done.")
def add_or_update_individuals_and_families(project, individual_records, user=None): """Add or update individual and family records in the given project. Args: project (object): Django ORM model for the project to add families to individual_records (list): A list of JSON records representing individuals. See the return value of pedigree_info_utils#convert_fam_file_rows_to_json(..) Return: 2-tuple: updated_families, updated_individuals containing Django ORM models """ families = {} updated_individuals = [] for i, record in enumerate(individual_records): # family id will be in different places in the json depending on whether it comes from a flat uploaded file or from the nested individual object family_id = record.get('familyId') or record.get('family', {}).get('familyId') if not family_id: raise ValueError( "record #%s doesn't contain a 'familyId' key: %s" % (i, record)) if 'individualId' not in record and 'individualGuid' not in record: raise ValueError( "record #%s doesn't contain an 'individualId' key: %s" % (i, record)) family, created = get_or_create_seqr_model(Family, project=project, family_id=family_id) if created: logger.info("Created family: %s", family) if not family.display_name: update_seqr_model(family, display_name=family.family_id) # uploaded files do not have unique guid's so fall back to a combination of family and individualId if record.get('individualGuid'): individual_filters = {'guid': record['individualGuid']} else: individual_filters = { 'family': family, 'individual_id': record['individualId'] } individual, created = get_or_create_seqr_model(Individual, **individual_filters) record['family'] = family record.pop('familyId', None) update_individual_from_json(individual, record, allow_unknown_keys=True, user=user) updated_individuals.append(individual) # apply additional json fields which don't directly map to Individual model fields individual.phenotips_eid = individual.guid # use guid instead of indiv_id to avoid collisions if created: # create new PhenoTips patient record patient_record = create_patient(project, individual) update_seqr_model(individual, phenotips_patient_id=patient_record['id'], case_review_status='I') logger.info( "Created PhenoTips record with patient id %s and external id %s" % (str(individual.phenotips_patient_id), str(individual.phenotips_eid))) if record.get(JsonConstants.HPO_TERMS_PRESENT_COLUMN) or record.get( JsonConstants.FINAL_DIAGNOSIS_OMIM_COLUMN): # update phenotips hpo ids logger.info("Setting PhenoTips HPO Terms to: %s" % (record.get(JsonConstants.HPO_TERMS_PRESENT_COLUMN), )) set_patient_hpo_terms( project, individual, hpo_terms_present=record.get( JsonConstants.HPO_TERMS_PRESENT_COLUMN, []), hpo_terms_absent=record.get( JsonConstants.HPO_TERMS_ABSENT_COLUMN, []), final_diagnosis_mim_ids=record.get( JsonConstants.FINAL_DIAGNOSIS_OMIM_COLUMN, [])) if not individual.display_name: update_seqr_model(individual, display_name=individual.individual_id) families[family.family_id] = family updated_families = list(families.values()) # update pedigree images update_pedigree_images(updated_families) return updated_families, updated_individuals
def update_pedigree_image(family): """Uses HaploPainter to (re)generate the pedigree image for the given family. Args: family (object): seqr Family model. """ family_id = family.family_id individuals = Individual.objects.filter(family=family) if len(individuals) < 2: update_seqr_model(family, pedigree_image=None) return # convert individuals to json individual_records = {} for i in individuals: individual_records[i.individual_id] = _get_json_for_individual(i) individual_records[i.individual_id]['familyId'] = i.family.family_id # compute a map of parent ids to list of children parent_ids_to_children_map = collections.defaultdict(list) for individual_id, individual_json in individual_records.items(): if not individual_json['paternalId'] and not individual_json[ 'maternalId']: continue key = (individual_json['paternalId'], individual_json['maternalId']) parent_ids_to_children_map[key].append(individual_json) # generate placeholder individuals as needed, since HaploPainter1.043.pl doesn't support families with only 1 parent for ((paternal_id, maternal_id), children) in parent_ids_to_children_map.items(): for parent_id_key, parent_id, sex in [('paternalId', paternal_id, 'M'), ('maternalId', maternal_id, 'F') ]: if not parent_id or parent_id not in individual_records: placeholder_parent_id = 'placeholder_%s' % _random_string(10) placeholder_parent_json = { 'familyId': family_id, 'individualId': placeholder_parent_id, # fake indiv id 'paternalId': '', 'maternalId': '', 'sex': sex, 'affected': 'INVISIBLE', # use a special value to tell HaploPainter to draw this individual as '?' } for child_json in children: child_json[parent_id_key] = placeholder_parent_id individual_records[ placeholder_parent_id] = placeholder_parent_json # convert to FAM file values SEX_TO_FAM_FILE_VALUE = {"M": "1", "F": "2", "U": "0"} AFFECTED_STATUS_TO_FAM_FILE_VALUE = { "A": "2", "N": "1", "U": "0", "INVISIBLE": "9" } # HaploPainter1.043.pl has been modified to hide individuals with affected-status='9' for individual_json in individual_records.values(): if not individual_json['paternalId']: individual_json['paternalId'] = '0' if not individual_json['maternalId']: individual_json['maternalId'] = '0' individual_json['sex'] = SEX_TO_FAM_FILE_VALUE[individual_json['sex']] individual_json['affected'] = AFFECTED_STATUS_TO_FAM_FILE_VALUE[ individual_json['affected']] # run HaploPainter to generate the pedigree image png_file_path = os.path.join(tempfile.gettempdir(), "pedigree_image_%s.png" % _random_string(10)) with tempfile.NamedTemporaryFile('w', suffix=".fam", delete=True) as fam_file: # columns: family, individual id, paternal id, maternal id, sex, affected for i in individual_records.values(): row = [ i[key] for key in [ 'familyId', 'individualId', 'paternalId', 'maternalId', 'sex', 'affected' ] ] fam_file.write("\t".join(row).encode('UTF-8')) fam_file.write("\n") fam_file.flush() fam_file_path = fam_file.name haplopainter_command = "/usr/bin/perl " + os.path.join( BASE_DIR, "xbrowse_server/base/management/commands/HaploPainter1.043.pl") haplopainter_command += " -b -outformat png -pedfile %(fam_file_path)s -family %(family_id)s -outfile %(png_file_path)s" % locals( ) os.system(haplopainter_command) if not os.path.isfile(png_file_path): logger.error("Failed to generated pedigree image for family: %s" % family_id) update_seqr_model(family, pedigree_image=None) return _save_pedigree_image_file(family, png_file_path) os.remove(png_file_path)
def update_mme_submission(request, individual_guid): """ Create or update the submission for the given individual. """ individual = Individual.objects.get(guid=individual_guid) project = individual.family.project check_permissions(project, request.user) submission_json = json.loads(request.body) submission_json.pop('individualGuid', {}) phenotypes = submission_json.pop('phenotypes', []) gene_variants = submission_json.pop('geneVariants', []) if not phenotypes and not gene_variants: return create_json_response({}, status=400, reason='Genotypes or phentoypes are required') if not submission_json.get('patient', {}).get('id'): return create_json_response({}, status=400, reason='Patient id is required') genomic_features = [] for gene_variant in gene_variants: if not gene_variant.get('geneId'): return create_json_response({}, status=400, reason='Gene id is required for genomic features') feature = {'gene': {'id': gene_variant['geneId']}} if 'numAlt' in gene_variant: feature['zygosity'] = gene_variant['numAlt'] % 2 if gene_variant.get('pos'): feature['variant'] = { 'alternateBases': gene_variant['alt'], 'referenceBases': gene_variant['ref'], 'referenceName': gene_variant['chrom'], 'start': gene_variant['pos'], 'assembly': gene_variant['genomeVersion'], } genomic_features.append(feature) submission_json['patient']['genomicFeatures'] = genomic_features submission_json['patient']['features'] = phenotypes response = requests.post(url=MME_ADD_INDIVIDUAL_URL, headers=MME_HEADERS, data=json.dumps(submission_json)) if response.status_code not in (200, 409): try: response_json = response.json() except Exception: response_json = {} return create_json_response(response_json, status=response.status_code, reason=response.content) submitted_date = datetime.now() individual.mme_submitted_data = submission_json individual.mme_submitted_date = submitted_date individual.mme_deleted_date = None individual.mme_deleted_by = None individual.save() # update the project contact information if anything new was added new_contact_names = set(submission_json['patient']['contact']['name'].split(',')) - set(project.mme_primary_data_owner.split(',')) new_contact_urls = set(submission_json['patient']['contact']['href'].replace('mailto:', '').split(',')) - set(project.mme_contact_url.replace('mailto:', '').split(',')) updates = {} if new_contact_names: updates['mme_primary_data_owner'] = '{},{}'.format(project.mme_primary_data_owner, ','.join(new_contact_names)) if new_contact_urls: updates['mme_contact_url'] = '{},{}'.format(project.mme_contact_url, ','.join(new_contact_urls)) if updates: update_seqr_model(project, **updates) # search for new matches return _search_individual_matches(individual, request.user)
def add_dataset_handler(request, project_guid): """Create or update samples for the given dataset Args: request: Django request object project_guid (string): GUID of the project that should be updated HTTP POST Request body - should contain the following json structure: { 'sampleType': <"WGS", "WES", or "RNA"> (required) 'datasetType': <"VARIANTS", or "ALIGN"> (required) 'elasticsearchIndex': <String> 'datasetPath': <String> 'datasetName': <String> 'ignoreExtraSamplesInCallset': <Boolean> 'mappingFile': { 'uploadedFileId': <Id for temporary uploaded file> } } Response body - will contain the following structure: """ logger.info("add_dataset_handler: " + str(request)) project = get_project_and_check_permissions(project_guid, request.user, permission_level=CAN_EDIT) request_json = json.loads(request.body) logger.info("add_dataset_handler: received %s" % pformat(request_json)) required_fields = ['sampleType', 'datasetType'] if any(field not in request_json for field in required_fields): raise ValueError("request must contain fields: {}".format( ', '.join(required_fields))) sample_type = request_json['sampleType'] dataset_type = request_json['datasetType'] elasticsearch_index = request_json.get('elasticsearchIndex') if elasticsearch_index: elasticsearch_index = elasticsearch_index.strip() dataset_path = request_json.get('datasetPath') if dataset_path: dataset_path = dataset_path.strip() dataset_name = request_json.get('datasetName') if dataset_name: dataset_name = dataset_name.strip() ignore_extra_samples_in_callset = request_json.get( 'ignoreExtraSamplesInCallset') ignore_missing_family_members = request_json.get( 'ignoreMissingFamilyMembers') mapping_file_id = request_json.get('mappingFile', {}).get('uploadedFileId') mapping_file_path = request_json.get('mappingFilePath') try: updated_samples, created_sample_ids = add_dataset( project=project, sample_type=sample_type, dataset_type=dataset_type, elasticsearch_index=elasticsearch_index, dataset_path=dataset_path, dataset_name=dataset_name, max_edit_distance=0, ignore_extra_samples_in_callset=ignore_extra_samples_in_callset, ignore_missing_family_members=ignore_missing_family_members, mapping_file_path=mapping_file_path, mapping_file_id=mapping_file_id, ) # update VCFFile records if updated_samples: if dataset_type == Sample.DATASET_TYPE_VARIANT_CALLS: base_project = BaseProject.objects.get(seqr_project=project) get_datastore(base_project).bust_project_cache( base_project.project_id) clear_project_results_cache(base_project.project_id) vcf_file = VCFFile.objects.filter( project=base_project, dataset_type=dataset_type, sample_type=sample_type, elasticsearch_index=elasticsearch_index).order_by( '-pk').first() if not vcf_file: vcf_file = VCFFile.objects.create( project=base_project, dataset_type=dataset_type, sample_type=sample_type, elasticsearch_index=elasticsearch_index, ) logger.info("Created vcf file: " + str(vcf_file.__dict__)) vcf_file.file_path = dataset_path or "{}.vcf.gz".format( elasticsearch_index ) # legacy VCFFile model requires non-empty vcf path vcf_file.loaded_date = iter(updated_samples).next().loaded_date vcf_file.save() for indiv in [s.individual for s in updated_samples]: for base_indiv in BaseIndividual.objects.filter( seqr_individual=indiv).only('id'): base_indiv.vcf_files.add(vcf_file) elif dataset_type == Sample.DATASET_TYPE_READ_ALIGNMENTS: for sample in updated_samples: for base_indiv in BaseIndividual.objects.filter( seqr_individual=sample.individual).only('id'): base_indiv.bam_file_path = sample.dataset_file_path base_indiv.save() updated_sample_json = get_json_for_samples(updated_samples, project_guid=project_guid) response = { 'samplesByGuid': {s['sampleGuid']: s for s in updated_sample_json} } updated_individuals = { s['individualGuid'] for s in updated_sample_json if s['sampleId'] in created_sample_ids } if updated_individuals: individuals = Individual.objects.filter( guid__in=updated_individuals).prefetch_related( 'sample_set', 'family').only('guid') response['individualsByGuid'] = { ind.guid: { 'sampleGuids': [s.guid for s in ind.sample_set.only('guid').all()] } for ind in individuals } for ind in individuals: family = ind.family if family.analysis_status == Family.ANALYSIS_STATUS_WAITING_FOR_DATA: update_seqr_model(family, analysis_status=Family. ANALYSIS_STATUS_ANALYSIS_IN_PROGRESS) return create_json_response(response) except Exception as e: traceback.print_exc() return create_json_response({'errors': [e.message or str(e)]}, status=400)