def delete_families_handler(request, project_guid): """Edit or delete one or more Individual records. Args: project_guid (string): GUID of project that contains these individuals. """ project = get_project_and_check_pm_permissions(project_guid, request.user) request_json = json.loads(request.body) families_to_delete = request_json.get('families') if families_to_delete is None: return create_json_response( {}, status=400, reason="'recordIdsToDelete' not specified") family_guids_to_delete = [f['familyGuid'] for f in families_to_delete] # delete individuals 1st individual_guids_to_delete = [i.guid for i in Individual.objects.filter( family__project=project, family__guid__in=family_guids_to_delete)] delete_individuals(project, individual_guids_to_delete, request.user) # delete families Family.bulk_delete(request.user, project=project, guid__in=family_guids_to_delete) # send response return create_json_response({ 'individualsByGuid': { individual_guid: None for individual_guid in individual_guids_to_delete }, 'familiesByGuid': { family_guid: None for family_guid in family_guids_to_delete }, })
def save_individuals_table_handler(request, project_guid, upload_file_id): """Handler for 'save' requests to apply Individual tables previously uploaded through receive_individuals_table(..) Args: request (object): Django request object project_guid (string): project GUID uploadedFileId (string): a token sent to the client by receive_individuals_table(..) """ project = get_project_and_check_pm_permissions(project_guid, request.user) json_records = load_uploaded_file(upload_file_id) updated_families, updated_individuals = add_or_update_individuals_and_families( project, individual_records=json_records, user=request.user) # edit individuals individuals = _get_json_for_individuals(updated_individuals, request.user, add_sample_guids_field=True) individuals_by_guid = { individual['individualGuid']: individual for individual in individuals } families = _get_json_for_families(updated_families, request.user, add_individual_guids_field=True) families_by_guid = {family['familyGuid']: family for family in families} updated_families_and_individuals_by_guid = { 'individualsByGuid': individuals_by_guid, 'familiesByGuid': families_by_guid, } return create_json_response(updated_families_and_individuals_by_guid)
def edit_families_handler(request, project_guid): """Edit or one or more Family records. Args: project_guid (string): GUID of project that contains these individuals. """ project = get_project_and_check_pm_permissions(project_guid, request.user) request_json = json.loads(request.body) if request_json.get('uploadedFileId'): modified_families = load_uploaded_file( request_json.get('uploadedFileId')) else: modified_families = request_json.get('families') if modified_families is None: return create_json_response({}, status=400, reason="'families' not specified") updated_families = [] for fields in modified_families: if fields.get('familyGuid'): family = Family.objects.get(project=project, guid=fields['familyGuid']) elif fields.get(PREVIOUS_FAMILY_ID_FIELD): family = Family.objects.get( project=project, family_id=fields[PREVIOUS_FAMILY_ID_FIELD]) else: family, _ = get_or_create_model_from_json( Family, { 'project': project, 'family_id': fields[FAMILY_ID_FIELD] }, update_json=None, user=request.user) update_family_from_json(family, fields, user=request.user, allow_unknown_keys=True) updated_families.append(family) updated_families_by_guid = { 'familiesByGuid': { family.guid: _get_json_for_family(family, request.user, add_individual_guids_field=True) for family in updated_families } } return create_json_response(updated_families_by_guid)
def receive_families_table_handler(request, project_guid): """Handler for the initial upload of an Excel or .tsv table of families. This handler parses the records, but doesn't save them in the database. Instead, it saves them to a temporary file and sends a 'uploadedFileId' representing this file back to the client. Args: request (object): Django request object project_guid (string): project GUID """ project = get_project_and_check_pm_permissions(project_guid, request.user) def _process_records(records, filename=''): column_map = {} for i, field in enumerate(records[0]): key = field.lower() if 'family' in key: if 'prev' in key: column_map[PREVIOUS_FAMILY_ID_FIELD] = i else: column_map[FAMILY_ID_FIELD] = i elif 'display' in key: column_map['displayName'] = i elif 'description' in key: column_map['description'] = i elif 'phenotype' in key: column_map['codedPhenotype'] = i if FAMILY_ID_FIELD not in column_map: raise ValueError('Invalid header, missing family id column') return [{ column: row[index] if isinstance(index, int) else next( (row[i] for i in index if row[i]), None) for column, index in column_map.items() } for row in records[1:]] try: uploaded_file_id, filename, json_records = save_uploaded_file( request, process_records=_process_records) except Exception as e: return create_json_response({ 'errors': [str(e)], 'warnings': [] }, status=400, reason=str(e)) prev_fam_ids = { r[PREVIOUS_FAMILY_ID_FIELD] for r in json_records if r.get(PREVIOUS_FAMILY_ID_FIELD) } existing_prev_fam_ids = { f.family_id for f in Family.objects.filter(family_id__in=prev_fam_ids, project=project).only('family_id') } if len(prev_fam_ids) != len(existing_prev_fam_ids): missing_prev_ids = [ family_id for family_id in prev_fam_ids if family_id not in existing_prev_fam_ids ] return create_json_response( { 'errors': [ 'Could not find families with the following previous IDs: {}' .format(', '.join(missing_prev_ids)) ], 'warnings': [] }, status=400, reason='Invalid input') fam_ids = { r[FAMILY_ID_FIELD] for r in json_records if not r.get(PREVIOUS_FAMILY_ID_FIELD) } num_families_to_update = len(prev_fam_ids) + Family.objects.filter( family_id__in=fam_ids, project=project).count() num_families = len(json_records) num_families_to_create = num_families - num_families_to_update info = [ "{num_families} families parsed from {filename}".format( num_families=num_families, filename=filename), "{} new families will be added, {} existing families will be updated". format(num_families_to_create, num_families_to_update), ] return create_json_response({ 'uploadedFileId': uploaded_file_id, 'errors': [], 'warnings': [], 'info': info, })
def receive_individuals_table_handler(request, project_guid): """Handler for the initial upload of an Excel or .tsv table of individuals. This handler parses the records, but doesn't save them in the database. Instead, it saves them to a temporary file and sends a 'uploadedFileId' representing this file back to the client. If/when the client then wants to 'apply' this table, it can send the uploadedFileId to the save_individuals_table(..) handler to actually save the data in the database. Args: request (object): Django request object project_guid (string): project GUID """ project = get_project_and_check_pm_permissions(project_guid, request.user) warnings = [] def process_records(json_records, filename='ped_file'): pedigree_records, errors, ped_warnings = parse_pedigree_table( json_records, filename, user=request.user, project=project) if errors: raise ErrorsWarningsException(errors, ped_warnings) nonlocal warnings warnings += ped_warnings return pedigree_records try: uploaded_file_id, filename, json_records = save_uploaded_file( request, process_records=process_records) except ErrorsWarningsException as e: return create_json_response( { 'errors': e.errors, 'warnings': e.warnings }, status=400, reason=e.errors) except Exception as e: return create_json_response({ 'errors': [str(e)], 'warnings': [] }, status=400, reason=str(e)) if warnings: # If there are warnings, it might be because the upload referenced valid existing individuals and there is no # issue, or because it referenced individuals that actually don't exist, so re-validate with all individuals family_ids = {r[JsonConstants.FAMILY_ID_COLUMN] for r in json_records} individual_ids = { r[JsonConstants.INDIVIDUAL_ID_COLUMN] for r in json_records } related_individuals = Individual.objects.filter( family__family_id__in=family_ids, family__project=project).exclude(individual_id__in=individual_ids) related_individuals_json = _get_json_for_individuals( related_individuals, project_guid=project_guid, family_fields=['family_id']) errors, _ = validate_fam_file_records(json_records + related_individuals_json, fail_on_warnings=True) if errors: return create_json_response({ 'errors': errors, 'warnings': [] }, status=400, reason=errors) # send back some stats individual_ids_by_family = defaultdict(list) for r in json_records: if r.get(JsonConstants.PREVIOUS_INDIVIDUAL_ID_COLUMN): individual_ids_by_family[r[JsonConstants.FAMILY_ID_COLUMN]].append( (r[JsonConstants.PREVIOUS_INDIVIDUAL_ID_COLUMN], True)) else: individual_ids_by_family[r[JsonConstants.FAMILY_ID_COLUMN]].append( (r[JsonConstants.INDIVIDUAL_ID_COLUMN], False)) num_individuals = sum( [len(indiv_ids) for indiv_ids in individual_ids_by_family.values()]) num_existing_individuals = 0 missing_prev_ids = [] for family_id, indiv_ids in individual_ids_by_family.items(): existing_individuals = { i.individual_id for i in Individual.objects.filter( individual_id__in=[indiv_id for (indiv_id, _) in indiv_ids], family__family_id=family_id, family__project=project).only('individual_id') } num_existing_individuals += len(existing_individuals) missing_prev_ids += [ indiv_id for (indiv_id, is_previous) in indiv_ids if is_previous and indiv_id not in existing_individuals ] num_individuals_to_create = num_individuals - num_existing_individuals if missing_prev_ids: return create_json_response( { 'errors': [ 'Could not find individuals with the following previous IDs: {}' .format(', '.join(missing_prev_ids)) ], 'warnings': [] }, status=400, reason='Invalid input') family_ids = set(r[JsonConstants.FAMILY_ID_COLUMN] for r in json_records) num_families = len(family_ids) num_existing_families = Family.objects.filter(family_id__in=family_ids, project=project).count() num_families_to_create = num_families - num_existing_families info = [ "{num_families} families, {num_individuals} individuals parsed from {filename}" .format(num_families=num_families, num_individuals=num_individuals, filename=filename), "{} new families, {} new individuals will be added to the project". format(num_families_to_create, num_individuals_to_create), "{} existing individuals will be updated".format( num_existing_individuals), ] response = { 'uploadedFileId': uploaded_file_id, 'errors': [], 'warnings': [], 'info': info, } logger.info(response) return create_json_response(response)
def delete_individuals_handler(request, project_guid): """Delete one or more Individual records. Args: request (object): Django HTTP Request object. project_guid (string): GUID of project that contains these individuals. Request: body should be a json dictionary that contains a 'recordIdsToDelete' list of individual GUIDs to delete - for example: { 'form': { 'recordIdsToDelete': [ <individualGuid1>, <individualGuid2>, ... } } } Response: json dictionary with the deleted GUIDs mapped to None: { <individualGuid1> : None, <individualGuid2> : None, ... } """ # validate request project = get_project_and_check_pm_permissions(project_guid, request.user) request_json = json.loads(request.body) individuals_list = request_json.get('individuals') if individuals_list is None: return create_json_response( {}, status=400, reason="Invalid request: 'individuals' not in request_json") logger.info("delete_individuals_handler %s", request_json) individual_guids_to_delete = [ ind['individualGuid'] for ind in individuals_list ] # delete the individuals families_with_deleted_individuals = delete_individuals( project, individual_guids_to_delete, request.user) deleted_individuals_by_guid = { individual_guid: None for individual_guid in individual_guids_to_delete } families_by_guid = { family.guid: _get_json_for_family(family, request.user, add_individual_guids_field=True) for family in families_with_deleted_individuals } # families whose list of individuals may have changed # send response return create_json_response({ 'individualsByGuid': deleted_individuals_by_guid, 'familiesByGuid': families_by_guid, })
def edit_individuals_handler(request, project_guid): """Modify one or more Individual records. Args: request (object): Django HTTP Request object. project_guid (string): GUID of project that contains these individuals. Request: body should be a json dictionary that contains a 'individuals' list that includes the individuals to update, represented by dictionaries of their guid and fields to update - for example: { 'individuals': [ { 'individualGuid': <individualGuid1>, 'paternalId': <paternalId>, 'affected': 'A' }, { 'individualGuid': <individualGuid1>, 'sex': 'U' }, ... [ } Response: json dictionary representing the updated individual(s) like: { <individualGuid1> : { individualId: xxx, sex: xxx, affected: xxx, ...}, <individualGuid2> : { individualId: xxx, sex: xxx, affected: xxx, ...}, ... } """ project = get_project_and_check_pm_permissions(project_guid, request.user) request_json = json.loads(request.body) modified_individuals_list = request_json.get('individuals') if modified_individuals_list is None: return create_json_response({}, status=400, reason="'individuals' not specified") update_individuals = { ind['individualGuid']: ind for ind in modified_individuals_list } update_individual_models = { ind.guid: ind for ind in Individual.objects.filter( guid__in=update_individuals.keys()) } for modified_ind in modified_individuals_list: model = update_individual_models[modified_ind['individualGuid']] if modified_ind[ JsonConstants.INDIVIDUAL_ID_COLUMN] != model.individual_id: modified_ind[JsonConstants. PREVIOUS_INDIVIDUAL_ID_COLUMN] = model.individual_id modified_family_ids = { ind.get('familyId') or ind['family']['familyId'] for ind in modified_individuals_list } modified_family_ids.update( {ind.family.family_id for ind in update_individual_models.values()}) related_individuals = Individual.objects.filter( family__family_id__in=modified_family_ids, family__project=project).exclude(guid__in=update_individuals.keys()) related_individuals_json = _get_json_for_individuals( related_individuals, project_guid=project_guid, family_fields=['family_id']) individuals_list = modified_individuals_list + related_individuals_json errors, warnings = validate_fam_file_records(individuals_list, fail_on_warnings=True) if errors: return create_json_response({ 'errors': errors, 'warnings': warnings }, status=400, reason='Invalid updates') updated_families, updated_individuals = add_or_update_individuals_and_families( project, modified_individuals_list, user=request.user) individuals_by_guid = { individual.guid: _get_json_for_individual(individual, request.user) for individual in updated_individuals } families_by_guid = { family.guid: _get_json_for_family(family, request.user, add_individual_guids_field=True) for family in updated_families } return create_json_response({ 'individualsByGuid': individuals_by_guid, 'familiesByGuid': families_by_guid, })