def update_individual_igv_sample(request, individual_guid): individual = Individual.objects.get(guid=individual_guid) project = individual.family.project check_project_permissions(project, request.user, can_edit=True) request_json = json.loads(request.body) try: file_path = request_json.get('filePath') if not file_path: raise ValueError('request must contain fields: filePath') suffix = '.'.join(file_path.split('.')[1:]) sample_type = SAMPLE_TYPE_MAP.get(suffix) if not sample_type: raise Exception( 'Invalid file extension for "{}" - valid extensions are {}'. format(file_path, ', '.join(SAMPLE_TYPE_MAP.keys()))) validate_alignment_dataset_path(file_path) sample, created = get_or_create_model_from_json( IgvSample, create_json={ 'individual': individual, 'sample_type': sample_type }, update_json={ 'file_path': file_path, 'sample_id': request_json.get('sampleId') }, user=request.user) response = { 'igvSamplesByGuid': { sample.guid: get_json_for_sample(sample, individual_guid=individual_guid, project_guid=project.guid) } } if created: response['individualsByGuid'] = { individual.guid: { 'igvSampleGuids': [s.guid for s in individual.igvsample_set.all()] } } return create_json_response(response) except Exception as e: error = str(e) return create_json_response({'error': error}, status=400, reason=error)
def handle(self, *args, **options): samples = IgvSample.objects.filter( individual__family__project__name__in=args, ).prefetch_related('individual', 'individual__family') failed = [] for sample in samples: try: validate_alignment_dataset_path(sample.file_path) except Exception as e: individual_id = sample.individual.individual_id failed.append(individual_id) logger.info('Error at {} (Individual: {}): {} '.format(sample.file_path, individual_id, str(e))) logger.info('---- DONE ----') logger.info('Checked {} samples'.format(len(samples))) logger.info('{} failed samples: {}'.format(len(failed), ', '.join(failed)))
def update_individual_igv_sample(request, individual_guid): individual = Individual.objects.get(guid=individual_guid) project = individual.family.project check_permissions(project, request.user, CAN_EDIT) request_json = json.loads(request.body) try: required_fields = ['filePath'] if any(field not in request_json for field in required_fields): raise ValueError("request must contain fields: {}".format( ', '.join(required_fields))) file_path = request_json['filePath'] if not (file_path.endswith(".bam") or file_path.endswith(".cram")): raise Exception( 'BAM / CRAM file "{}" must have a .bam or .cram extension'. format(file_path)) validate_alignment_dataset_path(file_path) sample, created = IgvSample.objects.get_or_create( individual=individual) sample.file_path = file_path sample.save() response = { 'igvSamplesByGuid': { sample.guid: get_json_for_sample(sample, individual_guid=individual_guid, project_guid=project.guid) } } if created: response['individualsByGuid'] = { individual.guid: { 'igvSampleGuids': [s.guid for s in individual.igvsample_set.all()] } } return create_json_response(response) except Exception as e: error = e.message or str(e) return create_json_response({'error': error}, status=400, reason=error)
def handle(self, *args, **options): samples = Sample.objects.filter( dataset_type=Sample.DATASET_TYPE_READ_ALIGNMENTS, is_active=True, dataset_file_path__isnull=False, individual__family__project__name__in=args, ).prefetch_related('individual', 'individual__family') failed = [] for sample in samples: try: validate_alignment_dataset_path(sample.dataset_file_path) except Exception as e: individual_id = sample.individual.individual_id failed.append(individual_id) print('Error at {} (Individual: {}): {} '.format( sample.dataset_file_path, individual_id, e.message)) print('---- DONE ----') print('Checked {} samples'.format(len(samples))) print('{} failed samples: {}'.format(len(failed), ', '.join(failed)))
def update_individual_alignment_sample(request, individual_guid): """Create or update samples for the given dataset Args: request: Django request object individual_guid (string): GUID of the individual that should be updated HTTP POST Request body - should contain the following json structure: { 'sampleType': <"WGS", "WES", or "RNA"> (required) 'datasetType': <"VARIANTS", or "ALIGN"> (required) 'elasticsearchIndex': <String> 'datasetPath': <String> 'datasetName': <String> 'ignoreExtraSamplesInCallset': <Boolean> 'mappingFile': { 'uploadedFileId': <Id for temporary uploaded file> } } Response body - will contain the following structure: """ individual = Individual.objects.get(guid=individual_guid) project = individual.family.project check_permissions(project, request.user, CAN_EDIT) request_json = json.loads(request.body) try: required_fields = ['sampleType', 'datasetFilePath'] if any(field not in request_json for field in required_fields): raise ValueError( "request must contain fields: {}".format(', '.join(required_fields))) sample_type = request_json['sampleType'] if sample_type not in {choice[0] for choice in Sample.SAMPLE_TYPE_CHOICES}: raise Exception("Sample type not supported: {}".format(sample_type)) dataset_path = request_json['datasetFilePath'] if not (dataset_path.endswith(".bam") or dataset_path.endswith(".cram")): raise Exception('BAM / CRAM file "{}" must have a .bam or .cram extension'.format(dataset_path)) validate_alignment_dataset_path(dataset_path) sample, created = Sample.objects.get_or_create( individual=individual, dataset_type=Sample.DATASET_TYPE_READ_ALIGNMENTS, is_active=True, ) sample.dataset_file_path = dataset_path sample.sample_type = sample_type sample.sample_id = dataset_path.split('/')[-1].split('.')[0] if created: sample.loaded_date = timezone.now() sample.save() response = { 'samplesByGuid': { sample.guid: get_json_for_sample(sample, individual_guid=individual_guid, project_guid=project.guid)} } if created: response['individualsByGuid'] = { individual.guid: {'sampleGuids': [s.guid for s in individual.sample_set.all()]} } return create_json_response(response) except Exception as e: return create_json_response({}, status=400, reason=e.message or str(e))
def add_alignment_dataset_handler(request, project_guid): """Create or update samples for the given dataset Args: request: Django request object project_guid (string): GUID of the project that should be updated HTTP POST Request body - should contain the following json structure: { 'sampleType': <"WGS", "WES", or "RNA"> (required) 'datasetType': <"VARIANTS", or "ALIGN"> (required) 'elasticsearchIndex': <String> 'datasetPath': <String> 'datasetName': <String> 'ignoreExtraSamplesInCallset': <Boolean> 'mappingFile': { 'uploadedFileId': <Id for temporary uploaded file> } } Response body - will contain the following structure: """ project = get_project_and_check_permissions(project_guid, request.user, permission_level=CAN_EDIT) request_json = json.loads(request.body) try: required_fields = ['sampleType', 'mappingFile'] if any(field not in request_json for field in required_fields): raise ValueError("request must contain fields: {}".format( ', '.join(required_fields))) sample_type = request_json['sampleType'] if sample_type not in { choice[0] for choice in Sample.SAMPLE_TYPE_CHOICES }: raise Exception( "Sample type not supported: {}".format(sample_type)) mapping_file_id = request_json['mappingFile']['uploadedFileId'] sample_id_to_individual_id_mapping = {} sample_dataset_path_mapping = {} for individual_id, dataset_path in load_uploaded_mapping_file( mapping_file_id).items(): if not (dataset_path.endswith(".bam") or dataset_path.endswith(".cram")): raise Exception( 'BAM / CRAM file "{}" must have a .bam or .cram extension'. format(dataset_path)) validate_alignment_dataset_path(dataset_path) sample_id = dataset_path.split('/')[-1].split('.')[0] sample_id_to_individual_id_mapping[sample_id] = individual_id sample_dataset_path_mapping[sample_id] = dataset_path matched_sample_id_to_sample_record = match_sample_ids_to_sample_records( project=project, sample_ids=sample_id_to_individual_id_mapping.keys(), sample_type=sample_type, dataset_type=Sample.DATASET_TYPE_READ_ALIGNMENTS, sample_id_to_individual_id_mapping= sample_id_to_individual_id_mapping, ) unmatched_samples = set( sample_id_to_individual_id_mapping.keys()) - set( matched_sample_id_to_sample_record.keys()) if len(unmatched_samples) > 0: raise Exception( 'The following Individual IDs do not exist: {}'.format( ", ".join(unmatched_samples))) _update_samples( matched_sample_id_to_sample_record, sample_dataset_path_mapping=sample_dataset_path_mapping) except Exception as e: traceback.print_exc() return create_json_response({'errors': [e.message or str(e)]}, status=400) if not matched_sample_id_to_sample_record: return create_json_response({'samplesByGuid': {}}) # Deprecated update VCFFile records for sample in matched_sample_id_to_sample_record.values(): base_indiv = find_matching_xbrowse_model(sample.individual) if base_indiv: base_indiv.bam_file_path = sample.dataset_file_path base_indiv.save() return create_json_response( _get_samples_json(matched_sample_id_to_sample_record, project_guid))
def add_alignment_dataset_handler(request, project_guid): """Create or update samples for the given dataset Args: request: Django request object project_guid (string): GUID of the project that should be updated HTTP POST Request body - should contain the following json structure: { 'sampleType': <"WGS", "WES", or "RNA"> (required) 'datasetType': <"VARIANTS", or "ALIGN"> (required) 'elasticsearchIndex': <String> 'datasetPath': <String> 'datasetName': <String> 'ignoreExtraSamplesInCallset': <Boolean> 'mappingFile': { 'uploadedFileId': <Id for temporary uploaded file> } } Response body - will contain the following structure: """ project = get_project_and_check_permissions(project_guid, request.user, permission_level=CAN_EDIT) request_json = json.loads(request.body) try: required_fields = ['sampleType', 'mappingFile'] if any(field not in request_json for field in required_fields): raise ValueError( "request must contain fields: {}".format(', '.join(required_fields))) sample_type = request_json['sampleType'] if sample_type not in {choice[0] for choice in Sample.SAMPLE_TYPE_CHOICES}: raise Exception("Sample type not supported: {}".format(sample_type)) mapping_file_id = request_json['mappingFile']['uploadedFileId'] sample_id_to_individual_id_mapping = {} sample_dataset_path_mapping = {} for individual_id, dataset_path in load_uploaded_mapping_file(mapping_file_id).items(): if not (dataset_path.endswith(".bam") or dataset_path.endswith(".cram")): raise Exception('BAM / CRAM file "{}" must have a .bam or .cram extension'.format(dataset_path)) validate_alignment_dataset_path(dataset_path) sample_id = dataset_path.split('/')[-1].split('.')[0] sample_id_to_individual_id_mapping[sample_id] = individual_id sample_dataset_path_mapping[sample_id] = dataset_path matched_sample_id_to_sample_record = match_sample_ids_to_sample_records( project=project, sample_ids=sample_id_to_individual_id_mapping.keys(), sample_type=sample_type, dataset_type=Sample.DATASET_TYPE_READ_ALIGNMENTS, sample_id_to_individual_id_mapping=sample_id_to_individual_id_mapping, ) unmatched_samples = set(sample_id_to_individual_id_mapping.keys()) - set(matched_sample_id_to_sample_record.keys()) if len(unmatched_samples) > 0: raise Exception('The following Individual IDs do not exist: {}'.format(", ".join(unmatched_samples))) _update_samples(matched_sample_id_to_sample_record, sample_dataset_path_mapping=sample_dataset_path_mapping) except Exception as e: traceback.print_exc() return create_json_response({'errors': [e.message or str(e)]}, status=400) if not matched_sample_id_to_sample_record: return create_json_response({'samplesByGuid': {}}) # Deprecated update VCFFile records for sample in matched_sample_id_to_sample_record.values(): for base_indiv in BaseIndividual.objects.filter(seqr_individual=sample.individual).only('id'): base_indiv.bam_file_path = sample.dataset_file_path base_indiv.save() return create_json_response(_get_samples_json(matched_sample_id_to_sample_record, project_guid))