def __init__(self, metadata_file): self.metadata_file = metadata_file self.reader = EvaXlsxReader(metadata_file) self.metadata = {} for worksheet in self.reader.reader.valid_worksheets(): self.metadata[worksheet] = self.reader._get_all_rows(worksheet) self.error_list = []
def compare_spreadsheet_and_vcf(eva_files_sheet, vcf_dir, expected_vcf_files=None): """ Take a spreadsheet following EVA standard and compare the samples in it to the ones found in the VCF files """ eva_xls_reader = EvaXlsxReader(eva_files_sheet) vcf_files = [row['File Name'] for row in eva_xls_reader.files] if expected_vcf_files: expected_vcf_files = [ os.path.basename(vcf_file) for vcf_file in expected_vcf_files ] if sorted(vcf_files) != sorted(expected_vcf_files): logger.warning( 'VCF files found in the spreadsheet does not match the ones submitted. ' 'Submitted VCF will be added to the spreadsheet') analysis_alias = '' if len(eva_xls_reader.analysis) > 0: analysis_alias = eva_xls_reader.analysis[0].get( 'Analysis Alias') or '' eva_xls_writer = EvaXlsxWriter(eva_files_sheet) eva_xls_writer.set_files([ { 'File Name': os.path.basename(vcf_file), 'File Type': 'vcf', 'Analysis Alias': analysis_alias, 'MD5': '' # Dummy md5 for now } for vcf_file in expected_vcf_files ]) eva_xls_writer.save() eva_xls_reader = EvaXlsxReader(eva_files_sheet) samples_per_analysis = eva_xls_reader.samples_per_analysis files_per_analysis = eva_xls_reader.files_per_analysis overall_differences = False results_per_analysis_alias = {} if len(samples_per_analysis) == 1 and None in samples_per_analysis or \ len(files_per_analysis) == 1 and None in files_per_analysis: # No analysis alias available in samples or in files. Prepare the samples and files to be used together samples_per_analysis[None] = eva_xls_reader.samples files_per_analysis[None] = eva_xls_reader.files for analysis_alias in samples_per_analysis: has_differences, diff_submitted_file_submission, diff_submission_submitted_file = compare_names_in_files_and_samples( get_vcf_file_paths(files_per_analysis[analysis_alias], vcf_dir), samples_per_analysis[analysis_alias], analysis_alias) results_per_analysis_alias[analysis_alias] = ( has_differences, diff_submitted_file_submission, diff_submission_submitted_file) overall_differences = overall_differences or has_differences if not overall_differences: logger.info( 'No differences found between the samples in the Metadata sheet and the submitted VCF file(s)!' ) logger.info('Samples checking completed!') return overall_differences, results_per_analysis_alias
def test_replace_values_in_metadata(self): metadata = self.create_metadata() reader = EvaXlsxReader(metadata) assert reader.project['Tax ID'] == 9606 assert reader.analysis[0]['Reference'] == 'GCA_000001405.1' self.eload.replace_values_in_metadata(taxid=10000, reference_accession='GCA_000009999.9') reader = EvaXlsxReader(metadata) assert reader.project['Tax ID'] == 10000 assert reader.analysis[0]['Reference'] == 'GCA_000009999.9'
def test_replace_values_in_metadata(self): source_metadata = os.path.join(self.resources_folder, 'metadata.xlsx') metadata = os.path.join(self.eload.eload_dir, '10_submitted', 'metadata_file', 'metadata.xlsx') shutil.copyfile(source_metadata, metadata) reader = EvaXlsxReader(metadata) assert reader.project['Tax ID'] == 9606 assert reader.analysis[0]['Reference'] == 'GCA_000001405.1' self.eload.replace_values_in_metadata( taxid=10000, reference_accession='GCA_000009999.9') reader = EvaXlsxReader(metadata) assert reader.project['Tax ID'] == 10000 assert reader.analysis[0]['Reference'] == 'GCA_000009999.9'
def _report_metadata(self, metadata_file): report_params = { 'filepath': metadata_file, 'metadata_last_modified': self._last_modified_of([ (f, s, t) for f, s, t in self._metadata_files if f == metadata_file ]) or 'NA' } try: reader = EvaXlsxReader(metadata_file) report_params['project_title'] = reader.project_title report_params['number_analysis'] = len(reader.analysis) if reader.references: report_params['reference genome'] = ', '.join(reader.references) else: report_params['reference genome'] = 'None' report_params['number_samples'] = len(reader.samples) except Exception: self.error(traceback.format_exc()) report_params['project_title'] = 'NA' report_params['number_analysis'] = 0 report_params['reference genome'] = 'NA' report_params['number_samples'] = 0 report_template = """metadata file path: {filepath} last modified: {metadata_last_modified} Project title: {project_title} Number of analysis: {number_analysis} Reference sequence: {reference genome} Number of sample: {number_samples} """ return report_template.format(**report_params)
def compare_spreadsheet_and_vcf(eva_files_sheet, vcf_dir): """ Take a spreadsheet following EVA standard and compare the samples in it to the ones found in the VCF files """ eva_xls_reader = EvaXlsxReader(eva_files_sheet) samples_per_analysis = eva_xls_reader.samples_per_analysis files_per_analysis = eva_xls_reader.files_per_analysis overall_differences = False results_per_analysis_alias = {} if len(samples_per_analysis) == 1 and None in samples_per_analysis or \ len(files_per_analysis) == 1 and None in files_per_analysis: # No analysis alias available in samples or in files. Prepare the samples and files to be used together samples_per_analysis[None] = eva_xls_reader.samples files_per_analysis[None] = eva_xls_reader.files for analysis_alias in samples_per_analysis: has_differences, diff_submitted_file_submission, diff_submission_submitted_file = compare_names_in_files_and_samples( get_vcf_file_paths(files_per_analysis[analysis_alias], vcf_dir), samples_per_analysis[analysis_alias], analysis_alias) results_per_analysis_alias[analysis_alias] = ( has_differences, diff_submitted_file_submission, diff_submission_submitted_file) overall_differences = overall_differences or has_differences if not overall_differences: logger.info( 'No differences found between the samples in the Metadata sheet and the submitted VCF file(s)!' ) logger.info('Samples checking completed!') return overall_differences, results_per_analysis_alias
def __init__(self, metadata_file, output_folder, name): self.metadata_file = metadata_file self.output_folder = output_folder self.name = name self.reader = EvaXlsxReader(self.metadata_file) self.project_file = os.path.join(self.output_folder, self.name + '.Project.xml') self.analysis_file = os.path.join(self.output_folder, self.name + '.Analysis.xml') self.submission_file = os.path.join(self.output_folder, self.name + '.Submission.xml')
def detect_metadata_attributes(self): eva_metadata = EvaXlsxReader( self.eload_cfg.query('submission', 'metadata_spreadsheet')) analysis_reference = {} for analysis in eva_metadata.analysis: reference_txt = analysis.get('Reference') assembly_accessions = resolve_accession_from_text( reference_txt) if reference_txt else None if not assembly_accessions: assembly_accession = None elif len(assembly_accessions) == 1: assembly_accession = assembly_accessions[0] else: self.warning( f"Multiple assemblies found for {analysis.get('Analysis Alias')}: {', '.join(assembly_accessions)} " ) assembly_accession = sorted(assembly_accessions)[-1] self.warning( f"Will use the most recent assembly: {assembly_accession}") if assembly_accession: analysis_reference[analysis.get('Analysis Alias')] = { 'assembly_accession': assembly_accession, 'vcf_files': [] } else: self.error( f"Reference is missing for Analysis {analysis.get('Analysis Alias')}" ) for file in eva_metadata.files: if file.get("File Type") == 'vcf': file_full = os.path.join(self.eload_dir, directory_structure['vcf'], file.get("File Name")) analysis_alias = file.get("Analysis Alias") analysis_reference[analysis_alias]['vcf_files'].append( file_full) self.eload_cfg.set('submission', 'analyses', value=analysis_reference) taxonomy_id = eva_metadata.project.get('Tax ID') if taxonomy_id and (isinstance(taxonomy_id, int) or taxonomy_id.isdigit()): self.eload_cfg.set('submission', 'taxonomy_id', value=int(taxonomy_id)) scientific_name = get_scientific_name_from_ensembl(taxonomy_id) self.eload_cfg.set('submission', 'scientific_name', value=scientific_name) else: if taxonomy_id: self.error('Taxonomy id %s is invalid:', taxonomy_id) else: self.error('Taxonomy id is missing for the submission')
def check_submitted_filenames(self): """Compares submitted vcf filenames with those in metadata sheet, and amends the metadata when possible.""" vcf_dir = os.path.join(self.eload_dir, directory_structure['vcf']) uncompressed_vcf = glob.glob(os.path.join(vcf_dir, '*.vcf')) compressed_vcf = glob.glob(os.path.join(vcf_dir, '*.vcf.gz')) submitted_vcfs = [ os.path.basename(vcf) for vcf in uncompressed_vcf + compressed_vcf ] if len(submitted_vcfs) < 1: raise FileNotFoundError('Could not locate vcf file in %s', vcf_dir) eva_files_sheet = self.eload_cfg.query('submission', 'metadata_spreadsheet') eva_xls_reader = EvaXlsxReader(eva_files_sheet) spreadsheet_vcfs = [ os.path.basename(row['File Name']) for row in eva_xls_reader.files if row['File Type'] == 'vcf' or row['File Name'].endswith('.vcf') or row['File Name'].endswith('.vcf.gz') ] if sorted(spreadsheet_vcfs) != sorted(submitted_vcfs): self.warning( 'VCF files found in the spreadsheet does not match the ones submitted. ' 'Submitted VCF will be added to the spreadsheet') self.debug( f'Difference between spreadsheet vcfs and submitted vcfs: ' f'{", ".join(set(spreadsheet_vcfs).difference(set(submitted_vcfs)))}' ) self.debug( f'Difference between submitted vcfs and spreadsheet vcfs: ' f'{", ".join(set(submitted_vcfs).difference(set(spreadsheet_vcfs)))}' ) analysis_alias = '' if len(eva_xls_reader.analysis) == 1: analysis_alias = eva_xls_reader.analysis[0].get( 'Analysis Alias') or '' elif len(eva_xls_reader.analysis) > 1: self.error( "Multiple analyses found, can't add submitted VCF to spreadsheet" ) raise ValueError( "Multiple analyses found, can't add submitted VCF to spreadsheet" ) eva_xls_writer = EvaXlsxWriter(eva_files_sheet) eva_xls_writer.set_files([ { 'File Name': vcf_file, 'File Type': 'vcf', 'Analysis Alias': analysis_alias, 'MD5': '' # Dummy md5 for now } for vcf_file in submitted_vcfs ]) eva_xls_writer.save()
def test_set_rows(self): self.xls_writer.active_worksheet = 'Sample' rows = [{ 'Analysis Alias': 'GAE', 'Sample ID': 'S' + str(sample_num), 'Sample Accession': 'SABCDEFGHIJKLMNOP' + str(sample_num), } for sample_num in range(1, 51)] self.xls_writer.set_rows(rows, empty_remaining_rows=True) self.xls_writer.save(self.metadata_copy_file) reader = EvaXlsxReader(self.metadata_copy_file) assert len(reader.samples) == 50 assert reader.samples[0]['Sample Accession'] == 'SABCDEFGHIJKLMNOP1'
def test_edit_row(self): self.xls_writer.active_worksheet = 'Sample' for sample_num in range(1, 101): self.xls_writer.edit_row( { 'Analysis Alias': 'GAE', 'Sample ID': 'S' + str(sample_num), 'Sample Accession': 'SABCDEF' + str(sample_num), 'row_num': sample_num + 3 }, remove_when_missing_values=True) self.xls_writer.save(self.metadata_copy_file) reader = EvaXlsxReader(self.metadata_copy_file) assert len(reader.samples) == 100 assert reader.samples[0]['Sample Accession'] == 'SABCDEF1'
def replace_values_in_metadata(self, taxid=None, reference_accession=None): """Find and Replace the value in the metadata spreadsheet with the one provided """ input_spreadsheet = self.eload_cfg.query('submission', 'metadata_spreadsheet') reader = EvaXlsxReader(input_spreadsheet) # This will write the spreadsheet in place of the existing one eva_xls_writer = EvaXlsxWriter(input_spreadsheet) if taxid: project = reader.project project['Tax ID'] = taxid eva_xls_writer.set_project(project) if reference_accession: analysis_rows = [] for analysis in reader.analysis: analysis['Reference'] = reference_accession analysis_rows.append(analysis) eva_xls_writer.set_analysis(analysis_rows) eva_xls_writer.save()
def get_analysis_alias_from_metadata(eload_cfg): """ Returns analysis alias only if we find a metadata spreadsheet and it has exactly one analysis. Otherwise provides an error message and raise an error. """ metadata_spreadsheet = eload_cfg.query('submission', 'metadata_spreadsheet') if metadata_spreadsheet: reader = EvaXlsxReader(metadata_spreadsheet) if len(reader.analysis) == 1: return reader.analysis[0].get('Analysis Alias') if len(reader.analysis) > 1: logger.error( "Can't assign analysis alias: multiple analyses found in metadata!" ) else: logger.error( "Can't assign analysis alias: no analyses found in metadata!") else: logger.error("Can't assign analysis alias: no metadata found!") logger.error( "Try running upgrade_config and passing an analysis alias explicitly.") raise ValueError("Can't find an analysis alias for config upgrade.")
class EvaXlsxValidator(AppLogger): def __init__(self, metadata_file): self.metadata_file = metadata_file self.reader = EvaXlsxReader(metadata_file) self.metadata = {} for worksheet in self.reader.reader.valid_worksheets(): self.metadata[worksheet] = self.reader._get_all_rows(worksheet) self.error_list = [] def validate(self): self.cerberus_validation() self.complex_validation() self.semantic_validation() def cerberus_validation(self): """ Leverage cerberus validation to check the format of the metadata. This function adds error statements to the errors attribute """ config_file = os.path.join(ETC_DIR, 'eva_project_validation.yaml') with open(config_file) as open_file: validation_schema = yaml.safe_load(open_file) validator = Validator(validation_schema) validator.allow_unknown = True validator.validate(self.metadata) for sheet in validator.errors: for error1 in validator.errors[sheet]: for data_pos in error1: # data_pos is 0 based position in the data that was provided to cerberus # Convert this position to the excel row number row_num = data_pos + self.reader.reader.base_row_offset( sheet) + 1 for error2 in error1[data_pos]: for field_name in error2: for error3 in error2[field_name]: self.error_list.append( f'In Sheet {sheet}, Row {row_num}, field {field_name}: {error3}' ) def complex_validation(self): """ More complex validation steps that cannot be expressed in Cerberus This function adds error statements to the errors attribute """ analysis_aliases = [ analysis_row['Analysis Alias'] for analysis_row in self.metadata['Analysis'] ] self.same_set(analysis_aliases, [ sample_row['Analysis Alias'] for sample_row in self.metadata['Sample'] ], 'Analysis Alias', 'Samples') self.same_set(analysis_aliases, [ file_row['Analysis Alias'] for file_row in self.metadata['Files'] ], 'Analysis Alias', 'Files') project_titles = [ analysis_row['Project Title'] for analysis_row in self.metadata['Project'] ] self.same_set(project_titles, [ analysis_row['Project Title'] for analysis_row in self.metadata['Analysis'] ], 'Project Title', 'Analysis') for row in self.metadata['Sample']: self.group_of_fields_required( 'Sample', row, ['Analysis Alias', 'Sample Accession', 'Sample ID'], [ 'Analysis Alias', 'Sample Name', 'Title', 'Tax Id', 'Scientific Name' ]) def semantic_validation(self): """ Validation of the data that involve checking its meaning This function adds error statements to the errors attribute """ # Check if the references can be retrieved references = set([ row['Reference'] for row in self.metadata['Analysis'] if row['Reference'] ]) for reference in references: accessions = retrieve_genbank_assembly_accessions_from_ncbi( reference) if len(accessions) == 0: self.error_list.append( f'In Analysis, Reference {reference} did not resolve to any accession' ) elif len(accessions) > 1: self.error_list.append( f'In Analysis, Reference {reference} resolve to more than one accession: {accessions}' ) # Check taxonomy scientific name pair taxid_and_species_list = set([(row['Tax Id'], row['Scientific Name']) for row in self.metadata['Sample'] if row['Tax Id']]) for taxid, species in taxid_and_species_list: try: scientific_name = get_scientific_name_from_ensembl(int(taxid)) if species != scientific_name: self.error_list.append( f'In Samples, Taxonomy {taxid} and scientific name {species} are inconsistent' ) except ValueError as e: self.error(str(e)) self.error_list.append(str(e)) except HTTPError as e: self.error(str(e)) self.error_list.append(str(e)) def group_of_fields_required(self, sheet_name, row, *args): if not any([all(row.get(key) for key in group) for group in args]): self.error_list.append( 'In %s, row %s, one of this group of fields must be filled: %s -- %s' % ( sheet_name, row.get('row_num'), ' or '.join([', '.join(group) for group in args]), ' -- '.join((', '.join(('%s:%s' % (key, row[key]) for key in group)) for group in args)), )) def same_set(self, list1, list2, list1_desc, list2_desc): if not set(list1) == set(list2): list1_list2 = sorted(cast_list(set(list1).difference(list2))) list2_list1 = sorted(cast_list(set(list2).difference(list1))) errors = [] if list1_list2: errors.append('%s present in %s not in %s' % (','.join(list1_list2), list1_desc, list2_desc)) if list2_list1: errors.append('%s present in %s not in %s' % (','.join(list2_list1), list2_desc, list1_desc)) self.error_list.append( 'Check %s vs %s: %s' % (list1_desc, list2_desc, ' -- '.join(errors)))
def __init__(self, metadata_spreadsheet): super().__init__() self.metadata_spreadsheet = metadata_spreadsheet self.reader = EvaXlsxReader(self.metadata_spreadsheet) self.sample_data = self.map_metadata_to_bsd_data()
def test_update_metadata_from_config_for_files(self): # This metadata file contains multiple analysis each containing multiple files. # the file get merged into 1 file per analysis metadata_file = os.path.join(self.resources_folder, 'metadata_2_analysis.xlsx') ena_metadata_file = os.path.join( self.eload.eload_dir, 'metadata_2_analysis_for_brokering.xlsx') analyses = { 'GAE': { 'assembly_accession': 'GCA_000001405.1', 'vcf_files': { 'path/to/GAE.vcf.gz': { 'csi': 'path/to/GAE.vcf.gz.csi', 'csi_md5': '', 'index': 'path/to/GAE.vcf.gz.tbi', 'index_md5': '', 'md5': '', 'original_vcf': 'path/to/original_GAE.vcf.gz', 'output_vcf_file': None } } }, 'GAE2': { 'assembly_accession': 'GCA_000001405.1', 'vcf_files': { 'path/to/GAE2.vcf.gz': { 'csi': 'path/to/GAE2.vcf.gz.csi', 'csi_md5': '', 'index': 'path/to/GAE2.vcf.gz.tbi', 'index_md5': '', 'md5': '', 'original_vcf': 'path/to/original_GAE2.vcf.gz', 'output_vcf_file': None } } } } self.eload.eload_cfg.set('brokering', 'analyses', value=analyses) self.eload.update_metadata_from_config(metadata_file, ena_metadata_file) # Check that the Files get set to the merged file name and that the analysis alias is modified reader = EvaXlsxReader(ena_metadata_file) assert reader.files == [{ 'Analysis Alias': 'GAE', 'File Name': 'ELOAD_3/GAE.vcf.gz', 'File Type': 'vcf', 'MD5': None, 'row_num': 2 }, { 'Analysis Alias': 'GAE', 'File Name': 'ELOAD_3/GAE.vcf.gz.tbi', 'File Type': 'tabix', 'MD5': None, 'row_num': 3 }, { 'Analysis Alias': 'GAE2', 'File Name': 'ELOAD_3/GAE2.vcf.gz', 'File Type': 'vcf', 'MD5': None, 'row_num': 4 }, { 'Analysis Alias': 'GAE2', 'File Name': 'ELOAD_3/GAE2.vcf.gz.tbi', 'File Type': 'tabix', 'MD5': None, 'row_num': 5 }]
def setUp(self): self.xls_writer = XlsxWriter(self.metadata_file, self.eva_xls_reader_conf) self.reader = EvaXlsxReader(self.metadata_file)
def update_metadata_from_config(self, input_spreadsheet, output_spreadsheet=None): reader = EvaXlsxReader(input_spreadsheet) single_analysis_alias = None if len(reader.analysis) == 1: single_analysis_alias = reader.analysis[0].get('Analysis Alias') sample_rows = [] for sample_row in reader.samples: if self.eload_cfg.query('brokering', 'Biosamples', 'Samples', sample_row.get('Sample Name')): sample_rows.append({ 'row_num': sample_row.get('row_num'), 'Analysis Alias': sample_row.get('Analysis Alias') or single_analysis_alias, 'Sample ID': sample_row.get('Sample Name'), 'Sample Accession': self.eload_cfg['brokering']['Biosamples']['Samples'][ sample_row.get('Sample Name')] }) else: sample_rows.append(sample_row) file_rows = [] analyses = self.eload_cfg['brokering']['analyses'] for analysis in analyses: for vcf_file_name in analyses[analysis]['vcf_files']: vcf_file_info = self.eload_cfg['brokering']['analyses'][ analysis]['vcf_files'][vcf_file_name] # Add the vcf file file_rows.append({ 'Analysis Alias': analysis, 'File Name': self.eload + '/' + os.path.basename(vcf_file_name), 'File Type': 'vcf', 'MD5': vcf_file_info['md5'] }) # Add the index file if vcf_file_info['index'].endswith('.csi'): file_type = 'csi' else: file_type = 'tabix' file_rows.append({ 'Analysis Alias': analysis, 'File Name': self.eload + '/' + os.path.basename(vcf_file_info['index']), 'File Type': file_type, 'MD5': vcf_file_info['index_md5'] }) project_row = reader.project if self.eload_cfg.query('brokering', 'ena', 'existing_project'): project_row['Project Alias'] = self.eload_cfg.query( 'brokering', 'ena', 'PROJECT') if output_spreadsheet: eva_xls_writer = EvaXlsxWriter(input_spreadsheet, output_spreadsheet) else: eva_xls_writer = EvaXlsxWriter(input_spreadsheet) eva_xls_writer.set_project(project_row) eva_xls_writer.set_samples(sample_rows) eva_xls_writer.set_files(file_rows) eva_xls_writer.save() return output_spreadsheet
def test_get_all_rows(self): reader = EvaXlsxReader(self.metadata_file) rows = reader._get_all_rows('Analysis') assert len(rows) == 1 assert rows[0]['Analysis Title'] == 'Greatest analysis ever'