def __init__(self, metadata_file):
self.metadata_file = metadata_file
self.reader = EvaXlsxReader(metadata_file)
self.metadata = {}
for worksheet in self.reader.reader.valid_worksheets():
self.metadata[worksheet] = self.reader._get_all_rows(worksheet)
self.error_list = []
def compare_spreadsheet_and_vcf(eva_files_sheet,
vcf_dir,
expected_vcf_files=None):
"""
Take a spreadsheet following EVA standard and compare the samples in it to the ones found in the VCF files
"""
eva_xls_reader = EvaXlsxReader(eva_files_sheet)
vcf_files = [row['File Name'] for row in eva_xls_reader.files]
if expected_vcf_files:
expected_vcf_files = [
os.path.basename(vcf_file) for vcf_file in expected_vcf_files
]
if sorted(vcf_files) != sorted(expected_vcf_files):
logger.warning(
'VCF files found in the spreadsheet does not match the ones submitted. '
'Submitted VCF will be added to the spreadsheet')
analysis_alias = ''
if len(eva_xls_reader.analysis) > 0:
analysis_alias = eva_xls_reader.analysis[0].get(
'Analysis Alias') or ''
eva_xls_writer = EvaXlsxWriter(eva_files_sheet)
eva_xls_writer.set_files([
{
'File Name': os.path.basename(vcf_file),
'File Type': 'vcf',
'Analysis Alias': analysis_alias,
'MD5': '' # Dummy md5 for now
} for vcf_file in expected_vcf_files
])
eva_xls_writer.save()
eva_xls_reader = EvaXlsxReader(eva_files_sheet)
samples_per_analysis = eva_xls_reader.samples_per_analysis
files_per_analysis = eva_xls_reader.files_per_analysis
overall_differences = False
results_per_analysis_alias = {}
if len(samples_per_analysis) == 1 and None in samples_per_analysis or \
len(files_per_analysis) == 1 and None in files_per_analysis:
# No analysis alias available in samples or in files. Prepare the samples and files to be used together
samples_per_analysis[None] = eva_xls_reader.samples
files_per_analysis[None] = eva_xls_reader.files
for analysis_alias in samples_per_analysis:
has_differences, diff_submitted_file_submission, diff_submission_submitted_file = compare_names_in_files_and_samples(
get_vcf_file_paths(files_per_analysis[analysis_alias], vcf_dir),
samples_per_analysis[analysis_alias], analysis_alias)
results_per_analysis_alias[analysis_alias] = (
has_differences, diff_submitted_file_submission,
diff_submission_submitted_file)
overall_differences = overall_differences or has_differences
if not overall_differences:
logger.info(
'No differences found between the samples in the Metadata sheet and the submitted VCF file(s)!'
)
logger.info('Samples checking completed!')
return overall_differences, results_per_analysis_alias
def test_replace_values_in_metadata(self):
metadata = self.create_metadata()
reader = EvaXlsxReader(metadata)
assert reader.project['Tax ID'] == 9606
assert reader.analysis[0]['Reference'] == 'GCA_000001405.1'
self.eload.replace_values_in_metadata(taxid=10000, reference_accession='GCA_000009999.9')
reader = EvaXlsxReader(metadata)
assert reader.project['Tax ID'] == 10000
assert reader.analysis[0]['Reference'] == 'GCA_000009999.9'
def test_replace_values_in_metadata(self):
source_metadata = os.path.join(self.resources_folder, 'metadata.xlsx')
metadata = os.path.join(self.eload.eload_dir, '10_submitted',
'metadata_file', 'metadata.xlsx')
shutil.copyfile(source_metadata, metadata)
reader = EvaXlsxReader(metadata)
assert reader.project['Tax ID'] == 9606
assert reader.analysis[0]['Reference'] == 'GCA_000001405.1'
self.eload.replace_values_in_metadata(
taxid=10000, reference_accession='GCA_000009999.9')
reader = EvaXlsxReader(metadata)
assert reader.project['Tax ID'] == 10000
assert reader.analysis[0]['Reference'] == 'GCA_000009999.9'
def _report_metadata(self, metadata_file):
report_params = {
'filepath': metadata_file,
'metadata_last_modified': self._last_modified_of([
(f, s, t) for f, s, t in self._metadata_files if f == metadata_file
]) or 'NA'
}
try:
reader = EvaXlsxReader(metadata_file)
report_params['project_title'] = reader.project_title
report_params['number_analysis'] = len(reader.analysis)
if reader.references:
report_params['reference genome'] = ', '.join(reader.references)
else:
report_params['reference genome'] = 'None'
report_params['number_samples'] = len(reader.samples)
except Exception:
self.error(traceback.format_exc())
report_params['project_title'] = 'NA'
report_params['number_analysis'] = 0
report_params['reference genome'] = 'NA'
report_params['number_samples'] = 0
report_template = """metadata file path: {filepath}
last modified: {metadata_last_modified}
Project title: {project_title}
Number of analysis: {number_analysis}
Reference sequence: {reference genome}
Number of sample: {number_samples}
"""
return report_template.format(**report_params)
def compare_spreadsheet_and_vcf(eva_files_sheet, vcf_dir):
"""
Take a spreadsheet following EVA standard and compare the samples in it to the ones found in the VCF files
"""
eva_xls_reader = EvaXlsxReader(eva_files_sheet)
samples_per_analysis = eva_xls_reader.samples_per_analysis
files_per_analysis = eva_xls_reader.files_per_analysis
overall_differences = False
results_per_analysis_alias = {}
if len(samples_per_analysis) == 1 and None in samples_per_analysis or \
len(files_per_analysis) == 1 and None in files_per_analysis:
# No analysis alias available in samples or in files. Prepare the samples and files to be used together
samples_per_analysis[None] = eva_xls_reader.samples
files_per_analysis[None] = eva_xls_reader.files
for analysis_alias in samples_per_analysis:
has_differences, diff_submitted_file_submission, diff_submission_submitted_file = compare_names_in_files_and_samples(
get_vcf_file_paths(files_per_analysis[analysis_alias], vcf_dir),
samples_per_analysis[analysis_alias], analysis_alias)
results_per_analysis_alias[analysis_alias] = (
has_differences, diff_submitted_file_submission,
diff_submission_submitted_file)
overall_differences = overall_differences or has_differences
if not overall_differences:
logger.info(
'No differences found between the samples in the Metadata sheet and the submitted VCF file(s)!'
)
logger.info('Samples checking completed!')
return overall_differences, results_per_analysis_alias
def __init__(self, metadata_file, output_folder, name):
self.metadata_file = metadata_file
self.output_folder = output_folder
self.name = name
self.reader = EvaXlsxReader(self.metadata_file)
self.project_file = os.path.join(self.output_folder, self.name + '.Project.xml')
self.analysis_file = os.path.join(self.output_folder, self.name + '.Analysis.xml')
self.submission_file = os.path.join(self.output_folder, self.name + '.Submission.xml')
def detect_metadata_attributes(self):
eva_metadata = EvaXlsxReader(
self.eload_cfg.query('submission', 'metadata_spreadsheet'))
analysis_reference = {}
for analysis in eva_metadata.analysis:
reference_txt = analysis.get('Reference')
assembly_accessions = resolve_accession_from_text(
reference_txt) if reference_txt else None
if not assembly_accessions:
assembly_accession = None
elif len(assembly_accessions) == 1:
assembly_accession = assembly_accessions[0]
else:
self.warning(
f"Multiple assemblies found for {analysis.get('Analysis Alias')}: {', '.join(assembly_accessions)} "
)
assembly_accession = sorted(assembly_accessions)[-1]
self.warning(
f"Will use the most recent assembly: {assembly_accession}")
if assembly_accession:
analysis_reference[analysis.get('Analysis Alias')] = {
'assembly_accession': assembly_accession,
'vcf_files': []
}
else:
self.error(
f"Reference is missing for Analysis {analysis.get('Analysis Alias')}"
)
for file in eva_metadata.files:
if file.get("File Type") == 'vcf':
file_full = os.path.join(self.eload_dir,
directory_structure['vcf'],
file.get("File Name"))
analysis_alias = file.get("Analysis Alias")
analysis_reference[analysis_alias]['vcf_files'].append(
file_full)
self.eload_cfg.set('submission', 'analyses', value=analysis_reference)
taxonomy_id = eva_metadata.project.get('Tax ID')
if taxonomy_id and (isinstance(taxonomy_id, int)
or taxonomy_id.isdigit()):
self.eload_cfg.set('submission',
'taxonomy_id',
value=int(taxonomy_id))
scientific_name = get_scientific_name_from_ensembl(taxonomy_id)
self.eload_cfg.set('submission',
'scientific_name',
value=scientific_name)
else:
if taxonomy_id:
self.error('Taxonomy id %s is invalid:', taxonomy_id)
else:
self.error('Taxonomy id is missing for the submission')
def check_submitted_filenames(self):
"""Compares submitted vcf filenames with those in metadata sheet, and amends the metadata when possible."""
vcf_dir = os.path.join(self.eload_dir, directory_structure['vcf'])
uncompressed_vcf = glob.glob(os.path.join(vcf_dir, '*.vcf'))
compressed_vcf = glob.glob(os.path.join(vcf_dir, '*.vcf.gz'))
submitted_vcfs = [
os.path.basename(vcf) for vcf in uncompressed_vcf + compressed_vcf
]
if len(submitted_vcfs) < 1:
raise FileNotFoundError('Could not locate vcf file in %s', vcf_dir)
eva_files_sheet = self.eload_cfg.query('submission',
'metadata_spreadsheet')
eva_xls_reader = EvaXlsxReader(eva_files_sheet)
spreadsheet_vcfs = [
os.path.basename(row['File Name']) for row in eva_xls_reader.files
if row['File Type'] == 'vcf' or row['File Name'].endswith('.vcf')
or row['File Name'].endswith('.vcf.gz')
]
if sorted(spreadsheet_vcfs) != sorted(submitted_vcfs):
self.warning(
'VCF files found in the spreadsheet does not match the ones submitted. '
'Submitted VCF will be added to the spreadsheet')
self.debug(
f'Difference between spreadsheet vcfs and submitted vcfs: '
f'{", ".join(set(spreadsheet_vcfs).difference(set(submitted_vcfs)))}'
)
self.debug(
f'Difference between submitted vcfs and spreadsheet vcfs: '
f'{", ".join(set(submitted_vcfs).difference(set(spreadsheet_vcfs)))}'
)
analysis_alias = ''
if len(eva_xls_reader.analysis) == 1:
analysis_alias = eva_xls_reader.analysis[0].get(
'Analysis Alias') or ''
elif len(eva_xls_reader.analysis) > 1:
self.error(
"Multiple analyses found, can't add submitted VCF to spreadsheet"
)
raise ValueError(
"Multiple analyses found, can't add submitted VCF to spreadsheet"
)
eva_xls_writer = EvaXlsxWriter(eva_files_sheet)
eva_xls_writer.set_files([
{
'File Name': vcf_file,
'File Type': 'vcf',
'Analysis Alias': analysis_alias,
'MD5': '' # Dummy md5 for now
} for vcf_file in submitted_vcfs
])
eva_xls_writer.save()
def test_set_rows(self):
self.xls_writer.active_worksheet = 'Sample'
rows = [{
'Analysis Alias': 'GAE',
'Sample ID': 'S' + str(sample_num),
'Sample Accession': 'SABCDEFGHIJKLMNOP' + str(sample_num),
} for sample_num in range(1, 51)]
self.xls_writer.set_rows(rows, empty_remaining_rows=True)
self.xls_writer.save(self.metadata_copy_file)
reader = EvaXlsxReader(self.metadata_copy_file)
assert len(reader.samples) == 50
assert reader.samples[0]['Sample Accession'] == 'SABCDEFGHIJKLMNOP1'
def test_edit_row(self):
self.xls_writer.active_worksheet = 'Sample'
for sample_num in range(1, 101):
self.xls_writer.edit_row(
{
'Analysis Alias': 'GAE',
'Sample ID': 'S' + str(sample_num),
'Sample Accession': 'SABCDEF' + str(sample_num),
'row_num': sample_num + 3
},
remove_when_missing_values=True)
self.xls_writer.save(self.metadata_copy_file)
reader = EvaXlsxReader(self.metadata_copy_file)
assert len(reader.samples) == 100
assert reader.samples[0]['Sample Accession'] == 'SABCDEF1'
def replace_values_in_metadata(self, taxid=None, reference_accession=None):
"""Find and Replace the value in the metadata spreadsheet with the one provided """
input_spreadsheet = self.eload_cfg.query('submission',
'metadata_spreadsheet')
reader = EvaXlsxReader(input_spreadsheet)
# This will write the spreadsheet in place of the existing one
eva_xls_writer = EvaXlsxWriter(input_spreadsheet)
if taxid:
project = reader.project
project['Tax ID'] = taxid
eva_xls_writer.set_project(project)
if reference_accession:
analysis_rows = []
for analysis in reader.analysis:
analysis['Reference'] = reference_accession
analysis_rows.append(analysis)
eva_xls_writer.set_analysis(analysis_rows)
eva_xls_writer.save()
def get_analysis_alias_from_metadata(eload_cfg):
"""
Returns analysis alias only if we find a metadata spreadsheet and it has exactly one analysis.
Otherwise provides an error message and raise an error.
"""
metadata_spreadsheet = eload_cfg.query('submission',
'metadata_spreadsheet')
if metadata_spreadsheet:
reader = EvaXlsxReader(metadata_spreadsheet)
if len(reader.analysis) == 1:
return reader.analysis[0].get('Analysis Alias')
if len(reader.analysis) > 1:
logger.error(
"Can't assign analysis alias: multiple analyses found in metadata!"
)
else:
logger.error(
"Can't assign analysis alias: no analyses found in metadata!")
else:
logger.error("Can't assign analysis alias: no metadata found!")
logger.error(
"Try running upgrade_config and passing an analysis alias explicitly.")
raise ValueError("Can't find an analysis alias for config upgrade.")
class EvaXlsxValidator(AppLogger):
def __init__(self, metadata_file):
self.metadata_file = metadata_file
self.reader = EvaXlsxReader(metadata_file)
self.metadata = {}
for worksheet in self.reader.reader.valid_worksheets():
self.metadata[worksheet] = self.reader._get_all_rows(worksheet)
self.error_list = []
def validate(self):
self.cerberus_validation()
self.complex_validation()
self.semantic_validation()
def cerberus_validation(self):
"""
Leverage cerberus validation to check the format of the metadata.
This function adds error statements to the errors attribute
"""
config_file = os.path.join(ETC_DIR, 'eva_project_validation.yaml')
with open(config_file) as open_file:
validation_schema = yaml.safe_load(open_file)
validator = Validator(validation_schema)
validator.allow_unknown = True
validator.validate(self.metadata)
for sheet in validator.errors:
for error1 in validator.errors[sheet]:
for data_pos in error1:
# data_pos is 0 based position in the data that was provided to cerberus
# Convert this position to the excel row number
row_num = data_pos + self.reader.reader.base_row_offset(
sheet) + 1
for error2 in error1[data_pos]:
for field_name in error2:
for error3 in error2[field_name]:
self.error_list.append(
f'In Sheet {sheet}, Row {row_num}, field {field_name}: {error3}'
)
def complex_validation(self):
"""
More complex validation steps that cannot be expressed in Cerberus
This function adds error statements to the errors attribute
"""
analysis_aliases = [
analysis_row['Analysis Alias']
for analysis_row in self.metadata['Analysis']
]
self.same_set(analysis_aliases, [
sample_row['Analysis Alias']
for sample_row in self.metadata['Sample']
], 'Analysis Alias', 'Samples')
self.same_set(analysis_aliases, [
file_row['Analysis Alias'] for file_row in self.metadata['Files']
], 'Analysis Alias', 'Files')
project_titles = [
analysis_row['Project Title']
for analysis_row in self.metadata['Project']
]
self.same_set(project_titles, [
analysis_row['Project Title']
for analysis_row in self.metadata['Analysis']
], 'Project Title', 'Analysis')
for row in self.metadata['Sample']:
self.group_of_fields_required(
'Sample', row,
['Analysis Alias', 'Sample Accession', 'Sample ID'], [
'Analysis Alias', 'Sample Name', 'Title', 'Tax Id',
'Scientific Name'
])
def semantic_validation(self):
"""
Validation of the data that involve checking its meaning
This function adds error statements to the errors attribute
"""
# Check if the references can be retrieved
references = set([
row['Reference'] for row in self.metadata['Analysis']
if row['Reference']
])
for reference in references:
accessions = retrieve_genbank_assembly_accessions_from_ncbi(
reference)
if len(accessions) == 0:
self.error_list.append(
f'In Analysis, Reference {reference} did not resolve to any accession'
)
elif len(accessions) > 1:
self.error_list.append(
f'In Analysis, Reference {reference} resolve to more than one accession: {accessions}'
)
# Check taxonomy scientific name pair
taxid_and_species_list = set([(row['Tax Id'], row['Scientific Name'])
for row in self.metadata['Sample']
if row['Tax Id']])
for taxid, species in taxid_and_species_list:
try:
scientific_name = get_scientific_name_from_ensembl(int(taxid))
if species != scientific_name:
self.error_list.append(
f'In Samples, Taxonomy {taxid} and scientific name {species} are inconsistent'
)
except ValueError as e:
self.error(str(e))
self.error_list.append(str(e))
except HTTPError as e:
self.error(str(e))
self.error_list.append(str(e))
def group_of_fields_required(self, sheet_name, row, *args):
if not any([all(row.get(key) for key in group) for group in args]):
self.error_list.append(
'In %s, row %s, one of this group of fields must be filled: %s -- %s'
% (
sheet_name,
row.get('row_num'),
' or '.join([', '.join(group) for group in args]),
' -- '.join((', '.join(('%s:%s' % (key, row[key])
for key in group))
for group in args)),
))
def same_set(self, list1, list2, list1_desc, list2_desc):
if not set(list1) == set(list2):
list1_list2 = sorted(cast_list(set(list1).difference(list2)))
list2_list1 = sorted(cast_list(set(list2).difference(list1)))
errors = []
if list1_list2:
errors.append('%s present in %s not in %s' %
(','.join(list1_list2), list1_desc, list2_desc))
if list2_list1:
errors.append('%s present in %s not in %s' %
(','.join(list2_list1), list2_desc, list1_desc))
self.error_list.append(
'Check %s vs %s: %s' %
(list1_desc, list2_desc, ' -- '.join(errors)))
def __init__(self, metadata_spreadsheet):
super().__init__()
self.metadata_spreadsheet = metadata_spreadsheet
self.reader = EvaXlsxReader(self.metadata_spreadsheet)
self.sample_data = self.map_metadata_to_bsd_data()
def test_update_metadata_from_config_for_files(self):
# This metadata file contains multiple analysis each containing multiple files.
# the file get merged into 1 file per analysis
metadata_file = os.path.join(self.resources_folder,
'metadata_2_analysis.xlsx')
ena_metadata_file = os.path.join(
self.eload.eload_dir, 'metadata_2_analysis_for_brokering.xlsx')
analyses = {
'GAE': {
'assembly_accession': 'GCA_000001405.1',
'vcf_files': {
'path/to/GAE.vcf.gz': {
'csi': 'path/to/GAE.vcf.gz.csi',
'csi_md5': '',
'index': 'path/to/GAE.vcf.gz.tbi',
'index_md5': '',
'md5': '',
'original_vcf': 'path/to/original_GAE.vcf.gz',
'output_vcf_file': None
}
}
},
'GAE2': {
'assembly_accession': 'GCA_000001405.1',
'vcf_files': {
'path/to/GAE2.vcf.gz': {
'csi': 'path/to/GAE2.vcf.gz.csi',
'csi_md5': '',
'index': 'path/to/GAE2.vcf.gz.tbi',
'index_md5': '',
'md5': '',
'original_vcf': 'path/to/original_GAE2.vcf.gz',
'output_vcf_file': None
}
}
}
}
self.eload.eload_cfg.set('brokering', 'analyses', value=analyses)
self.eload.update_metadata_from_config(metadata_file,
ena_metadata_file)
# Check that the Files get set to the merged file name and that the analysis alias is modified
reader = EvaXlsxReader(ena_metadata_file)
assert reader.files == [{
'Analysis Alias': 'GAE',
'File Name': 'ELOAD_3/GAE.vcf.gz',
'File Type': 'vcf',
'MD5': None,
'row_num': 2
}, {
'Analysis Alias': 'GAE',
'File Name': 'ELOAD_3/GAE.vcf.gz.tbi',
'File Type': 'tabix',
'MD5': None,
'row_num': 3
}, {
'Analysis Alias': 'GAE2',
'File Name': 'ELOAD_3/GAE2.vcf.gz',
'File Type': 'vcf',
'MD5': None,
'row_num': 4
}, {
'Analysis Alias': 'GAE2',
'File Name': 'ELOAD_3/GAE2.vcf.gz.tbi',
'File Type': 'tabix',
'MD5': None,
'row_num': 5
}]
def setUp(self):
self.xls_writer = XlsxWriter(self.metadata_file,
self.eva_xls_reader_conf)
self.reader = EvaXlsxReader(self.metadata_file)
def update_metadata_from_config(self,
input_spreadsheet,
output_spreadsheet=None):
reader = EvaXlsxReader(input_spreadsheet)
single_analysis_alias = None
if len(reader.analysis) == 1:
single_analysis_alias = reader.analysis[0].get('Analysis Alias')
sample_rows = []
for sample_row in reader.samples:
if self.eload_cfg.query('brokering', 'Biosamples', 'Samples',
sample_row.get('Sample Name')):
sample_rows.append({
'row_num':
sample_row.get('row_num'),
'Analysis Alias':
sample_row.get('Analysis Alias') or single_analysis_alias,
'Sample ID':
sample_row.get('Sample Name'),
'Sample Accession':
self.eload_cfg['brokering']['Biosamples']['Samples'][
sample_row.get('Sample Name')]
})
else:
sample_rows.append(sample_row)
file_rows = []
analyses = self.eload_cfg['brokering']['analyses']
for analysis in analyses:
for vcf_file_name in analyses[analysis]['vcf_files']:
vcf_file_info = self.eload_cfg['brokering']['analyses'][
analysis]['vcf_files'][vcf_file_name]
# Add the vcf file
file_rows.append({
'Analysis Alias':
analysis,
'File Name':
self.eload + '/' + os.path.basename(vcf_file_name),
'File Type':
'vcf',
'MD5':
vcf_file_info['md5']
})
# Add the index file
if vcf_file_info['index'].endswith('.csi'):
file_type = 'csi'
else:
file_type = 'tabix'
file_rows.append({
'Analysis Alias':
analysis,
'File Name':
self.eload + '/' +
os.path.basename(vcf_file_info['index']),
'File Type':
file_type,
'MD5':
vcf_file_info['index_md5']
})
project_row = reader.project
if self.eload_cfg.query('brokering', 'ena', 'existing_project'):
project_row['Project Alias'] = self.eload_cfg.query(
'brokering', 'ena', 'PROJECT')
if output_spreadsheet:
eva_xls_writer = EvaXlsxWriter(input_spreadsheet,
output_spreadsheet)
else:
eva_xls_writer = EvaXlsxWriter(input_spreadsheet)
eva_xls_writer.set_project(project_row)
eva_xls_writer.set_samples(sample_rows)
eva_xls_writer.set_files(file_rows)
eva_xls_writer.save()
return output_spreadsheet
def test_get_all_rows(self):
reader = EvaXlsxReader(self.metadata_file)
rows = reader._get_all_rows('Analysis')
assert len(rows) == 1
assert rows[0]['Analysis Title'] == 'Greatest analysis ever'