def semantic_validation(self):
"""
Validation of the data that involve checking its meaning
This function adds error statements to the errors attribute
"""
# Check if the references can be retrieved
references = set([row['Reference'] for row in self.metadata['Analysis'] if row['Reference']])
for reference in references:
accessions = retrieve_genbank_assembly_accessions_from_ncbi(reference)
if len(accessions) == 0:
self.error_list.append(f'In Analysis, Reference {reference} did not resolve to any accession')
elif len(accessions) > 1:
self.error_list.append(f'In Analysis, Reference {reference} resolve to more than one accession: {accessions}')
# Check taxonomy scientific name pair
taxid_and_species_list = set([(row['Tax Id'], row['Scientific Name']) for row in self.metadata['Sample'] if row['Tax Id']])
for taxid, species in taxid_and_species_list:
try:
scientific_name = get_scientific_name_from_ensembl(int(taxid))
if species != scientific_name:
self.error_list.append(
f'In Samples, Taxonomy {taxid} and scientific name {species} are inconsistent')
except ValueError as e:
self.error(str(e))
self.error_list.append(str(e))
except HTTPError as e:
self.error(str(e))
self.error_list.append(str(e))
def detect_metadata_attributes(self):
eva_metadata = EvaXlsxReader(
self.eload_cfg.query('submission', 'metadata_spreadsheet'))
analysis_reference = {}
for analysis in eva_metadata.analysis:
reference_txt = analysis.get('Reference')
assembly_accessions = resolve_accession_from_text(
reference_txt) if reference_txt else None
if not assembly_accessions:
assembly_accession = None
elif len(assembly_accessions) == 1:
assembly_accession = assembly_accessions[0]
else:
self.warning(
f"Multiple assemblies found for {analysis.get('Analysis Alias')}: {', '.join(assembly_accessions)} "
)
assembly_accession = sorted(assembly_accessions)[-1]
self.warning(
f"Will use the most recent assembly: {assembly_accession}")
if assembly_accession:
analysis_reference[analysis.get('Analysis Alias')] = {
'assembly_accession': assembly_accession,
'vcf_files': []
}
else:
self.error(
f"Reference is missing for Analysis {analysis.get('Analysis Alias')}"
)
for file in eva_metadata.files:
if file.get("File Type") == 'vcf':
file_full = os.path.join(self.eload_dir,
directory_structure['vcf'],
file.get("File Name"))
analysis_alias = file.get("Analysis Alias")
analysis_reference[analysis_alias]['vcf_files'].append(
file_full)
self.eload_cfg.set('submission', 'analyses', value=analysis_reference)
taxonomy_id = eva_metadata.project.get('Tax ID')
if taxonomy_id and (isinstance(taxonomy_id, int)
or taxonomy_id.isdigit()):
self.eload_cfg.set('submission',
'taxonomy_id',
value=int(taxonomy_id))
scientific_name = get_scientific_name_from_ensembl(taxonomy_id)
self.eload_cfg.set('submission',
'scientific_name',
value=scientific_name)
else:
if taxonomy_id:
self.error('Taxonomy id %s is invalid:', taxonomy_id)
else:
self.error('Taxonomy id is missing for the submission')
def get_tax_asm_from_ensembl(tax_id):
try:
logger.info(f'Query Ensembl for species name using taxonomy {tax_id}')
sp_name = get_scientific_name_from_ensembl(tax_id)
except Exception:
logger.warning(
f'Could not get species name for taxonomy {tax_id} in Ensembl')
return None
# Get assembly from Ensembl
logger.info(
f'Query Ensembl for supported assembly for taxonomy {tax_id} using species_name "{sp_name}"'
)
assembly = get_supported_asm_from_ensembl(sp_name)
if assembly != 'None':
return {'assembly': assembly, 'source': 'Ensembl'}
else:
logger.warning(
f'Could not find supported assembly for taxonomy_id {tax_id} using species_name "{sp_name}" in Ensembl'
)
return None
def retrieve_species_name_from_taxid_ensembl(taxid):
logger.info(f'Query Ensembl for taxonomy {taxid}', )
return get_scientific_name_from_ensembl(taxid)
def _create_project_xml(self):
"""
This function read the project row from the XLS parser then create and populate an XML element following ENA
data model.
:return: The top XML element
"""
project_row = self.reader.project
root = Element('PROJECT_SET')
project_elemt = add_element(
root,
'PROJECT',
alias=project_row.get('Project Alias'),
accession=project_row.get('Project Accession'),
center_name=project_row.get('Center'))
add_element(project_elemt,
'TITLE',
project_row.get('Project Title'),
content_required=True)
add_element(project_elemt,
'DESCRIPTION',
project_row.get('Description'),
content_required=True)
if 'Publication(s)' in project_row and project_row.get(
'Publication(s)'):
publications_elemt = add_element(project_elemt, 'PUBLICATIONS')
publications = project_row.get('Publication(s)').strip().split(',')
for publication in publications:
pub_elemt = add_element(publications_elemt, 'PUBLICATION')
pub_links_elemt = add_element(pub_elemt, 'PUBLICATION_LINKS')
pub_link_elemt = add_element(pub_links_elemt,
'PUBLICATION_LINK')
xref_link_elemt = add_element(pub_link_elemt, 'XREF_LINK')
# Assuming format like PubMed:123456
# TODO: requirement in the format
pub_db, pub_id = publication.split(':')
add_element(xref_link_elemt, 'DB', element_text=pub_db)
add_element(xref_link_elemt, 'ID', element_text=pub_id)
if 'Collaborator(s)' in project_row and project_row.get(
'Collaborator(s)'):
collaborators = project_row.get('Collaborator(s)').strip().split(
',')
collaborators_elemt = add_element(project_elemt, 'COLLABORATORS')
for collaborator in collaborators:
add_element(collaborators_elemt,
'COLLABORATOR',
element_text=collaborator)
sub_project_elemt = add_element(project_elemt, 'SUBMISSION_PROJECT')
add_element(sub_project_elemt, 'SEQUENCING_PROJECT')
if 'Tax ID' in project_row:
org_elemt = add_element(sub_project_elemt, 'ORGANISM')
add_element(org_elemt,
'TAXON_ID',
element_text=str(project_row.get('Tax ID')).strip())
scientific_name = get_scientific_name_from_ensembl(
str(project_row.get('Tax ID')).strip())
add_element(org_elemt,
'SCIENTIFIC_NAME',
element_text=scientific_name)
add_element(org_elemt,
'STRAIN',
element_text=project_row.get('Strain', ''),
content_required=True)
add_element(org_elemt,
'BREED',
element_text=project_row.get('Breed', ''),
content_required=True)
if project_row.get('Parent Project(s)') or \
project_row.get('Child Project(s)') or \
project_row.get('Peer Project(s)'):
related_prjs_elemt = add_element(project_elemt, 'RELATED_PROJECTS')
if 'Parent Project(s)' in project_row and project_row.get(
'Parent Project(s)'):
parent_prjs = project_row.get('Parent Project(s)').split(',')
for parent_prj in parent_prjs:
related_prj_elemt = add_element(related_prjs_elemt,
'RELATED_PROJECT')
add_element(related_prj_elemt,
'PARENT_PROJECT',
accession=parent_prj)
if 'Child Project(s)' in project_row and project_row.get(
'Child Project(s)'):
children_prjs = project_row.get('Child Project(s)').split(',')
for child_prj in children_prjs:
related_prj_elemt = add_element(related_prjs_elemt,
'RELATED_PROJECT')
add_element(related_prj_elemt,
'CHILD_PROJECT',
accession=child_prj)
if 'Peer Project(s)' in project_row and project_row.get(
'Peer Project(s)'):
peer_prjs = project_row.get('Peer Project(s)').split(',')
for peer_prj in peer_prjs:
related_prj_elemt = add_element(related_prjs_elemt,
'RELATED_PROJECT')
add_element(related_prj_elemt,
'PEER_PROJECT',
accession=peer_prj)
if 'Link(s)' in project_row and project_row.get('Link(s)'):
links_elemt = add_element(project_elemt, 'PROJECT_LINKS')
project_links = project_row.get('Link(s)').split(',')
add_links(links_elemt, project_links, link_type='PROJECT_LINK')
# TODO: Is this still relevant because it is not documented in the metadata template
add_attribute_elements(project_elemt,
project_row,
object_type='PROJECT')
return root
def get_scientific_name_from_taxonomy(taxonomy):
if taxonomy not in cache['scientific_name_from_taxonomy']:
species_scientific_name = get_scientific_name_from_ensembl(taxonomy)
cache['scientific_name_from_taxonomy'][taxonomy] =species_scientific_name
return cache['scientific_name_from_taxonomy'][taxonomy]