def resolve_gtdb_species(self, gid_anis, ncbi_sp, type_gids, cur_genomes):
"""Resolve by identifying genomes with a conflicting GTDB species assignments to different type material."""
ncbi_sp_epithet = specific_epithet(ncbi_sp)
untrustworthy_gids = {}
matched_sp_epithet = 0
for gid in type_gids:
if ncbi_sp_epithet == cur_genomes[gid].gtdb_taxa.specific_epithet:
matched_sp_epithet += 1
else:
# check if genome is classified to a GTDB species cluster supported
# by a type strain genome in which case we should consider this
# genome untrustworthy
gtdb_sp = cur_genomes[gid].gtdb_taxa.species
if gtdb_sp != 's__':
gtdb_sp_rid = cur_genomes.gtdb_sp_rep(gtdb_sp)
if cur_genomes[gtdb_sp_rid].is_effective_type_strain():
# genome has been assigned to another species
# defined by a type strain genome
ani, af = self.fastani.symmetric_ani_cached(
gid, gtdb_sp_rid, cur_genomes[gid].genomic_file,
cur_genomes[gtdb_sp_rid].genomic_file)
untrustworthy_gids[
gid] = f'Conflicting GTDB species assignment of {cur_genomes[gid].gtdb_taxa.species} [ANI={ani:.2f}%; AF={af:.2f}%]'
all_similar = self.check_strain_ani(gid_anis, untrustworthy_gids)
# conflict is resolved if remaining genomes pass ANI similarity test,
if all_similar and len(
untrustworthy_gids) > 0 and matched_sp_epithet > 0:
return True, untrustworthy_gids
return False, {}
def add_suffixed_species(self, species):
"""Account for species names when generating further placeholder names.
This is required as species can be transferred between genera and will
retain their existing suffix. As such, we must track that this genera
now has a species name with a given suffix.
(e.g., Lactobacillus_G kunkeei_A is transferred to Apilactobacillus
as A. kunkeei_A, so the next suffixed A. kunkeei representative
must have a 'B' suffix.)
"""
canonical_species = canonical_taxon(species)
if canonical_species not in self.taxon_suffix:
self.taxon_suffix[canonical_species] = 'A'
else:
specific = specific_epithet(species)
suffix = taxon_suffix(specific)
if canonical_species in self.taxon_suffix:
if self.is_higher_suffix(suffix, self.taxon_suffix[canonical_species]):
self.taxon_suffix[canonical_species] = suffix
else:
# add new canonical taxon to suffix map
self.taxon_suffix[canonical_species] = suffix
def identify_consensus_synonyms(self, ncbi_misclassified_gids):
"""Identify synonyms arising from all genomes with an NCBI species classification
which lack a type strain genome being contained in a GTDB cluster defined by
a type strain genome."""
# get genomes with NCBI species assignment
ncbi_species_gids = defaultdict(list)
ncbi_all_sp_gids = set()
for gid in self.cur_genomes:
if gid in ncbi_misclassified_gids:
continue
ncbi_species = self.cur_genomes[gid].ncbi_taxa.species
ncbi_specific = specific_epithet(ncbi_species)
if ncbi_species != 's__' and ncbi_specific not in self.forbidden_specific_names:
ncbi_species_gids[ncbi_species].append(gid)
ncbi_all_sp_gids.add(gid)
# identify consensus synonyms
consensus_synonyms = defaultdict(list)
for ncbi_species, ncbi_sp_gids in ncbi_species_gids.items():
if ncbi_species in self.ncbi_sp_type_strain_genomes:
# NCBI species define by type strain genome so should
# be either the representative of a GTDB species cluster
# or a type strain synonym
continue
# determine if all genomes in NCBI species are contained
# in a single GTDB species represented by a type strain genome
for gtdb_rid, cids in self.cur_clusters.items():
assert gtdb_rid in cids
if not self.cur_genomes[gtdb_rid].is_effective_type_strain():
continue
ncbi_cur_sp_in_cluster = cids.intersection(ncbi_sp_gids)
ncbi_cur_sp_in_cluster_perc = len(
ncbi_cur_sp_in_cluster) * 100.0 / len(ncbi_sp_gids)
if ncbi_cur_sp_in_cluster_perc == 100:
# using the best quality genome in NCBI species to establish
# synonym statistics such as ANI and AF
q = {
gid: self.cur_genomes[gid].score_type_strain()
for gid in ncbi_sp_gids
}
q_sorted = sorted(q.items(),
key=lambda kv: (kv[1], kv[0]),
reverse=True)
consensus_synonyms[gtdb_rid].append(q_sorted[0][0])
break
self.logger.info(
' - identified {:,} GTDB representatives resulting in {:,} majority vote synonyms.'
.format(len(consensus_synonyms),
sum([len(gids) for gids in consensus_synonyms.values()])))
return consensus_synonyms
def validate_type_strain_clustering(self, mc_species):
"""Validate that all type strain genomes for an NCBI species occur in a single GTDB cluster."""
self.logger.info(
'Verifying that all type strain genomes for a NCBI species occur in a single GTDB cluster.'
)
rid_map = {}
for rid, gids in self.cur_clusters.items():
rid_map[rid] = rid
for gid in gids:
rid_map[gid] = rid
for ncbi_sp, type_gids in self.ncbi_sp_type_strain_genomes.items():
gtdb_rids = set([rid_map[gid] for gid in type_gids])
gtdb_rids = set()
for gid in type_gids:
rid = rid_map[gid]
ncbi_specific = specific_epithet(
self.cur_genomes[rid].ncbi_taxa.species)
if (rid in mc_species and
specific_epithet(mc_species[rid]) != ncbi_specific):
# skip this genome as it has been manually changed, likely
# to resolve this NCBI species having multiple type strain genomes
continue
gtdb_rids.add(rid)
if len(gtdb_rids) > 1:
self.logger.error(
'Type strain genomes from NCBI species {} were assigned to {:,} GTDB species clusters: {}.'
.format(ncbi_sp, len(gtdb_rids),
[(rid, self.cur_genomes[rid].gtdb_taxa.species)
for rid in gtdb_rids]))
sys.exit(-1)
def ncbi_sp_gtdb_cluster_table(self, final_taxonomy):
"""Create table indicating GTDB species clusters for each NCBI species."""
# get map between genomes and representatives
gid_to_rid = {}
for rid, cids in self.cur_clusters.items():
for cid in cids:
gid_to_rid[cid] = rid
# get genomes with NCBI species assignment
ncbi_species_gids = defaultdict(list)
for rid, cids in self.cur_clusters.items():
if rid not in final_taxonomy:
continue # other domain
for cid in cids:
ncbi_species = self.cur_genomes[cid].ncbi_taxa.species
ncbi_specific = specific_epithet(ncbi_species)
if ncbi_species != 's__' and ncbi_specific not in self.forbidden_specific_names:
ncbi_species_gids[ncbi_species].append(cid)
# write out table
fout = open(
os.path.join(self.output_dir, 'ncbi_sp_to_gtdb_cluster_map.tsv'),
'w')
fout.write(
'NCBI species\tNo. genomes\tNo. GTDB clusters\tHighest prevalence (%)\tGTDB species clusters\n'
)
for ncbi_sp, gids in ncbi_species_gids.items():
gtdb_rep_list = [gid_to_rid[gid] for gid in gids]
gtdb_rep_count = Counter(gtdb_rep_list)
gtdb_rep_str = []
for idx, (rid, count) in enumerate(gtdb_rep_count.most_common()):
gtdb_rep_str.append('{} ({}): {}'.format(
final_taxonomy[rid][Taxonomy.SPECIES_INDEX], rid, count))
if idx == 0:
highest_prevalence = count * 100.0 / len(gids)
fout.write('{}\t{}\t{}\t{:.2f}\t{}\n'.format(
ncbi_sp, len(gids), len(set(gtdb_rep_list)),
highest_prevalence, '; '.join(gtdb_rep_str)))
fout.close()
def run(self, cur_gtdb_metadata_file, cur_genomic_path_file,
qc_passed_file, ncbi_genbank_assembly_file, ltp_taxonomy_file,
gtdb_type_strains_ledger, untrustworthy_type_ledger,
ncbi_env_bioproject_ledger):
"""Resolve cases where a species has multiple genomes assembled from the type strain."""
# get species in LTP reference database
self.logger.info(
'Determining species defined in LTP reference database.')
ltp_defined_species = self.ltp_defined_species(ltp_taxonomy_file)
self.logger.info(
f' - identified {len(ltp_defined_species):,} species.')
# create current GTDB genome sets
self.logger.info('Creating current GTDB genome set.')
cur_genomes = Genomes()
cur_genomes.load_from_metadata_file(
cur_gtdb_metadata_file,
gtdb_type_strains_ledger=gtdb_type_strains_ledger,
create_sp_clusters=False,
qc_passed_file=qc_passed_file,
ncbi_genbank_assembly_file=ncbi_genbank_assembly_file,
untrustworthy_type_ledger=untrustworthy_type_ledger,
ncbi_env_bioproject_ledger=ncbi_env_bioproject_ledger)
cur_genomes.load_genomic_file_paths(cur_genomic_path_file)
# parsing genomes manually established to be untrustworthy as type
self.logger.info(
'Determining genomes manually annotated as untrustworthy as type.')
manual_untrustworthy_types = self.parse_untrustworthy_type_ledger(
untrustworthy_type_ledger)
self.logger.info(
f' - identified {len(manual_untrustworthy_types):,} genomes manually annotated as untrustworthy as type.'
)
# Identify NCBI species with multiple genomes assembled from type strain of species. This
# is done using a series of heuristics that aim to ensure that the selected type strain
# genome is reliable. More formal evaluation and a manuscript descirbing this selection
# process is ultimately required. Ideally, the community will eventually adopt a
# database that indicates a single `type genome assembly` for each species instead
# of just indicating a type strain from which many (sometimes dissimilar) assemblies exist.
self.logger.info(
'Determining number of type strain genomes in each NCBI species.')
multi_type_strains_sp = self.sp_with_mult_type_strains(cur_genomes)
self.logger.info(
f' - identified {len(multi_type_strains_sp):,} NCBI species with multiple assemblies indicated as being type strain genomes.'
)
# resolve species with multiple type strain genomes
fout = open(
os.path.join(self.output_dir, 'multi_type_strain_species.tsv'),
'w')
fout.write(
'NCBI species\tNo. type strain genomes\t>=99% ANI\tMean ANI\tStd ANI\tMean AF\tStd AF\tResolution\tGenome IDs\n'
)
fout_genomes = open(
os.path.join(self.output_dir, 'type_strain_genomes.tsv'), 'w')
fout_genomes.write(
'Genome ID\tUntrustworthy\tNCBI species\tGTDB genus\tGTDB species\tLTP species\tConflict with prior GTDB assignment'
)
fout_genomes.write(
'\tMean ANI\tStd ANI\tMean AF\tStd AF\tExclude from RefSeq\tNCBI taxonomy\tGTDB taxonomy\tReason for GTDB untrustworthy as type\n'
)
fout_unresolved = open(
os.path.join(self.output_dir,
'unresolved_type_strain_genomes.tsv'), 'w')
fout_unresolved.write(
'Genome ID\tNCBI species\tGTDB genus\tGTDB species\tLTP species')
fout_unresolved.write(
'\tMean ANI\tStd ANI\tMean AF\tStd AF\tExclude from RefSeq\tNCBI taxonomy\tGTDB taxonomy\n'
)
fout_high_divergence = open(
os.path.join(self.output_dir,
'highly_divergent_type_strain_genomes.tsv'), 'w')
fout_high_divergence.write(
'Genome ID\tNCBI species\tGTDB genus\tGTDB species\tLTP species\tMean ANI\tStd ANI\tMean AF\tStd AF\tExclude from RefSeq\tNCBI taxonomy\tGTDB taxonomy\n'
)
fout_untrustworthy = open(
os.path.join(self.output_dir, 'untrustworthy_type_material.tsv'),
'w')
fout_untrustworthy.write(
'Genome ID\tNCBI species\tGTDB species\tLTP species\tReason for declaring untrustworthy\n'
)
for gid in manual_untrustworthy_types:
ncbi_sp, reason = manual_untrustworthy_types[gid]
fout_untrustworthy.write('{}\t{}\t{}\t{}\t{}\t{}\n'.format(
gid, ncbi_sp, cur_genomes[gid].gtdb_taxa.species,
'<not tested>', 'n/a', 'Manual curation: ' + reason))
processed = 0
num_divergent = 0
unresolved_sp_count = 0
ncbi_ltp_resolved = 0
intra_ani_resolved = 0
ncbi_type_resolved = 0
ncbi_rep_resolved = 0
gtdb_family_resolved = 0
gtdb_genus_resolved = 0
gtdb_sp_resolved = 0
ltp_resolved = 0
# *** Perhaps should be an external flag, but used right now to speed up debugging
use_pickled_results = False
if use_pickled_results:
self.logger.warning(
'Using previously calculated ANI results in: {}'.format(
self.ani_pickle_dir))
prev_gtdb_sp_conflicts = 0
self.logger.info(
'Resolving species with multiple type strain genomes:')
for ncbi_sp, type_gids in sorted(multi_type_strains_sp.items(),
key=lambda kv: len(kv[1])):
assert len(type_gids) > 1
status_str = '-> Processing {} with {:,} type strain genomes [{:,} of {:,} ({:.2f}%)].'.format(
ncbi_sp, len(type_gids), processed + 1,
len(multi_type_strains_sp), (processed + 1) * 100.0 /
len(multi_type_strains_sp)).ljust(128)
sys.stdout.write('{}\r'.format(status_str))
sys.stdout.flush()
processed += 1
# calculate ANI between type strain genomes
all_similar, anis, afs, gid_anis, gid_afs = self.calculate_type_strain_ani(
ncbi_sp, type_gids, cur_genomes, use_pickled_results)
# read LTP metadata for genomes
ltp_metadata = self.parse_ltp_metadata(type_gids, cur_genomes)
untrustworthy_gids = {}
gtdb_resolved_sp_conflict = False
unresolved_species = False
note = 'All type strain genomes have ANI >99% and AF >65%.'
if not all_similar:
note = ''
# need to establish which genomes are untrustworthy as type
num_divergent += 1
unresolved_species = True
# write out highly divergent cases for manual inspection;
# these should be compared to the automated selection
if np_mean(anis) < 95:
for gid in type_gids:
ltp_species = self.ltp_species(gid, ltp_metadata)
fout_high_divergence.write(
'{}\t{}\t{}\t{}\t{}\t{:.2f}\t{:.3f}\t{:.3f}\t{:.4f}\t{}\t{}\t{}\n'
.format(gid, ncbi_sp,
cur_genomes[gid].gtdb_taxa.genus,
cur_genomes[gid].gtdb_taxa.species,
' / '.join(ltp_species),
np_mean(list(gid_anis[gid].values())),
np_std(list(gid_anis[gid].values())),
np_mean(list(gid_afs[gid].values())),
np_std(list(gid_afs[gid].values())),
cur_genomes[gid].excluded_from_refseq_note,
cur_genomes[gid].ncbi_taxa,
cur_genomes[gid].gtdb_taxa))
# filter genomes marked as `untrustworthy as type` at NCBI and where the LTP
# assignment also suggest the asserted type material is incorrect
resolved, untrustworthy_gids = self.resolve_validated_untrustworthy_ncbi_genomes(
gid_anis, ncbi_sp, type_gids, ltp_metadata,
ltp_defined_species, cur_genomes)
if resolved:
note = "Species resolved by removing genomes considered `untrustworthy as type` and with a LTP BLAST hit confirming the assembly is likely untrustworthy"
ncbi_ltp_resolved += 1
# try to resolve by LTP 16S BLAST results
if not resolved:
resolved, untrustworthy_gids = self.resolve_ltp_conflict(
gid_anis, ncbi_sp, type_gids, ltp_metadata, 0)
if resolved:
note = 'Species resolved by identifying conflicting or lack of LTP BLAST results'
ltp_resolved += 1
# try to resolve species using intra-specific ANI test
if not resolved:
resolved, untrustworthy_gids = self.resolve_by_intra_specific_ani(
gid_anis)
if resolved:
note = 'Species resolved by intra-specific ANI test'
intra_ani_resolved += 1
# try to resolve by GTDB family assignment
if not resolved:
resolved, untrustworthy_gids = self.resolve_gtdb_family(
gid_anis, ncbi_sp, type_gids, cur_genomes)
if resolved:
note = 'Species resolved by consulting GTDB family classifications'
gtdb_family_resolved += 1
# try to resolve by GTDB genus assignment
if not resolved:
resolved, untrustworthy_gids = self.resolve_gtdb_genus(
gid_anis, ncbi_sp, type_gids, cur_genomes)
if resolved:
note = 'Species resolved by consulting GTDB genus classifications'
gtdb_genus_resolved += 1
# try to resolve by GTDB species assignment
if not resolved:
resolved, untrustworthy_gids = self.resolve_gtdb_species(
gid_anis, ncbi_sp, type_gids, cur_genomes)
if resolved:
note = 'Species resolved by consulting GTDB species classifications'
gtdb_sp_resolved += 1
# try to resolve by considering genomes annotated as type material at NCBI,
# which includes considering if genomes are marked as untrustworthy as type
if not resolved:
resolved, untrustworthy_gids = self.resolve_by_ncbi_types(
gid_anis, type_gids, cur_genomes)
if resolved:
note = 'Species resolved by consulting NCBI assembled from type metadata'
ncbi_type_resolved += 1
# try to resovle by considering genomes annotated as representative genomes at NCBI
if not resolved:
resolved, untrustworthy_gids = self.resolve_by_ncbi_reps(
gid_anis, type_gids, cur_genomes)
if resolved:
note = 'Species resolved by considering NCBI representative genomes'
ncbi_rep_resolved += 1
if resolved:
unresolved_species = False
# check if type strain genomes marked as trusted or untrusted conflict
# with current GTDB species assignment
untrustworthy_gtdb_sp_match = False
trusted_gtdb_sp_match = False
for gid in type_gids:
gtdb_canonical_epithet = canonical_taxon(
specific_epithet(
cur_genomes[gid].gtdb_taxa.species))
if gtdb_canonical_epithet == specific_epithet(ncbi_sp):
if gid in untrustworthy_gids:
untrustworthy_gtdb_sp_match = True
else:
trusted_gtdb_sp_match = True
if untrustworthy_gtdb_sp_match and not trusted_gtdb_sp_match:
prev_gtdb_sp_conflicts += 1
gtdb_resolved_sp_conflict = True
else:
note = 'Species is unresolved; manual curation is required!'
unresolved_sp_count += 1
if unresolved_species:
for gid in type_gids:
ltp_species = self.ltp_species(gid, ltp_metadata)
fout_unresolved.write(
'{}\t{}\t{}\t{}\t{}\t{:.2f}\t{:.3f}\t{:.3f}\t{:.4f}\t{}\t{}\t{}\n'
.format(gid, ncbi_sp,
cur_genomes[gid].gtdb_taxa.genus,
cur_genomes[gid].gtdb_taxa.species,
' / '.join(ltp_species),
np_mean(list(gid_anis[gid].values())),
np_std(list(gid_anis[gid].values())),
np_mean(list(gid_afs[gid].values())),
np_std(list(gid_afs[gid].values())),
cur_genomes[gid].excluded_from_refseq_note,
cur_genomes[gid].ncbi_taxa,
cur_genomes[gid].gtdb_taxa))
# remove genomes marked as untrustworthy as type at NCBI if one or more potential type strain genomes remaining
ncbi_untrustworthy_gids = set([
gid for gid in type_gids if 'untrustworthy as type' in
cur_genomes[gid].excluded_from_refseq_note
])
if len(type_gids - set(untrustworthy_gids) -
ncbi_untrustworthy_gids) >= 1:
for gid in ncbi_untrustworthy_gids:
untrustworthy_gids[
gid] = "Genome annotated as `untrustworthy as type` at NCBI and there are other potential type strain genomes available"
# report cases where genomes marked as untrustworthy as type at NCBI are being retained as potential type strain genomes
num_ncbi_untrustworthy = len(ncbi_untrustworthy_gids)
for gid in type_gids:
if (gid not in untrustworthy_gids and 'untrustworthy as type'
in cur_genomes[gid].excluded_from_refseq_note):
self.logger.warning(
"Retaining genome {} from {} despite being marked as `untrustworthy as type` at NCBI [{:,} of {:,} considered untrustworthy]."
.format(gid, ncbi_sp, num_ncbi_untrustworthy,
len(type_gids)))
# write out genomes identified as being untrustworthy
for gid, reason in untrustworthy_gids.items():
ltp_species = self.ltp_species(gid, ltp_metadata)
if 'untrustworthy as type' in cur_genomes[
gid].excluded_from_refseq_note:
reason += "; considered `untrustworthy as type` at NCBI"
fout_untrustworthy.write('{}\t{}\t{}\t{}\t{}\n'.format(
gid, ncbi_sp, cur_genomes[gid].gtdb_taxa.species,
' / '.join(ltp_species), reason))
# Sanity check that if the untrustworthy genome has an LTP to only the
# expected species, that all other genomes also have a hit to the
# expected species (or potentially no hit). Otherwise, more consideration
# should be given to the genome with the conflicting LTP hit.
if len(ltp_species) == 1 and ncbi_sp in ltp_species:
other_sp = set()
for test_gid in type_gids:
ltp_species = self.ltp_species(test_gid, ltp_metadata)
if ltp_species and ncbi_sp not in ltp_species:
other_sp.update(ltp_species)
if other_sp:
self.logger.warning(
f'Genome {gid} marked as untrustworthy, but this conflicts with high confidence LTP 16S rRNA assignment.'
)
# write out information about all type genomes
for gid in type_gids:
ltp_species = self.ltp_species(gid, ltp_metadata)
fout_genomes.write(
'{}\t{}\t{}\t{}\t{}\t{}\t{}\t{:.2f}\t{:.3f}\t{:.3f}\t{:.4f}\t{}\t{}\t{}\t{}\n'
.format(gid, gid in untrustworthy_gids, ncbi_sp,
cur_genomes[gid].gtdb_taxa.genus,
cur_genomes[gid].gtdb_taxa.species,
' / '.join(ltp_species), gtdb_resolved_sp_conflict,
np_mean(list(gid_anis[gid].values())),
np_std(list(gid_anis[gid].values())),
np_mean(list(gid_afs[gid].values())),
np_std(list(gid_afs[gid].values())),
cur_genomes[gid].excluded_from_refseq_note,
cur_genomes[gid].ncbi_taxa,
cur_genomes[gid].gtdb_taxa,
untrustworthy_gids.get(gid, '')))
fout.write(
'{}\t{}\t{}\t{:.2f}\t{:.3f}\t{:.3f}\t{:.4f}\t{}\t{}\n'.format(
ncbi_sp, len(type_gids), all_similar, np_mean(anis),
np_std(anis), np_mean(afs), np_std(afs), note,
', '.join(type_gids)))
sys.stdout.write('\n')
fout.close()
fout_unresolved.close()
fout_high_divergence.close()
fout_genomes.close()
fout_untrustworthy.close()
self.logger.info(
f'Identified {num_divergent:,} species with 1 or more divergent type strain genomes.'
)
self.logger.info(
f' - resolved {ncbi_ltp_resolved:,} species by removing NCBI `untrustworthy as type` genomes with a conflicting LTP 16S rRNA classifications.'
)
self.logger.info(
f' - resolved {ltp_resolved:,} species by considering conflicting LTP 16S rRNA classifications.'
)
self.logger.info(
f' - resolved {intra_ani_resolved:,} species by considering intra-specific ANI values.'
)
self.logger.info(
f' - resolved {gtdb_family_resolved:,} species by considering conflicting GTDB family classifications.'
)
self.logger.info(
f' - resolved {gtdb_genus_resolved:,} species by considering conflicting GTDB genus classifications.'
)
self.logger.info(
f' - resolved {gtdb_sp_resolved:,} species by considering conflicting GTDB species classifications.'
)
self.logger.info(
f' - resolved {ncbi_type_resolved:,} species by considering type material designations at NCBI.'
)
self.logger.info(
f' - resolved {ncbi_rep_resolved:,} species by considering RefSeq reference and representative designations at NCBI.'
)
if unresolved_sp_count > 0:
self.logger.warning(
f'There are {unresolved_sp_count:,} unresolved species with multiple type strain genomes.'
)
self.logger.warning(
'These should be handled before proceeding with the next step of GTDB species updating.'
)
self.logger.warning(
"This can be done by manual curation and adding genomes to 'untrustworthy_type_ledger'."
)
self.logger.info(
f'Identified {prev_gtdb_sp_conflicts:,} cases where resolved type strain conflicts with prior GTDB assignment.'
)
def infer_epithet_map(self, gids_of_interest, mc_species, cur_genomes,
cur_clusters):
"""Infer mapping of NCBI epithet to GTDB epithet which may be different due to gender of genus."""
# **************************************
# This should be updated so it only includes valid transfers, and not
# results due to misclassifications at NCBI. For example, right now this
# code reports Enterobacter cancerogenus being transferred to Pantoea, but
# really this is just a misclassified NCBI genome.
# get species in GTDB genus
generic_rids = defaultdict(list)
for rid in cur_clusters:
if rid not in gids_of_interest:
continue
gtdb_generic = cur_genomes[rid].gtdb_taxa.genus.replace('g__', '')
if rid in mc_species:
gtdb_generic = generic_name(mc_species[rid])
generic_rids[gtdb_generic].append(rid)
# establish epithets that are nearly identical
# except for small change to suffix which is
# assumed to be due to a gender change
for gtdb_generic, rids in generic_rids.items():
ncbi_sp_epithet_list = defaultdict(list)
for rid in rids:
ncbi_species = cur_genomes[rid].ncbi_taxa.species
if ncbi_species == 's__':
continue
ncbi_generic = generic_name(ncbi_species)
ncbi_specific = specific_epithet(ncbi_species)
if rid in mc_species:
gtdb_species = mc_species[rid]
else:
gtdb_species = cur_genomes[rid].gtdb_taxa.species
gtdb_specific = canonical_taxon(specific_epithet(gtdb_species))
self.gtdb_ncbi_generic_map[gtdb_generic][gtdb_specific].append(
ncbi_generic)
if test_same_epithet(ncbi_specific, gtdb_specific):
ncbi_sp_epithet_list[ncbi_specific].append(gtdb_specific)
for ncbi_specific, gtdb_specific_list in ncbi_sp_epithet_list.items(
):
gtdb_specific_counter = Counter(gtdb_specific_list)
top_gtdb_specific, count = gtdb_specific_counter.most_common(
1)[0]
map_perc = count * 100.0 / len(gtdb_specific_list)
if map_perc >= 50:
self.sp_epithet_map[gtdb_generic][
ncbi_specific] = top_gtdb_specific
if map_perc != 100:
self.logger.warning(
'Imperfect suffix mapping between from {} {} to {} at {:.1f}%.'
.format(gtdb_generic, top_gtdb_specific, ncbi_specific,
count * 100.0 / len(gtdb_specific_list)))
def classify_ncbi_species_consensus(self, ncbi_synonyms, misclassified_gids):
"""Classify each NCBI species as either unambiguously, ambiguously, or synonym."""
# get genomes with NCBI species assignment
ncbi_all_species = set()
ncbi_species_gids = defaultdict(list)
ncbi_all_species_gids = set()
for gid in self.cur_genomes:
ncbi_species = self.cur_genomes[gid].ncbi_taxa.species
ncbi_specific = specific_epithet(ncbi_species)
if ncbi_species != 's__' and ncbi_specific not in self.forbidden_specific_names:
ncbi_all_species.add(ncbi_species)
if gid not in misclassified_gids:
ncbi_species_gids[ncbi_species].append(gid)
ncbi_all_species_gids.add(gid)
# establish if NCBI species can be unambiguously assigned to a GTDB species cluster
fout = open(os.path.join(self.output_dir, 'ncbi_sp_classification.tsv'), 'w')
fout.write('NCBI species\tClassification\tAssignment type\tGTDB representative\t% NCBI species in cluster\t% cluster with NCBI species\tNCBI classifications in cluster\tNote\n')
unambiguous_ncbi_sp = {}
ambiguous_ncbi_sp = set()
for ncbi_species, ncbi_sp_gids in ncbi_species_gids.items():
if ncbi_species in ncbi_synonyms:
continue
# get type strains for NCBI species that are GTDB species representatives
type_strain_rids = [gid for gid in ncbi_sp_gids
if (self.cur_genomes[gid].is_effective_type_strain() and gid in self.cur_clusters)]
if len(type_strain_rids) == 0:
highest_sp_in_cluster_perc = 0
highest_cluster_perc = 0
highest_gtdb_rid = None
for gtdb_rid, cids in self.cur_clusters.items():
assert gtdb_rid in cids
ncbi_cur_sp_in_cluster = cids.intersection(ncbi_sp_gids)
ncbi_any_sp_in_cluster = cids.intersection(ncbi_all_species_gids)
if len(ncbi_any_sp_in_cluster) > 0:
ncbi_cur_sp_in_cluster_perc = len(ncbi_cur_sp_in_cluster)*100.0/len(ncbi_sp_gids)
if ncbi_cur_sp_in_cluster_perc > highest_sp_in_cluster_perc:
highest_sp_in_cluster_perc = ncbi_cur_sp_in_cluster_perc
highest_cluster_perc = len(ncbi_cur_sp_in_cluster)*100.0/len(ncbi_any_sp_in_cluster)
highest_gtdb_rid = gtdb_rid
if highest_sp_in_cluster_perc == 100:
break
if highest_sp_in_cluster_perc == 100 and highest_cluster_perc == 100:
unambiguous_ncbi_sp[ncbi_species] = (highest_gtdb_rid, 'UNANIMOUS_CONSENSUS')
else:
ambiguous_ncbi_sp.add(ncbi_species)
ncbi_sp_str = []
for sp, count in Counter([self.cur_genomes[gid].ncbi_taxa.species for gid in self.cur_clusters[highest_gtdb_rid]]).most_common():
ncbi_sp_str.append('{} ({:,})'.format(sp, count))
ncbi_sp_str = '; '.join(ncbi_sp_str)
fout.write('{}\t{}\t{}\t{}\t{:.2f}\t{:.2f}\t{}\t{}\n'.format(
ncbi_species,
'UNAMBIGUOUS' if ncbi_species in unambiguous_ncbi_sp else 'AMBIGUOUS',
'UNANIMOUS_CONSENSUS' if ncbi_species in unambiguous_ncbi_sp else 'AMBIGUOUS',
highest_gtdb_rid,
highest_sp_in_cluster_perc,
highest_cluster_perc,
ncbi_sp_str,
''))
elif len(type_strain_rids) == 1:
gtdb_rid = type_strain_rids[0]
unambiguous_ncbi_sp[ncbi_species] = (gtdb_rid, 'TYPE_STRAIN_GENOME')
cids = self.cur_clusters[gtdb_rid]
ncbi_cur_sp_in_cluster = cids.intersection(ncbi_sp_gids)
ncbi_any_sp_in_cluster = cids.intersection(ncbi_all_species_gids)
cur_sp_in_cluster_perc = len(ncbi_cur_sp_in_cluster)*100.0/len(ncbi_sp_gids)
cluster_perc = len(ncbi_cur_sp_in_cluster)*100.0/len(ncbi_any_sp_in_cluster)
ncbi_sp_str = []
for sp, count in Counter([self.cur_genomes[gid].ncbi_taxa.species for gid in self.cur_clusters[gtdb_rid]]).most_common():
ncbi_sp_str.append('{} ({:,})'.format(sp, count))
ncbi_sp_str = '; '.join(ncbi_sp_str)
fout.write('{}\t{}\t{}\t{}\t{:.2f}\t{:.2f}\t{}\t{}\n'.format(
ncbi_species,
'UNAMBIGUOUS',
'TYPE_STRAIN_GENOME',
gtdb_rid,
cur_sp_in_cluster_perc,
cluster_perc,
ncbi_sp_str,
''))
else:
self.logger.error('Multiple GTDB species clusters represented by type strain genomes of {}: {}'.format(
ncbi_species, type_strain_rids))
sys.exit(-1)
# get NCBI synonyms
for ncbi_species, ncbi_sp_gids in ncbi_species_gids.items():
if ncbi_species not in ncbi_synonyms:
continue
fout.write('{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\n'.format(
ncbi_species,
'SYNONYM',
'SYNONYM',
'n/a',
'n/a',
'n/a',
'n/a',
'Synonym of {} under GTDB'.format(ncbi_synonyms[ncbi_species])))
fout.close()
# sanity check results
assert set(unambiguous_ncbi_sp).intersection(ncbi_synonyms) == set()
assert set(ambiguous_ncbi_sp).intersection(ncbi_synonyms) == set()
assert set(unambiguous_ncbi_sp).intersection(ambiguous_ncbi_sp) == set()
assert set(ncbi_all_species) - set(unambiguous_ncbi_sp) - set(ambiguous_ncbi_sp) - set(ncbi_synonyms) == set()
gtdb_rep_ncbi_sp_assignments = {}
for ncbi_species, (gtdb_rid, classification) in unambiguous_ncbi_sp.items():
if gtdb_rid in gtdb_rep_ncbi_sp_assignments:
self.logger.error('GTDB representative {} assigned to multiple unambiguous NCBI species: {} {}'.format(
gtdb_rid,
gtdb_rep_ncbi_sp_assignments[gtdb_rid],
ncbi_species))
sys.exit(-1)
gtdb_rep_ncbi_sp_assignments[gtdb_rid] = ncbi_species
return ncbi_all_species, unambiguous_ncbi_sp, ambiguous_ncbi_sp
def run(self, manual_taxonomy, cur_gtdb_metadata_file, uba_genome_paths,
qc_passed_file, ncbi_genbank_assembly_file,
untrustworthy_type_file, synonym_file, gtdb_type_strains_ledger,
sp_priority_ledger, genus_priority_ledger, dsmz_bacnames_file):
"""Finalize species names based on results of manual curation."""
# initialize species priority manager
sp_priority_mngr = SpeciesPriorityManager(sp_priority_ledger,
genus_priority_ledger,
dsmz_bacnames_file)
# identify species and genus names updated during manual curation
self.logger.info('Parsing manually curated taxonomy.')
mc_taxonomy = Taxonomy().read(manual_taxonomy, use_canonical_gid=True)
self.logger.info(' - read taxonomy for {:,} genomes.'.format(
len(mc_taxonomy)))
# create current GTDB genome sets
self.logger.info('Creating current GTDB genome set.')
cur_genomes = Genomes()
cur_genomes.load_from_metadata_file(
cur_gtdb_metadata_file,
gtdb_type_strains_ledger=gtdb_type_strains_ledger,
create_sp_clusters=False,
uba_genome_file=uba_genome_paths,
qc_passed_file=qc_passed_file,
ncbi_genbank_assembly_file=ncbi_genbank_assembly_file,
untrustworthy_type_ledger=untrustworthy_type_file)
self.logger.info(
f' ... current genome set contains {len(cur_genomes):,} genomes.')
# get all GTDB species represented by a type strain:
gtdb_type_species = set()
for rid in mc_taxonomy:
if cur_genomes[rid].is_effective_type_strain():
gtdb_type_species.add(mc_taxonomy[rid][Taxonomy.SPECIES_INDEX])
# establish appropriate species names for GTDB clusters with new representatives
self.logger.info(
'Identifying type strain genomes with incongruent GTDB species assignments.'
)
fout = open(
os.path.join(self.output_dir, 'type_strains_incongruencies.tsv'),
'w')
fout.write(
'Genome ID\tGTDB species\tNCBI species\tGTDB type strain\tNCBI type strain\tNCBI RefSeq note\n'
)
num_incongruent = 0
for rid, taxa in mc_taxonomy.items():
if cur_genomes[rid].is_effective_type_strain():
gtdb_sp = taxa[Taxonomy.SPECIES_INDEX]
gtdb_generic = generic_name(gtdb_sp)
ncbi_sp = cur_genomes[rid].ncbi_taxa.species
ncbi_generic = generic_name(ncbi_sp)
if ncbi_sp == 's__':
# NCBI taxonomy is sometimes behind the genome annotation pages,
# and do not have a species assignment even for type strain genome
continue
# check if genome is a valid genus transfer into a genus
# that already contains a species with the specific
# name which results in a polyphyletic suffix being required
# e.g. G002240355 is Prauserella marina at NCBI and is
# transferred into Saccharomonospora under the GTDB. However,
# Saccharomonospora marina already exists so this genome
# needs to be S. marina_A.
if (is_placeholder_taxon(gtdb_sp)
and gtdb_generic != ncbi_generic
and canonical_species(gtdb_sp) in gtdb_type_species):
continue
if not test_same_epithet(specific_epithet(gtdb_sp),
specific_epithet(ncbi_sp)):
num_incongruent += 1
fout.write('{}\t{}\t{}\t{}\t{}\t{}\n'.format(
rid, gtdb_sp, ncbi_sp,
cur_genomes[rid].is_gtdb_type_strain(),
cur_genomes[rid].is_ncbi_type_strain(),
cur_genomes[rid].excluded_from_refseq_note))
self.logger.info(
' - identified {:,} genomes with incongruent species assignments.'.
format(num_incongruent))
fout.close()
def identify_misclassified_genomes_ani(self, cur_genomes, cur_clusters):
"""Identify genomes with erroneous NCBI species assignments, based on ANI to type strain genomes."""
forbidden_names = set(['cyanobacterium'])
# get mapping from genomes to their representatives
gid_to_rid = {}
for rid, cids in cur_clusters.items():
for cid in cids:
gid_to_rid[cid] = rid
# get genomes with NCBI species assignment
ncbi_sp_gids = defaultdict(list)
for gid in cur_genomes:
ncbi_species = cur_genomes[gid].ncbi_taxa.species
ncbi_specific = specific_epithet(ncbi_species)
if ncbi_species != 's__' and ncbi_specific not in forbidden_names:
ncbi_sp_gids[ncbi_species].append(gid)
# get NCBI species anchored by a type strain genome
ncbi_type_anchored_species = {}
for rid, cids in cur_clusters.items():
if cur_genomes[rid].is_effective_type_strain():
ncbi_type_species = cur_genomes[rid].ncbi_taxa.species
if ncbi_type_species != 's__':
ncbi_type_anchored_species[ncbi_type_species] = rid
self.logger.info(
' - identified {:,} NCBI species anchored by a type strain genome.'
.format(len(ncbi_type_anchored_species)))
# identify genomes with erroneous NCBI species assignments
fout = open(
os.path.join(
self.output_dir, 'ncbi_misclassified_sp.ani_{}.tsv'.format(
self.ani_ncbi_erroneous)), 'w')
fout.write(
'Genome ID\tNCBI species\tGenome cluster\tType species cluster\tANI to type strain\tAF to type strain\n'
)
misclassified_gids = set()
for idx, (ncbi_species,
species_gids) in enumerate(ncbi_sp_gids.items()):
if ncbi_species not in ncbi_type_anchored_species:
continue
type_rid = ncbi_type_anchored_species[ncbi_species]
gids_to_check = []
for gid in species_gids:
cur_rid = gid_to_rid[gid]
if type_rid != cur_rid:
# need to check genome as it has the same NCBI species name
# as a type strain genome, but resides in a different GTDB
# species cluster
gids_to_check.append(gid)
if len(gids_to_check) > 0:
gid_pairs = []
for gid in gids_to_check:
gid_pairs.append((type_rid, gid))
gid_pairs.append((gid, type_rid))
statusStr = '-> Establishing erroneous assignments for {} [ANI pairs: {:,}; {:,} of {:,} species].'.format(
ncbi_species, len(gid_pairs), idx + 1,
len(ncbi_sp_gids)).ljust(96)
sys.stdout.write('{}\r'.format(statusStr))
sys.stdout.flush()
ani_af = self.fastani.pairs(gid_pairs,
cur_genomes.genomic_files,
report_progress=False,
check_cache=True)
for gid in gids_to_check:
ani, af = symmetric_ani(ani_af, type_rid, gid)
if ani < self.ani_ncbi_erroneous:
misclassified_gids.add(gid)
fout.write('{}\t{}\t{}\t{}\t{:.2f}\t{:.3f}\n'.format(
gid, ncbi_species, gid_to_rid[gid], type_rid, ani,
af))
sys.stdout.write('\n')
fout.close()
misclassified_species = set(
[cur_genomes[gid].ncbi_taxa.species for gid in misclassified_gids])
self.logger.info(
' - identified {:,} genomes from {:,} species as having misclassified NCBI species assignments.'
.format(len(misclassified_gids), len(misclassified_species)))
return misclassified_gids
def identify_misclassified_genomes_cluster(self, cur_genomes,
cur_clusters):
"""Identify genomes with erroneous NCBI species assignments, based on GTDB clustering of type strain genomes."""
forbidden_names = set(['cyanobacterium'])
# get mapping from genomes to their representatives
gid_to_rid = {}
for rid, cids in cur_clusters.items():
for cid in cids:
gid_to_rid[cid] = rid
# get genomes with NCBI species assignment
ncbi_sp_gids = defaultdict(list)
for gid in cur_genomes:
ncbi_species = cur_genomes[gid].ncbi_taxa.species
ncbi_specific = specific_epithet(ncbi_species)
if ncbi_species != 's__' and ncbi_specific not in forbidden_names:
ncbi_sp_gids[ncbi_species].append(gid)
# get NCBI species anchored by a type strain genome
ncbi_type_anchored_species = {}
for rid, cids in cur_clusters.items():
for cid in cids:
if cur_genomes[cid].is_effective_type_strain():
ncbi_type_species = cur_genomes[cid].ncbi_taxa.species
ncbi_specific = specific_epithet(ncbi_species)
if ncbi_type_species != 's__' and ncbi_specific not in forbidden_names:
if (ncbi_type_species in ncbi_type_anchored_species
and rid !=
ncbi_type_anchored_species[ncbi_type_species]):
self.logger.error(
'NCBI species {} has multiple effective type strain genomes in different clusters.'
.format(ncbi_type_species))
sys.exit(-1)
ncbi_type_anchored_species[ncbi_type_species] = rid
self.logger.info(
' - identified {:,} NCBI species anchored by a type strain genome.'
.format(len(ncbi_type_anchored_species)))
# identify genomes with erroneous NCBI species assignments
fout = open(
os.path.join(self.output_dir,
'ncbi_misclassified_sp.gtdb_clustering.tsv'), 'w')
fout.write(
'Genome ID\tNCBI species\tGenome cluster\tType species cluster\n')
misclassified_gids = set()
for idx, (ncbi_species,
species_gids) in enumerate(ncbi_sp_gids.items()):
if ncbi_species not in ncbi_type_anchored_species:
continue
# find genomes with NCBI species assignments that are in a
# different cluster than the type strain genome
type_rid = ncbi_type_anchored_species[ncbi_species]
for gid in species_gids:
cur_rid = gid_to_rid[gid]
if type_rid != cur_rid:
misclassified_gids.add(gid)
fout.write('{}\t{}\t{}\t{}\t\n'.format(
gid, ncbi_species, cur_rid, type_rid))
sys.stdout.write('\n')
fout.close()
misclassified_species = set(
[cur_genomes[gid].ncbi_taxa.species for gid in misclassified_gids])
self.logger.info(
' - identified {:,} genomes from {:,} species as having misclassified NCBI species assignments.'
.format(len(misclassified_gids), len(misclassified_species)))
return misclassified_gids
def classify_ncbi_species(self, ncbi_synonyms, misclassified_gids):
"""Classify each NCBI species as either unambiguously, ambiguously, or synonym."""
# get genomes with NCBI species assignment
ncbi_all_species = set()
ncbi_species_gids = defaultdict(list)
ncbi_all_species_gids = set()
for gid in self.cur_genomes:
if gid in misclassified_gids:
continue
ncbi_species = self.cur_genomes[gid].ncbi_taxa.species
ncbi_specific = specific_epithet(ncbi_species)
if ncbi_species != 's__' and ncbi_specific not in self.forbidden_specific_names:
ncbi_all_species.add(ncbi_species)
ncbi_species_gids[ncbi_species].append(gid)
ncbi_all_species_gids.add(gid)
# process NCBI species with type strain genomes followed by
# remaining NCBI species
ncbi_type_strain_species = []
ncbi_nontype_species = []
for ncbi_species, ncbi_sp_gids in ncbi_species_gids.items():
has_type_strain_genome = False
for gid in ncbi_sp_gids:
if self.cur_genomes[gid].is_effective_type_strain():
has_type_strain_genome = True
break
if has_type_strain_genome:
ncbi_type_strain_species.append(ncbi_species)
else:
ncbi_nontype_species.append(ncbi_species)
self.logger.info(
' - identified {:,} NCBI species with effective type strain genome and {:,} NCBI species without a type strain genome.'
.format(len(ncbi_type_strain_species), len(ncbi_nontype_species)))
# establish if NCBI species can be unambiguously assigned to a GTDB species cluster
fout = open(
os.path.join(self.output_dir, 'ncbi_sp_classification.tsv'), 'w')
fout.write(
'NCBI taxon\tClassification\tAssignment type\tGTDB representative\t% NCBI species in cluster\t% cluster with NCBI species\tNCBI classifications in cluster\tNote\n'
)
unambiguous_ncbi_sp = {}
ambiguous_ncbi_sp = set()
unambiguous_rids = set()
for ncbi_species in ncbi_type_strain_species + ncbi_nontype_species:
if ncbi_species in ncbi_synonyms:
continue
ncbi_sp_gids = ncbi_species_gids[ncbi_species]
# get type strains for NCBI species that are GTDB species representatives
type_strain_rids = [
gid for gid in ncbi_sp_gids
if (self.cur_genomes[gid].is_effective_type_strain()
and gid in self.cur_clusters)
]
if len(type_strain_rids) == 0:
highest_sp_in_cluster_perc = 0
highest_cluster_perc = 0
highest_gtdb_rid = None
for gtdb_rid, cids in self.cur_clusters.items():
assert gtdb_rid in cids
ncbi_cur_sp_in_cluster = cids.intersection(ncbi_sp_gids)
ncbi_any_sp_in_cluster = cids.intersection(
ncbi_all_species_gids)
if len(ncbi_any_sp_in_cluster) > 0:
ncbi_cur_sp_in_cluster_perc = len(
ncbi_cur_sp_in_cluster) * 100.0 / len(ncbi_sp_gids)
cluster_perc = len(
ncbi_cur_sp_in_cluster) * 100.0 / len(
ncbi_any_sp_in_cluster)
if (ncbi_cur_sp_in_cluster_perc >
highest_sp_in_cluster_perc
or (ncbi_cur_sp_in_cluster_perc
== highest_sp_in_cluster_perc
and cluster_perc > highest_cluster_perc)):
highest_sp_in_cluster_perc = ncbi_cur_sp_in_cluster_perc
highest_cluster_perc = cluster_perc
highest_gtdb_rid = gtdb_rid
sp_in_cluster_threshold = 50
cluster_threshold = 50
if (highest_gtdb_rid not in unambiguous_rids and
highest_sp_in_cluster_perc > sp_in_cluster_threshold
and highest_cluster_perc > cluster_threshold):
unambiguous_ncbi_sp[ncbi_species] = (
highest_gtdb_rid, NCBI_SpeciesManager.MAJORITY_VOTE)
else:
ambiguous_ncbi_sp.add(ncbi_species)
ncbi_sp_str = []
for sp, count in Counter([
self.cur_genomes[gid].ncbi_taxa.species
for gid in self.cur_clusters[highest_gtdb_rid]
]).most_common():
ncbi_sp_str.append('{} ({:,})'.format(sp, count))
ncbi_sp_str = '; '.join(ncbi_sp_str)
fout.write('{}\t{}\t{}\t{}\t{:.2f}\t{:.2f}\t{}\t{}\n'.format(
ncbi_species,
NCBI_SpeciesManager.UNAMBIGUOUS if ncbi_species
in unambiguous_ncbi_sp else NCBI_SpeciesManager.AMBIGUOUS,
NCBI_SpeciesManager.MAJORITY_VOTE if ncbi_species
in unambiguous_ncbi_sp else NCBI_SpeciesManager.AMBIGUOUS,
highest_gtdb_rid, highest_sp_in_cluster_perc,
highest_cluster_perc, ncbi_sp_str, ''))
elif len(type_strain_rids) == 1:
gtdb_rid = type_strain_rids[0]
unambiguous_ncbi_sp[ncbi_species] = (
gtdb_rid, NCBI_SpeciesManager.TYPE_STRAIN_GENOME)
unambiguous_rids.add(gtdb_rid)
cids = self.cur_clusters[gtdb_rid]
ncbi_cur_sp_in_cluster = cids.intersection(ncbi_sp_gids)
ncbi_any_sp_in_cluster = cids.intersection(
ncbi_all_species_gids)
cur_sp_in_cluster_perc = len(
ncbi_cur_sp_in_cluster) * 100.0 / len(ncbi_sp_gids)
cluster_perc = len(ncbi_cur_sp_in_cluster) * \
100.0/len(ncbi_any_sp_in_cluster)
ncbi_sp_str = []
for sp, count in Counter([
self.cur_genomes[gid].ncbi_taxa.species
for gid in self.cur_clusters[gtdb_rid]
]).most_common():
ncbi_sp_str.append('{} ({:,})'.format(sp, count))
ncbi_sp_str = '; '.join(ncbi_sp_str)
fout.write('{}\t{}\t{}\t{}\t{:.2f}\t{:.2f}\t{}\t{}\n'.format(
ncbi_species, NCBI_SpeciesManager.UNAMBIGUOUS,
NCBI_SpeciesManager.TYPE_STRAIN_GENOME, gtdb_rid,
cur_sp_in_cluster_perc, cluster_perc, ncbi_sp_str, ''))
else:
self.logger.error(
'Multiple GTDB species clusters represented by type strain genomes of {}: {}'
.format(ncbi_species, type_strain_rids))
sys.exit(-1)
# get NCBI synonyms
for ncbi_species, ncbi_sp_gids in ncbi_species_gids.items():
if ncbi_species not in ncbi_synonyms:
continue
fout.write('{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\n'.format(
ncbi_species, 'SYNONYM', 'SYNONYM', 'n/a', 'n/a', 'n/a', 'n/a',
'Synonym of {} under GTDB'.format(
ncbi_synonyms[ncbi_species])))
fout.close()
# sanity check results
assert set(unambiguous_ncbi_sp).intersection(ncbi_synonyms) == set()
assert set(ambiguous_ncbi_sp).intersection(ncbi_synonyms) == set()
assert set(unambiguous_ncbi_sp).intersection(
ambiguous_ncbi_sp) == set()
assert set(ncbi_all_species) - set(unambiguous_ncbi_sp) - \
set(ambiguous_ncbi_sp) - set(ncbi_synonyms) == set()
gtdb_rep_ncbi_sp_assignments = {}
for ncbi_species, (gtdb_rid,
_classification) in unambiguous_ncbi_sp.items():
if gtdb_rid in gtdb_rep_ncbi_sp_assignments:
self.logger.error(
'GTDB representative {} assigned to multiple unambiguous NCBI species: {} {}'
.format(gtdb_rid, gtdb_rep_ncbi_sp_assignments[gtdb_rid],
ncbi_species))
# ***sys.exit(-1)
gtdb_rep_ncbi_sp_assignments[gtdb_rid] = ncbi_species
return ncbi_all_species, unambiguous_ncbi_sp, ambiguous_ncbi_sp
def run(self,
cur_gtdb_metadata_file,
cur_genomic_path_file,
qc_passed_file,
ncbi_genbank_assembly_file,
ltp_taxonomy_file,
gtdb_type_strains_ledger,
untrustworthy_type_ledger):
"""Resolve cases where a species has multiple genomes assembled from the type strain."""
# get species in LTP reference database
self.logger.info('Determining species defined in LTP reference database.')
ltp_defined_species = self.ltp_defined_species(ltp_taxonomy_file)
self.logger.info(f' ... identified {len(ltp_defined_species):,} species.')
# create current GTDB genome sets
self.logger.info('Creating current GTDB genome set.')
cur_genomes = Genomes()
cur_genomes.load_from_metadata_file(cur_gtdb_metadata_file,
gtdb_type_strains_ledger=gtdb_type_strains_ledger,
create_sp_clusters=False,
uba_genome_file=None,
qc_passed_file=qc_passed_file,
ncbi_genbank_assembly_file=ncbi_genbank_assembly_file,
untrustworthy_type_ledger=untrustworthy_type_ledger)
cur_genomes.load_genomic_file_paths(cur_genomic_path_file)
self.logger.info(f' ... current genome set contains {len(cur_genomes):,} genomes.')
# update current genomes with GTDB-Tk classifications
self.logger.info('Updating current genomes with GTDB-Tk classifications.')
num_updated, num_ncbi_sp = cur_genomes.set_gtdbtk_classification(gtdbtk_classify_file, prev_genomes)
self.logger.info(f' ... set GTDB taxa for {num_updated:,} genomes with {num_ncbi_sp:,} genomes using NCBI genus and species name.')
# parsing genomes manually established to be untrustworthy as type
self.logger.info('Determining genomes manually annotated as untrustworthy as type.')
manual_untrustworthy_types = {}
with open(untrustworthy_type_ledger) as f:
header = f.readline().strip().split('\t')
ncbi_sp_index = header.index('NCBI species')
reason_index = header.index('Reason for declaring untrustworthy')
for line in f:
tokens = line.strip().split('\t')
gid = canonical_gid(tokens[0])
manual_untrustworthy_types[gid] = (tokens[ncbi_sp_index], tokens[reason_index])
self.logger.info(f' ... identified {len(manual_untrustworthy_types):,} genomes manually annotated as untrustworthy as type.')
# identify NCBI species with multiple genomes assembled from type strain of species
self.logger.info('Determining number of type strain genomes in each NCBI species.')
sp_type_strain_genomes = defaultdict(set)
for gid in cur_genomes:
if cur_genomes[gid].is_effective_type_strain():
ncbi_sp = cur_genomes[gid].ncbi_taxa.species
if ncbi_sp != 's__':
# yes, NCBI has genomes marked as assembled from type material
# that do not actually have a binomial species name
sp_type_strain_genomes[ncbi_sp].add(gid)
multi_type_strains_sp = [ncbi_sp for ncbi_sp, gids in sp_type_strain_genomes.items() if len(gids) > 1]
self.logger.info(f' ... identified {len(multi_type_strains_sp):,} NCBI species with multiple assemblies indicated as being type strain genomes.')
# sort by number of genome assemblies
self.logger.info('Calculating ANI between type strain genomes in each species.')
fout = open(os.path.join(self.output_dir, 'multi_type_strain_species.tsv'), 'w')
fout.write('NCBI species\tNo. type strain genomes\t>=99% ANI\tMean ANI\tStd ANI\tMean AF\tStd AF\tResolution\tGenome IDs\n')
fout_genomes = open(os.path.join(self.output_dir, 'type_strain_genomes.tsv'), 'w')
fout_genomes.write('Genome ID\tUntrustworthy\tNCBI species\tGTDB genus\tGTDB species\tLTP species\tConflict with prior GTDB assignment')
fout_genomes.write('\tMean ANI\tStd ANI\tMean AF\tStd AF\tExclude from RefSeq\tNCBI taxonomy\tGTDB taxonomy\n')
fout_unresolved = open(os.path.join(self.output_dir, 'unresolved_type_strain_genomes.tsv'), 'w')
fout_unresolved.write('Genome ID\tNCBI species\tGTDB genus\tGTDB species\tLTP species')
fout_unresolved.write('\tMean ANI\tStd ANI\tMean AF\tStd AF\tExclude from RefSeq\tNCBI taxonomy\tGTDB taxonomy\n')
fout_high_divergence = open(os.path.join(self.output_dir, 'highly_divergent_type_strain_genomes.tsv'), 'w')
fout_high_divergence.write('Genome ID\tNCBI species\tGTDB genus\tGTDB species\tLTP species\tMean ANI\tStd ANI\tMean AF\tStd AF\tExclude from RefSeq\tNCBI taxonomy\tGTDB taxonomy\n')
fout_untrustworthy = open(os.path.join(self.output_dir, 'untrustworthy_type_material.tsv'), 'w')
fout_untrustworthy.write('Genome ID\tNCBI species\tGTDB species\tLTP species\tReason for declaring untrustworthy\n')
for gid in manual_untrustworthy_types:
ncbi_sp, reason = manual_untrustworthy_types[gid]
fout_untrustworthy.write('{}\t{}\t{}\t{}\t{}\n'.format(
gid,
ncbi_sp,
cur_genomes[gid].gtdb_taxa.species,
'<not tested>',
'n/a',
'Manual curation: ' + reason))
processed = 0
num_divergent = 0
unresolved_sp_count = 0
ncbi_ltp_resolved = 0
intra_ani_resolved = 0
ncbi_type_resolved = 0
gtdb_family_resolved = 0
gtdb_genus_resolved = 0
gtdb_sp_resolved = 0
ltp_resolved = 0
use_pickled_results = False #***
if use_pickled_results:
self.logger.warning('Using previously calculated ANI results in: {}'.format(self.ani_pickle_dir))
prev_gtdb_sp_conflicts = 0
for ncbi_sp, type_gids in sorted(sp_type_strain_genomes.items(), key=lambda kv: len(kv[1])):
if len(type_gids) == 1:
continue
status_str = '-> Processing {} with {:,} type strain genomes [{:,} of {:,} ({:.2f}%)].'.format(
ncbi_sp,
len(type_gids),
processed+1,
len(multi_type_strains_sp),
(processed+1)*100.0/len(multi_type_strains_sp)).ljust(128)
sys.stdout.write('{}\r'.format(status_str))
sys.stdout.flush()
processed += 1
# calculate ANI between type strain genomes
ncbi_sp_str = ncbi_sp[3:].lower().replace(' ', '_')
if not use_pickled_results: #***
ani_af = self.fastani.pairwise(type_gids, cur_genomes.genomic_files)
pickle.dump(ani_af, open(os.path.join(self.ani_pickle_dir, f'{ncbi_sp_str}.pkl'), 'wb'))
else:
ani_af = pickle.load(open(os.path.join(self.ani_pickle_dir, f'{ncbi_sp_str}.pkl'), 'rb'))
anis = []
afs = []
gid_anis = defaultdict(lambda: {})
gid_afs = defaultdict(lambda: {})
all_similar = True
for gid1, gid2 in combinations(type_gids, 2):
ani, af = symmetric_ani(ani_af, gid1, gid2)
if ani < 99 or af < 0.65:
all_similar = False
anis.append(ani)
afs.append(af)
gid_anis[gid1][gid2] = ani
gid_anis[gid2][gid1] = ani
gid_afs[gid1][gid2] = af
gid_afs[gid2][gid1] = af
note = 'All type strain genomes have ANI >99% and AF >65%.'
unresolved_species = False
# read LTP metadata for genomes
ltp_metadata = self.parse_ltp_metadata(type_gids, cur_genomes)
untrustworthy_gids = {}
gtdb_resolved_sp_conflict = False
if not all_similar:
# need to establish which genomes are untrustworthy as type
num_divergent += 1
unresolved_species = True
# write out highly divergent cases for manual inspection;
# these should be compared to the automated selection
if np_mean(anis) < 95:
for gid in type_gids:
ltp_species = self.ltp_species(gid, ltp_metadata)
fout_high_divergence.write('{}\t{}\t{}\t{}\t{}\t{:.2f}\t{:.3f}\t{:.3f}\t{:.4f}\t{}\t{}\t{}\n'.format(
gid,
ncbi_sp,
cur_genomes[gid].gtdb_taxa.genus,
cur_genomes[gid].gtdb_taxa.species,
' / '.join(ltp_species),
np_mean(list(gid_anis[gid].values())),
np_std(list(gid_anis[gid].values())),
np_mean(list(gid_afs[gid].values())),
np_std(list(gid_afs[gid].values())),
cur_genomes[gid].excluded_from_refseq_note,
cur_genomes[gid].ncbi_taxa,
cur_genomes[gid].gtdb_taxa))
# filter genomes marked as `untrustworthy as type` at NCBI and where the LTP
# assignment also suggest the asserted type material is incorrect
resolved, untrustworthy_gids = self.resolve_validated_untrustworthy_ncbi_genomes(gid_anis,
ncbi_sp,
type_gids,
ltp_metadata,
ltp_defined_species,
cur_genomes)
if resolved:
note = "Species resolved by removing genomes considered `untrustworthy as type` and with a LTP BLAST hit confirming the assembly is likely untrustworthy"
ncbi_ltp_resolved += 1
# try to resolve by LTP 16S BLAST results
if not resolved:
resolved, untrustworthy_gids = self.resolve_ltp_conflict(gid_anis, ncbi_sp, type_gids, ltp_metadata, 0)
if resolved:
note = 'Species resolved by identifying conflicting or lack of LTP BLAST results'
ltp_resolved += 1
# try to resolve species using intra-specific ANI test
if not resolved:
resolved, untrustworthy_gids = self.resolve_by_intra_specific_ani(gid_anis)
if resolved:
note = 'Species resolved by intra-specific ANI test'
intra_ani_resolved += 1
# try to resolve by GTDB family assignment
if not resolved:
resolved, untrustworthy_gids = self.resolve_gtdb_family(gid_anis, ncbi_sp, type_gids, cur_genomes)
if resolved:
note = 'Species resolved by consulting GTDB family classifications'
gtdb_family_resolved += 1
# try to resolve by GTDB genus assignment
if not resolved:
resolved, untrustworthy_gids = self.resolve_gtdb_genus(gid_anis, ncbi_sp, type_gids, cur_genomes)
if resolved:
note = 'Species resolved by consulting GTDB genus classifications'
gtdb_genus_resolved += 1
# try to resolve by GTDB species assignment
if not resolved:
resolved, untrustworthy_gids = self.resolve_gtdb_species(gid_anis, ncbi_sp, type_gids, cur_genomes)
if resolved:
note = 'Species resolved by consulting GTDB species classifications'
gtdb_sp_resolved += 1
# try to resolve by considering genomes annotated as type material at NCBI,
# which includes considering if genomes are marked as untrustworthy as type
if not resolved:
resolved, untrustworthy_gids = self.resolve_by_ncbi_types(gid_anis, type_gids, cur_genomes)
if resolved:
note = 'Species resolved by consulting NCBI assembled from type metadata'
ncbi_type_resolved += 1
if resolved:
unresolved_species = False
# check if type strain genomes marked as trusted or untrusted conflict
# with current GTDB species assignment
untrustworthy_gtdb_sp_match = False
trusted_gtdb_sp_match = False
for gid in type_gids:
gtdb_canonical_epithet = canonical_taxon(specific_epithet(cur_genomes[gid].gtdb_taxa.species))
if gtdb_canonical_epithet == specific_epithet(ncbi_sp):
if gid in untrustworthy_gids:
untrustworthy_gtdb_sp_match = True
else:
trusted_gtdb_sp_match = True
if untrustworthy_gtdb_sp_match and not trusted_gtdb_sp_match:
prev_gtdb_sp_conflicts += 1
gtdb_resolved_sp_conflict = True
# write results to file
for gid, reason in untrustworthy_gids.items():
ltp_species = self.ltp_species(gid, ltp_metadata)
if 'untrustworthy as type' in cur_genomes[gid].excluded_from_refseq_note:
reason += "; considered `untrustworthy as type` at NCBI"
fout_untrustworthy.write('{}\t{}\t{}\t{}\t{}\n'.format(gid,
ncbi_sp,
cur_genomes[gid].gtdb_taxa.species,
' / '.join(ltp_species),
reason))
# Sanity check that if the untrustworthy genome has an LTP to only the
# expected species, that all other genomes also have a hit to the
# expected species (or potentially no hit). Otherwise, more consideration
# should be given to the genome with the conflicting LTP hit.
if len(ltp_species) == 1 and ncbi_sp in ltp_species:
other_sp = set()
for test_gid in type_gids:
ltp_species = self.ltp_species(test_gid, ltp_metadata)
if ltp_species and ncbi_sp not in ltp_species:
other_sp.update(ltp_species)
if other_sp:
self.logger.warning(f'Genome {gid} marked as untrustworthy, but this conflicts with high confidence LTP 16S rRNA assignment.')
num_ncbi_untrustworthy = sum([1 for gid in type_gids if 'untrustworthy as type' in cur_genomes[gid].excluded_from_refseq_note])
if num_ncbi_untrustworthy != len(type_gids):
for gid in type_gids:
if (gid not in untrustworthy_gids
and 'untrustworthy as type' in cur_genomes[gid].excluded_from_refseq_note):
self.logger.warning("Retaining genome {} from {} despite being marked as `untrustworthy as type` at NCBI [{:,} of {:,} considered untrustworthy].".format(
gid,
ncbi_sp,
num_ncbi_untrustworthy,
len(type_gids)))
else:
note = 'Species is unresolved; manual curation is required!'
unresolved_sp_count += 1
if unresolved_species:
for gid in type_gids:
ltp_species = self.ltp_species(gid, ltp_metadata)
fout_unresolved.write('{}\t{}\t{}\t{}\t{}\t{:.2f}\t{:.3f}\t{:.3f}\t{:.4f}\t{}\t{}\t{}\n'.format(
gid,
ncbi_sp,
cur_genomes[gid].gtdb_taxa.genus,
cur_genomes[gid].gtdb_taxa.species,
' / '.join(ltp_species),
np_mean(list(gid_anis[gid].values())),
np_std(list(gid_anis[gid].values())),
np_mean(list(gid_afs[gid].values())),
np_std(list(gid_afs[gid].values())),
cur_genomes[gid].excluded_from_refseq_note,
cur_genomes[gid].ncbi_taxa,
cur_genomes[gid].gtdb_taxa))
for gid in type_gids:
ltp_species = self.ltp_species(gid, ltp_metadata)
fout_genomes.write('{}\t{}\t{}\t{}\t{}\t{}\t{}\t{:.2f}\t{:.3f}\t{:.3f}\t{:.4f}\t{}\t{}\t{}\n'.format(
gid,
gid in untrustworthy_gids,
ncbi_sp,
cur_genomes[gid].gtdb_taxa.genus,
cur_genomes[gid].gtdb_taxa.species,
' / '.join(ltp_species),
gtdb_resolved_sp_conflict,
np_mean(list(gid_anis[gid].values())),
np_std(list(gid_anis[gid].values())),
np_mean(list(gid_afs[gid].values())),
np_std(list(gid_afs[gid].values())),
cur_genomes[gid].excluded_from_refseq_note,
cur_genomes[gid].ncbi_taxa,
cur_genomes[gid].gtdb_taxa))
fout.write('{}\t{}\t{}\t{:.2f}\t{:.3f}\t{:.3f}\t{:.4f}\t{}\t{}\n'.format(
ncbi_sp,
len(type_gids),
all_similar,
np_mean(anis),
np_std(anis),
np_mean(afs),
np_std(afs),
note,
', '.join(type_gids)))
sys.stdout.write('\n')
fout.close()
fout_unresolved.close()
fout_high_divergence.close()
fout_genomes.close()
fout_untrustworthy.close()
self.logger.info(f'Identified {num_divergent:,} species with 1 or more divergent type strain genomes.')
self.logger.info(f' ... resolved {ncbi_ltp_resolved:,} species by removing NCBI `untrustworthy as type` genomes with a conflicting LTP 16S rRNA classifications.')
self.logger.info(f' ... resolved {ltp_resolved:,} species by considering conflicting LTP 16S rRNA classifications.')
self.logger.info(f' ... resolved {intra_ani_resolved:,} species by considering intra-specific ANI values.')
self.logger.info(f' ... resolved {gtdb_family_resolved:,} species by considering conflicting GTDB family classifications.')
self.logger.info(f' ... resolved {gtdb_genus_resolved:,} species by considering conflicting GTDB genus classifications.')
self.logger.info(f' ... resolved {gtdb_sp_resolved:,} species by considering conflicting GTDB species classifications.')
self.logger.info(f' ... resolved {ncbi_type_resolved:,} species by considering type material designations at NCBI.')
if unresolved_sp_count > 0:
self.logger.warning(f'There are {unresolved_sp_count:,} unresolved species with multiple type strain genomes.')
self.logger.warning('These should be handled before proceeding with the next step of GTDB species updating.')
self.logger.warning("This can be done by manual curation and adding genomes to 'untrustworthy_type_ledger'.")
self.logger.info(f'Identified {prev_gtdb_sp_conflicts:,} cases where resolved type strain conflicts with prior GTDB assignment.')
