def load_from_sp_cluster_file(self, cluster_file): """Create species clusters from file.""" with open(cluster_file) as f: headers = f.readline().strip().split('\t') sp_index = headers.index('GTDB species') rid_index = headers.index('Representative genome') num_clustered_index = headers.index('No. clustered genomes') cluster_index = headers.index('Clustered genomes') for line in f: line_split = line.strip().split('\t') sp = line_split[sp_index] rid = canonical_gid(line_split[rid_index]) self.genome_rid[rid] = rid self.sp_clusters[rid] = set([rid]) self.species_names[rid] = sp num_clustered = int(line_split[num_clustered_index]) if num_clustered > 0: cids = [ canonical_gid(cid.strip()) for cid in line_split[cluster_index].split(',') ] self.sp_clusters[rid].update([cid for cid in cids]) for cid in cids: self.genome_rid[cid] = rid
def _apply_ncbi_taxonomy_ledgers(self, species_exception_file, genus_exception_file): """Apply corrections to NCBI taxonomy.""" species_updates = {} if species_exception_file: with open(species_exception_file, encoding='utf-8') as f: f.readline() for line in f: line_split = [ token.strip() for token in line.strip().split('\t') ] gid = canonical_gid(line_split[0].strip()) sp = line_split[1].strip().replace('Candidatus ', '') if gid not in self.genomes: self.logger.warning( f'Genome {gid} in species exception list not defined in genome set.' ) continue if not sp.startswith('s__'): sp = 's__' + sp self.genomes[gid].ncbi_taxa.species = sp species_updates[gid] = sp if genus_exception_file: with open(genus_exception_file, encoding='utf-8') as f: f.readline() for line in f: line_split = [ token.strip() for token in line.strip().split('\t') ] gid = canonical_gid(line_split[0].strip()) genus = line_split[1].strip() if gid not in self.genomes: self.logger.warning( f'Genome {gid} in genus exception list not defined in genome set.' ) continue if genus.startswith('g__'): genus = genus[3:] self.genomes[gid].ncbi_taxa.genus = f'g__{genus}' species = self.genomes[gid].ncbi_taxa.species if species != 's__': specific = self.genomes[gid].ncbi_taxa.specific_epithet self.genomes[ gid].ncbi_taxa.species = f's__{genus} {specific}' # sanity check ledgers if gid in species_updates and genus not in species_updates[ gid]: self.logger.error( f'Species and genus ledgers have conflicting assignments for {gid}.' ) sys.exit(-1)
def read_gtdb_taxonomy(metadata_file): """Parse GTDB taxonomy from GTDB metadata. Parameters ---------- metadata_file : str Metadata for all genomes. Return ------ dict : d[genome_id] -> taxonomy list """ taxonomy = {} with open(metadata_file, encoding='utf-8') as f: headers = f.readline().strip().split('\t') genome_index = headers.index('accession') taxonomy_index = headers.index('gtdb_taxonomy') for line in f: line_split = line.strip().split('\t') genome_id = canonical_gid(line_split[genome_index]) taxa_str = line_split[taxonomy_index].strip() if taxa_str and taxa_str != 'none': taxonomy[genome_id] = [t.strip() for t in taxa_str.split(';')] else: taxonomy[genome_id] = list(Taxonomy.rank_prefixes) return taxonomy
def check_domain_assignments(self, gtdb_domain_report, cur_genomes, pass_qc_gids): """Check GTDB domain assignment.""" with open(gtdb_domain_report, encoding='utf-8') as f: header = f.readline().rstrip().split('\t') domain_index = header.index('Predicted domain') bac_marker_perc_index = header.index('Bacterial Marker Percentage') ar_marker_perc_index = header.index('Archaeal Marker Percentage') ncbi_taxonomy_index = header.index('NCBI taxonomy') gtdb_taxonomy_index = header.index('GTDB taxonomy') for line in f: line_split = line.strip().split('\t') gid = canonical_gid(line_split[0]) gid = cur_genomes.user_uba_id_map.get(gid, gid) if gid not in pass_qc_gids: continue domain = line_split[domain_index] bac_perc = float(line_split[bac_marker_perc_index]) ar_perc = float(line_split[ar_marker_perc_index]) ncbi_domain = [t.strip() for t in line_split[ncbi_taxonomy_index].split(';')][0] gtdb_domain = [t.strip() for t in line_split[gtdb_taxonomy_index].split(';')][0] if not gid.startswith('U'): if ncbi_domain != gtdb_domain and ncbi_domain != 'None': print(f'[WARNING] NCBI ({ncbi_domain}) and GTDB ({gtdb_domain}) domains disagree in domain report (Bac = {bac_perc:.1f}%; Ar = {ar_perc:.1f}%): {gid}') if domain != gtdb_domain and domain != 'None': print(f'[WARNING] GTDB and predicted domain (Bac = {bac_perc:.1f}%; Ar = {ar_perc:.1f}%) disagree in domain report: {gid}')
def _mash_genome_id(self, mash_genome_id): """Extract canonical GTDB genome ID from Mash results.""" # get filename and remove information past genome accession # (e.g., GCA_002498385.1_ASM249838v1_genomic.fna => GCA_002498385.1) genome_file = ntpath.basename(mash_genome_id) gid = genome_file[0:genome_file.find('_', 4)] gid = canonical_gid(gid) return gid
def parse_untrustworthy_type_ledger(self, untrustworth_type_ledger): """Determine genomes that should be considered untrustworthy as type material.""" untrustworthy_as_type = set() with open(untrustworth_type_ledger) as f: f.readline() for line in f: tokens = line.strip().split('\t') untrustworthy_as_type.add(canonical_gid(tokens[0])) return untrustworthy_as_type
def parse_qc_exception_file(self, qc_exception_file): """Parse file indicating genomes flagged as exceptions from QC.""" qc_exceptions = set() with open(qc_exception_file, encoding='utf-8') as f: f.readline() for line in f: gid = canonical_gid(line.split('\t')[0].strip()) qc_exceptions.add(gid) return qc_exceptions
def parse_ncbi_untrustworthy_sp_ledger(self, ncbi_untrustworthy_sp_ledger): """Determine genomes that should be considered as having untrustworthy NCBI species assignments.""" untrustworthy_ncbi_sp = set() with open(ncbi_untrustworthy_sp_ledger) as f: f.readline() for line in f: tokens = line.strip().split('\t') untrustworthy_ncbi_sp.add(canonical_gid(tokens[0])) return untrustworthy_ncbi_sp
def same_genome_accn(self, accn1, accn2, identical_accns): """Check if NCBI genome accessions are the same.""" if accn1 == accn2: return True if canonical_gid(accn1) != canonical_gid(accn2): self.logger.error( 'Genomes have different canonical genome IDs: {}, {}, {}, {}'. format(accn1, canonical_gid(accn1), accn2, canonical_gid(accn2))) sys.exit(-1) accn1 = accn1.replace('RS_', '').replace('GB_', '') accn2 = accn2.replace('RS_', '').replace('GB_', '') if identical_accns.get(accn1, None) == accn2: return True return False
def parse_disbanded_cluster_ledger(disbanded_cluster_ledger): """Parse file indicating GTDB species clusters to be disbanded.""" disbanded = set() with open(disbanded_cluster_ledger) as f: f.readline() for line in f: tokens = line.strip().split('\t') disbanded.add(canonical_gid(tokens[0])) return disbanded
def mash_sp_ani(self, gids, genomes, output_prefix): """Calculate pairwise Mash ANI estimates between genomes.""" INIT_MASH_ANI_FILTER = 95.0 # create Mash sketch for all genomes mash_sketch_file = f'{output_prefix}.msh' genome_list_file = f'{output_prefix}.lst' self.mash.sketch(gids, genomes.genomic_files, genome_list_file, mash_sketch_file, silence=True) # get Mash distances mash_dist_file = f'{output_prefix}.dst' self.mash.dist_pairwise(float(100 - INIT_MASH_ANI_FILTER) / 100, mash_sketch_file, mash_dist_file, silence=True) # read Mash distances mash_ani = self.mash.read_ani(mash_dist_file) # report pairs above Mash threshold revised_mash_ani = defaultdict(lambda: {}) count = 0 for qid in mash_ani: for rid in mash_ani[qid]: if qid != rid: new_qid = canonical_gid(self.user_id_map.get(qid, qid)) new_rid = canonical_gid(self.user_id_map.get(rid, rid)) revised_mash_ani[new_qid][new_rid] = mash_ani[qid][rid] count += 1 self.logger.info( ' - identified {:,} pairs passing Mash filtering of ANI >= {:.1f}%.' .format(count, INIT_MASH_ANI_FILTER)) return revised_mash_ani
def parse_clusters(self, cluster_file): """Parse species clustering information.""" species = {} clusters = {} cluster_radius = {} with open(cluster_file) as f: headers = f.readline().strip().split('\t') type_sp_index = headers.index('NCBI species') type_genome_index = headers.index('Type genome') num_clustered_index = headers.index('No. clustered genomes') clustered_genomes_index = headers.index('Clustered genomes') closest_type_index = headers.index('Closest type genome') ani_radius_index = headers.index('ANI radius') af_index = headers.index('AF closest') for line in f: line_split = line.strip().split('\t') rid = line_split[type_genome_index] rid = canonical_gid(rid) species[rid] = line_split[type_sp_index] clusters[rid] = set() num_clustered = int(line_split[num_clustered_index]) if num_clustered > 0: for gid in [g.strip() for g in line_split[clustered_genomes_index].split(',')]: gid = canonical_gid(gid) clusters[rid].add(gid) cluster_radius[rid] = GenomeRadius(ani=float(line_split[ani_radius_index]), af=float( line_split[af_index]), neighbour_gid=line_split[closest_type_index]) return clusters, species, cluster_radius
def _get_genome_id(self, genome_path): """Extract genome ID from path to genomic file.""" genome_id = ntpath.basename(genome_path) if genome_id.startswith('GCA_') or genome_id.startswith('GCF_'): genome_id = '_'.join(genome_id.split('_')[0:2]) if genome_id.startswith('GCA_'): genome_id = 'GB_' + genome_id else: genome_id = 'RS_' + genome_id else: genome_id = '_'.join(genome_id.split('_')[0:2]) return canonical_gid(genome_id)
def read_gtdb_metadata(metadata_file, fields): """Parse genome quality from GTDB metadata. Parameters ---------- metadata_file : str Metadata for all genomes in CSV file. fields : iterable Fields to read. Return ------ dict : d[genome_id] -> namedtuple Value for fields indicted by genome IDs. """ gtdb_metadata = namedtuple('gtdb_metadata', ' '.join(fields)) m = {} with open(metadata_file, encoding='utf-8') as f: headers = f.readline().strip().split('\t') genome_index = headers.index('accession') indices = [] for field in fields: indices.append(headers.index(field)) for line in f: line_split = line.strip().split('\t') genome_id = canonical_gid(line_split[genome_index]) values = [] for i in indices: # save values as floats or strings v = line_split[i] try: values.append(float(v)) except ValueError: if v is None or v == '' or v == 'none': values.append(None) elif v == 'f' or v.lower() == 'false': values.append(False) elif v == 't' or v.lower() == 'true': values.append(True) else: values.append(v) m[genome_id] = gtdb_metadata._make(values) return m
def load_genomic_file_paths(self, genome_path_file): """Determine path to genomic FASTA file for each genome.""" for line in open(genome_path_file): line_split = line.strip().split('\t') gid = line_split[0] gid = canonical_gid(gid) if gid in self.genomes: genome_path = line_split[1] accession = os.path.basename(os.path.normpath(genome_path)) genomic_file = os.path.join(genome_path, accession + '_genomic.fna') self.genomes[gid].genomic_file = genomic_file self.genomic_files[gid] = genomic_file #*** Need to handle UBA genomes that can reference via their U_ ID when run through MASH self.genomic_files[accession] = genomic_file
def parse_untrustworthy_type_ledger(self, untrustworthy_type_ledger): """Parse file indicating genomes considered to be untrustworthy as type material.""" manual_untrustworthy_types = {} with open(untrustworthy_type_ledger) as f: header = f.readline().strip().split('\t') ncbi_sp_index = header.index('NCBI species') reason_index = header.index('Reason for declaring untrustworthy') for line in f: tokens = line.strip().split('\t') gid = canonical_gid(tokens[0]) manual_untrustworthy_types[gid] = (tokens[ncbi_sp_index], tokens[reason_index]) return manual_untrustworthy_types
def type_status(self, cur_gtdb_metadata_file, qc_passed_file, ncbi_genbank_assembly_file, untrustworthy_type_file, gtdb_type_strains_ledger, ncbi_env_bioproject_ledger, genome_ids): """Report information related to a genome being type material.""" # create current GTDB genome sets self.logger.info('Creating current GTDB genome set.') cur_genomes = Genomes() cur_genomes.load_from_metadata_file(cur_gtdb_metadata_file, gtdb_type_strains_ledger=gtdb_type_strains_ledger, create_sp_clusters=False, qc_passed_file=qc_passed_file, ncbi_genbank_assembly_file=ncbi_genbank_assembly_file, untrustworthy_type_ledger=untrustworthy_type_file, ncbi_env_bioproject_ledger=ncbi_env_bioproject_ledger) self.logger.info( f' - current genome set contains {len(cur_genomes):,} genomes.') # report information pt = PrettyTable() pt.field_names = ['Genome ID', 'GTDB representative', 'GTDB type strain', 'GTDB untrustworthy as type', 'NCBI type strain', 'NCBI untrustworthy as type', 'GTDB species', 'NCBI species', 'NCBI strain IDs\n'] for gid in genome_ids: gid = canonical_gid(gid) if gid not in cur_genomes: self.logger.warning(f'Genome {gid} not in current genome set.') continue pt.add_row([gid, cur_genomes[gid].is_gtdb_sp_rep(), cur_genomes[gid].is_gtdb_type_strain(), cur_genomes[gid].is_gtdb_untrustworthy_as_type(), cur_genomes[gid].is_ncbi_type_strain(), cur_genomes[gid].is_ncbi_untrustworthy_as_type(), cur_genomes[gid].gtdb_taxa.species, cur_genomes[gid].ncbi_taxa.species, cur_genomes[gid].ncbi_strain_identifiers]) print(pt)
def parse_marker_percentages(self, gtdb_domain_report): """Parse percentage of marker genes for each genome.""" marker_perc = {} with open(gtdb_domain_report, encoding='utf-8') as f: header = f.readline().rstrip().split('\t') bac_marker_perc_index = header.index('Bacterial Marker Percentage') ar_marker_perc_index = header.index('Archaeal Marker Percentage') for line in f: line_split = line.strip().split('\t') gid = canonical_gid(line_split[0]) bac_perc = float(line_split[bac_marker_perc_index]) ar_perc = float(line_split[ar_marker_perc_index]) marker_perc[gid] = max(bac_perc, ar_perc) return marker_perc
def check_domain_assignments(self, gtdb_domain_report, passed_qc_gids): """Check GTDB domain assignment.""" with open(gtdb_domain_report, encoding='utf-8') as f: header = f.readline().rstrip().split('\t') domain_index = header.index('Predicted domain') bac_marker_perc_index = header.index('Bacterial Marker Percentage') ar_marker_perc_index = header.index('Archaeal Marker Percentage') ncbi_taxonomy_index = header.index('NCBI taxonomy') gtdb_taxonomy_index = header.index('GTDB taxonomy') for line in f: line_split = line.strip().split('\t') gid = canonical_gid(line_split[0]) if gid not in passed_qc_gids: continue domain = line_split[domain_index] bac_perc = float(line_split[bac_marker_perc_index]) ar_perc = float(line_split[ar_marker_perc_index]) ncbi_domain = [ t.strip() for t in line_split[ncbi_taxonomy_index].split(';') ][0] gtdb_domain = [ t.strip() for t in line_split[gtdb_taxonomy_index].split(';') ][0] if not gid.startswith('U'): if ncbi_domain != gtdb_domain and ncbi_domain != 'None': self.logger.warning( f'NCBI ({ncbi_domain}) and GTDB ({gtdb_domain}) domains disagree in domain report (Bac = {bac_perc:.1f}%; Ar = {ar_perc:.1f}%): {gid}' ) if domain != gtdb_domain and domain != 'None': self.logger.error( f'GTDB and predicted domain (Bac = {bac_perc:.1f}%; Ar = {ar_perc:.1f}%) disagree in domain report: {gid} [THIS MUST BE FIXED BEFORE PROCEEDING].' )
def _get_genome_id(self, genome_path): """Extract genome ID from path to genomic file.""" genome_id = ntpath.basename(genome_path) return canonical_gid(genome_id)
def load_from_metadata_file(self, metadata_file, species_exception_file=None, genus_exception_file=None, gtdb_type_strains_ledger=None, create_sp_clusters=True, uba_genome_file=None, qc_passed_file=None, ncbi_genbank_assembly_file=None, untrustworthy_type_ledger=None): """Create genome set from file(s).""" pass_qc_gids = set() if qc_passed_file: with open(qc_passed_file) as f: f.readline() for line in f: line_split = line.strip().split('\t') pass_qc_gids.add(line_split[0].strip()) self.logger.info(f' - identified {len(pass_qc_gids):,} genomes passing QC.') valid_uba_ids = set() if uba_genome_file: with open(uba_genome_file) as f: for line in f: line_split = line.strip().split('\t') valid_uba_ids.add(line_split[0].strip()) self.logger.info(f' - identified {len(valid_uba_ids):,} UBA genomes to retain.') gtdb_type_strains = set() if gtdb_type_strains_ledger: with open(gtdb_type_strains_ledger) as f: f.readline() for line in f: tokens = line.strip().split('\t') gid = canonical_gid(tokens[0].strip()) gtdb_type_strains.add(gid) self.logger.info(f' - identified {len(gtdb_type_strains):,} manually annotated as type strain genomes.') excluded_from_refseq_note = {} if ncbi_genbank_assembly_file: excluded_from_refseq_note = exclude_from_refseq(ncbi_genbank_assembly_file) untrustworthy_as_type = set() if untrustworthy_type_ledger: untrustworthy_as_type = self.parse_untrustworthy_type_ledger(untrustworthy_type_ledger) self.logger.info(f' - identified {len(untrustworthy_as_type):,} genomes annotated as untrustworthy as type.') with open(metadata_file, encoding='utf-8') as f: headers = f.readline().strip().split('\t') genome_index = headers.index('accession') gtdb_taxonomy_index = headers.index('gtdb_taxonomy') ncbi_taxonomy_index = headers.index('ncbi_taxonomy') ncbi_taxonomy_unfiltered_index = headers.index('ncbi_taxonomy_unfiltered') gtdb_type_index = headers.index('gtdb_type_designation') gtdb_type_sources_index = headers.index('gtdb_type_designation_sources') gtdb_type_species_of_genus_index = headers.index('gtdb_type_species_of_genus') ncbi_strain_identifiers_index = headers.index('ncbi_strain_identifiers') ncbi_type_index = headers.index('ncbi_type_material_designation') ncbi_asm_level_index = headers.index('ncbi_assembly_level') ncbi_genome_representation_index = headers.index('ncbi_genome_representation') ncbi_refseq_cat_index = headers.index('ncbi_refseq_category') ncbi_genome_cat_index = headers.index('ncbi_genome_category') comp_index = headers.index('checkm_completeness') cont_index = headers.index('checkm_contamination') sh_100_index = None if 'checkm_strain_heterogeneity_100' in headers: sh_100_index = headers.index('checkm_strain_heterogeneity_100') gs_index = headers.index('genome_size') contig_count_index = headers.index('contig_count') n50_index = headers.index('n50_contigs') scaffold_count_index = headers.index('scaffold_count') ambiguous_bases_index = headers.index('ambiguous_bases') total_gap_len_index = headers.index('total_gap_length') ssu_count_index = headers.index('ssu_count') ssu_length_index = headers.index('ssu_length') ncbi_molecule_count_index = headers.index('ncbi_molecule_count') ncbi_unspanned_gaps_index = headers.index('ncbi_unspanned_gaps') ncbi_spanned_gaps_index = headers.index('ncbi_spanned_gaps') gtdb_genome_rep_index = headers.index('gtdb_genome_representative') gtdb_rep_index = headers.index('gtdb_representative') if 'lpsn_priority_year' in headers: # this information will be missing from the previous # GTDB metadata file as we strip this out due to # concerns over republishing this information lpsn_priority_index = headers.index('lpsn_priority_year') dsmz_priority_index = headers.index('dsmz_priority_year') straininfo_priority_index = headers.index('straininfo_priority_year') for line in f: line_split = line.strip().split('\t') ncbi_accn = line_split[genome_index] gid = canonical_gid(ncbi_accn) if gid.startswith('U_'): # check if genome has a UBA identifier org_name_index = headers.index('organism_name') org_name = line_split[org_name_index] if '(UBA' in org_name: uba_id = org_name[org_name.find('(')+1:-1] if uba_id in valid_uba_ids: self.user_uba_id_map[gid] = uba_id self.uba_user_id_map[uba_id] = gid gid = uba_id else: continue # retain only valid UBA genomes else: continue # skip non-UBA user genomes if pass_qc_gids and gid not in pass_qc_gids: continue gtdb_taxonomy = Taxa(line_split[gtdb_taxonomy_index]) ncbi_taxonomy = Taxa(line_split[ncbi_taxonomy_index]) ncbi_taxonomy_unfiltered = Taxa(line_split[ncbi_taxonomy_unfiltered_index]) gtdb_type = line_split[gtdb_type_index] gtdb_type_sources = line_split[gtdb_type_sources_index] if gid in gtdb_type_strains: gtdb_type = 'type strain of species' gtdb_type_sources = 'GTDB curator' gtdb_type_species_of_genus = line_split[gtdb_type_species_of_genus_index] == 't' ncbi_type = line_split[ncbi_type_index] ncbi_strain_identifiers = line_split[ncbi_strain_identifiers_index] ncbi_asm_level = line_split[ncbi_asm_level_index] ncbi_genome_representation = line_split[ncbi_genome_representation_index] ncbi_refseq_cat = line_split[ncbi_refseq_cat_index] ncbi_genome_cat = line_split[ncbi_genome_cat_index] comp = float(line_split[comp_index]) cont = float(line_split[cont_index]) sh_100 = 0 if sh_100_index: sh_100 = self._convert_float(line_split[sh_100_index]) gs = int(line_split[gs_index]) contig_count = int(line_split[contig_count_index]) n50 = int(line_split[n50_index]) scaffold_count = int(line_split[scaffold_count_index]) ambiguous_bases = int(line_split[ambiguous_bases_index]) total_gap_len = int(line_split[total_gap_len_index]) ssu_count = int(line_split[ssu_count_index]) ssu_length = self._convert_int(line_split[ssu_length_index]) ncbi_molecule_count = self._convert_int(line_split[ncbi_molecule_count_index]) ncbi_unspanned_gaps = self._convert_int(line_split[ncbi_unspanned_gaps_index]) ncbi_spanned_gaps = self._convert_int(line_split[ncbi_spanned_gaps_index]) gtdb_is_rep = line_split[gtdb_rep_index] == 't' gtdb_rid = canonical_gid(line_split[gtdb_genome_rep_index]) if create_sp_clusters: self.sp_clusters.update_sp_cluster(gtdb_rid, gid, gtdb_taxonomy.species) if 'lpsn_priority_year' in headers: lpsn_priority_year = self._convert_int(line_split[lpsn_priority_index], Genome.NO_PRIORITY_YEAR) dsmz_priority_year = self._convert_int(line_split[dsmz_priority_index], Genome.NO_PRIORITY_YEAR) straininfo_priority_year = self._convert_int(line_split[straininfo_priority_index], Genome.NO_PRIORITY_YEAR) else: lpsn_priority_year = Genome.NO_PRIORITY_YEAR dsmz_priority_year = Genome.NO_PRIORITY_YEAR straininfo_priority_year = Genome.NO_PRIORITY_YEAR self.genomes[gid] = Genome(gid, ncbi_accn, gtdb_rid, gtdb_is_rep, gtdb_taxonomy, ncbi_taxonomy, ncbi_taxonomy_unfiltered, gtdb_type, gtdb_type_sources, gtdb_type_species_of_genus, gid in untrustworthy_as_type, ncbi_type, ncbi_strain_identifiers, ncbi_asm_level, ncbi_genome_representation, ncbi_refseq_cat, ncbi_genome_cat, excluded_from_refseq_note.get(gid, ''), comp, cont, sh_100, gs, contig_count, n50, scaffold_count, ambiguous_bases, total_gap_len, ssu_count, ssu_length, ncbi_molecule_count, ncbi_unspanned_gaps, ncbi_spanned_gaps, lpsn_priority_year, dsmz_priority_year, straininfo_priority_year) self._apply_ncbi_taxonomy_ledgers(species_exception_file, genus_exception_file)
def run(self, cur_gtdb_metadata_file, cur_genomic_path_file, qc_passed_file, ncbi_genbank_assembly_file, ltp_taxonomy_file, gtdb_type_strains_ledger, untrustworthy_type_ledger): """Resolve cases where a species has multiple genomes assembled from the type strain.""" # get species in LTP reference database self.logger.info('Determining species defined in LTP reference database.') ltp_defined_species = self.ltp_defined_species(ltp_taxonomy_file) self.logger.info(f' ... identified {len(ltp_defined_species):,} species.') # create current GTDB genome sets self.logger.info('Creating current GTDB genome set.') cur_genomes = Genomes() cur_genomes.load_from_metadata_file(cur_gtdb_metadata_file, gtdb_type_strains_ledger=gtdb_type_strains_ledger, create_sp_clusters=False, uba_genome_file=None, qc_passed_file=qc_passed_file, ncbi_genbank_assembly_file=ncbi_genbank_assembly_file, untrustworthy_type_ledger=untrustworthy_type_ledger) cur_genomes.load_genomic_file_paths(cur_genomic_path_file) self.logger.info(f' ... current genome set contains {len(cur_genomes):,} genomes.') # update current genomes with GTDB-Tk classifications self.logger.info('Updating current genomes with GTDB-Tk classifications.') num_updated, num_ncbi_sp = cur_genomes.set_gtdbtk_classification(gtdbtk_classify_file, prev_genomes) self.logger.info(f' ... set GTDB taxa for {num_updated:,} genomes with {num_ncbi_sp:,} genomes using NCBI genus and species name.') # parsing genomes manually established to be untrustworthy as type self.logger.info('Determining genomes manually annotated as untrustworthy as type.') manual_untrustworthy_types = {} with open(untrustworthy_type_ledger) as f: header = f.readline().strip().split('\t') ncbi_sp_index = header.index('NCBI species') reason_index = header.index('Reason for declaring untrustworthy') for line in f: tokens = line.strip().split('\t') gid = canonical_gid(tokens[0]) manual_untrustworthy_types[gid] = (tokens[ncbi_sp_index], tokens[reason_index]) self.logger.info(f' ... identified {len(manual_untrustworthy_types):,} genomes manually annotated as untrustworthy as type.') # identify NCBI species with multiple genomes assembled from type strain of species self.logger.info('Determining number of type strain genomes in each NCBI species.') sp_type_strain_genomes = defaultdict(set) for gid in cur_genomes: if cur_genomes[gid].is_effective_type_strain(): ncbi_sp = cur_genomes[gid].ncbi_taxa.species if ncbi_sp != 's__': # yes, NCBI has genomes marked as assembled from type material # that do not actually have a binomial species name sp_type_strain_genomes[ncbi_sp].add(gid) multi_type_strains_sp = [ncbi_sp for ncbi_sp, gids in sp_type_strain_genomes.items() if len(gids) > 1] self.logger.info(f' ... identified {len(multi_type_strains_sp):,} NCBI species with multiple assemblies indicated as being type strain genomes.') # sort by number of genome assemblies self.logger.info('Calculating ANI between type strain genomes in each species.') fout = open(os.path.join(self.output_dir, 'multi_type_strain_species.tsv'), 'w') fout.write('NCBI species\tNo. type strain genomes\t>=99% ANI\tMean ANI\tStd ANI\tMean AF\tStd AF\tResolution\tGenome IDs\n') fout_genomes = open(os.path.join(self.output_dir, 'type_strain_genomes.tsv'), 'w') fout_genomes.write('Genome ID\tUntrustworthy\tNCBI species\tGTDB genus\tGTDB species\tLTP species\tConflict with prior GTDB assignment') fout_genomes.write('\tMean ANI\tStd ANI\tMean AF\tStd AF\tExclude from RefSeq\tNCBI taxonomy\tGTDB taxonomy\n') fout_unresolved = open(os.path.join(self.output_dir, 'unresolved_type_strain_genomes.tsv'), 'w') fout_unresolved.write('Genome ID\tNCBI species\tGTDB genus\tGTDB species\tLTP species') fout_unresolved.write('\tMean ANI\tStd ANI\tMean AF\tStd AF\tExclude from RefSeq\tNCBI taxonomy\tGTDB taxonomy\n') fout_high_divergence = open(os.path.join(self.output_dir, 'highly_divergent_type_strain_genomes.tsv'), 'w') fout_high_divergence.write('Genome ID\tNCBI species\tGTDB genus\tGTDB species\tLTP species\tMean ANI\tStd ANI\tMean AF\tStd AF\tExclude from RefSeq\tNCBI taxonomy\tGTDB taxonomy\n') fout_untrustworthy = open(os.path.join(self.output_dir, 'untrustworthy_type_material.tsv'), 'w') fout_untrustworthy.write('Genome ID\tNCBI species\tGTDB species\tLTP species\tReason for declaring untrustworthy\n') for gid in manual_untrustworthy_types: ncbi_sp, reason = manual_untrustworthy_types[gid] fout_untrustworthy.write('{}\t{}\t{}\t{}\t{}\n'.format( gid, ncbi_sp, cur_genomes[gid].gtdb_taxa.species, '<not tested>', 'n/a', 'Manual curation: ' + reason)) processed = 0 num_divergent = 0 unresolved_sp_count = 0 ncbi_ltp_resolved = 0 intra_ani_resolved = 0 ncbi_type_resolved = 0 gtdb_family_resolved = 0 gtdb_genus_resolved = 0 gtdb_sp_resolved = 0 ltp_resolved = 0 use_pickled_results = False #*** if use_pickled_results: self.logger.warning('Using previously calculated ANI results in: {}'.format(self.ani_pickle_dir)) prev_gtdb_sp_conflicts = 0 for ncbi_sp, type_gids in sorted(sp_type_strain_genomes.items(), key=lambda kv: len(kv[1])): if len(type_gids) == 1: continue status_str = '-> Processing {} with {:,} type strain genomes [{:,} of {:,} ({:.2f}%)].'.format( ncbi_sp, len(type_gids), processed+1, len(multi_type_strains_sp), (processed+1)*100.0/len(multi_type_strains_sp)).ljust(128) sys.stdout.write('{}\r'.format(status_str)) sys.stdout.flush() processed += 1 # calculate ANI between type strain genomes ncbi_sp_str = ncbi_sp[3:].lower().replace(' ', '_') if not use_pickled_results: #*** ani_af = self.fastani.pairwise(type_gids, cur_genomes.genomic_files) pickle.dump(ani_af, open(os.path.join(self.ani_pickle_dir, f'{ncbi_sp_str}.pkl'), 'wb')) else: ani_af = pickle.load(open(os.path.join(self.ani_pickle_dir, f'{ncbi_sp_str}.pkl'), 'rb')) anis = [] afs = [] gid_anis = defaultdict(lambda: {}) gid_afs = defaultdict(lambda: {}) all_similar = True for gid1, gid2 in combinations(type_gids, 2): ani, af = symmetric_ani(ani_af, gid1, gid2) if ani < 99 or af < 0.65: all_similar = False anis.append(ani) afs.append(af) gid_anis[gid1][gid2] = ani gid_anis[gid2][gid1] = ani gid_afs[gid1][gid2] = af gid_afs[gid2][gid1] = af note = 'All type strain genomes have ANI >99% and AF >65%.' unresolved_species = False # read LTP metadata for genomes ltp_metadata = self.parse_ltp_metadata(type_gids, cur_genomes) untrustworthy_gids = {} gtdb_resolved_sp_conflict = False if not all_similar: # need to establish which genomes are untrustworthy as type num_divergent += 1 unresolved_species = True # write out highly divergent cases for manual inspection; # these should be compared to the automated selection if np_mean(anis) < 95: for gid in type_gids: ltp_species = self.ltp_species(gid, ltp_metadata) fout_high_divergence.write('{}\t{}\t{}\t{}\t{}\t{:.2f}\t{:.3f}\t{:.3f}\t{:.4f}\t{}\t{}\t{}\n'.format( gid, ncbi_sp, cur_genomes[gid].gtdb_taxa.genus, cur_genomes[gid].gtdb_taxa.species, ' / '.join(ltp_species), np_mean(list(gid_anis[gid].values())), np_std(list(gid_anis[gid].values())), np_mean(list(gid_afs[gid].values())), np_std(list(gid_afs[gid].values())), cur_genomes[gid].excluded_from_refseq_note, cur_genomes[gid].ncbi_taxa, cur_genomes[gid].gtdb_taxa)) # filter genomes marked as `untrustworthy as type` at NCBI and where the LTP # assignment also suggest the asserted type material is incorrect resolved, untrustworthy_gids = self.resolve_validated_untrustworthy_ncbi_genomes(gid_anis, ncbi_sp, type_gids, ltp_metadata, ltp_defined_species, cur_genomes) if resolved: note = "Species resolved by removing genomes considered `untrustworthy as type` and with a LTP BLAST hit confirming the assembly is likely untrustworthy" ncbi_ltp_resolved += 1 # try to resolve by LTP 16S BLAST results if not resolved: resolved, untrustworthy_gids = self.resolve_ltp_conflict(gid_anis, ncbi_sp, type_gids, ltp_metadata, 0) if resolved: note = 'Species resolved by identifying conflicting or lack of LTP BLAST results' ltp_resolved += 1 # try to resolve species using intra-specific ANI test if not resolved: resolved, untrustworthy_gids = self.resolve_by_intra_specific_ani(gid_anis) if resolved: note = 'Species resolved by intra-specific ANI test' intra_ani_resolved += 1 # try to resolve by GTDB family assignment if not resolved: resolved, untrustworthy_gids = self.resolve_gtdb_family(gid_anis, ncbi_sp, type_gids, cur_genomes) if resolved: note = 'Species resolved by consulting GTDB family classifications' gtdb_family_resolved += 1 # try to resolve by GTDB genus assignment if not resolved: resolved, untrustworthy_gids = self.resolve_gtdb_genus(gid_anis, ncbi_sp, type_gids, cur_genomes) if resolved: note = 'Species resolved by consulting GTDB genus classifications' gtdb_genus_resolved += 1 # try to resolve by GTDB species assignment if not resolved: resolved, untrustworthy_gids = self.resolve_gtdb_species(gid_anis, ncbi_sp, type_gids, cur_genomes) if resolved: note = 'Species resolved by consulting GTDB species classifications' gtdb_sp_resolved += 1 # try to resolve by considering genomes annotated as type material at NCBI, # which includes considering if genomes are marked as untrustworthy as type if not resolved: resolved, untrustworthy_gids = self.resolve_by_ncbi_types(gid_anis, type_gids, cur_genomes) if resolved: note = 'Species resolved by consulting NCBI assembled from type metadata' ncbi_type_resolved += 1 if resolved: unresolved_species = False # check if type strain genomes marked as trusted or untrusted conflict # with current GTDB species assignment untrustworthy_gtdb_sp_match = False trusted_gtdb_sp_match = False for gid in type_gids: gtdb_canonical_epithet = canonical_taxon(specific_epithet(cur_genomes[gid].gtdb_taxa.species)) if gtdb_canonical_epithet == specific_epithet(ncbi_sp): if gid in untrustworthy_gids: untrustworthy_gtdb_sp_match = True else: trusted_gtdb_sp_match = True if untrustworthy_gtdb_sp_match and not trusted_gtdb_sp_match: prev_gtdb_sp_conflicts += 1 gtdb_resolved_sp_conflict = True # write results to file for gid, reason in untrustworthy_gids.items(): ltp_species = self.ltp_species(gid, ltp_metadata) if 'untrustworthy as type' in cur_genomes[gid].excluded_from_refseq_note: reason += "; considered `untrustworthy as type` at NCBI" fout_untrustworthy.write('{}\t{}\t{}\t{}\t{}\n'.format(gid, ncbi_sp, cur_genomes[gid].gtdb_taxa.species, ' / '.join(ltp_species), reason)) # Sanity check that if the untrustworthy genome has an LTP to only the # expected species, that all other genomes also have a hit to the # expected species (or potentially no hit). Otherwise, more consideration # should be given to the genome with the conflicting LTP hit. if len(ltp_species) == 1 and ncbi_sp in ltp_species: other_sp = set() for test_gid in type_gids: ltp_species = self.ltp_species(test_gid, ltp_metadata) if ltp_species and ncbi_sp not in ltp_species: other_sp.update(ltp_species) if other_sp: self.logger.warning(f'Genome {gid} marked as untrustworthy, but this conflicts with high confidence LTP 16S rRNA assignment.') num_ncbi_untrustworthy = sum([1 for gid in type_gids if 'untrustworthy as type' in cur_genomes[gid].excluded_from_refseq_note]) if num_ncbi_untrustworthy != len(type_gids): for gid in type_gids: if (gid not in untrustworthy_gids and 'untrustworthy as type' in cur_genomes[gid].excluded_from_refseq_note): self.logger.warning("Retaining genome {} from {} despite being marked as `untrustworthy as type` at NCBI [{:,} of {:,} considered untrustworthy].".format( gid, ncbi_sp, num_ncbi_untrustworthy, len(type_gids))) else: note = 'Species is unresolved; manual curation is required!' unresolved_sp_count += 1 if unresolved_species: for gid in type_gids: ltp_species = self.ltp_species(gid, ltp_metadata) fout_unresolved.write('{}\t{}\t{}\t{}\t{}\t{:.2f}\t{:.3f}\t{:.3f}\t{:.4f}\t{}\t{}\t{}\n'.format( gid, ncbi_sp, cur_genomes[gid].gtdb_taxa.genus, cur_genomes[gid].gtdb_taxa.species, ' / '.join(ltp_species), np_mean(list(gid_anis[gid].values())), np_std(list(gid_anis[gid].values())), np_mean(list(gid_afs[gid].values())), np_std(list(gid_afs[gid].values())), cur_genomes[gid].excluded_from_refseq_note, cur_genomes[gid].ncbi_taxa, cur_genomes[gid].gtdb_taxa)) for gid in type_gids: ltp_species = self.ltp_species(gid, ltp_metadata) fout_genomes.write('{}\t{}\t{}\t{}\t{}\t{}\t{}\t{:.2f}\t{:.3f}\t{:.3f}\t{:.4f}\t{}\t{}\t{}\n'.format( gid, gid in untrustworthy_gids, ncbi_sp, cur_genomes[gid].gtdb_taxa.genus, cur_genomes[gid].gtdb_taxa.species, ' / '.join(ltp_species), gtdb_resolved_sp_conflict, np_mean(list(gid_anis[gid].values())), np_std(list(gid_anis[gid].values())), np_mean(list(gid_afs[gid].values())), np_std(list(gid_afs[gid].values())), cur_genomes[gid].excluded_from_refseq_note, cur_genomes[gid].ncbi_taxa, cur_genomes[gid].gtdb_taxa)) fout.write('{}\t{}\t{}\t{:.2f}\t{:.3f}\t{:.3f}\t{:.4f}\t{}\t{}\n'.format( ncbi_sp, len(type_gids), all_similar, np_mean(anis), np_std(anis), np_mean(afs), np_std(afs), note, ', '.join(type_gids))) sys.stdout.write('\n') fout.close() fout_unresolved.close() fout_high_divergence.close() fout_genomes.close() fout_untrustworthy.close() self.logger.info(f'Identified {num_divergent:,} species with 1 or more divergent type strain genomes.') self.logger.info(f' ... resolved {ncbi_ltp_resolved:,} species by removing NCBI `untrustworthy as type` genomes with a conflicting LTP 16S rRNA classifications.') self.logger.info(f' ... resolved {ltp_resolved:,} species by considering conflicting LTP 16S rRNA classifications.') self.logger.info(f' ... resolved {intra_ani_resolved:,} species by considering intra-specific ANI values.') self.logger.info(f' ... resolved {gtdb_family_resolved:,} species by considering conflicting GTDB family classifications.') self.logger.info(f' ... resolved {gtdb_genus_resolved:,} species by considering conflicting GTDB genus classifications.') self.logger.info(f' ... resolved {gtdb_sp_resolved:,} species by considering conflicting GTDB species classifications.') self.logger.info(f' ... resolved {ncbi_type_resolved:,} species by considering type material designations at NCBI.') if unresolved_sp_count > 0: self.logger.warning(f'There are {unresolved_sp_count:,} unresolved species with multiple type strain genomes.') self.logger.warning('These should be handled before proceeding with the next step of GTDB species updating.') self.logger.warning("This can be done by manual curation and adding genomes to 'untrustworthy_type_ledger'.") self.logger.info(f'Identified {prev_gtdb_sp_conflicts:,} cases where resolved type strain conflicts with prior GTDB assignment.')
def identify_misclassified_genomes_cluster(self, cur_genomes, cur_clusters, ncbi_untrustworthy_sp_ledger): """Identify genomes with erroneous NCBI species assignments, based on GTDB clustering of type strain genomes.""" forbidden_names = set(['cyanobacterium']) # get mapping from genomes to their representatives gid_to_rid = {} for rid, cids in cur_clusters.items(): for cid in cids: gid_to_rid[cid] = rid # get genomes with NCBI species assignment ncbi_sp_gids = defaultdict(list) for gid in cur_genomes: ncbi_species = cur_genomes[gid].ncbi_taxa.species ncbi_specific = specific_epithet(ncbi_species) if ncbi_species != 's__' and ncbi_specific not in forbidden_names: ncbi_sp_gids[ncbi_species].append(gid) # get NCBI species anchored by a type strain genome ncbi_type_anchored_species = {} for rid, cids in cur_clusters.items(): for cid in cids: if cur_genomes[cid].is_effective_type_strain(): ncbi_type_species = cur_genomes[cid].ncbi_taxa.species ncbi_specific = specific_epithet(ncbi_species) if ncbi_type_species != 's__' and ncbi_specific not in forbidden_names: if (ncbi_type_species in ncbi_type_anchored_species and rid != ncbi_type_anchored_species[ncbi_type_species]): self.logger.error( 'NCBI species {} has multiple effective type strain genomes in different clusters.' .format(ncbi_type_species)) sys.exit(-1) ncbi_type_anchored_species[ncbi_type_species] = rid self.logger.info( ' - identified {:,} NCBI species anchored by a type strain genome.' .format(len(ncbi_type_anchored_species))) # identify genomes with erroneous NCBI species assignments fout = open( os.path.join(self.output_dir, 'ncbi_misclassified_sp.gtdb_clustering.tsv'), 'w') fout.write( 'Genome ID\tNCBI species\tGenome cluster\tType species cluster\n') misclassified_gids = set() for ncbi_species, species_gids in ncbi_sp_gids.items(): if ncbi_species not in ncbi_type_anchored_species: continue # find genomes with NCBI species assignments that are in a # different cluster than the type strain genome type_rid = ncbi_type_anchored_species[ncbi_species] for gid in species_gids: cur_rid = gid_to_rid[gid] if type_rid != cur_rid: misclassified_gids.add(gid) fout.write('{}\t{}\t{}\t{}\t\n'.format( gid, ncbi_species, cur_rid, type_rid)) # add in genomes manually indicated as having erroneous NCBI species assignments with open(ncbi_untrustworthy_sp_ledger) as f: f.readline() for line in f: tokens = line.strip().split('\t') gid = canonical_gid(tokens[0]) fout.write('{}\t{}\t{}\t{}\t\n'.format( gid, cur_genomes[gid].ncbi_taxa.species, gid_to_rid[gid], "Manually marked as being erroneous")) fout.close() misclassified_species = { cur_genomes[gid].ncbi_taxa.species for gid in misclassified_gids } self.logger.info( ' - identified {:,} genomes from {:,} species as having misclassified NCBI species assignments.' .format(len(misclassified_gids), len(misclassified_species))) return misclassified_gids
def new_ncbi_genera(self, prev_genomes, cur_genomes, cur_clusters, gtdbtk_classify_file): """Determine new NCBI genera that likely need to be considered by curators.""" self.logger.info( 'Determining new NCBI genera for consideration by curators.') # read GTDB-Tk classification information gtdbtk = {} with open(gtdbtk_classify_file) as f: header = f.readline().strip().split('\t') classification_index = header.index('classification') red_index = header.index('red_value') note_index = header.index('note') for line in f: tokens = line.strip().split('\t') gid = tokens[0] gid = canonical_gid(gid) classification = [ t.strip() for t in tokens[classification_index].split(';') ] red = tokens[red_index] if red != 'N/A': red = float(tokens[red_index]) gtdbtk[gid] = (classification, red, tokens[note_index]) # get NCBI genera in previous GTDB release prev_ncbi_genera = set() for gid in prev_genomes: ncbi_genera = prev_genomes[gid].ncbi_taxa.genus prev_ncbi_genera.add(ncbi_genera) self.logger.info( ' ... identified {:,} NCBI genera in previous GTDB release.'. format(len(prev_ncbi_genera))) # get NCBI genera in current GTDB release cur_ncbi_genera = set() for rid in cur_clusters: ncbi_genera = cur_genomes[rid].ncbi_taxa.genus cur_ncbi_genera.add(ncbi_genera) self.logger.info( ' ... identified {:,} NCBI genera in current GTDB release.'.format( len(cur_ncbi_genera))) # determine new NCBI genera new_ncbi_genera = cur_ncbi_genera - prev_ncbi_genera self.logger.info( ' ... identified {:,} NCBI genera that are new to the current GTDB release.' .format(len(new_ncbi_genera))) # determine genomes from new NCBI genera new_genera_rids = defaultdict(list) for rid in cur_clusters: ncbi_genera = cur_genomes[rid].ncbi_taxa.genus if ncbi_genera in new_ncbi_genera: new_genera_rids[ncbi_genera].append(rid) # report each new NCBI genera fout = open(os.path.join(self.output_dir, 'new_ncbi_genera.tsv'), 'w') fout.write( 'NCBI genus\tRepresentative ID(s)\tNew genomes\tGTDB-Tk classification\tNCBI genus conflict\tRED\tGTDB-Tk note\n' ) for new_genus in new_ncbi_genera: new_genomes = [] red_values = [] gtdbtk_classification = [] gtdbtk_notes = [] ncbi_genus_conflict = False for rid in new_genera_rids[new_genus]: new_genomes.append(str(rid not in prev_genomes)) if rid in gtdbtk: classification, red, note = gtdbtk[rid] gtdbtk_notes.append(note) if red != 'N/A': red_values.append('{:.2f}'.format(red)) else: red_values.append('n/a') if classification[6] != 's__': gtdbtk_classification.append(classification[6]) else: gtdbtk_classification.append(classification[5]) gtdbtk_genus = classification[5] if gtdbtk_genus != 'g__' and gtdbtk_genus != new_genus: ncbi_genus_conflict = True else: red_values.append('n/a') gtdbtk_classification.append('n/a') gtdbtk_notes.append('n/a') fout.write('{}\t{}\t{}\t{}\t{}\t{}\t{}\n'.format( new_genus, ', '.join(new_genera_rids[new_genus]), ', '.join(new_genomes), ', '.join(gtdbtk_classification), str(ncbi_genus_conflict), ', '.join(red_values), ', '.join(gtdbtk_notes))) fout.close()
def run(self, metadata_file, cur_uba_gid_file, ncbi_genbank_assembly_file, gtdb_domain_report, qc_exception_file, min_comp, max_cont, min_quality, sh_exception, min_perc_markers, max_contigs, min_N50, max_ambiguous, output_dir): """Quality check all potential GTDB genomes.""" # create current GTDB genome sets self.logger.info('Creating current GTDB genome set.') cur_genomes = Genomes() cur_genomes.load_from_metadata_file(metadata_file, create_sp_clusters=False, uba_genome_file=cur_uba_gid_file) self.logger.info(f' ...current genome set contains {len(cur_genomes):,} genomes.') # parse genomes flagged as exceptions from QC qc_exceptions = set() with open(qc_exception_file, encoding='utf-8') as f: f.readline() for line in f: gid = canonical_gid(line.split('\t')[0].strip()) qc_exceptions.add(gid) self.logger.info(f'Identified {len(qc_exceptions):,} genomes flagged as exceptions from QC.') # get percentage of bac120 or ar122 marker genes marker_perc = self.read_marker_percentages(gtdb_domain_report, cur_genomes) # parse NCBI assembly files self.logger.info('Parsing NCBI assembly files.') excluded_from_refseq_note = exclude_from_refseq(ncbi_genbank_assembly_file) # QC all genomes self.logger.info('Validating genomes.') fout_retained = open(os.path.join(output_dir, 'qc_passed.tsv'), 'w') fout_failed = open(os.path.join(output_dir, 'qc_failed.tsv'), 'w') header = 'Accession\tNCBI species\tGTDB taxonomy' header += '\tCompleteness (%)\tContamination (%)\tQuality\tStrain heterogeneity at 100%' header += '\tMarkers (%)\tNo. contigs\tN50 contigs\tAmbiguous bases' fout_retained.write(header + '\tNote\n') fout_failed.write(header) fout_failed.write('\tFailed completeness\tFailed contamination\tFailed quality') fout_failed.write('\tFailed marker percentage\tFailed no. contigs\tFailed N50 contigs\tFailed ambiguous bases\n') pass_qc_gids = set() failed_qc_gids = set() for gid in cur_genomes: failed_tests = defaultdict(int) passed_qc = cur_genomes[gid].pass_qc(marker_perc[gid], min_comp, max_cont, min_quality, sh_exception, min_perc_markers, max_contigs, min_N50, max_ambiguous, failed_tests) if passed_qc or gid in qc_exceptions: pass_qc_gids.add(gid) fout_retained.write('%s\t%s\t%s' % (gid, cur_genomes[gid].ncbi_taxa.species, cur_genomes[gid].gtdb_taxa)) fout_retained.write('\t%.2f\t%.2f\t%.2f\t%s\t%.2f\t%d\t%d\t%d\t%s\n' % ( cur_genomes[gid].comp, cur_genomes[gid].cont, cur_genomes[gid].comp-5*cur_genomes[gid].cont, ('%.2f' % cur_genomes[gid].strain_heterogeneity_100) if cur_genomes[gid].strain_heterogeneity_100 else '-', marker_perc[gid], cur_genomes[gid].contig_count, cur_genomes[gid].contig_n50, cur_genomes[gid].ambiguous_bases, 'Passed QC' if passed_qc else 'Flagged as exception')) else: failed_qc_gids.add(gid) fout_failed.write('%s\t%s\t%s' % (gid, cur_genomes[gid].ncbi_taxa.species, cur_genomes[gid].gtdb_taxa)) fout_failed.write('\t%.2f\t%.2f\t%.2f\t%s\t%.2f\t%d\t%d\t%d' % ( cur_genomes[gid].comp, cur_genomes[gid].cont, cur_genomes[gid].comp-5*cur_genomes[gid].cont, ('%.2f' % cur_genomes[gid].strain_heterogeneity_100) if cur_genomes[gid].strain_heterogeneity_100 else '-', marker_perc[gid], cur_genomes[gid].contig_count, cur_genomes[gid].contig_n50, cur_genomes[gid].ambiguous_bases)) fout_failed.write('\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n' % ( failed_tests['comp'], failed_tests['cont'], failed_tests['qual'], failed_tests['marker_perc'], failed_tests['contig_count'], failed_tests['N50'], failed_tests['ambig'])) fout_retained.close() fout_failed.close() self.logger.info('Retained {:,} ({:.2f}%) genomes and filtered {:,} ({:.2f}%) genomes.'.format( len(pass_qc_gids), len(pass_qc_gids)*100.0/len(cur_genomes), len(failed_qc_gids), len(failed_qc_gids)*100.0/len(cur_genomes))) # check domain assignment of genomes passing QC # report potential issues self.check_domain_assignments(gtdb_domain_report, cur_genomes, pass_qc_gids) # QC genomes in each named species named_ncbi_species = cur_genomes.named_ncbi_species() self.logger.info(f'Performing QC of type genome for each of the {len(named_ncbi_species):,} NCBI species.') fout_type_fail = open(os.path.join(output_dir, 'type_genomes_fail_qc.tsv'), 'w') fout_type_fail.write('NCBI species\tAccession\tGTDB taxonomy\tNCBI taxonomy\tType sources\tNCBI assembly type\tGenome size (bp)') fout_type_fail.write('\tCompleteness (%)\tContamination (%)\tQuality\tStrain heterogeneity at 100%') fout_type_fail.write('\tMarkers (%)\tNo. contigs\tN50 contigs\tAmbiguous bases\tNCBI exclude from RefSeq\tLost species\n') fout_fail_sp = open(os.path.join(output_dir, 'species_fail_qc.tsv'), 'w') fout_fail_sp.write('NCBI species\tAccession\tGTDB taxonomy\tNCBI taxonomy\tAssembled from type material\tGenome size (bp)') fout_fail_sp.write('\tCompleteness (%)\tContamination (%)\tQuality\tStrain heterogeneity at 100%') fout_fail_sp.write('\tMarkers (%)\tNo. contigs\tN50 contigs\tAmbiguous bases') fout_fail_sp.write('\tFailed completeness\tFailed contamination\tFailed quality') fout_fail_sp.write('\tFailed marker percentage\tFailed no. contigs\tFailed N50 contigs\tFailed ambiguous bases') fout_fail_sp.write('\tNCBI exclude from RefSeq\n') fout_sp_lost = open(os.path.join(output_dir, 'species_lost.tsv'), 'w') fout_sp_lost.write('NCBI species\tNo. genomes\tNo. type genomes') fout_sp_lost.write('\tFail completeness\tFail contamination\tFail quality\tFailed percent markers') fout_sp_lost.write('\tFail no. contigs\tFail N50 contigs\tFail ambiguous bases\n') lost_type = 0 lost_sp = 0 filtered_genomes = 0 failed_tests_cumulative = defaultdict(int) for sp, gids in named_ncbi_species.items(): type_pass = set() type_fail = set() other_pass = set() other_fail = set() failed_tests_gids = {} for gid in gids: failed_tests = defaultdict(int) passed_qc = cur_genomes[gid].pass_qc(marker_perc[gid], min_comp, max_cont, min_quality, sh_exception, min_perc_markers, max_contigs, min_N50, max_ambiguous, failed_tests) failed_tests_gids[gid] = failed_tests if cur_genomes[gid].is_gtdb_type_strain() or cur_genomes[gid].is_ncbi_type_strain(): if passed_qc or gid in qc_exceptions: type_pass.add(gid) else: type_fail.add(gid) filtered_genomes += 1 else: if passed_qc or gid in qc_exceptions: other_pass.add(gid) else: other_fail.add(gid) filtered_genomes += 1 # tally failed species for test, count in failed_tests.items(): failed_tests_cumulative[test] += count if len(type_pass) >= 1: # great: one or more type genomes pass QC and will be selected as the type genome continue if len(type_fail): # all potential type genomes for species failed QC so report these for manual inspection lost_type += 1 for gid in type_fail: fout_type_fail.write('%s\t%s\t%s\t%s\t%s\t%s\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%d\t%d\t%d\t%s\t%s\n' % ( sp, gid, cur_genomes[gid].gtdb_taxa, cur_genomes[gid].ncbi_taxa, cur_genomes[gid].gtdb_type_designation_sources, cur_genomes[gid].ncbi_type_material, float(cur_genomes[gid].length)/1e6, cur_genomes[gid].comp, cur_genomes[gid].cont, cur_genomes[gid].comp-5*cur_genomes[gid].cont, cur_genomes[gid].strain_heterogeneity_100, marker_perc[gid], cur_genomes[gid].contig_count, cur_genomes[gid].contig_n50, cur_genomes[gid].ambiguous_bases, excluded_from_refseq_note[gid], len(other_pass) == 0)) if len(other_pass) == 0: # no genomes for species pass QC so report loss of species lost_sp += 1 fout_sp_lost.write('%s\t%d\t%d' % (sp, len(gids), len(type_fail))) fout_sp_lost.write('\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n' % ( sum([failed_tests_gids[gid]['comp'] for gid in gids]), sum([failed_tests_gids[gid]['cont'] for gid in gids]), sum([failed_tests_gids[gid]['qual'] for gid in gids]), sum([failed_tests_gids[gid]['marker_perc'] for gid in gids]), sum([failed_tests_gids[gid]['contig_count'] for gid in gids]), sum([failed_tests_gids[gid]['N50'] for gid in gids]), sum([failed_tests_gids[gid]['ambig'] for gid in gids]))) for gid in type_fail.union(other_fail): fout_fail_sp.write('%s\t%s\t%s\t%s\t%s\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%d\t%d\t%d' % ( sp, gid, cur_genomes[gid].gtdb_taxa, cur_genomes[gid].ncbi_taxa, gid in type_fail, float(cur_genomes[gid].length)/1e6, cur_genomes[gid].comp, cur_genomes[gid].cont, cur_genomes[gid].comp-5*cur_genomes[gid].cont, cur_genomes[gid].strain_heterogeneity_100, marker_perc[gid], cur_genomes[gid].contig_count, cur_genomes[gid].contig_n50, cur_genomes[gid].ambiguous_bases)) fout_fail_sp.write('\t%d\t%d\t%d\t%d\t%d\t%d\t%d' % ( failed_tests_gids[gid]['comp'], failed_tests_gids[gid]['cont'], failed_tests_gids[gid]['qual'], failed_tests_gids[gid]['marker_perc'], failed_tests_gids[gid]['contig_count'], failed_tests_gids[gid]['N50'], failed_tests_gids[gid]['ambig'])) fout_fail_sp.write('\t%s\n' % excluded_from_refseq_note[gid]) fout_type_fail.close() fout_fail_sp.close() fout_sp_lost.close() self.logger.info(f'Filtered {filtered_genomes:,} genomes assigned to NCBI species.') self.logger.info(f'Identified {lost_type:,} species with type genomes failing QC and {lost_sp:,} total species failing QC.') self.logger.info('Genomes from NCBI species filtered by each criterion:') for test in sorted(failed_tests_cumulative): self.logger.info(f'{test}: {failed_tests_cumulative[test]:,}')
def read_gtdb_ncbi_taxonomy(metadata_file, species_exception_file, genus_exception_file): """Parse NCBI taxonomy from GTDB metadata. Parameters ---------- metadata_file : str Metadata for all genomes. Return ------ dict : d[genome_id] -> taxonomy list """ taxonomy = {} with open(metadata_file, encoding='utf-8') as f: headers = f.readline().strip().split('\t') genome_index = headers.index('accession') taxonomy_index = headers.index('ncbi_taxonomy') for line in f: line_split = [token.strip() for token in line.strip().split('\t')] gid = canonical_gid(line_split[genome_index]) taxa_str = line_split[taxonomy_index].strip() taxa_str = taxa_str.replace('Candidatus ', '') if taxa_str and taxa_str != 'none': taxonomy[gid] = [t.strip() for t in taxa_str.split(';')] else: taxonomy[gid] = list(Taxonomy.rank_prefixes) ncbi_update_count = 0 species_updates = {} if species_exception_file: with open(species_exception_file, encoding='utf-8') as f: f.readline() for line in f: line_split = [token.strip() for token in line.strip().split('\t')] gid = canonical_gid(line_split[0]) sp = line_split[1].replace('Candidatus ', '') if gid not in taxonomy: print('Genome in species exception list not defined at NCBI: %s' % gid) sys.exit(-1) if not sp.startswith('s__'): sp = 's__' + sp taxonomy[gid][6] = sp ncbi_update_count += 1 species_updates[gid] = sp if genus_exception_file: with open(genus_exception_file, encoding='utf-8') as f: f.readline() for line in f: line_split = [token.strip() for token in line.strip().split('\t')] gid = canonical_gid(line_split[0]) genus = line_split[1] if gid not in taxonomy: print('Genome in genus exception list not defined at NCBI: %s' % gid) sys.exit(-1) if genus.startswith('g__'): genus = genus[3:] taxonomy[gid][5] = f'g__{genus}' species = taxonomy[gid][6] if species != 's__': generic, specific = generic_specific_names(species) taxonomy[gid][6] = f's__{genus} {specific}' ncbi_update_count += 1 # sanity check ledgers if gid in species_updates and genus not in species_updates[gid]: self.logger.error(f'Species and genus ledgers have conflicting assignments for {gid}.') sys.exit(-1) return taxonomy, ncbi_update_count
def load_from_metadata_file(self, metadata_file, species_exception_file=None, genus_exception_file=None, gtdb_type_strains_ledger=None, create_sp_clusters=True, qc_passed_file=None, ncbi_genbank_assembly_file=None, untrustworthy_type_ledger=None, ncbi_untrustworthy_sp_ledger=None, ncbi_env_bioproject_ledger=None): """Create genome set from file(s).""" pass_qc_gids = set() if qc_passed_file: with open(qc_passed_file) as f: f.readline() for line in f: line_split = line.strip().split('\t') pass_qc_gids.add(line_split[0].strip()) self.logger.info( f' - identified {len(pass_qc_gids):,} genomes passing QC.') gtdb_type_strains = set() if gtdb_type_strains_ledger: with open(gtdb_type_strains_ledger) as f: f.readline() for line in f: tokens = line.strip().split('\t') gid = canonical_gid(tokens[0].strip()) gtdb_type_strains.add(gid) self.logger.info( f' - identified {len(gtdb_type_strains):,} manually annotated as type strain genomes.' ) excluded_from_refseq_note = {} ncbi_bioproject = {} if ncbi_genbank_assembly_file: ncbi_bioproject = parse_ncbi_bioproject(ncbi_genbank_assembly_file) excluded_from_refseq_note = exclude_from_refseq( ncbi_genbank_assembly_file) ncbi_env_bioproject = set() if ncbi_env_bioproject_ledger: with open(ncbi_env_bioproject_ledger) as f: f.readline() for line in f: tokens = line.strip().split('\t') ncbi_env_bioproject.add(tokens[0].strip()) untrustworthy_as_type = set() if untrustworthy_type_ledger: untrustworthy_as_type = self.parse_untrustworthy_type_ledger( untrustworthy_type_ledger) self.logger.info( f' - identified {len(untrustworthy_as_type):,} genomes annotated as untrustworthy as type by GTDB.' ) untrustworthy_ncbi_sp = set() if ncbi_untrustworthy_sp_ledger: untrustworthy_ncbi_sp = self.parse_ncbi_untrustworthy_sp_ledger( ncbi_untrustworthy_sp_ledger) self.logger.info( f' - identified {len(untrustworthy_ncbi_sp):,} genomes annotated as having untrustworthy NCBI species assignments.' ) with open(metadata_file, encoding='utf-8') as f: headers = f.readline().strip().split('\t') genome_index = headers.index('accession') gtdb_taxonomy_index = headers.index('gtdb_taxonomy') ncbi_taxonomy_index = headers.index('ncbi_taxonomy') ncbi_taxonomy_unfiltered_index = headers.index( 'ncbi_taxonomy_unfiltered') gtdb_type_index = headers.index('gtdb_type_designation') gtdb_type_sources_index = headers.index( 'gtdb_type_designation_sources') gtdb_type_species_of_genus_index = headers.index( 'gtdb_type_species_of_genus') ncbi_strain_identifiers_index = headers.index( 'ncbi_strain_identifiers') ncbi_type_index = headers.index('ncbi_type_material_designation') ncbi_asm_level_index = headers.index('ncbi_assembly_level') ncbi_genome_representation_index = headers.index( 'ncbi_genome_representation') ncbi_refseq_cat_index = headers.index('ncbi_refseq_category') ncbi_genome_cat_index = headers.index('ncbi_genome_category') comp_index = headers.index('checkm_completeness') cont_index = headers.index('checkm_contamination') sh_100_index = None if 'checkm_strain_heterogeneity_100' in headers: sh_100_index = headers.index('checkm_strain_heterogeneity_100') gs_index = headers.index('genome_size') contig_count_index = headers.index('contig_count') n50_index = headers.index('n50_contigs') scaffold_count_index = headers.index('scaffold_count') ambiguous_bases_index = headers.index('ambiguous_bases') total_gap_len_index = headers.index('total_gap_length') ssu_count_index = headers.index('ssu_count') ssu_length_index = headers.index('ssu_length') ncbi_molecule_count_index = headers.index('ncbi_molecule_count') ncbi_unspanned_gaps_index = headers.index('ncbi_unspanned_gaps') ncbi_spanned_gaps_index = headers.index('ncbi_spanned_gaps') gtdb_genome_rep_index = headers.index('gtdb_genome_representative') gtdb_rep_index = headers.index('gtdb_representative') if 'lpsn_priority_year' in headers: # this information will be missing from the previous # GTDB metadata file as we strip this out due to # concerns over republishing this information lpsn_priority_index = headers.index('lpsn_priority_year') for line in f: line_split = line.strip().split('\t') ncbi_accn = line_split[genome_index] gid = canonical_gid(ncbi_accn) self.full_gid[gid] = ncbi_accn if gid.startswith('U_'): continue if pass_qc_gids and gid not in pass_qc_gids: continue gtdb_taxonomy = Taxa(line_split[gtdb_taxonomy_index]) ncbi_taxonomy = Taxa(line_split[ncbi_taxonomy_index]) ncbi_taxonomy_unfiltered = Taxa( line_split[ncbi_taxonomy_unfiltered_index], filtered=False) gtdb_type = line_split[gtdb_type_index] gtdb_type_sources = line_split[gtdb_type_sources_index] if gid in gtdb_type_strains: gtdb_type = 'type strain of species' gtdb_type_sources = 'GTDB curator' gtdb_type_species_of_genus = line_split[ gtdb_type_species_of_genus_index] == 't' ncbi_type = line_split[ncbi_type_index] ncbi_strain_identifiers = line_split[ ncbi_strain_identifiers_index] ncbi_asm_level = line_split[ncbi_asm_level_index] ncbi_genome_representation = line_split[ ncbi_genome_representation_index] ncbi_refseq_cat = line_split[ncbi_refseq_cat_index] ncbi_genome_cat = line_split[ncbi_genome_cat_index] if ncbi_bioproject.get(gid, None) in ncbi_env_bioproject: # *** # HACK to force genomes from MAG mining projects # to be indicated as MAGs which are currently # not correctly annotated at NCBI ncbi_genome_cat = 'derived from environmental source' comp = float(line_split[comp_index]) cont = float(line_split[cont_index]) sh_100 = 0 if sh_100_index: sh_100 = self._convert_float(line_split[sh_100_index]) gs = int(line_split[gs_index]) contig_count = int(line_split[contig_count_index]) n50 = int(line_split[n50_index]) scaffold_count = int(line_split[scaffold_count_index]) ambiguous_bases = int(line_split[ambiguous_bases_index]) total_gap_len = int(line_split[total_gap_len_index]) ssu_count = int(line_split[ssu_count_index]) ssu_length = self._convert_int(line_split[ssu_length_index]) ncbi_molecule_count = self._convert_int( line_split[ncbi_molecule_count_index]) ncbi_unspanned_gaps = self._convert_int( line_split[ncbi_unspanned_gaps_index]) ncbi_spanned_gaps = self._convert_int( line_split[ncbi_spanned_gaps_index]) gtdb_is_rep = line_split[gtdb_rep_index] == 't' gtdb_rid = canonical_gid(line_split[gtdb_genome_rep_index]) if create_sp_clusters: self.sp_clusters.update_sp_cluster(gtdb_rid, gid, gtdb_taxonomy.species) lpsn_priority_year = Genome.NO_PRIORITY_YEAR if 'lpsn_priority_year' in headers: lpsn_priority_year = self._convert_int( line_split[lpsn_priority_index], Genome.NO_PRIORITY_YEAR) self.genomes[gid] = Genome( gid, ncbi_accn, gtdb_rid, gtdb_is_rep, gtdb_taxonomy, ncbi_taxonomy, ncbi_taxonomy_unfiltered, gtdb_type, gtdb_type_sources, gtdb_type_species_of_genus, gid in untrustworthy_as_type, gid in untrustworthy_ncbi_sp, ncbi_type, ncbi_strain_identifiers, ncbi_asm_level, ncbi_genome_representation, ncbi_refseq_cat, ncbi_genome_cat, excluded_from_refseq_note.get(gid, ''), comp, cont, sh_100, gs, contig_count, n50, scaffold_count, ambiguous_bases, total_gap_len, ssu_count, ssu_length, ncbi_molecule_count, ncbi_unspanned_gaps, ncbi_spanned_gaps, lpsn_priority_year) self._apply_ncbi_taxonomy_ledgers(species_exception_file, genus_exception_file)
def run(self, prev_gtdb_metadata_file, cur_gtdb_metadata_file, cur_genome_paths, ncbi_assembly_summary_genbank): """Identify new or modified genomes.""" self.logger.info('Reading previous GTDB genomes.') prev_accns = {} gtdb_taxonomy = {} gtdb_rep = {} ncbi_genome_category = {} with open(prev_gtdb_metadata_file, encoding='utf-8') as f: header = f.readline().strip().split('\t') gtdb_index = header.index('gtdb_taxonomy') gtdb_rep_index = header.index('gtdb_representative') ncbi_genome_cat_index = header.index('ncbi_genome_category') for line in f: tokens = line.strip().split('\t') gid = tokens[0] if gid.startswith('U'): # only concerned with genomes at NCBI continue cid = canonical_gid(gid) prev_accns[cid] = gid gtdb_taxonomy[cid] = tokens[gtdb_index] gtdb_rep[cid] = tokens[gtdb_rep_index] ncbi_genome_category[cid] = tokens[ncbi_genome_cat_index] self.logger.info(f' - identified {len(prev_accns):,} genomes.') # get genomes in current release self.logger.info('Reading current GTDB genomes.') cur_accns = {} with open(cur_gtdb_metadata_file, encoding='utf-8') as f: f.readline() for line in f: tokens = line.strip().split('\t') gid = tokens[0] if gid.startswith('U'): # only concerned with genomes at NCBI continue cur_accns[canonical_gid(gid)] = gid self.logger.info(f' - identified {len(cur_accns):,} genomes.') # get equivalent GenBank and RefSeq genome assemblies self.logger.info( 'Determining identical GenBank and RefSeq accessions.') identical_accns = {} with open(ncbi_assembly_summary_genbank, encoding='utf-8') as f: for line in f: if line.startswith('#'): if 'assembly_accession' in line: header = line.strip().split('\t') gb_accn_index = header.index('# assembly_accession') rs_accn_index = header.index('gbrs_paired_asm') paired_asm_index = header.index('paired_asm_comp') else: tokens = line.strip().split('\t') paired_asm = tokens[paired_asm_index] if paired_asm.lower() == 'identical': gb_accn = tokens[gb_accn_index] rs_accn = tokens[rs_accn_index] identical_accns[gb_accn] = rs_accn identical_accns[rs_accn] = gb_accn # identify new and modified genome IDs self.logger.info('Identifying new or modified genome IDs.') new_gids = set() updated_gids = set() for cur_gid in cur_accns: if cur_gid in prev_accns: if not self.same_genome_accn(cur_accns[cur_gid], prev_accns[cur_gid], identical_accns): updated_gids.add(cur_gid) else: # genome not present in previous GTDB release new_gids.add(cur_gid) lost_gids = set(prev_accns) - set(cur_accns) num_lost_gtdb_reps = sum( [1 for gid in lost_gids if gtdb_rep[gid] == 't']) self.logger.info( f' - identified {len(new_gids):,} new, {len(updated_gids):,} updated, and {len(lost_gids):,} lost genomes.' ) self.logger.info( f' - {num_lost_gtdb_reps:,} lost genomes were GTDB representatives.' ) # get path to current GTDB genome directories self.logger.info( 'Identifying path to genomic files for current GTDB genomes.') cur_genome_files = {} skipped_genomes = 0 fout = open(os.path.join(self.output_dir, 'skipped_genomes.tsv'), 'w') with open(cur_genome_paths) as f: for line in f: tokens = line.strip().split('\t') accn = tokens[0] genome_path = tokens[1] gid = tokens[2] assert canonical_gid(accn) == gid if gid not in cur_accns: # a genome may not be part of the GTDB release # (e.g., genome has no NCBI taxonomy information # or Prodigal failed to call genes such as for # GCA_000716285.1) skipped_genomes += 1 fout.write(gid + '\n') continue assembly_id = os.path.basename(os.path.normpath(genome_path)) genomic_file = os.path.join(genome_path, assembly_id + '_genomic.fna') cur_genome_files[gid] = genomic_file fout.close() self.logger.info( f' - identified genomic file for {len(cur_genome_files):,} genomes.' ) self.logger.info( f' - skipped {skipped_genomes:,} genomes without GTDB metadata.') # write out new or modified genome IDs self.logger.info( 'Writing out path to new and updated genomic FASTA files.') output_file = os.path.join(self.output_dir, 'genomes_new_updated.tsv') fout = open(output_file, 'w') fout.write('Genome ID\tNCBI accession\tStatus\tGenomic file\n') for type_str, gids in [('NEW', new_gids), ('UPDATED', updated_gids)]: for gid in gids: genomic_file = cur_genome_files[gid] fout.write('{}\t{}\t{}\t{}\n'.format(gid, cur_accns[gid], type_str, genomic_file)) fout.close() # write out lost genomes output_file = os.path.join(self.output_dir, 'genomes_lost.tsv') fout = open(output_file, 'w') fout.write( 'Genome ID\tGTDB taxonomy\tGTDB representative\tNCBI genome category\n' ) for gid in lost_gids: fout.write('{}\t{}\t{}\t{}\n'.format(gid, gtdb_taxonomy[gid], gtdb_rep[gid], ncbi_genome_category[gid])) fout.close()