def bulk_query_ncbi(accessions, args): """Bulk query NCBI and retrieve webenvironment and history tags :param accessions: list of GenBank protein accessions :param args: cmd-line args parser Return webenv and query key """ # perform batch query of Entrez try: accessions_string = ",".join(accessions) except TypeError: accessions_string = accessions # Runtime error captured by try/except function call epost_result = Entrez.read( entrez_retry( args.retries, Entrez.epost, db="Protein", id=accessions_string, )) # retrieve the web environment and query key from the Entrez post epost_webenv = epost_result["WebEnv"] epost_query_key = epost_result["QueryKey"] return epost_webenv, epost_query_key
def get_ncbi_counts(args): """Retrieve the number of genomic assemblies per Kingdom from NCBI. :param args: cmd-line args parser Return dict {kingdom: count} """ counts = {} for kingdom in KINGDOMS: with entrez_retry( args.retries, Entrez.esearch, db="Assembly", term=kingdom, retmode="xml", ) as record_handle: record = Entrez.read(record_handle, validate=False) number_of_assemblies = record['Count'] counts[kingdom] = number_of_assemblies return counts
def link_nucleotide_ids_individually(accessions_to_parse, no_accession_logger, args): """Use Entrez.elink to get the ID of the NCBI.Nucleotide db record linked to the NCBI.Protein db record. In the resulting dict, the protein record ID is used to group nucleotide records that are linked to the same protein record, and from which it will need to be determined which is the longest record, so that only one (and the longest) Nucleotide record is parsed for this protein not all retrieved Nucleotide records. :param accessions_to_parse: list of protein accessions :param no_accession_logger: path to log file for logging protein accessions for which no linked nucleotide record was retrieved :param args: cmd-line args parser Return dict {protein_accession: {nucleotide records IDs}} no_nucleotides: set of protein accessions for which no nucleotide record could be retrieved """ logger = logging.getLogger(__name__) nucleotide_ids = {} # {protein_record_id: {nucleotide_ids}} no_nucleotides = set( ) # protein accessions for which no nucleotide record could be retrieved for protein_accession in tqdm(accessions_to_parse, "Retrieving Nucleotide IDs individually"): # used for identifying which nucleotide record to retrieve protein accessions from with entrez_retry( args.retries, Entrez.elink, dbfrom="Protein", db="nuccore", id=protein_accession, ) as handle: try: batch_nuccore = Entrez.read(handle) except Exception as err: logger.warning( f"Could not retrieve linked Nucletoide record for {protein_accession}" ) with open(no_accession_logger, "a") as fh: fh.write( f"Could not retrieve linked Nucletoide record for {protein_accession}\t" f"Error: {err}") no_nucleotides.add(protein_accession) continue for record in tqdm( batch_nuccore, desc="Retrieving Nucleotide db records IDs from nuccore"): protein_record_id = record['IdList'][0] record_nucleotide_ids = set() # Linked db records are contained in the 'LinkSetDb' field # multiple linked records may be retrieved if len(record['LinkSetDb']) == 0: logger.warning( f"Could not retrieve linked Nucletoide record for {protein_accession}" ) err = "No data containiend in 'LinkSetDb' field from eLink output" with open(no_accession_logger, "a") as fh: fh.write( f"Could not retrieve linked Nucletoide record for {protein_accession}\t" f"Error: {err}") no_nucleotides.add(protein_accession) continue for link_dict in record['LinkSetDb']: # record['LinkSetDb'] = [{'Link': [{'Id': '1127815138'}], 'DbTo': 'nuccore', 'LinkName': 'protein_nuccore'}] # link_list = {'Link': [{'Id': '1127815138'}], 'DbTo': 'nuccore', 'LinkName': 'protein_nuccore'} links = link_dict['Link'] for link in links: # links = [{'Id': '1127815138'}] # link = {'Id': '1127815138'} nucleotide_id = link['Id'] record_nucleotide_ids.add(nucleotide_id) try: existing_ids = nucleotide_ids[protein_record_id] all_ids = existing_ids.union(record_nucleotide_ids) nucleotide_ids[protein_record_id] = all_ids except KeyError: nucleotide_ids[protein_record_id] = record_nucleotide_ids return nucleotide_ids, no_nucleotides
def parse_longest_record_individually(nucleotide_record_ids, retrieved_proteins, gbk_accessions, args): """Identify the longest NCBI.Nucleotide record, and extract Protein GenBank accessions :param nucleotide_record_ids: set, NCBI.Nucleotide records IDs retrieved for one Protein record :param retrieved_proteins: dict, {protein_accession: nucleotide record accession} :param gbk_accessions: list of protein GenBank accessions from the local CAZyme database :param args: cmd-line args parser Return retrieved_proteins (dict) newly_retrieved_proteins: set of CAZyme protein accessions retrieved from parsed records bool: True if successful Entrez connection, False is connection fails """ newly_retrieved_proteins = set() record_lengths = { } # {Nucleotide record accession: {len: Number of features (int), record: record} # longest (most features) record interpretted as the most complete record for nucleotide_id in tqdm(nucleotide_record_ids, desc="Parsing nucleotide records individually"): with entrez_retry( args.retries, Entrez.efetch, db="Nucleotide", id=nucleotide_id, retmode="xml", ) as handle: try: nucleotide_records = Entrez.read(handle) except Exception: pass for record in nucleotide_records: nucleotide_accession = record['GBSeq_accession-version'] number_of_features = 0 for feature_dict in record['GBSeq_feature-table']: for feature_qual in feature_dict['GBFeature_quals']: if feature_qual['GBQualifier_name'] == 'protein_id': number_of_features += 1 record_lengths[nucleotide_accession] = { 'length': number_of_features, 'record': record } list_of_lengths = [acc['length'] for acc in list(record_lengths.keys())] list_of_lengths.sort(reverse=True) longest_length = list_of_lengths[0] for nucleotide_accession in record_lengths: if record_lengths[nucleotide_accession]['length'] == longest_length: # found the longest record # extract protein accessions for CAZymes in the local CAZyme database # method explained in extract_protein_accessions() for feature_qual in feature_dict['GBFeature_quals']: if feature_qual['GBQualifier_name'] == 'protein_id': protein_accession = feature_qual['GBQualifier_value'] if protein_accession in gbk_accessions: try: retrieved_proteins[protein_accession].add( nucleotide_accession) except KeyError: retrieved_proteins[protein_accession] = { nucleotide_accession } newly_retrieved_proteins.add(protein_accession) break return retrieved_proteins, newly_retrieved_proteins, True
def parse_longest_record(nucleotide_record_ids, retrieved_proteins, gbk_accessions, args): """Identify the longest NCBI.Nucleotide record, and extract Protein GenBank accessions :param nucleotide_record_ids: set, NCBI.Nucleotide records IDs retrieved for one Protein record :param retrieved_proteins: dict, {protein_accession: nucleotide record accession} :param gbk_accessions: list of protein GenBank accessions from the local CAZyme database :param args: cmd-line args parser Return retrieved_proteins (dict) newly_retrieved_proteins: set of CAZyme protein accessions retrieved from parsed records bool: True if successful Entrez connection, False is connection fails """ logger = logging.getLogger(__name__) newly_retrieved_proteins = set() batch_query_ids = ",".join(list(nucleotide_record_ids)) with entrez_retry( args.retries, Entrez.epost, "Nucleotide", id=batch_query_ids, ) as handle: batch_post = Entrez.read(handle) print("posted") with entrez_retry( args.retries, Entrez.efetch, db="Nucleotide", query_key=batch_post['QueryKey'], WebEnv=batch_post['WebEnv'], retmode="xml", ) as handle: try: batch_nucleotide = Entrez.read(handle) except Exception as err: logger.warning( f"Failed Entrez connection for fetching Nucleotide records: {err}" ) return retrieved_proteins, newly_retrieved_proteins, False print("fetched") record_lengths = { } # {Nucleotide record accession: {len: Number of features (int), record: record} # longest (most features) record interpretted as the most complete record for record in tqdm(batch_nucleotide, desc="Selecting longest Nucleotide record"): nucleotide_accession = record['GBSeq_accession-version'] number_of_features = 0 for feature_dict in record['GBSeq_feature-table']: for feature_qual in feature_dict['GBFeature_quals']: if feature_qual['GBQualifier_name'] == 'protein_id': number_of_features += 1 record_lengths[nucleotide_accession] = { 'length': number_of_features, 'record': record } list_of_lengths = [acc['length'] for acc in list(record_lengths.keys())] list_of_lengths.sort(reverse=True) longest_length = list_of_lengths[0] for nucleotide_accession in record_lengths: if record_lengths[nucleotide_accession]['length'] == longest_length: # found the longest record # extract protein accessions for CAZymes in the local CAZyme database # method explained in extract_protein_accessions() for feature_qual in feature_dict['GBFeature_quals']: if feature_qual['GBQualifier_name'] == 'protein_id': protein_accession = feature_qual['GBQualifier_value'] if protein_accession in gbk_accessions: try: retrieved_proteins[protein_accession].add( nucleotide_accession) except KeyError: retrieved_proteins[protein_accession] = { nucleotide_accession } newly_retrieved_proteins.add(protein_accession) break return retrieved_proteins, newly_retrieved_proteins, True
def extract_protein_accessions_individually(single_nucleotide_ids, retrieved_proteins, gbk_accessions, args): """Retrieve and parse Nucleotide db records, for NCBI.Protein records from which only one NCBI.Nucleotide db record ID was retrieved. :param retrieved_proteins: dict, {protein_accession: nucleotide record accession} :param single_nucleotide_ids: list of nucloetide record IDs :param gbk_accessions: list of protein GenBank accessions from the local CAZyme database :param args: cmd-line args parser Return retrieved_proteins (dict) newly_retrieved_proteins: set of CAZyme protein accessions retrieved from parsed records """ logger = logging.getLogger(__name__) newly_retrieved_proteins = set() for nucleotide_id in tqdm(single_nucleotide_ids, desc="Parsing nucelotide records individually"): with entrez_retry( args.retries, Entrez.efetch, db="Nucleotide", id=nucleotide_id, retmode="xml", ) as handle: try: batch_nucleotide = Entrez.read(handle) except Exception as err: logger.warning( f"Failed Entrez connection for fetching Nucleotide records: {err}" ) pass for record in tqdm(batch_nucleotide, desc="Extracting data from Nucleotide records"): nucleotide_accession = record['GBSeq_accession-version'] # retrieve protein accessions of proteins features in the nucletide record for feature_dict in record['GBSeq_feature-table']: # feature-table contains a list of features, one feature is one feature_dict for feature_qual in feature_dict['GBFeature_quals']: # feature_quals contains a list of dicts, one dict is feature_qual # looking for dict containing protein accession (protein_id) # e.g. {'GBQualifier_name': 'protein_id', 'GBQualifier_value': 'APS93952.1'} if feature_qual['GBQualifier_name'] == 'protein_id': protein_accession = feature_qual['GBQualifier_value'] if protein_accession in gbk_accessions: # protein is in the local CAZyme database try: retrieved_proteins[protein_accession].add( nucleotide_accession) except KeyError: retrieved_proteins[protein_accession] = { nucleotide_accession } newly_retrieved_proteins.add(protein_accession) return retrieved_proteins, newly_retrieved_proteins
def get_genomic_accessions(nucleotide_accessions_dict, no_accession_logger, args): """Retrieve genomic accessions for the genomic assemblies :param nucleotide_accessions_dict: dict {kingdom: {genus: {species: {nucleotide record accession: {protein_accessions},},},},} :param no_accession_logger: Path, path to log file to write out assembly names for which no genomic accession was retrieved :param args: cmd-line args parser Return dict, {kingdom: {genus: {species: {genomic_accession: {proteins: {protein_accessions}, count=int},},},},} """ logger = logging.getLogger(__name__) genomic_accession_dict = {} # {kingdom: {genus: {species: {genomic_accession: {proteins: {protein_accessions}, count=int},},},},} for kingdom in tqdm(nucleotide_accessions_dict, desc='Retrieving genomic accessions per kingdom'): genera = nucleotide_accessions_dict[kingdom] for genus in genera: organisms = genera[genus] for species in organisms: # retrieve all genomic assembly names for the given species assembly_names = list(organisms[species].keys()) # break up the list into a series of smaller lists that can be batched querried batch_queries = get_chunks_list(args.batch_size, assembly_names) for batch_query in tqdm( batch_queries, desc=f"Batch querying for {genus} {species}"): batch_query_ids = ",".join(batch_query) # retrieve the records IDs for the assembly names with entrez_retry( args.retries, Entrez.esearch, "Assembly", id=batch_query_ids, ) as handle: batch_post = Entrez.read(handle) with entrez_retry( args.retries, Entrez.efetch, db="Assembly", query_key=batch_post['QueryKey'], WebEnv=batch_post['WebEnv'], retmode="xml", ) as handle: batch_fetch = Entrez.read(handle) genomic_accessions = {} for genome_record in tqdm(batch_fetch, desc="Retrieving assembly IDs"): index = 0 accessions = set() for index in range(len(genome_record['IdList'])): with entrez_retry( 10, Entrez.efetch, db="Assembly", id=genome_record['IdList'][index], retmode="xml", rettype="docsum", ) as handle: result = Entrez.read(handle) genomic_accession = result['DocumentSummarySet'][ 'DocumentSummary'][0]['AssemblyAccession'] assembly_name = result['DocumentSummarySet'][ 'DocumentSummary'][0]['AssemblyName'] genomic_accessions[genomic_accession] = assembly_name accessions = list(genomic_accessions.keys) accessions.sort(reverse=True) latest_accession = accessions[0] latest_assembly_name = genomic_accessions[latest_accession] # replace assemlby name for genomic accession try: protein_accessions = nucleotide_accessions_dict[ kingdom][genus][species][latest_assembly_name] except KeyError: logger.warning( f"Retrieved assembly name {latest_assembly_name}, but not retrieved previously" ) with open(no_accession_logger, 'a') as fh: fh.write( f"{latest_assembly_name}\tRetrieved assembly name, but not retrieved previously\t" f"{latest_accession}\t{genus} {species}\n") continue try: genomic_accession_dict[kingdom] try: genomic_accession_dict[kingdom][genus] try: genomic_accession_dict[kingdom][genus][species] except KeyError: genomic_accession_dict[kingdom][genus][ species] = { genomic_accession: { 'proteins': protein_accessions, 'count': len(protein_accessions), }, } except KeyError: genomic_accession_dict[kingdom][genus] = { species: { genomic_accession: { 'proteins': protein_accessions, 'count': len(protein_accessions), }, }, } except KeyError: genomic_accession_dict[kingdom] = { genus: { species: { genomic_accession: { 'proteins': protein_accessions, 'count': len(protein_accessions), }, }, }, } return genomic_accession_dict
def get_nucleotide_accessions(genbank_kingdom_dict, no_accession_logger, cache_dir, args): """Retrieve the NCBI Nucleotide db records ID's containing the GenBank protein accessions. :param genbank_kingdom_dict: dict of Genbank and Kingdom records from db {kingdom: {genus: {species: {protein_accessions}}} :param no_accession_logger: Path, path to log file to save protein accessions for which no genomic accession was retrieved :param cache_dir: Path to cache directory :param args: cmd-line args parser Return dict {kingdom: {genus: {species: {nucleotide_accession: {protein_accessions},},},},} """ logger = logging.getLogger(__name__) # convert the structure of genbank_kingdom_dict # { protein_accession: {species: str, genus: str, kingdom} } if args.gbk_organisms_relationships is not None: logger.warning( f"Retriving protein-organisms relationships from file: {args.gbk_organisms_relationships}" ) try: with open(args.gbk_organisms_relationships, "r") as fh: gbk_organism_dict = json.load(fh) except FileNotFoundError: logger.warning( f"Could not find JSON file containing protein-organisms relationships at: {args.gbk_organisms_relationships}\n" "Check the correct path was given.\n" "Terminating program") sys.exit(1) else: logger.warning( "Retrieving protein-organism relationships from the local database" ) gbk_organism_dict = get_gbk_organism_relationships( genbank_kingdom_dict) # create dict to store the nucleotide records accessions nucleotide_accessions_dict = {} # {kingdom: {genus: {species: {nucleotide record accessio: {protein_accessions},},},},} # create a dict to keep track of protein accessions for which a nucleotide record was retrieved # store all retrieved ncucleotide accessions to prevent retrieval of the # same assembly record multiple times retrieved_proteins = {} # {protein_accession: nucleotide record accession} # create set of protein accessions for which no nucleotide ID was retrievied no_retrieved_nucleotide_ids = set() # gbk accessions waiting for a nucleotide id to be retrieved remaining_accessions = list(gbk_organism_dict.keys()) # break up the list into batches gbk_accessions = list(gbk_organism_dict.keys()) gbk_batches = get_chunks_list(gbk_accessions, args.batch_size) failed_batches = set() for batch in tqdm(gbk_batches, desc="Batching quering NCBI for nucleotide IDs"): # eLink Protein to Nuccore db and retrieve IDs of linked nucleotide records batch_query_ids = ",".join(batch) try: with entrez_retry( args.retries, Entrez.epost, "Protein", id=batch_query_ids, ) as handle: epost_result = Entrez.read(handle) except RuntimeError: logger.warning( "Runtime error raised when batch quering\n" "Possible result of a accessions not being in NCBI\n" "Attempt identification of the causal accession later\n") failed_batches.add(batch) continue epost_webenv = epost_result["WebEnv"] epost_query_key = epost_result["QueryKey"] try: with entrez_retry( args.retries, Entrez.elink, dbfrom="Protein", db="nuccore", query_key=epost_webenv, WebEnv=epost_query_key, linkname='protein_nuccore', ) as handle: batch_nuccore = Entrez.read(handle) except Exception as err: logger.warning(f"Failed Entrez connection: {err}\n" "No nucletoide IDs retrieved for batch query\n" "Will try again later") failed_batches.add(batch) continue # parse the query result from nuccore # {protein record ID: {nucleotide records IDs}} nucleotide_ids = entrez.get_linked_nucleotide_record_ids(batch_nuccore) if nucleotide_ids is None: # issue with at least one accession in the batch # e.g. it is not longer stored in NCBI # pass individually to find/parse the bad accession(s) nucleotide_ids, no_nucleotides = entrez.link_nucleotide_ids_individually( accessions_to_parse, no_accession_logger, args, ) # no nucleotide IDs retrieved for at least one protein if len(no_nucleotides) != 0: logger.warning( f"{len(no_nucleotides)} proteins found with no linked Nucleotide records" ) for protein_accession in no_nucleotides: # already logged in link_nucleotide_ids_individually() try: remaining_accessions.remove(protein_accession) no_retrieved_nucleotide_ids.add(protein_accession) except ValueError: pass if len(list(nucleotide_ids.values())) == 0: # no linked Nucleotide records retrieved for the current batch of protein accessions for protein_accession in accessions_to_parse: logger.warning( f"Could not reitreve linked Nucleotide record for {protein_accession}" ) no_retrieved_nucleotide_ids.add(protein_accession) # do not try to retrieve record again try: remaining_accessions.remove(protein_accession) except ValueError: pass continue # retrieves the nucleotide records IDs for protein records for whcih only one # nuclotide ID was retrieved single_nucleotide_ids = set() # retrieve the protein_record_ids of protein records from which multiple nucletoide # record IDs were retrieved protein_records_multi_nuc = set() for protein_record_id in tqdm( nucleotide_ids, desc= "Idenitfying proteins with multiple linked nucleotide records" ): if len(nucleotide_ids[protein_record_id]) == 1: single_nucleotide_ids.add( list(nucleotide_ids[protein_record_id])[0]) else: logger.warning( f"Found {len(nucleotide_ids[protein_record_id])} linked nucletoide records " f"for protein record {protein_record_id}") protein_records_multi_nuc.add(protein_record_id) # batch query to fetch nucletoide records for protein records # from which only a sinlge nucleotide ID was retrieved if len(single_nucleotide_ids) != 0: retrieved_proteins, newly_retrieved_proteins, succcess = entrez.extract_protein_accessions( single_nucleotide_ids, retrieved_proteins, gbk_accessions, args, ) if succcess is False: # issue with at least one accession in the batch # e.g. it is not longer stored in NCBI # pass individually to find/parse the bad accession(s) retrieved_proteins, newly_retrieved_proteins = entrez.extract_protein_accessions_individually( single_nucleotide_ids, retrieved_proteins, gbk_accessions, args, ) # add the nucleotide accessions to the nucleotide_accessions_dict # {kingdom: {genus: {species: {nucleotide record accessio: {protein_accessions},},},},} nucleotide_accessions_dict = add_nucleotide_accessions( nucleotide_accessions_dict, gbk_organism_dict, retrieved_proteins, newly_retrieved_proteins, kingdom, no_accession_logger, ) # for Protein records for which multiple Nucleotide record IDs were retrieved # Identify the longest Nucleotide record and retrieve protein accessions from it # The longest record is most likely to be the most complete record for protein_record_id in tqdm( protein_records_multi_nuc, "Parsing protein records with multiple linked Nucletide records" ): nucleotide_record_ids = nucleotide_ids[protein_record_id] retrieved_proteins, newly_retrieved_proteins, success = entrez.parse_longest_record( nucleotide_record_ids, retrieved_proteins, gbk_accessions, args, ) if success is False: # issue with at least one accession in the batch # e.g. it is not longer stored in NCBI # pass individually to find/parse the bad accession(s) retrieved_proteins, newly_retrieved_proteins, success = entrez.parse_longest_record_individually( nucleotide_record_ids, retrieved_proteins, gbk_accessions, args, ) nucleotide_accessions_dict = add_nucleotide_accessions( nucleotide_accessions_dict, gbk_organism_dict, retrieved_proteins, newly_retrieved_proteins, kingdom, no_accession_logger, ) # remove protein accessions from remaining_accessions because the linked Nucleotide # record ID has already been retrieved for protein_accession in retrieved_proteins: try: remaining_accessions.remove(protein_accession) except ValueError: pass if starting_loop_length == len( remaining_accessions) and len(remaining_accessions) != 0: # failing to retrieve data for protein accessions nucleotide_ids, no_nucleotides = entrez.link_nucleotide_ids_individually( accessions_to_parse, no_accession_logger, args, ) if nucleotide_ids is None: for protein_accession in remaining_accessions: logger.warning( f"Could not retrieve Nucletoide record for {protein_accession}" ) try: remaining_accessions.remove(protein_accession) except ValueError: pass continue if len(list(nucleotide_ids.values())) == 0: # no linked Nucleotide records retrieved for the current batch of protein accessions for protein_accession in accessions_to_parse: logger.warning( f"Could not reitreve linked Nucleotide record for {protein_accession}" ) # do not try to retrieve record again try: remaining_accessions.remove(protein_accession) except ValueError: pass if len(no_nucleotides) != 0: for protein_accession in no_nucleotides: # already logged in link_nucleotide_ids_individually() try: remaining_accessions.remove(protein_accession) except ValueError: pass return nucleotide_accessions_dict
def get_sequences(genbank_accessions, args, retry=False): """Retrieve protein sequences from Entrez. :param genbank_accessions: list, GenBank accessions :param args: cmb-line args parser :param retry: bool, default False, if get_sequences is being called for retrying a previously failed query Return dict keyed by GenBank accession and valued by Seq instance, and a list of all GenBank accessions for which no record from NCBI was retrieved. """ logger = logging.getLogger(__name__) seq_dict = {} # {gbk_accession: SeqRecord} # the list of accessions is to long, break down into smaller chunks for batch querying all_queries = get_chunks_list(genbank_accessions, args.batch_size) failed_queries = [ ] # lists which raised an error, likely because contain an accession not in NCBI irregular_accessions = [] success_accessions = set() # accessions for which seqs were retrieved for query_list in tqdm(all_queries, desc="Batch querying NCBI.Entrez"): try: epost_webenv, epost_query_key = bulk_query_ncbi(query_list, args) except RuntimeError: logger.warning( "Runtime error raised when batch quering\n" "Possible result of a accessions not being in NCBI\n" "Attempt identification of the causal accession later\n") if retry: return None, None failed_queries.append(query_list) continue try: # retrieve the protein sequences with entrez_retry( args.retries, Entrez.efetch, db="Protein", query_key=epost_query_key, WebEnv=epost_webenv, rettype="fasta", retmode="text", ) as seq_handle: for record in SeqIO.parse(seq_handle, "fasta"): temp_accession = record.id # check if multiple items returned in ID temp_accession = temp_accession.split("|") retrieved_accession = None for acc in temp_accession: if acc.strip() in genbank_accessions: retrieved_accession = acc if retrieved_accession is None: # if could not retrieve GenBank accession from the record logger.error( "Could not retrieve a GenBank protein accession matching an accession from the local database from:\n" f"{record.id}\n" "The sequence from this record will not be added to the db" ) irregular_accessions.append(temp_accession) continue seq_dict[retrieved_accession] = record.seq success_accessions.add(retrieved_accession) except IncompleteRead as err: logger.warning("IncompleteRead error raised:\n" f"{err}\n" "Will reattempt NCBI query later") if retry: return None, None failed_queries.append(all_queries) continue # list of GenBank accessions for which no protein sequence was retrieved no_seq = [ acc for acc in genbank_accessions if acc not in success_accessions ] no_seq += irregular_accessions if retry: return seq_dict, no_seq if len(failed_queries) != 0: for failed_query in tqdm(failed_queries, desc="Reparsing failed queries"): first_half = failed_query[:int((len(failed_query) / 2))] seq_dict, success_accessions, failed_accessions = retry_failed_queries( first_half, seq_dict, success_accessions, args, ) no_seq += failed_accessions second_half = failed_query[int((len(failed_query) / 2)):] seq_dict, success_accessions, failed_accessions = retry_failed_queries( second_half, seq_dict, success_accessions, args, ) no_seq += failed_accessions logger.warning( f"Retrieved sequences for {len(success_accessions)} proteins") return seq_dict, no_seq
def replace_multiple_tax(cazy_data, genbank_accessions, replaced_taxa_logger, args, invalid_ids): """Identify GenBank accessions which have multiple source organisms listedi in CAZy. Replace with the latest source organism from NCBI. :param cazy_data: dict of CAZy data :param genbank_accessions: list of genbank accessions with multiple taxa in CAZy :param replaced_taxa_logger: logger, used for logging GenBank accessions with multiple taxa in CAZy, and the data this replaced and what is replaced by :param args: cmd-line args parser :param invalid_ids: boolean, potential presence of invalid GenBank accessions Set as true when func is called by replace_multiple_tax_with_invalid_ids() Return dict updated cazy_data dict and boolean, if multiple taxa were replaced by single taxa """ logger = logging.getLogger(__name__) id_post_list = str(",".join(genbank_accessions)) success = False try: epost_results = Entrez.read( entrez_retry( args.retries, Entrez.epost, "Protein", id=id_post_list, ) ) success = True except (TypeError, AttributeError): # if no record is returned from call to Entrez logger.error( f"Entrez failed to post assembly IDs.\n" "Not retrieving taxonomy classification from NCBI.\n" "Selecting the first organism retrieved from CAZy as the source organism" ) cazy_data = select_first_organism(cazy_data, genbank_accessions) success = True except RuntimeError: logger.warning("Found GenBank accessions in CAZy data that are no longer in NCBI") if invalid_ids: # replace_multiple_tax was called by replace_multiple_tax_with_invalid_ids # return results, don't use recursive programming return cazy_data, success else: # first time replace_multiple_tax was called cazy_data, success = replace_multiple_tax_with_invalid_ids(cazy_data, args) if success is False: logger.error( "Could not retrieve taxonomy data from NCBI,\n" "Using the first source organism retrieved from CAZy for each GenBank accession" ) cazy_data = select_first_organism(cazy_data, genbank_accessions, replaced_taxa_logger) success = True else: logger.info("Parsing data retrieved from NCBI") cazy_data = get_ncbi_tax(epost_results, cazy_data, replaced_taxa_logger, args) return cazy_data, success
def get_ncbi_tax(epost_results, cazy_data, replaced_taxa_logger, args): """Parse the ePost output from Entrez and add the NCBI tax classifications to the CAZy data :param epost_results: Entrez ePost output :param cazy_data: dict, data retrieved from CAZy :param args: cmd-line args parser :param replaced_taxa_logger: logger, used for logging GenBank accessions with multiple taxa in CAZy Return cazy_data (dict) """ logger = logging.getLogger(__name__) # Retrieve web environment and query key from Entrez epost epost_webenv = epost_results["WebEnv"] epost_query_key = epost_results["QueryKey"] try: with entrez_retry( args.retries, Entrez.efetch, db="Protein", query_key=epost_query_key, WebEnv=epost_webenv, retmode="xml", ) as record_handle: protein_records = Entrez.read(record_handle, validate=False) # if no record is returned from call to Entrez except (TypeError, AttributeError) as error: logger.error( f"Entrez failed to retireve accession numbers." "Exiting retrieval of accession numbers, and returning null value 'NA'" ) for protein in tqdm(protein_records, desc="Retrieving organism from NCBI"): # retrieve NCBI taxonomy data accession = protein['GBSeq_accession-version'] organism = protein['GBSeq_organism'] kingdom = protein['GBSeq_taxonomy'].split(';')[0] # retrieve CAZy taxonomy data cazy_kingdom = cazy_data[accession]["kingdom"] cazy_organisms = cazy_data[accession]["organism"] cazy_kingdom_str = ",".join(cazy_kingdom) cazy_organism_str = ','.join(cazy_organisms) try: cazy_data[accession]['kingdom'] = {kingdom} cazy_data[accession]['organism'] = {organism} # log the difference replaced_taxa_logger.warning( f"{accession}\t{cazy_kingdom_str}: {cazy_organism_str}\t{kingdom}: {organism}" ) except KeyError: err = f'GenBank accession {accession} retrieved from NCBI, but it is not present in CAZy' logger.error(err) replaced_taxa_logger.warning( f"{accession}\t{cazy_kingdom_str}: {cazy_organism_str}\t{err}" ) return cazy_data