def lineage_id(self, acc_number_list):
"""Get taxonomic lineage name for accession ids
Given a list of accession numbers, yield the accession number and their
associated lineage (in the form of taxids) as tuples
Args:
acc_number_list (:obj:`list`): a list of accession numbers
Yields:
tuple: (accession id, lineage list)
"""
self.check_list_ids(acc_number_list)
with self.db.atomic():
query = Accession.select().where(
Accession.accession << acc_number_list)
for i in query:
lineage_list = []
current_lineage = i.taxid.tax_name
current_lineage_id = i.taxid.ncbi_taxid
parent = i.taxid.parent_taxid
while current_lineage != 'root':
lineage_list.append(current_lineage_id)
new_query = Taxa.get(Taxa.ncbi_taxid == parent)
current_lineage = new_query.tax_name
current_lineage_id = new_query.ncbi_taxid
parent = new_query.parent_taxid
yield (i.accession, lineage_list)
def lineage_id(self, acc_number_list):
"""Get taxonomic lineage name for accession ids
Given a list of accession numbers, yield the accession number and their
associated lineage (in the form of taxids) as tuples
Args:
acc_number_list (:obj:`list`): a list of accession numbers
Yields:
tuple: (accession id, lineage list)
"""
self.check_list_ids(acc_number_list)
with self.db.atomic():
query = Accession.select().where(
Accession.accession << acc_number_list)
for i in query:
lineage_list = []
current_lineage = i.taxid.tax_name
current_lineage_id = i.taxid.ncbi_taxid
parent = i.taxid.parent_taxid
while current_lineage != 'root':
lineage_list.append(current_lineage_id)
new_query = Taxa.get(Taxa.ncbi_taxid == parent)
current_lineage = new_query.tax_name
current_lineage_id = new_query.ncbi_taxid
parent = new_query.parent_taxid
yield (i.accession, lineage_list)
def sci_name(self, acc_number_list):
"""Get taxonomic scientific name for accession ids
Given a list of accession numbers, yield the accession number and their
associated scientific name as tuples
Args:
acc_number_list (:obj:`list`): a list of accession numbers
Yields:
tuple: (accession id, taxonomy id)
"""
self.check_list_ids(acc_number_list)
with self.db.atomic():
query = Accession.select().where(
Accession.accession << acc_number_list)
for i in query:
yield (i.accession, i.taxid.tax_name)
def sci_name(self, acc_number_list):
"""Get taxonomic scientific name for accession ids
Given a list of accession numbers, yield the accession number and their
associated scientific name as tuples
Args:
acc_number_list (:obj:`list`): a list of accession numbers
Yields:
tuple: (accession id, taxonomy id)
"""
self.check_list_ids(acc_number_list)
with self.db.atomic():
query = Accession.select().where(
Accession.accession << acc_number_list)
for i in query:
yield (i.accession, i.taxid.tax_name)
def accession2taxid(self, acc2taxid=None, chunk=None):
"""Parses the accession2taxid files
This method parses the accession2taxid file, build a dictionary,
stores it in a list and yield for insertion in the database.
::
{
'accession': accession_id_from_file,
'taxid': associated_taxonomic_id
}
Args:
acc2taxid (:obj:`str`): Path to acc2taxid input file (gzipped)
chunk (:obj:`int`): Chunk size of entries to gather before
yielding. Default 500 (set at object construction)
Yields:
list: Chunk size of read entries
"""
# Some accessions (e.g.: AAA22826) have a taxid = 0
entries = []
counter = 0
taxids = self.cache_taxids()
if not self.fast:
accessions = {}
if acc2taxid is None:
acc2taxid = self.acc_file
self.check_file(acc2taxid)
if chunk is None:
chunk = self.chunk
self.logger.debug("Parsing %s" % str(acc2taxid))
self.logger.debug("Fast mode %s" % "ON" if self.fast else "OFF")
with gzip.open(acc2taxid, 'rb') as f:
f.readline() # discard the header
for line in f:
line_list = line.decode().rstrip('\n').split('\t')
# Check the taxid already exists and get its id
if line_list[2] not in taxids:
continue
# In case of an update or parsing an already inserted list of
# accessions
if not self.fast:
if line_list[0] in accessions:
continue
try:
Accession.get(Accession.accession == line_list[0])
except Accession.DoesNotExist:
accessions[line_list[0]] = True
data_dict = {
'accession': line_list[0],
'taxid': line_list[2]
}
else:
data_dict = {
'accession': line_list[0],
'taxid': line_list[2]
}
entries.append(data_dict)
counter += 1
if counter == chunk:
yield(entries)
entries = []
counter = 0
if len(entries):
yield(entries)
def accession2taxid(self, acc2taxid=None, chunk=None):
"""Parses the accession2taxid files
This method parses the accession2taxid file, build a dictionary,
stores it in a list and yield for insertion in the database.
::
{
'accession': accession_id_from_file,
'taxid': associated_taxonomic_id
}
Args:
acc2taxid (:obj:`str`): Path to acc2taxid input file (gzipped)
chunk (:obj:`int`): Chunk size of entries to gather before
yielding. Default 500 (set at object construction)
Yields:
list: Chunk size of read entries
"""
# Some accessions (e.g.: AAA22826) have a taxid = 0
entries = []
counter = 0
taxids = self.cache_taxids()
if not self.fast:
accessions = {}
if acc2taxid is None:
acc2taxid = self.acc_file
self.check_file(acc2taxid)
if chunk is None:
chunk = self.chunk
self.verbose("Parsing %s" % str(acc2taxid))
self.verbose("Fast mode %s" % "ON" if self.fast else "OFF")
with gzip.open(acc2taxid, 'rb') as f:
f.readline() # discard the header
for line in f:
line_list = line.decode().rstrip('\n').split('\t')
# Check the taxid already exists and get its id
if line_list[2] not in taxids:
continue
# In case of an update or parsing an already inserted list of
# accessions
if not self.fast:
if line_list[0] in accessions:
continue
try:
Accession.get(Accession.accession == line_list[0])
except Accession.DoesNotExist:
accessions[line_list[0]] = True
data_dict = {
'accession': line_list[0],
'taxid': line_list[2]
}
else:
data_dict = {
'accession': line_list[0],
'taxid': line_list[2]
}
entries.append(data_dict)
counter += 1
if counter == chunk:
yield (entries)
entries = []
counter = 0
if len(entries):
yield (entries)
def create_db(args):
"""Main function for the 'taxadb create' sub-command.
This function creates a taxonomy database with 2 tables: Taxa and Sequence.
Args:
args.input (:obj:`str`): input directory. It is the directory created by
`taxadb download`
args.dbname (:obj:`str`): name of the database to be created
args.dbtype (:obj:`str`): type of database to be used.
args.division (:obj:`str`): division to create the db for.
args.fast (:obj:`bool`): Disables checks for faster db creation. Use
with caution!
"""
database = DatabaseFactory(**args.__dict__).get_database()
div = args.division # am lazy at typing
db.initialize(database)
nucl_est = 'nucl_est.accession2taxid.gz'
nucl_gb = 'nucl_gb.accession2taxid.gz'
nucl_gss = 'nucl_gss.accession2taxid.gz'
nucl_wgs = 'nucl_wgs.accession2taxid.gz'
prot = 'prot.accession2taxid.gz'
acc_dl_list = []
db.connect()
parser = TaxaDumpParser(nodes_file=os.path.join(args.input, 'nodes.dmp'),
names_file=os.path.join(args.input, 'names.dmp'),
verbose=args.verbose)
parser.verbose("Connected to database ...")
# If taxa table already exists, do not recreate and fill it
# safe=True prevent not to create the table if it already exists
if not Taxa.table_exists():
parser.verbose("Creating table %s" % str(Taxa.get_table_name()))
db.create_table(Taxa, safe=True)
parser.verbose("Parsing files")
taxa_info_list = parser.taxdump()
parser.verbose("Inserting taxa data")
with db.atomic():
for i in range(0, len(taxa_info_list), args.chunk):
Taxa.insert_many(taxa_info_list[i:i + args.chunk]).execute()
print('Taxa: completed')
parser.verbose("Checking table accession ...")
# At first load, table accession does not exist yet, we create it
db.create_table(Accession, safe=True)
if div in ['full', 'nucl', 'est']:
acc_dl_list.append(nucl_est)
if div in ['full', 'nucl', 'gb']:
acc_dl_list.append(nucl_gb)
if div in ['full', 'nucl', 'gss']:
acc_dl_list.append(nucl_gss)
if div in ['full', 'nucl', 'wgs']:
acc_dl_list.append(nucl_wgs)
if div in ['full', 'prot']:
acc_dl_list.append(prot)
parser = Accession2TaxidParser(verbose=args.verbose, fast=args.fast)
with db.atomic():
for acc_file in acc_dl_list:
inserted_rows = 0
parser.verbose("Parsing %s" % str(acc_file))
for data_dict in parser.accession2taxid(acc2taxid=os.path.join(
args.input, acc_file),
chunk=args.chunk):
Accession.insert_many(data_dict[0:args.chunk]).execute()
inserted_rows += len(data_dict)
print('%s: %s added to database (%d rows inserted)' %
(Accession.get_table_name(), acc_file, inserted_rows))
if not Accession.has_index(name='accession_accession'):
print('Creating index for %s' % Accession.get_table_name())
try:
db.create_index(Accession, ['accession'], unique=True)
except PeeweeException as err:
raise Exception("Could not create Accession index: %s" %
str(err))
print('Accession: completed')
db.close()
def create_db(args):
"""Main function for the 'taxadb create' sub-command.
This function creates a taxonomy database with 2 tables: Taxa and Sequence.
Args:
args.input (:obj:`str`): input directory. It is the directory created
by `taxadb download`
args.dbname (:obj:`str`): name of the database to be created
args.dbtype (:obj:`str`): type of database to be used.
args.division (:obj:`str`): division to create the db for.
args.fast (:obj:`bool`): Disables checks for faster db creation. Use
with caution!
"""
logger = logging.getLogger(__name__)
database = DatabaseFactory(**args.__dict__).get_database()
div = args.division # am lazy at typing
db.initialize(database)
nucl_gb = 'nucl_gb.accession2taxid.gz'
nucl_wgs = 'nucl_wgs.accession2taxid.gz'
prot = 'prot.accession2taxid.gz'
acc_dl_list = []
db.connect()
parser = TaxaDumpParser(nodes_file=os.path.join(args.input, 'nodes.dmp'),
names_file=os.path.join(args.input, 'names.dmp'),
verbose=args.verbose)
logger.debug('Connected to database')
# If taxa table already exists, do not recreate and fill it
# safe=True prevent not to create the table if it already exists
if not Taxa.table_exists():
logger.info('Creating table %s' % str(Taxa.get_table_name()))
db.create_tables([Taxa])
logger.info("Parsing files")
taxa_info_list = parser.taxdump()
logger.info("Inserting taxonomy data")
total_size = len(taxa_info_list)
try:
with db.atomic():
for i in tqdm(range(0, total_size, args.chunk),
unit=' chunks',
desc='INFO:taxadb.app',
total=''):
Taxa.insert_many(taxa_info_list[i:i + args.chunk]).execute()
except OperationalError as e:
print("\n") # needed because the above counter has none
logger.error("sqlite3 error: %s" % e)
logger.error("Maybe retry with a lower chunk size.")
sys.exit(1)
logger.info('Table Taxa completed')
# At first load, table accession does not exist yet, we create it
db.create_tables([Accession])
if div in ['full', 'nucl', 'gb']:
acc_dl_list.append(nucl_gb)
if div in ['full', 'nucl', 'wgs']:
acc_dl_list.append(nucl_wgs)
if div in ['full', 'prot']:
acc_dl_list.append(prot)
parser = Accession2TaxidParser(verbose=args.verbose, fast=args.fast)
with db.atomic():
for acc_file in acc_dl_list:
inserted_rows = 0
logger.info("Parsing %s" % str(acc_file))
for data_dict in tqdm(parser.accession2taxid(
acc2taxid=os.path.join(args.input,
acc_file), chunk=args.chunk),
unit=' chunks',
desc='INFO:taxadb.app',
total=''):
Accession.insert_many(data_dict[0:args.chunk]).execute()
inserted_rows += len(data_dict)
logger.info('%s: %s added to database (%d rows inserted)' %
(Accession.get_table_name(), acc_file, inserted_rows))
if not Accession.has_index(name='accession_accession'):
logger.info('Creating index for %s' % Accession.get_table_name())
try:
# db.add_index(Accession, ['accession'], unique=True)
idx = db.index(db.Accession, name='accession', unique=True)
db.add_index(idx)
except PeeweeException as err:
raise Exception("Could not create Accession index: %s" %
str(err))
logger.info('Table Accession completed')
db.close()
def create_db(args):
"""Main function for the 'taxadb create' sub-command.
This function creates a taxonomy database with 2 tables: Taxa and Sequence.
Args:
args.input (:obj:`str`): input directory. It is the directory created
by `taxadb download`
args.dbname (:obj:`str`): name of the database to be created
args.dbtype (:obj:`str`): type of database to be used.
args.division (:obj:`str`): division to create the db for.
args.fast (:obj:`bool`): Disables checks for faster db creation. Use
with caution!
"""
logger = logging.getLogger(__name__)
database = DatabaseFactory(**args.__dict__).get_database()
div = args.division # am lazy at typing
db.initialize(database)
nucl_gb = 'nucl_gb.accession2taxid.gz'
nucl_wgs = 'nucl_wgs.accession2taxid.gz'
prot = 'prot.accession2taxid.gz'
acc_dl_list = []
db.connect()
parser = TaxaDumpParser(nodes_file=os.path.join(args.input, 'nodes.dmp'),
names_file=os.path.join(args.input, 'names.dmp'),
verbose=args.verbose)
logger.debug('Connected to database')
# If taxa table already exists, do not recreate and fill it
# safe=True prevent not to create the table if it already exists
if not Taxa.table_exists():
logger.info('Creating table %s' % str(Taxa.get_table_name()))
db.create_tables([Taxa])
logger.info("Parsing files")
taxa_info_list = parser.taxdump()
logger.info("Inserting taxonomy data")
total_size = len(taxa_info_list)
try:
with db.atomic():
for i in tqdm(range(0, total_size, args.chunk),
unit=' chunks', desc='INFO:taxadb.app',
total=''):
Taxa.insert_many(taxa_info_list[i:i+args.chunk]).execute()
except OperationalError as e:
print("\n") # needed because the above counter has none
logger.error("sqlite3 error: %s" % e)
logger.error("Maybe retry with a lower chunk size.")
sys.exit(1)
logger.info('Table Taxa completed')
# At first load, table accession does not exist yet, we create it
db.create_tables([Accession])
if div in ['full', 'nucl', 'gb']:
acc_dl_list.append(nucl_gb)
if div in ['full', 'nucl', 'wgs']:
acc_dl_list.append(nucl_wgs)
if div in ['full', 'prot']:
acc_dl_list.append(prot)
parser = Accession2TaxidParser(verbose=args.verbose, fast=args.fast)
with db.atomic():
for acc_file in acc_dl_list:
inserted_rows = 0
logger.info("Parsing %s" % str(acc_file))
for data_dict in tqdm(
parser.accession2taxid(
acc2taxid=os.path.join(args.input, acc_file),
chunk=args.chunk), unit=' chunks',
desc='INFO:taxadb.app',
total=''):
Accession.insert_many(data_dict[0:args.chunk]).execute()
inserted_rows += len(data_dict)
logger.info('%s: %s added to database (%d rows inserted)'
% (Accession.get_table_name(),
acc_file, inserted_rows))
if not Accession.has_index(name='accession_accession'):
logger.info('Creating index for %s'
% Accession.get_table_name())
try:
# db.add_index(Accession, ['accession'], unique=True)
idx = db.index(db.Accession, name='accession', unique=True)
db.add_index(idx)
except PeeweeException as err:
raise Exception("Could not create Accession index: %s"
% str(err))
logger.info('Table Accession completed')
db.close()