def make_lca_counts(dblist):
"""
Collect counts of all the LCAs in the list of databases.
CTB this could usefully be converted to a generator function.
"""
# gather all hashvalue assignments from across all the databases
assignments = defaultdict(set)
for lca_db in dblist:
for hashval, lid_list in lca_db.hashval_to_lineage_id.items():
lineages = [lca_db.lineage_dict[lid] for lid in lid_list]
assignments[hashval].update(lineages)
# now convert to trees -> do LCA & counts
counts = defaultdict(int)
for hashval, lineages in assignments.items():
# for each list of tuple_info [(rank, name), ...] build
# a tree that lets us discover lowest-common-ancestor.
debug(lineages)
tree = lca_utils.build_tree(lineages)
# now find either a leaf or the first node with multiple
# children; that's our lowest-common-ancestor node.
lca, reason = lca_utils.find_lca(tree)
counts[lca] += 1
return counts
def classify_signature(query_sig, dblist, threshold):
"""
Classify 'query_sig' using the given list of databases.
Insist on at least 'threshold' counts of a given lineage before taking
it seriously.
Return (lineage, status) where 'lineage' is a tuple of LineagePairs
and 'status' is either 'nomatch', 'found', or 'disagree'.
This function proceeds in two stages:
* first, build a list of assignments for all the lineages for each
hashval. (For e.g. kraken, this is done in the database preparation
step; here, we do it dynamically each time.
* then, across all the hashvals, count the number of times each linage
shows up, and filter out low-abundance ones (under threshold).
Then, determine the LCA of all of those.
"""
# gather assignments from across all the databases
assignments = lca_utils.gather_assignments(query_sig.minhash.get_mins(),
dblist)
# now convert to trees -> do LCA & counts
counts = lca_utils.count_lca_for_assignments(assignments)
debug(counts.most_common())
# ok, we now have the LCAs for each hashval, and their number of
# counts. Now build a tree across "significant" LCAs - those above
# threshold.
tree = {}
for lca, count in counts.most_common():
if count < threshold:
break
# update tree with this set of assignments
lca_utils.build_tree([lca], tree)
status = 'nomatch'
if not tree:
return [], status
# now find lowest-common-ancestor of the resulting tree.
lca, reason = lca_utils.find_lca(tree)
if reason == 0: # leaf node
debug('END', lca)
status = 'found'
else: # internal node => disagreement
debug('MULTI', lca)
status = 'disagree'
debug('lineage is:', lca)
return lca, status
def summarize(hashvals, dblist, threshold):
"""
Classify 'hashvals' using the given list of databases.
Insist on at least 'threshold' counts of a given lineage before taking
it seriously.
Return (lineage, counts) where 'lineage' is a tuple of LineagePairs.
"""
# gather assignments from across all the databases
assignments = lca_utils.gather_assignments(hashvals, dblist)
# now convert to trees -> do LCA & counts
counts = lca_utils.count_lca_for_assignments(assignments)
debug(counts.most_common())
# ok, we now have the LCAs for each hashval, and their number
# of counts. Now aggregate counts across the tree, going up from
# the leaves.
aggregated_counts = defaultdict(int)
for lca, count in counts.most_common():
if count < threshold:
break
if not lca:
aggregated_counts[lca] += count
# climb from the lca to the root.
while lca:
aggregated_counts[lca] += count
lca = lca[:-1]
debug(aggregated_counts)
return aggregated_counts
def gather_main(args):
"""
Do a greedy search for the hash components of a query against an LCA db.
Here we don't actually do a least-common-ancestor search of any kind; we
do essentially the same kind of search as we do in `sourmash gather`, with
the main difference that we are implicitly combining different genomes of
identical lineages.
This takes advantage of the structure of the LCA db, where we store the
full lineage information for each known hash, as opposed to storing only
the least-common-ancestor information for it.
"""
p = argparse.ArgumentParser(prog="sourmash lca gather")
p.add_argument('query')
p.add_argument('db', nargs='+')
p.add_argument('-d', '--debug', action='store_true')
p.add_argument('-o', '--output', type=argparse.FileType('wt'),
help='output CSV containing matches to this file')
p.add_argument('--output-unassigned', type=argparse.FileType('wt'),
help='output unassigned portions of the query as a signature to this file')
p.add_argument('--ignore-abundance', action='store_true',
help='do NOT use k-mer abundances if present')
args = p.parse_args(args)
if args.debug:
set_debug(args.debug)
# load all the databases
dblist, ksize, scaled = lca_utils.load_databases(args.db, None)
# for each query, gather all the matches across databases
query_sig = sourmash_args.load_query_signature(args.query, ksize, 'DNA')
debug('classifying', query_sig.name())
# make sure we're looking at the same scaled value as database
query_sig.minhash = query_sig.minhash.downsample_scaled(scaled)
# do the classification, output results
found = []
for result, f_unassigned, est_bp, remaining_mins in gather_signature(query_sig, dblist, args.ignore_abundance):
# is this our first time through the loop? print headers, if so.
if not len(found):
print_results("")
print_results("overlap p_query p_match ")
print_results("--------- ------- --------")
# output!
pct_query = '{:.1f}%'.format(result.f_unique_to_query*100)
pct_match = '{:.1f}%'.format(result.f_match*100)
str_bp = format_bp(result.intersect_bp)
name = format_lineage(result.lineage)
equal_match_str = ""
if result.n_equal_matches:
equal_match_str = " (** {} equal matches)".format(result.n_equal_matches)
print_results('{:9} {:>6} {:>6} {}{}', str_bp, pct_query,
pct_match, name, equal_match_str)
found.append(result)
if found:
print_results('')
if f_unassigned:
print_results('{:.1f}% ({}) of hashes have no assignment.', f_unassigned*100,
format_bp(est_bp))
else:
print_results('Query is completely assigned.')
print_results('')
# nothing found.
else:
est_bp = len(query_sig.minhash.get_mins()) * query_sig.minhash.scaled
print_results('')
print_results('No assignment for est {} of sequence.',
format_bp(est_bp))
print_results('')
if not found:
sys.exit(0)
if args.output:
fieldnames = ['intersect_bp', 'f_match', 'f_unique_to_query', 'f_unique_weighted',
'average_abund', 'name', 'n_equal_matches'] + list(lca_utils.taxlist())
w = csv.DictWriter(args.output, fieldnames=fieldnames)
w.writeheader()
for result in found:
lineage = result.lineage
d = dict(result._asdict())
del d['lineage']
for (rank, value) in lineage:
d[rank] = value
w.writerow(d)
if args.output_unassigned:
if not found:
notify('nothing found - entire query signature unassigned.')
elif not remaining_mins:
notify('no unassigned hashes! not saving.')
else:
outname = args.output_unassigned.name
notify('saving unassigned hashes to "{}"', outname)
e = query_sig.minhash.copy_and_clear()
e.add_many(remaining_mins)
sourmash_lib.save_signatures([ sourmash_lib.SourmashSignature(e) ],
args.output_unassigned)
def classify(args):
"""
main single-genome classification function.
"""
p = argparse.ArgumentParser()
p.add_argument('--db', nargs='+', action='append')
p.add_argument('--query', nargs='+', action='append')
p.add_argument('--threshold', type=int, default=DEFAULT_THRESHOLD)
p.add_argument('-o',
'--output',
type=argparse.FileType('wt'),
help='output CSV to this file instead of stdout')
p.add_argument('--scaled', type=float)
p.add_argument('--traverse-directory',
action='store_true',
help='load all signatures underneath directories.')
p.add_argument('-d', '--debug', action='store_true')
args = p.parse_args(args)
if not args.db:
error('Error! must specify at least one LCA database with --db')
sys.exit(-1)
if not args.query:
error('Error! must specify at least one query signature with --query')
sys.exit(-1)
if args.debug:
set_debug(args.debug)
# flatten --db and --query
args.db = [item for sublist in args.db for item in sublist]
args.query = [item for sublist in args.query for item in sublist]
# load all the databases
dblist, ksize, scaled = lca_utils.load_databases(args.db, args.scaled)
notify('ksize={} scaled={}', ksize, scaled)
# find all the queries
notify('finding query signatures...')
if args.traverse_directory:
inp_files = list(sourmash_args.traverse_find_sigs(args.query))
else:
inp_files = list(args.query)
# set up output
csvfp = csv.writer(sys.stdout)
if args.output:
notify("outputting classifications to '{}'", args.output.name)
csvfp = csv.writer(args.output)
else:
notify("outputting classifications to stdout")
csvfp.writerow(['ID', 'status'] + list(lca_utils.taxlist()))
# for each query, gather all the matches across databases
total_count = 0
n = 0
total_n = len(inp_files)
for query_filename in inp_files:
n += 1
for query_sig in sourmash_lib.load_signatures(query_filename,
ksize=ksize):
notify(u'\r\033[K', end=u'')
notify('... classifying {} (file {} of {})',
query_sig.name(),
n,
total_n,
end='\r')
debug('classifying', query_sig.name())
total_count += 1
# make sure we're looking at the same scaled value as database
query_sig.minhash = query_sig.minhash.downsample_scaled(scaled)
# do the classification
lineage, status = classify_signature(query_sig, dblist,
args.threshold)
debug(lineage)
# output each classification to the spreadsheet
row = [query_sig.name(), status]
row += lca_utils.zip_lineage(lineage)
# when outputting to stdout, make output intelligible
if not args.output:
notify(u'\r\033[K', end=u'')
csvfp.writerow(row)
notify(u'\r\033[K', end=u'')
notify('classified {} signatures total', total_count)
def index(args):
"""
main function for building an LCA database.
"""
p = argparse.ArgumentParser()
p.add_argument('csv', help='taxonomy spreadsheet')
p.add_argument('lca_db_out', help='name to save database to')
p.add_argument('signatures', nargs='+',
help='one or more sourmash signatures')
p.add_argument('--scaled', default=10000, type=float)
p.add_argument('-k', '--ksize', default=31, type=int)
p.add_argument('-d', '--debug', action='store_true')
p.add_argument('-C', '--start-column', default=2, type=int,
help='column at which taxonomic assignments start')
p.add_argument('--tabs', action='store_true',
help='input spreadsheet is tab-delimited (default: commas)')
p.add_argument('--no-headers', action='store_true',
help='no headers present in taxonomy spreadsheet')
p.add_argument('--split-identifiers', action='store_true',
help='split names in signatures on whitspace and period')
p.add_argument('-f', '--force', action='store_true')
p.add_argument('--traverse-directory', action='store_true',
help='load all signatures underneath directories.')
p.add_argument('--report', help='output a report on anomalies, if any.')
args = p.parse_args(args)
if args.start_column < 2:
error('error, --start-column cannot be less than 2')
sys.exit(-1)
if args.debug:
set_debug(args.debug)
args.scaled = int(args.scaled)
# first, load taxonomy spreadsheet
delimiter = ','
if args.tabs:
delimiter = '\t'
assignments, num_rows = load_taxonomy_assignments(args.csv,
delimiter=delimiter,
start_column=args.start_column,
use_headers=not args.no_headers,
force=args.force)
# convert lineages to numbers.
next_lineage_index = 0
lineage_dict = {}
assignments_idx = {}
lineage_to_idx = {}
for (ident, lineage) in assignments.items():
idx = lineage_to_idx.get(lineage)
if idx is None:
idx = next_lineage_index
next_lineage_index += 1
lineage_dict[idx] = lineage
lineage_to_idx[lineage] = idx
assignments_idx[ident] = idx
notify('{} distinct lineages in spreadsheet out of {} rows',
len(lineage_dict), num_rows)
# load signatures, construct index of hashvals to lineages
hashval_to_lineage = defaultdict(set)
md5_to_lineage = {}
notify('finding signatures...')
if args.traverse_directory:
yield_all_files = False # only pick up *.sig files?
if args.force:
yield_all_files = True
inp_files = list(sourmash_args.traverse_find_sigs(args.signatures,
yield_all_files=yield_all_files))
else:
inp_files = list(args.signatures)
n = 0
total_n = len(inp_files)
record_duplicates = set()
record_no_lineage = set()
record_remnants = set(assignments_idx.keys())
for filename in inp_files:
n += 1
for sig in sourmash_lib.load_signatures(filename, ksize=args.ksize):
notify(u'\r\033[K', end=u'')
notify('... loading signature {} (file {} of {})', sig.name()[:30], n, total_n, end='\r')
debug(filename, sig.name())
if sig.md5sum() in md5_to_lineage:
notify('\nWARNING: in file {}, duplicate md5sum: {}; skipping', filename, sig.md5sum())
record_duplicates.add(filename)
continue
name = sig.name()
if args.split_identifiers: # hack for NCBI-style names, etc.
name = name.split(' ')[0].split('.')[0]
# is this one for which we have a lineage assigned?
lineage_idx = assignments_idx.get(name)
if lineage_idx is None:
notify('\nWARNING: no lineage assignment for {}.', name)
record_no_lineage.add(name)
else:
# remove from our list of remnant lineages
record_remnants.remove(name)
# downsample to specified scaled; this has the side effect of
# making sure they're all at the same scaled value!
minhash = sig.minhash.downsample_scaled(args.scaled)
# connect hashvals to lineage
for hashval in minhash.get_mins():
hashval_to_lineage[hashval].add(lineage_idx)
# store md5 -> lineage too
md5_to_lineage[sig.md5sum()] = lineage_idx
notify(u'\r\033[K', end=u'')
notify('...found {} genomes with lineage assignments!!',
len(md5_to_lineage))
# remove those lineages with no genomes associated
assigned_lineages = set(md5_to_lineage.values())
lineage_dict_2 = {}
for idx in assigned_lineages:
lineage_dict_2[idx] = lineage_dict[idx]
unused_lineages = set(lineage_dict.values()) - set(lineage_dict_2.values())
notify('{} assigned lineages out of {} distinct lineages in spreadsheet',
len(lineage_dict_2), len(lineage_dict))
lineage_dict = lineage_dict_2
# now, save!
db_outfile = args.lca_db_out
if not (db_outfile.endswith('.lca.json') or db_outfile.endswith('.lca.json.gz')):
db_outfile += '.lca.json'
notify('saving to LCA DB: {}'.format(db_outfile))
db = lca_utils.LCA_Database()
db.lineage_dict = lineage_dict
db.hashval_to_lineage_id = hashval_to_lineage
db.ksize = int(args.ksize)
db.scaled = int(args.scaled)
db.signatures_to_lineage = md5_to_lineage
db.save(db_outfile)
if record_duplicates or record_no_lineage or record_remnants or unused_lineages:
if record_duplicates:
notify('WARNING: {} duplicate signatures.', len(record_duplicates))
if record_no_lineage:
notify('WARNING: no lineage provided for {} signatures.',
len(record_no_lineage))
if record_remnants:
notify('WARNING: no signatures for {} lineage assignments.',
len(record_remnants))
if unused_lineages:
notify('WARNING: {} unused lineages.', len(unused_lineages))
if args.report:
notify("generating a report and saving in '{}'", args.report)
generate_report(record_duplicates, record_no_lineage,
record_remnants, unused_lineages, args.report)
else:
notify('(You can use --report to generate a detailed report.)')
def main():
p = argparse.ArgumentParser()
p.add_argument('sigs', nargs='+')
p.add_argument('--traverse-directory',
action='store_true',
help='load all signatures underneath directories.')
p.add_argument('-k', '--ksize', default=31, type=int)
p.add_argument('-d', '--debug', action='store_true')
p.add_argument('-f', '--force', action='store_true')
p.add_argument('--scaled', type=float, default=10000)
p.add_argument('--plot', default=None)
p.add_argument('-o',
'--output',
type=argparse.FileType('wt'),
help='CSV output')
p.add_argument('--step', type=int, default=1000)
p.add_argument('--repeat', type=int, default=5)
p.add_argument('--db', nargs='+', action='append')
args = p.parse_args()
if args.debug:
set_debug(args.debug)
args.scaled = int(args.scaled)
dblist = []
known_hashes = set()
if args.db:
args.db = [item for sublist in args.db for item in sublist]
dblist, ksize, scaled = lca_utils.load_databases(args.db, args.scaled)
assert ksize == args.ksize
notify('loaded {} LCA databases', len(dblist))
for db in dblist:
known_hashes.update(db.hashval_to_lineage_id.keys())
notify('got {} known hashes!', len(known_hashes))
notify('finding signatures...')
if args.traverse_directory:
yield_all_files = False # only pick up *.sig files?
if args.force:
yield_all_files = True
inp_files = list(
sourmash_args.traverse_find_sigs(args.sigs,
yield_all_files=yield_all_files))
else:
inp_files = list(args.sigs)
n = 0
total_n = len(inp_files)
sigs = []
total_hashvals = list()
for filename in inp_files:
n += 1
for sig in sourmash_lib.load_signatures(filename, ksize=args.ksize):
notify(u'\r\033[K', end=u'')
notify('... loading signature {} (file {} of {})',
sig.name()[:30],
n,
total_n,
end='\r')
debug(filename, sig.name())
sig.minhash = sig.minhash.downsample_scaled(args.scaled)
total_hashvals.extend(sig.minhash.get_mins())
sigs.append(sig)
notify(u'\r\033[K', end=u'')
notify('...found {} signatures total in {} files.', len(sigs), total_n)
distinct_hashvals = set(total_hashvals)
notify('{} distinct out of {} total hashvals.', (len(distinct_hashvals)),
len(total_hashvals))
if known_hashes:
n_known = len(known_hashes.intersection(distinct_hashvals))
notify('{} of them known, or {:.1f}%', n_known,
n_known / float(len(distinct_hashvals)) * 100)
x = []
y = []
z = []
notify('subsampling...')
for n in range(0, len(total_hashvals), args.step):
notify(u'\r\033[K', end=u'')
notify('... {} of {}', n, len(total_hashvals), end='\r')
avg = 0
known = 0
for j in range(0, args.repeat):
subsample = random.sample(total_hashvals, n)
distinct = len(set(subsample))
if known_hashes:
known += len(set(subsample).intersection(known_hashes))
avg += distinct
x.append(n)
y.append(avg / args.repeat)
z.append(known / args.repeat)
notify('\n...done!')
if args.output:
w = csv.writer(args.output)
w.writerow(['n', 'k', 'known'])
for a, b, c in zip(x, y, z):
w.writerow([a, b, c])
if args.plot:
from matplotlib import pyplot
pyplot.plot(x, y)
pyplot.savefig(args.plot)
def summarize_main(args):
"""
main summarization function.
"""
p = argparse.ArgumentParser()
p.add_argument('--db', nargs='+', action='append')
p.add_argument('--query', nargs='+', action='append')
p.add_argument('--threshold', type=int, default=DEFAULT_THRESHOLD)
p.add_argument('--traverse-directory',
action='store_true',
help='load all signatures underneath directories.')
p.add_argument('-o',
'--output',
type=argparse.FileType('wt'),
help='CSV output')
p.add_argument('--scaled', type=float)
p.add_argument('-d', '--debug', action='store_true')
args = p.parse_args(args)
if not args.db:
error('Error! must specify at least one LCA database with --db')
sys.exit(-1)
if not args.query:
error('Error! must specify at least one query signature with --query')
sys.exit(-1)
if args.debug:
set_debug(args.debug)
if args.scaled:
args.scaled = int(args.scaled)
# flatten --db and --query
args.db = [item for sublist in args.db for item in sublist]
args.query = [item for sublist in args.query for item in sublist]
# load all the databases
dblist, ksize, scaled = lca_utils.load_databases(args.db, args.scaled)
notify('ksize={} scaled={}', ksize, scaled)
# find all the queries
notify('finding query signatures...')
if args.traverse_directory:
inp_files = list(sourmash_args.traverse_find_sigs(args.query))
else:
inp_files = list(args.query)
# for each query, gather all the hashvals across databases
total_count = 0
n = 0
total_n = len(inp_files)
hashvals = defaultdict(int)
for query_filename in inp_files:
n += 1
for query_sig in sourmash_lib.load_signatures(query_filename,
ksize=ksize):
notify(u'\r\033[K', end=u'')
notify('... loading {} (file {} of {})',
query_sig.name(),
n,
total_n,
end='\r')
total_count += 1
mh = query_sig.minhash.downsample_scaled(scaled)
for hashval in mh.get_mins():
hashvals[hashval] += 1
notify(u'\r\033[K', end=u'')
notify('loaded {} signatures from {} files total.', total_count, n)
# get the full counted list of lineage counts in this signature
lineage_counts = summarize(hashvals, dblist, args.threshold)
# output!
total = float(len(hashvals))
for (lineage, count) in lineage_counts.items():
if lineage:
lineage = lca_utils.zip_lineage(lineage, truncate_empty=True)
lineage = ';'.join(lineage)
else:
lineage = '(root)'
p = count / total * 100.
p = '{:.1f}%'.format(p)
print_results('{:5} {:>5} {}'.format(p, count, lineage))
# CSV:
if args.output:
w = csv.writer(args.output)
headers = ['count'] + list(lca_utils.taxlist())
w.writerow(headers)
for (lineage, count) in lineage_counts.items():
debug('lineage:', lineage)
row = [count] + lca_utils.zip_lineage(lineage)
w.writerow(row)