def _appraise_inexactly(self, metagenome_otu_table_collection,
found_otu_collection, sequence_identity):
'''Given a metagenome sample collection and OTUs 'found' either by binning or
assembly, return a AppraisalBuildingBlock representing the OTUs that
have been found, using inexact matching.
'''
found_otu_table = OtuTable()
found_otu_table.add(found_otu_collection)
found_collection = OtuTableCollection()
found_collection.otu_table_objects = [found_otu_table]
sample_to_building_block = {}
for uc in SequenceSearcher().global_search(
metagenome_otu_table_collection, found_otu_collection,
sequence_identity):
q = uc.query
if q.sample_name in sample_to_building_block:
appraisal = sample_to_building_block[q.sample_name]
else:
appraisal = AppraisalBuildingBlock()
sample_to_building_block[q.sample_name] = appraisal
if uc.target is not None:
appraisal.num_found += q.count
appraisal.found_otus.append(q)
return sample_to_building_block
def _appraise_inexactly(self, metagenome_otu_table_collection,
found_otu_collection,
sequence_identity):
'''Given a metagenome sample collection and OTUs 'found' either by binning or
assembly, return a AppraisalBuildingBlock representing the OTUs that
have been found, using inexact matching.
'''
found_otu_table = OtuTable()
found_otu_table.add(found_otu_collection)
found_collection = OtuTableCollection()
found_collection.otu_table_objects = [found_otu_table]
sample_to_building_block = {}
for uc in SequenceSearcher().global_search(metagenome_otu_table_collection,
found_otu_collection,
sequence_identity):
q = uc.query
if q.sample_name in sample_to_building_block:
appraisal = sample_to_building_block[q.sample_name]
else:
appraisal = AppraisalBuildingBlock()
sample_to_building_block[q.sample_name] = appraisal
if uc.target is not None:
appraisal.num_found += q.count
appraisal.found_otus.append(q)
return sample_to_building_block
def rarefy(self, otu_table_collection, num_to_sample, random_generator=random):
'''Return an OtuTable rarefied so that only num_to_sample sequences
are present in each sample. Samples not containing sufficient
sequences are ignored with a warning.
This is not a true rarefaction technique because sequences not
chosen in the rarefaction can still influence the output table
through the LCA or arbitrary choice operation that has been
carried out on the input table.
Also, the rarefier operates on counts rather than predicted
coverage, skeweing the results toward OTUs that lack
inserts. But not by a lot, presumably.
otu_table_collection: OtuTableCollection
OTU tables iterable
num_to_sample: int
number of sequences to sample from each
'''
sample_to_gene_to_otu = {}
to_return = OtuTable()
for otu in otu_table_collection:
sample_name = otu.sample_name
gene = otu.marker
if sample_name not in sample_to_gene_to_otu:
sample_to_gene_to_otu[sample_name] = {}
if gene not in sample_to_gene_to_otu[sample_name]:
sample_to_gene_to_otu[sample_name][gene] = {}
if otu.sequence in sample_to_gene_to_otu[sample_name][gene]:
raise Exception("Found duplicate sequence in OTU table in sample %s, gene %s" % sample_name, gene)
sample_to_gene_to_otu[sample_name][gene][otu.sequence] = otu
for sample_name in sample_to_gene_to_otu.keys():
for gene in sample_to_gene_to_otu[sample_name].keys():
sequences_to_sample = []
for sequence, otu in sample_to_gene_to_otu[sample_name][gene].items():
for _ in range(otu.count):
sequences_to_sample.append(sequence)
if len(sequences_to_sample) < num_to_sample:
logging.warn("Sample %s gene %s only contains %i sequences, so cannot be rarefied. Ignoring this sample/gene combination" % (sample_name, gene, len(sequences_to_sample)))
continue
else:
sequences_sampled = random_generator.sample(sequences_to_sample, num_to_sample)
sequence_counts = {}
for seq in sequences_sampled:
try:
sequence_counts[seq] += 1
except KeyError:
sequence_counts[seq] = 1
for seq, count in sequence_counts.items():
otu = sample_to_gene_to_otu[sample_name][gene][seq]
e = copy.copy(otu)
e.count = count
to_return.add([e])
return to_return
def collapse_coupled(self):
'''Return an OTU table that is collapsed in 2 ways: duplicate sequences are
collapsed together, and samples names are modified, removing r'.1$' and
r'.2$'.
'''
sample_to_sequence_to_otus = OrderedDict()
reg = re.compile(r'.[12]$')
for otu in self:
new_sample = reg.sub('',otu.sample_name)
otu.sample_name = new_sample
if new_sample not in sample_to_sequence_to_otus:
sample_to_sequence_to_otus[new_sample] = OrderedDict()
if otu.sequence not in sample_to_sequence_to_otus[new_sample]:
sample_to_sequence_to_otus[new_sample][otu.sequence] = []
sample_to_sequence_to_otus[new_sample][otu.sequence].append(otu)
otu_table = OtuTable()
for sample, seq_otus in sample_to_sequence_to_otus.items():
for seq, otus in seq_otus.items():
if len(otus) == 1:
otu_table.add(otus)
else:
o = OtuTableEntry()
o.marker = otus[0].marker
o.sample_name = sample
o.sequence = seq
o.count = sum([otu.count for otu in otus])
o.coverage = sum([otu.coverage for otu in otus])
o.taxonomy = otus[0].taxonomy #TODO: Make this more of a 'median' taxonomy.
otu_table.add([o])
return otu_table
def write_rarefied_otu_table(**kwargs):
output_table_io = kwargs.pop('output_table_io')
table_collection = kwargs.pop('table_collection')
number_to_choose = kwargs.pop('number_to_choose', None)
if len(kwargs) > 0:
raise Exception("Unexpected arguments detected: %s" % kwargs)
if number_to_choose is None:
counts = {}
for otu in table_collection:
key = "%s_singlem_RAND8_%s" % (otu.sample_name, otu.marker)
try:
counts[key] += otu.count
except KeyError:
counts[key] = otu.count
number_to_choose = min(counts.values())
logging.info(
"Minimum number of sequences detected is %i, rarefying all sample/gene combinations to this level"
% number_to_choose)
logging.info(
"Rarefying OTU table to max %i sequences per sample/gene combination and writing to %s"
% (number_to_choose, output_table_io.name))
OtuTable.write_otus_to(
Rarefier().rarefy(table_collection, number_to_choose),
output_table_io)
def rarefy(self, otu_table_collection, num_to_sample, random_generator=random):
'''Return an OtuTable rarefied so that only num_to_sample sequences
are present in each sample. Samples not containing sufficient
sequences are ignored with a warning.
This is not a true rarefaction technique because sequences not
chosen in the rarefaction can still influence the output table
through the LCA or arbitrary choice operation that has been
carried out on the input table.
Also, the rarefier operates on counts rather than predicted
coverage, skeweing the results toward OTUs that lack
inserts. But not by a lot, presumably.
otu_table_collection: OtuTableCollection
OTU tables iterable
num_to_sample: int
number of sequences to sample from each
'''
sample_to_gene_to_otu = {}
to_return = OtuTable()
for otu in otu_table_collection:
sample_name = otu.sample_name
gene = otu.marker
if sample_name not in sample_to_gene_to_otu:
sample_to_gene_to_otu[sample_name] = {}
if gene not in sample_to_gene_to_otu[sample_name]:
sample_to_gene_to_otu[sample_name][gene] = {}
if otu.sequence in sample_to_gene_to_otu[sample_name][gene]:
raise Exception("Found duplicate sequence in OTU table in sample %s, gene %s" % sample_name, gene)
sample_to_gene_to_otu[sample_name][gene][otu.sequence] = otu
for sample_name in sample_to_gene_to_otu.keys():
for gene in sample_to_gene_to_otu[sample_name].keys():
sequences_to_sample = []
for sequence, otu in sample_to_gene_to_otu[sample_name][gene].items():
for _ in range(otu.count):
sequences_to_sample.append(sequence)
if len(sequences_to_sample) < num_to_sample:
logging.warn("Sample %s gene %s only contains %i sequences, so cannot be rarefied. Ignoring this sample/gene combination" % (sample_name, gene, len(sequences_to_sample)))
continue
else:
sequences_sampled = random_generator.sample(sequences_to_sample, num_to_sample)
sequence_counts = {}
for seq in sequences_sampled:
try:
sequence_counts[seq] += 1
except KeyError:
sequence_counts[seq] = 1
for seq, count in sequence_counts.items():
otu = sample_to_gene_to_otu[sample_name][gene][seq]
e = copy.copy(otu)
e.count = count
to_return.add([e])
return to_return
def __iter__(self):
'''Iterate over all the OTUs from all the tables. This can only be done once
since the data is streamed in.
'''
for io in self._archive_table_io_objects:
for otu in ArchiveOtuTable.read(io):
yield otu
for io in self._otu_table_io_objects:
for otu in OtuTable.each(io):
yield otu
for file_path in self._archive_table_file_paths:
for otu in ArchiveOtuTable.read(open(file_path)):
yield otu
for file_path in self._otu_table_file_paths:
for otu in OtuTable.each(open(file_path)):
yield otu
def write_clustered_otu_table(**kwargs):
output_table_io = kwargs.pop('output_table_io')
table_collection = kwargs.pop('table_collection')
if len(kwargs) > 0:
raise Exception("Unexpected arguments detected: %s" % kwargs)
logging.info("Writing clustered OTU table")
output_table_io.write(
"\t".join(
OtuTable.DEFAULT_OUTPUT_FIELDS+
['representative',
'total_num_reads',
'total_coverage',
'num_sub_otus',
'max_sub_otu_abundance'])
+"\n")
for d in table_collection:
for otu in d.otus:
output_table_io.write("\t".join(
[OtuTable._to_printable(cell) for cell in [
otu.marker,
otu.sample_name,
otu.sequence,
otu.count,
otu.coverage,
otu.taxonomy,
d.sequence,
d.count,
d.coverage,
len(d.otus),
max([otu.count for otu in d.otus])
]])+"\n")
def write_clustered_otu_table(**kwargs):
output_table_io = kwargs.pop('output_table_io')
table_collection = kwargs.pop('table_collection')
if len(kwargs) > 0:
raise Exception("Unexpected arguments detected: %s" % kwargs)
logging.info("Writing clustered OTU table")
output_table_io.write(
"\t".join(
OtuTable.DEFAULT_OUTPUT_FIELDS+
['representative',
'total_num_reads',
'total_coverage',
'num_sub_otus',
'max_sub_otu_abundance'])
+"\n")
for d in table_collection:
for otu in d.otus:
output_table_io.write("\t".join(
[OtuTable._to_printable(cell) for cell in [
otu.marker,
otu.sample_name,
otu.sequence,
otu.count,
otu.coverage,
otu.taxonomy,
d.sequence,
d.count,
d.coverage,
len(d.otus),
max([otu.count for otu in d.otus])
]])+"\n")
def __iter__(self):
'''Iterate over all the OTUs from all the tables. This can only be done once
since the data is streamed in.
'''
for io in self._archive_table_io_objects:
for otu in ArchiveOtuTable.read(io):
yield otu
for io in self._otu_table_io_objects:
for otu in OtuTable.read(io):
yield otu
for file_path in self._archive_table_file_paths:
for otu in ArchiveOtuTable.read(open(file_path)):
yield otu
for file_path in self._otu_table_file_paths:
for otu in OtuTable.each(open(file_path)):
yield otu
def write_otu_table(**kwargs):
output_table_io = kwargs.pop('output_table_io')
table_collection = kwargs.pop('table_collection')
output_extras = kwargs.pop('output_extras')
if len(kwargs) > 0:
raise Exception("Unexpected arguments detected: %s" % kwargs)
if hasattr(output_table_io, 'name'):
logging.info("Writing %s" % output_table_io.name)
else:
logging.info("Writing an OTU table")
if output_extras:
OtuTable.write_otus_to(table_collection, output_table_io,
fields_to_print=table_collection.example_field_names())
else:
OtuTable.write_otus_to(table_collection, output_table_io)
def write_otu_table(**kwargs):
output_table_io = kwargs.pop('output_table_io')
table_collection = kwargs.pop('table_collection')
output_extras = kwargs.pop('output_extras')
if len(kwargs) > 0:
raise Exception("Unexpected arguments detected: %s" % kwargs)
if hasattr(output_table_io, 'name'):
logging.info("Writing %s" % output_table_io.name)
else:
logging.info("Writing an OTU table")
if output_extras:
OtuTable.write_otus_to(table_collection, output_table_io,
fields_to_print=table_collection.example_field_names())
else:
OtuTable.write_otus_to(table_collection, output_table_io)
def write_rarefied_otu_table(**kwargs):
output_table_io = kwargs.pop('output_table_io')
table_collection = kwargs.pop('table_collection')
number_to_choose = kwargs.pop('number_to_choose', None)
if len(kwargs) > 0:
raise Exception("Unexpected arguments detected: %s" % kwargs)
if number_to_choose is None:
counts = {}
for otu in table_collection:
key = "%s_singlem_RAND8_%s" % (otu.sample_name, otu.marker)
try:
counts[key] += otu.count
except KeyError:
counts[key] = otu.count
number_to_choose = min(counts.values())
logging.info("Minimum number of sequences detected is %i, rarefying all sample/gene combinations to this level" % number_to_choose)
logging.info("Rarefying OTU table to max %i sequences per sample/gene combination and writing to %s" % (number_to_choose, output_table_io.name))
OtuTable.write_otus_to(Rarefier().rarefy(table_collection, number_to_choose),
output_table_io)
def print_samples(self, **kwargs):
db = SequenceDatabase.acquire(kwargs.pop('db'))
sample_names = kwargs.pop('sample_names')
taxonomy = kwargs.pop('taxonomy')
output_io = kwargs.pop('output_io')
if len(kwargs) > 0:
raise Exception("Unexpected arguments detected: %s" % kwargs)
dbm = self._connect_to_sqlite(db)
max_set_size = 999 # Cannot query sqlite with > 999 '?' entries, so
# query in batches.
if sample_names:
query_chunks = set(sample_names)
else:
query_chunks = [taxonomy]
otus = OtuTable()
total_printed = 0
for chunk in SequenceDatabase.grouper(query_chunks, max_set_size):
if sample_names:
it = dbm.table('otus').where_in(
'sample_name', [sample for sample in chunk if sample is not None]).get()
elif taxonomy:
it = dbm.table('otus').where(
'taxonomy', 'like', "%%%s%%" % taxonomy).get()
else:
raise Exception("Programming error")
for entry in it:
otu = OtuTableEntry()
otu.marker = entry.marker
otu.sample_name = entry.sample_name
otu.sequence = entry.sequence
otu.count = entry.num_hits
otu.coverage = entry.coverage
otu.taxonomy = entry.taxonomy
otus.add([otu])
total_printed += 1
otus.write_to(output_io)
logging.info("Printed %i OTU table entries" % total_printed)
def collapse_coupled(self):
'''Return an OTU table that is collapsed in 2 ways: duplicate sequences are
collapsed together, and samples names are modified, removing r'.1$' and
r'.2$'.
'''
sample_to_sequence_to_otus = OrderedDict()
reg = re.compile(r'.[12]$')
for otu in self:
new_sample = reg.sub('',otu.sample_name)
otu.sample_name = new_sample
if new_sample not in sample_to_sequence_to_otus:
sample_to_sequence_to_otus[new_sample] = OrderedDict()
if otu.sequence not in sample_to_sequence_to_otus[new_sample]:
sample_to_sequence_to_otus[new_sample][otu.sequence] = []
sample_to_sequence_to_otus[new_sample][otu.sequence].append(otu)
otu_table = OtuTable()
for sample, seq_otus in sample_to_sequence_to_otus.items():
for seq, otus in seq_otus.items():
if len(otus) == 1:
otu_table.add(otus)
else:
o = OtuTableEntry()
o.marker = otus[0].marker
o.sample_name = sample
o.sequence = seq
o.count = sum([otu.count for otu in otus])
o.coverage = sum([otu.coverage for otu in otus])
o.taxonomy = otus[0].taxonomy #TODO: Make this more of a 'median' taxonomy.
otu_table.add([o])
return otu_table
def add_otu_table(self, input_otu_table_io):
'''Add a regular style OTU table to the collection.
Parameters
----------
input_otu_table_ios: list of IO
entries are open streams of OTU table data
Returns
-------
None
'''
self.otu_table_objects.append(OtuTable.read(input_otu_table_io))
def add_otu_table(self, input_otu_table_io):
'''Add a regular style OTU table to the collection.
Parameters
----------
input_otu_table_ios: list of IO
entries are open streams of OTU table data
Returns
-------
None
'''
self.otu_table_objects.append(OtuTable.read(input_otu_table_io))
def print_samples(self, **kwargs):
db = SequenceDatabase.acquire(kwargs.pop('db'))
sample_names = kwargs.pop('sample_names')
taxonomy = kwargs.pop('taxonomy')
output_io = kwargs.pop('output_io')
if len(kwargs) > 0:
raise Exception("Unexpected arguments detected: %s" % kwargs)
dbm = self._connect_to_sqlite(db)
max_set_size = 999 # Cannot query sqlite with > 999 '?' entries, so
# query in batches.
if sample_names:
query_chunks = set(sample_names)
else:
query_chunks = [taxonomy]
otus = OtuTable()
total_printed = 0
for chunk in SequenceDatabase.grouper(query_chunks, max_set_size):
if sample_names:
it = dbm.table('otus').where_in(
'sample_name',
[sample for sample in chunk if sample is not None]).get()
elif taxonomy:
it = dbm.table('otus').where('taxonomy', 'like',
"%%%s%%" % taxonomy).get()
else:
raise Exception("Programming error")
for entry in it:
otu = OtuTableEntry()
otu.marker = entry.marker
otu.sample_name = entry.sample_name
otu.sequence = entry.sequence
otu.count = entry.num_hits
otu.coverage = entry.coverage
otu.taxonomy = entry.taxonomy
otus.add([otu])
total_printed += 1
otus.write_to(output_io)
logging.info("Printed %i OTU table entries" % total_printed)
def run_to_otu_table(self, **kwargs):
'''Run the pipe, '''
forward_read_files = kwargs.pop('sequences')
num_threads = kwargs.pop('threads')
known_otu_tables = kwargs.pop('known_otu_tables')
singlem_assignment_method = kwargs.pop('assignment_method')
output_jplace = kwargs.pop('output_jplace')
evalue = kwargs.pop('evalue')
min_orf_length = kwargs.pop('min_orf_length')
restrict_read_length = kwargs.pop('restrict_read_length')
filter_minimum_protein = kwargs.pop('filter_minimum_protein')
filter_minimum_nucleotide = kwargs.pop('filter_minimum_nucleotide')
include_inserts = kwargs.pop('include_inserts')
singlem_packages = kwargs.pop('singlem_packages')
assign_taxonomy = kwargs.pop('assign_taxonomy')
known_sequence_taxonomy = kwargs.pop('known_sequence_taxonomy')
working_directory = kwargs.pop('working_directory')
force = kwargs.pop('force')
if len(kwargs) > 0:
raise Exception("Unexpected arguments detected: %s" % kwargs)
self._num_threads = num_threads
self._evalue = evalue
self._min_orf_length = min_orf_length
self._restrict_read_length = restrict_read_length
self._filter_minimum_protein = filter_minimum_protein
self._filter_minimum_nucleotide = filter_minimum_nucleotide
hmms = HmmDatabase(singlem_packages)
if singlem_assignment_method == DIAMOND_EXAMPLE_BEST_HIT_ASSIGNMENT_METHOD:
graftm_assignment_method = DIAMOND_ASSIGNMENT_METHOD
else:
graftm_assignment_method = singlem_assignment_method
if logging.getLevelName(logging.getLogger().level) == 'DEBUG':
self._graftm_verbosity = '5'
else:
self._graftm_verbosity = '2'
if not assign_taxonomy:
singlem_assignment_method = NO_ASSIGNMENT_METHOD
using_temporary_working_directory = working_directory is None
if using_temporary_working_directory:
shared_mem_directory = '/dev/shm'
if os.path.exists(shared_mem_directory):
logging.debug("Using shared memory as a base directory")
tmp = tempdir.TempDir(basedir=shared_mem_directory)
tempfiles_path = os.path.join(tmp.name, 'tempfiles')
os.mkdir(tempfiles_path)
os.environ['TEMP'] = tempfiles_path
else:
logging.debug(
"Shared memory directory not detected, using default temporary directory instead"
)
tmp = tempdir.TempDir()
working_directory = tmp.name
else:
working_directory = working_directory
if os.path.exists(working_directory):
if force:
logging.info("Overwriting directory %s" %
working_directory)
shutil.rmtree(working_directory)
os.mkdir(working_directory)
else:
raise Exception(
"Working directory '%s' already exists, not continuing"
% working_directory)
else:
os.mkdir(working_directory)
logging.debug("Using working directory %s" % working_directory)
self._working_directory = working_directory
extracted_reads = None
def return_cleanly():
if using_temporary_working_directory: tmp.dissolve()
logging.info("Finished")
#### Search
self._singlem_package_database = hmms
search_result = self._search(hmms, forward_read_files)
sample_names = search_result.samples_with_hits()
if len(sample_names) == 0:
logging.info("No reads identified in any samples, stopping")
return_cleanly()
return None
logging.debug("Recovered %i samples with at least one hit e.g. '%s'" \
% (len(sample_names), sample_names[0]))
#### Alignment
align_result = self._align(search_result)
### Extract reads that have already known taxonomy
if known_otu_tables:
logging.info("Parsing known taxonomy OTU tables")
known_taxes = KnownOtuTable()
known_taxes.parse_otu_tables(known_otu_tables)
logging.debug("Read in %i sequences with known taxonomy" %
len(known_taxes))
else:
known_taxes = []
if known_sequence_taxonomy:
logging.debug("Parsing sequence-wise taxonomy..")
tax1 = GreenGenesTaxonomy.read(
open(known_sequence_taxonomy)).taxonomy
known_sequence_tax = {}
for seq_id, tax in tax1.items():
known_sequence_tax[seq_id] = '; '.join(tax)
logging.info(
"Read in %i taxonomies from the GreenGenes format taxonomy file"
% len(known_sequence_tax))
### Extract other reads which do not have known taxonomy
extracted_reads = self._extract_relevant_reads(align_result,
include_inserts,
known_taxes)
logging.info("Finished extracting aligned sequences")
#### Taxonomic assignment
if assign_taxonomy:
logging.info("Running taxonomic assignment with GraftM..")
assignment_result = self._assign_taxonomy(
extracted_reads, graftm_assignment_method)
#### Process taxonomically assigned reads
# get the sequences out for each of them
otu_table_object = OtuTable()
if singlem_assignment_method == PPLACER_ASSIGNMENT_METHOD:
package_to_taxonomy_bihash = {}
for readset in extracted_reads:
sample_name = readset.sample_name
singlem_package = readset.singlem_package
known_sequences = readset.known_sequences
def add_info(infos, otu_table_object, known_tax):
for info in infos:
to_print = [
singlem_package.graftm_package_basename(), sample_name,
info.seq, info.count, info.coverage, info.taxonomy,
info.names, info.aligned_lengths, known_tax
]
otu_table_object.data.append(to_print)
known_infos = self._seqs_to_counts_and_taxonomy(
known_sequences, NO_ASSIGNMENT_METHOD, known_taxes,
known_sequence_taxonomy, None)
add_info(known_infos, otu_table_object, True)
if len(
readset.unknown_sequences
) > 0: # if any sequences were aligned (not just already known)
tmpbase = readset.tmpfile_basename
if assign_taxonomy:
is_known_taxonomy = False
aligned_seqs = list(
itertools.chain(readset.unknown_sequences,
readset.known_sequences))
if singlem_assignment_method == DIAMOND_EXAMPLE_BEST_HIT_ASSIGNMENT_METHOD:
tax_file = assignment_result.diamond_assignment_file(
sample_name, singlem_package, tmpbase)
taxonomies = DiamondResultParser(tax_file)
elif singlem_assignment_method == DIAMOND_ASSIGNMENT_METHOD:
tax_file = assignment_result.read_tax_file(
sample_name, singlem_package, tmpbase)
if not os.path.isfile(tax_file):
logging.warn(
"Unable to find tax file for gene %s from sample %s "
"(likely do to min length filtering), skipping"
% (os.path.basename(
singlem_package.base_directory()),
sample_name))
taxonomies = {}
else:
taxonomies = TaxonomyFile(tax_file)
elif singlem_assignment_method == PPLACER_ASSIGNMENT_METHOD:
bihash_key = singlem_package.base_directory()
if bihash_key in package_to_taxonomy_bihash:
taxonomy_bihash = package_to_taxonomy_bihash[
bihash_key]
else:
taxtastic_taxonomy = singlem_package.graftm_package(
).taxtastic_taxonomy_path()
logging.debug(
"Reading taxtastic taxonomy from %s" %
taxtastic_taxonomy)
with open(taxtastic_taxonomy) as f:
taxonomy_bihash = TaxonomyBihash.parse_taxtastic_taxonomy(
f)
package_to_taxonomy_bihash[
bihash_key] = taxonomy_bihash
base_dir = assignment_result._base_dir(
sample_name, singlem_package, tmpbase)
jplace_file = os.path.join(base_dir,
"placements.jplace")
logging.debug(
"Attempting to read jplace output from %s" %
jplace_file)
if os.path.exists(jplace_file):
with open(jplace_file) as f:
jplace_json = json.loads(f.read())
placement_parser = PlacementParser(
jplace_json, taxonomy_bihash, 0.5)
else:
# Sometimes alignments are filtered out.
placement_parser = None
taxonomies = {}
elif singlem_assignment_method == NO_ASSIGNMENT_METHOD:
taxonomies = {}
else:
raise Exception("Programming error")
else: # Taxonomy has not been assigned.
aligned_seqs = readset.unknown_sequences
if known_sequence_taxonomy:
taxonomies = known_sequence_tax
else:
taxonomies = {}
is_known_taxonomy = True
new_infos = list(
self._seqs_to_counts_and_taxonomy(
aligned_seqs, singlem_assignment_method,
known_sequence_tax if known_sequence_taxonomy else {},
taxonomies,
placement_parser if singlem_assignment_method
== PPLACER_ASSIGNMENT_METHOD else None))
add_info(new_infos, otu_table_object, is_known_taxonomy)
if output_jplace:
base_dir = assignment_result._base_dir(
sample_name, singlem_package, tmpbase)
input_jplace_file = os.path.join(base_dir,
"placements.jplace")
output_jplace_file = "%s_%s_%s.jplace" % (
output_jplace, sample_name,
singlem_package.graftm_package_basename())
logging.info("Writing jplace file '%s'" %
output_jplace_file)
logging.debug(
"Converting jplace file %s to singlem jplace file %s" %
(input_jplace_file, output_jplace_file))
with open(output_jplace_file, 'w') as output_jplace_io:
self._write_jplace_from_infos(open(input_jplace_file),
new_infos,
output_jplace_io)
return_cleanly()
return otu_table_object
def print_appraisal(self, appraisal,
doing_binning,
output_io=sys.stdout,
doing_assembly=False,
binned_otu_table_io=None,
unbinned_otu_table_io=None,
assembled_otu_table_io=None,
unaccounted_for_otu_table_io=None):
'''print the Appraisal object overview to STDOUT'''
headers = ['sample']
if doing_binning: headers.append('num_binned')
if doing_assembly: headers.append('num_assembled')
headers.append('num_not_found')
if doing_binning: headers.append('percent_binned')
if doing_assembly: headers.append('percent_assembled')
output_io.write("\t".join(headers)+"\n")
binned = []
assembled = []
assembled_not_binned = []
not_founds = []
def print_sample(num_binned, num_assembled, num_assembled_not_binned, num_not_found, sample,
mypercent_binned=None, mypercent_assembled=None):
if mypercent_binned is not None or mypercent_assembled is not None:
if doing_binning:
percent_binned = mypercent_binned
if doing_assembly:
percent_assembled = mypercent_assembled
else:
total = num_not_found
if doing_binning: total += num_binned
if doing_assembly: total += num_assembled_not_binned
if total == 0:
if doing_binning: percent_binned = 0.0
if doing_assembly: percent_assembled = 0.0
else:
if doing_binning:
percent_binned = float(num_binned)/total * 100
if doing_assembly:
percent_assembled = float(num_assembled)/total * 100
to_write = [sample]
if doing_binning: to_write.append(str(num_binned))
if doing_assembly: to_write.append(str(num_assembled))
to_write.append(str(num_not_found))
if doing_binning:
to_write.append("%2.1f" % percent_binned)
if doing_assembly:
to_write.append("%2.1f" % percent_assembled)
output_io.write("\t".join(to_write)+"\n")
def mean(l):
return float(sum(l))/len(l) if len(l) > 0 else float('nan')
if binned_otu_table_io:
binned_table = OtuTable()
if unbinned_otu_table_io:
unbinned_table = OtuTable()
if assembled_otu_table_io:
assembled_table = OtuTable()
if unaccounted_for_otu_table_io:
unaccounted_for_table = OtuTable()
for appraisal_result in appraisal.appraisal_results:
if doing_assembly:
num_assembled_not_binned = appraisal_result.num_assembled_not_binned()
print_sample(appraisal_result.num_binned if doing_binning else None,
appraisal_result.num_assembled if doing_assembly else None,
num_assembled_not_binned if doing_assembly else None,
appraisal_result.num_not_found,
appraisal_result.metagenome_sample_name)
if doing_binning:
binned.append(appraisal_result.num_binned)
if doing_assembly:
assembled.append(appraisal_result.num_assembled)
assembled_not_binned.append(num_assembled_not_binned)
not_founds.append(appraisal_result.num_not_found)
if binned_otu_table_io:
binned_table.add(appraisal_result.binned_otus)
if unbinned_otu_table_io:
unbinned_table.add(appraisal_result.assembled_not_binned_otus())
if assembled_otu_table_io:
assembled_table.add(appraisal_result.assembled_otus)
if unaccounted_for_otu_table_io:
unaccounted_for_table.add(appraisal_result.not_found_otus)
print_sample(sum(binned) if doing_binning else None,
sum(assembled) if doing_assembly else None,
sum(assembled_not_binned) if doing_assembly else None,
sum(not_founds),
'total')
binned_means = []
assembled_means = []
if doing_binning:
to_enumerate = binned
else:
to_enumerate = assembled
for i, _ in enumerate(to_enumerate):
num_binned = binned[i] if doing_binning else 0
num_assembled = assembled[i] if doing_assembly else 0
num_assembled_not_binned = assembled_not_binned[i] if doing_assembly else 0
num_not_found = not_founds[i]
total = num_assembled_not_binned+num_not_found
if doing_binning:
total += num_binned
binned_means.append(float(num_binned)/total)
if doing_assembly:
assembled_means.append(float(num_assembled)/total)
print_sample("%2.1f" % mean(binned) if doing_binning else None,
"%2.1f" % mean(assembled) if doing_assembly else None,
None,
"%2.1f" % mean(not_founds),
'average',
mypercent_binned=mean(binned_means)*100 if doing_binning else None,
mypercent_assembled=(mean(assembled_means)*100 if doing_assembly else None))
if binned_otu_table_io:
binned_table.write_to(binned_otu_table_io)
if unbinned_otu_table_io:
unbinned_table.write_to(unbinned_otu_table_io)
if assembled_otu_table_io:
assembled_table.write_to(assembled_otu_table_io)
if unaccounted_for_otu_table_io:
unaccounted_for_table.write_to(unaccounted_for_otu_table_io)
def print_appraisal(self, appraisal,
output_io=sys.stdout,
accounted_for_otu_table_io=None,
unaccounted_for_otu_table_io=None):
'''print the Appraisal object overview to STDOUT'''
output_io.write("\t".join(['sample','num_found','num_not_found','percent_found'])+"\n")
founds = []
not_founds = []
def print_sample(num_found, num_not_found, sample, mypercent=None):
if mypercent:
percent = mypercent
elif num_found + num_not_found == 0:
percent = 0.0
else:
percent = float(num_found)/(num_found+num_not_found) * 100
output_io.write("\t".join([sample, str(num_found), str(num_not_found), "%2.1f" % percent])+"\n")
def mean(l):
return float(sum(l))/len(l) if len(l) > 0 else float('nan')
if accounted_for_otu_table_io:
accounted_for_table = OtuTable()
if unaccounted_for_otu_table_io:
unaccounted_for_table = OtuTable()
for appraisal_result in appraisal.appraisal_results:
print_sample(appraisal_result.num_found,
appraisal_result.num_not_found,
appraisal_result.metagenome_sample_name)
founds.append(appraisal_result.num_found)
not_founds.append(appraisal_result.num_not_found)
if accounted_for_otu_table_io:
accounted_for_table.add(appraisal_result.found_otus)
if accounted_for_otu_table_io:
unaccounted_for_table.add(appraisal_result.not_found_otus)
print_sample(sum(founds), sum(not_founds), 'total')
means = []
for i, num_found in enumerate(founds):
num_not_found = not_founds[i]
means.append(float(num_found)/(num_found+num_not_found))
print_sample("%2.1f" % mean(founds), "%2.1f" % mean(not_founds), 'average',
mypercent=mean(means)*100)
if accounted_for_otu_table_io:
accounted_for_table.write_to(accounted_for_otu_table_io)
if unaccounted_for_otu_table_io:
unaccounted_for_table.write_to(unaccounted_for_otu_table_io)
def appraise(self, **kwargs):
'''Given a collection of OTU tables derived from samples, and OTU
table(s) corresponding to a collection of recovered genomes, how
much of the community has been recovered in those genomes?
Parameters
----------
kwargs:
sequence_identity: float for 'near enough', None when an exact match is required.
Returns
-------
An Appraisal object containing appraisals for each metagenome
'''
genome_otu_table_collection = kwargs.pop('genome_otu_table_collection')
metagenome_otu_table_collection = kwargs.pop('metagenome_otu_table_collection')
sequence_identity = kwargs.pop('sequence_identity', None)
if len(kwargs) > 0:
raise Exception("Unexpected arguments detected: %s" % kwargs)
logging.info("Read in %i markers from the different genomes" %\
len(genome_otu_table_collection))
filtered_genome_otus = \
list(genome_otu_table_collection.excluded_duplicate_distinct_genes())
logging.info("After excluding duplicate markers that may indicate "
"contamination, found %i markers" % len(filtered_genome_otus))
if sequence_identity is None:
genome_otu_sequences = set()
genome_names = set()
for otu in filtered_genome_otus:
genome_otu_sequences.add(otu.sequence)
genome_names.add(otu.sample_name)
logging.info("Read in %i unique sequences from the %i reference genomes" %\
(len(genome_otu_sequences), len(genome_names)))
# read in metagenome OTU sequences
sample_name_to_appraisal = {}
for otu in metagenome_otu_table_collection:
try:
appraisal = sample_name_to_appraisal[otu.sample_name]
except KeyError:
appraisal = AppraisalResult()
appraisal.metagenome_sample_name = otu.sample_name
sample_name_to_appraisal[otu.sample_name] = appraisal
count = otu.count
if otu.sequence in genome_otu_sequences:
appraisal.num_found += count
appraisal.found_otus.append(otu)
else:
appraisal.num_not_found += count
appraisal.not_found_otus.append(otu)
app = Appraisal()
app.appraisal_results = sample_name_to_appraisal.values()
return app
else:
sample_name_to_appraisal = {}
seen_otus = set()
genome_otu_table = OtuTable()
genome_otu_table.add(filtered_genome_otus)
filtered_collection = OtuTableCollection()
filtered_collection.otu_table_objects = [genome_otu_table]
for uc in SequenceSearcher().global_search(metagenome_otu_table_collection,
filtered_collection,
sequence_identity):
q = uc.query
key = str([q.sample_name, q.sequence])
if key in seen_otus:
logging.warn("Double-saw an OTU..")
continue
else:
seen_otus.add(key)
if q.sample_name not in sample_name_to_appraisal:
res = AppraisalResult()
res.metagenome_sample_name = q.sample_name
sample_name_to_appraisal[q.sample_name] = res
appraisal = sample_name_to_appraisal[q.sample_name]
if uc.target is None:
appraisal.num_not_found += q.count
appraisal.not_found_otus.append(q)
else:
appraisal.num_found += q.count
appraisal.found_otus.append(q)
app = Appraisal()
app.appraisal_results = sample_name_to_appraisal.values()
return app
def run(self, **kwargs):
forward_read_files = kwargs.pop('sequences')
output_otu_table = kwargs.pop('otu_table', None)
archive_otu_table = kwargs.pop('archive_otu_table', None)
num_threads = kwargs.pop('threads')
known_otu_tables = kwargs.pop('known_otu_tables')
singlem_assignment_method = kwargs.pop('assignment_method')
output_jplace = kwargs.pop('output_jplace')
output_extras = kwargs.pop('output_extras')
evalue = kwargs.pop('evalue')
min_orf_length = kwargs.pop('min_orf_length')
restrict_read_length = kwargs.pop('restrict_read_length')
filter_minimum_protein = kwargs.pop('filter_minimum_protein')
filter_minimum_nucleotide = kwargs.pop('filter_minimum_nucleotide')
include_inserts = kwargs.pop('include_inserts')
singlem_packages = kwargs.pop('singlem_packages')
window_size = kwargs.pop('window_size')
assign_taxonomy = kwargs.pop('assign_taxonomy')
known_sequence_taxonomy = kwargs.pop('known_sequence_taxonomy')
working_directory = kwargs.pop('working_directory')
force = kwargs.pop('force')
if len(kwargs) > 0:
raise Exception("Unexpected arguments detected: %s" % kwargs)
self._num_threads = num_threads
self._evalue = evalue
self._min_orf_length = min_orf_length
self._restrict_read_length = restrict_read_length
self._filter_minimum_protein = filter_minimum_protein
self._filter_minimum_nucleotide = filter_minimum_nucleotide
hmms = HmmDatabase(singlem_packages)
if singlem_assignment_method == DIAMOND_EXAMPLE_BEST_HIT_ASSIGNMENT_METHOD:
graftm_assignment_method = DIAMOND_ASSIGNMENT_METHOD
else:
graftm_assignment_method = singlem_assignment_method
if logging.getLevelName(logging.getLogger().level) == 'DEBUG':
self._graftm_verbosity = '5'
else:
self._graftm_verbosity = '2'
using_temporary_working_directory = working_directory is None
if using_temporary_working_directory:
shared_mem_directory = '/dev/shm'
if os.path.exists(shared_mem_directory):
logging.debug("Using shared memory as a base directory")
tmp = tempdir.TempDir(basedir=shared_mem_directory)
tempfiles_path = os.path.join(tmp.name, 'tempfiles')
os.mkdir(tempfiles_path)
os.environ['TEMP'] = tempfiles_path
else:
logging.debug("Shared memory directory not detected, using default temporary directory instead")
tmp = tempdir.TempDir()
working_directory = tmp.name
else:
working_directory = working_directory
if os.path.exists(working_directory):
if force:
logging.info("Overwriting directory %s" % working_directory)
shutil.rmtree(working_directory)
os.mkdir(working_directory)
else:
raise Exception("Working directory '%s' already exists, not continuing" % working_directory)
else:
os.mkdir(working_directory)
logging.debug("Using working directory %s" % working_directory)
self._working_directory = working_directory
extracted_reads = None
def return_cleanly():
if extracted_reads: extracted_reads.cleanup()
if using_temporary_working_directory: tmp.dissolve()
logging.info("Finished")
#### Search
self._singlem_package_database = hmms
search_result = self._search(hmms, forward_read_files)
sample_names = search_result.samples_with_hits()
if len(sample_names) == 0:
logging.info("No reads identified in any samples, stopping")
return_cleanly()
return
logging.debug("Recovered %i samples with at least one hit e.g. '%s'" \
% (len(sample_names), sample_names[0]))
#### Alignment
align_result = self._align(search_result)
### Extract reads that have already known taxonomy
if known_otu_tables:
logging.info("Parsing known taxonomy OTU tables")
known_taxes = KnownOtuTable()
known_taxes.parse_otu_tables(known_otu_tables)
logging.debug("Read in %i sequences with known taxonomy" % len(known_taxes))
else:
known_taxes = []
if known_sequence_taxonomy:
logging.debug("Parsing sequence-wise taxonomy..")
tax1 = GreenGenesTaxonomy.read(open(known_sequence_taxonomy)).taxonomy
known_sequence_tax = {}
for seq_id, tax in tax1.items():
known_sequence_tax[seq_id] = '; '.join(tax)
logging.info("Read in %i taxonomies from the GreenGenes format taxonomy file" % len(known_sequence_tax))
### Extract other reads which do not have known taxonomy
extracted_reads = self._extract_relevant_reads(
align_result, include_inserts, known_taxes)
logging.info("Finished extracting aligned sequences")
#### Taxonomic assignment
if assign_taxonomy:
logging.info("Running taxonomic assignment with graftm..")
assignment_result = self._assign_taxonomy(
extracted_reads, graftm_assignment_method)
#### Process taxonomically assigned reads
# get the sequences out for each of them
otu_table_object = OtuTable()
regular_output_fields = split('gene sample sequence num_hits coverage taxonomy')
otu_table_object.fields = regular_output_fields + \
split('read_names nucleotides_aligned taxonomy_by_known?')
for sample_name, singlem_package, tmp_graft, known_sequences, unknown_sequences in extracted_reads:
def add_info(infos, otu_table_object, known_tax):
for info in infos:
to_print = [
singlem_package.graftm_package_basename(),
sample_name,
info.seq,
info.count,
info.coverage,
info.taxonomy,
info.names,
info.aligned_lengths,
known_tax]
otu_table_object.data.append(to_print)
known_infos = self._seqs_to_counts_and_taxonomy(
known_sequences,
known_taxes,
False,
True)
add_info(known_infos, otu_table_object, True)
if tmp_graft: # if any sequences were aligned (not just already known)
tmpbase = os.path.basename(tmp_graft.name[:-6])#remove .fasta
if assign_taxonomy:
is_known_taxonomy = False
aligned_seqs = self._get_windowed_sequences(
assignment_result.prealigned_sequence_file(
sample_name, singlem_package, tmpbase),
assignment_result.nucleotide_hits_file(
sample_name, singlem_package, tmpbase),
singlem_package,
include_inserts)
if singlem_assignment_method == DIAMOND_EXAMPLE_BEST_HIT_ASSIGNMENT_METHOD:
tax_file = assignment_result.diamond_assignment_file(
sample_name, singlem_package, tmpbase)
else:
tax_file = assignment_result.read_tax_file(
sample_name, singlem_package, tmpbase)
logging.debug("Reading taxonomy from %s" % tax_file)
if singlem_assignment_method == DIAMOND_EXAMPLE_BEST_HIT_ASSIGNMENT_METHOD:
taxonomies = DiamondResultParser(tax_file)
use_first = True
else:
if not os.path.isfile(tax_file):
logging.warn("Unable to find tax file for gene %s from sample %s "
"(likely do to min length filtering), skipping" % (
os.path.basename(singlem_package.base_directory()),
sample_name))
taxonomies = {}
else:
taxonomies = TaxonomyFile(tax_file)
use_first = False
else: # Taxonomy has not been assigned.
aligned_seqs = unknown_sequences
if known_sequence_taxonomy:
taxonomies = known_sequence_tax
else:
taxonomies = {}
use_first = False # irrelevant
is_known_taxonomy = True
new_infos = list(self._seqs_to_counts_and_taxonomy(
aligned_seqs, taxonomies, use_first, False))
add_info(new_infos, otu_table_object, is_known_taxonomy)
if output_jplace:
base_dir = assignment_result._base_dir(
sample_name, singlem_package, tmpbase)
input_jplace_file = os.path.join(base_dir, "placements.jplace")
output_jplace_file = os.path.join(base_dir, "%s_%s_%s.jplace" % (
output_jplace, sample_name, singlem_package.graftm_package_basename()))
logging.debug("Converting jplace file %s to singlem jplace file %s" % (
input_jplace_file, output_jplace_file))
with open(output_jplace_file, 'w') as output_jplace_io:
self._write_jplace_from_infos(
open(input_jplace_file), new_infos, output_jplace_io)
if output_otu_table:
with open(output_otu_table, 'w') as f:
if output_extras:
otu_table_object.write_to(f, otu_table_object.fields)
else:
otu_table_object.write_to(f, regular_output_fields)
if archive_otu_table:
with open(archive_otu_table, 'w') as f:
otu_table_object.archive(hmms.singlem_packages).write_to(f)
return_cleanly()
def print_appraisal(self,
appraisal,
doing_binning,
output_io=sys.stdout,
doing_assembly=False,
binned_otu_table_io=None,
unbinned_otu_table_io=None,
assembled_otu_table_io=None,
unaccounted_for_otu_table_io=None):
'''print the Appraisal object overview to STDOUT'''
headers = ['sample']
if doing_binning: headers.append('num_binned')
if doing_assembly: headers.append('num_assembled')
headers.append('num_not_found')
if doing_binning: headers.append('percent_binned')
if doing_assembly: headers.append('percent_assembled')
output_io.write("\t".join(headers) + "\n")
binned = []
assembled = []
assembled_not_binned = []
not_founds = []
def print_sample(num_binned,
num_assembled,
num_assembled_not_binned,
num_not_found,
sample,
mypercent_binned=None,
mypercent_assembled=None):
if mypercent_binned is not None or mypercent_assembled is not None:
if doing_binning:
percent_binned = mypercent_binned
if doing_assembly:
percent_assembled = mypercent_assembled
else:
total = num_not_found
if doing_binning: total += num_binned
if doing_assembly: total += num_assembled_not_binned
if total == 0:
if doing_binning: percent_binned = 0.0
if doing_assembly: percent_assembled = 0.0
else:
if doing_binning:
percent_binned = float(num_binned) / total * 100
if doing_assembly:
percent_assembled = float(num_assembled) / total * 100
to_write = [sample]
if doing_binning: to_write.append(str(num_binned))
if doing_assembly: to_write.append(str(num_assembled))
to_write.append(str(num_not_found))
if doing_binning:
to_write.append("%2.1f" % percent_binned)
if doing_assembly:
to_write.append("%2.1f" % percent_assembled)
output_io.write("\t".join(to_write) + "\n")
def mean(l):
return float(sum(l)) / len(l) if len(l) > 0 else float('nan')
if binned_otu_table_io:
binned_table = OtuTable()
if unbinned_otu_table_io:
unbinned_table = OtuTable()
if assembled_otu_table_io:
assembled_table = OtuTable()
if unaccounted_for_otu_table_io:
unaccounted_for_table = OtuTable()
for appraisal_result in appraisal.appraisal_results:
if doing_assembly:
num_assembled_not_binned = appraisal_result.num_assembled_not_binned(
)
print_sample(
appraisal_result.num_binned if doing_binning else None,
appraisal_result.num_assembled if doing_assembly else None,
num_assembled_not_binned if doing_assembly else None,
appraisal_result.num_not_found,
appraisal_result.metagenome_sample_name)
if doing_binning:
binned.append(appraisal_result.num_binned)
if doing_assembly:
assembled.append(appraisal_result.num_assembled)
assembled_not_binned.append(num_assembled_not_binned)
not_founds.append(appraisal_result.num_not_found)
if binned_otu_table_io:
binned_table.add(appraisal_result.binned_otus)
if unbinned_otu_table_io:
unbinned_table.add(
appraisal_result.assembled_not_binned_otus())
if assembled_otu_table_io:
assembled_table.add(appraisal_result.assembled_otus)
if unaccounted_for_otu_table_io:
unaccounted_for_table.add(appraisal_result.not_found_otus)
print_sample(
sum(binned) if doing_binning else None,
sum(assembled) if doing_assembly else None,
sum(assembled_not_binned) if doing_assembly else None,
sum(not_founds), 'total')
binned_means = []
assembled_means = []
if doing_binning:
to_enumerate = binned
else:
to_enumerate = assembled
for i, _ in enumerate(to_enumerate):
num_binned = binned[i] if doing_binning else 0
num_assembled = assembled[i] if doing_assembly else 0
num_assembled_not_binned = assembled_not_binned[
i] if doing_assembly else 0
num_not_found = not_founds[i]
total = num_assembled_not_binned + num_not_found
if doing_binning:
total += num_binned
binned_means.append(float(num_binned) / total)
if doing_assembly:
assembled_means.append(float(num_assembled) / total)
print_sample("%2.1f" % mean(binned) if doing_binning else None,
"%2.1f" % mean(assembled) if doing_assembly else None,
None,
"%2.1f" % mean(not_founds),
'average',
mypercent_binned=mean(binned_means) *
100 if doing_binning else None,
mypercent_assembled=(mean(assembled_means) *
100 if doing_assembly else None))
if binned_otu_table_io:
binned_table.write_to(binned_otu_table_io)
if unbinned_otu_table_io:
unbinned_table.write_to(unbinned_otu_table_io)
if assembled_otu_table_io:
assembled_table.write_to(assembled_otu_table_io)
if unaccounted_for_otu_table_io:
unaccounted_for_table.write_to(unaccounted_for_otu_table_io)