def generate_info_from_sam(bowtie_sam_file, read2contig,
cdhitdup_cluster_sizes_path):
contig_stats = defaultdict(int)
contig_unique_counts = defaultdict(int)
cdhit_cluster_sizes = load_cdhit_cluster_sizes(
cdhitdup_cluster_sizes_path)
with open(bowtie_sam_file, "r", encoding='utf-8') as samf:
for line in samf:
if line[0] == '@':
continue
fields = line.split("\t")
read = fields[0]
contig = fields[2]
contig_stats[contig] += get_read_cluster_size(
cdhit_cluster_sizes,
read) # these are non-unique read counts now
contig_unique_counts[contig] += 1
if contig != '*':
read2contig[read] = contig
for contig, unique_count in contig_unique_counts.items(
): # TODO can't we just filter those out after spades, IN ONE PLACE
if unique_count < MIN_CONTIG_SIZE:
del contig_stats[contig]
elif READ_COUNTING_MODE == ReadCountingMode.COUNT_UNIQUE:
contig_stats[contig] = unique_count
return contig_stats
def _count_reads_work(self, cluster_key, counter_name, fasta_files):
# Count reads including duplicates (expanding cd-hit-dup clusters).
self.should_count_reads = True
self.counts_dict[counter_name] = count.reads_in_group(
file_group=fasta_files,
cluster_sizes=load_cdhit_cluster_sizes(self.input_cluster_sizes_path()),
cluster_key=cluster_key)
def generate_taxon_summary(read2contig, contig2lineage, read_dict,
added_reads_dict, db_type,
cdhit_cluster_sizes_path, should_keep):
# Return an array with
# { taxid: , tax_level:, contig_counts: { 'contig_name': <count>, .... } }
cdhit_cluster_sizes = load_cdhit_cluster_sizes(
cdhit_cluster_sizes_path)
def new_summary():
return defaultdict(lambda: defaultdict(lambda: [0, 0]))
genus_summary = new_summary()
species_summary = new_summary()
def record_read(species_taxid, genus_taxid, contig, read_id):
cluster_size = get_read_cluster_size(cdhit_cluster_sizes, read_id)
def increment(counters):
counters[0] += 1
counters[1] += cluster_size
increment(species_summary[species_taxid][contig])
increment(genus_summary[genus_taxid][contig])
for read_id, read_info in read_dict.items():
contig = read2contig.get(read_id, '*')
lineage = contig2lineage.get(contig)
if contig != '*' and lineage:
species_taxid, genus_taxid, _family_taxid = lineage
else:
# not mapping to a contig, or missing contig lineage
species_taxid, genus_taxid = read_info[4:6]
contig = '*'
if should_keep((species_taxid, genus_taxid)):
record_read(species_taxid, genus_taxid, contig, read_id)
for read_id, read_info in added_reads_dict.items():
contig = read2contig[read_id]
species_taxid, genus_taxid, _family_taxid = contig2lineage[contig]
if should_keep((species_taxid, genus_taxid)):
record_read(species_taxid, genus_taxid, contig, read_id)
# Filter out contigs that contain too few unique reads.
# This used to happen in db_loader in idseq-web. Any code left there that still appears to
# do this filtering is effectively a no-op and the filtering cannot be done there because
# the non-unique read counts are no longer output by the pipeline.
for summary in [species_summary, genus_summary]:
for taxid in list(summary.keys()):
contig_counts = summary[taxid]
for contig in list(contig_counts.keys()):
unique_count, nonunique_count = contig_counts[contig]
if unique_count < MIN_CONTIG_SIZE:
del contig_counts[contig]
else:
contig_counts[
contig] = nonunique_count if READ_COUNTING_MODE == ReadCountingMode.COUNT_ALL else unique_count
if not contig_counts:
del summary[taxid]
# construct the array for output
output_array = []
for idx, summary in enumerate([species_summary, genus_summary]):
tax_level = idx + 1
for taxid, contig_counts in summary.items():
entry = {
'taxid': taxid,
'tax_level': tax_level,
'count_type': db_type.upper(),
'contig_counts': contig_counts
}
output_array.append(entry)
return output_array
def generate_taxon_count_json_from_m8(m8_file, hit_level_file, e_value_type,
count_type, lineage_map_path,
deuterostome_path, taxon_whitelist_path,
taxon_blacklist_path,
cdhit_cluster_sizes_path,
output_json_file):
# Parse through hit file and m8 input file and format a JSON file with
# our desired attributes, including aggregated statistics.
cdhit_cluster_sizes = load_cdhit_cluster_sizes(cdhit_cluster_sizes_path)
should_keep = build_should_keep_filter(deuterostome_path,
taxon_whitelist_path,
taxon_blacklist_path)
# Setup
aggregation = {}
with open(hit_level_file, 'r', encoding='utf-8') as hit_f, \
open(m8_file, 'r', encoding='utf-8') as m8_f, \
open_file_db_by_extension(lineage_map_path, IdSeqDictValue.VALUE_TYPE_ARRAY) as lineage_map: # noqa
# Lines in m8_file and hit_level_file correspond (same read_id)
hit_line = hit_f.readline()
m8_line = m8_f.readline()
num_ranks = len(lineage.NULL_LINEAGE)
# See https://en.wikipedia.org/wiki/Double-precision_floating-point_format
MIN_NORMAL_POSITIVE_DOUBLE = 2.0**-1022
with log.log_context("generate_taxon_count_json_from_m8",
{"substep": "loop_1"}):
while hit_line and m8_line:
# Retrieve data values from files
hit_line_columns = hit_line.rstrip("\n").split("\t")
read_id = hit_line_columns[0]
hit_level = hit_line_columns[1]
hit_taxid = hit_line_columns[2]
if int(hit_level) < 0: # Skip negative levels and continue
hit_line = hit_f.readline()
m8_line = m8_f.readline()
continue
# m8 files correspond to BLAST tabular output format 6:
# Columns: read_id | _ref_id | percent_identity | alignment_length...
#
# * read_id = query (e.g., gene) sequence id
# * _ref_id = subject (e.g., reference genome) sequence id
# * percent_identity = percentage of identical matches
# * alignment_length = length of the alignments
# * e_value = the expect value
#
# See:
# * http://www.metagenomics.wiki/tools/blast/blastn-output-format-6
# * http://www.metagenomics.wiki/tools/blast/evalue
m8_line_columns = m8_line.split("\t")
msg = "read_ids in %s and %s do not match: %s vs. %s" % (
os.path.basename(m8_file),
os.path.basename(hit_level_file), m8_line_columns[0],
hit_line_columns[0])
assert m8_line_columns[0] == hit_line_columns[0], msg
percent_identity = float(m8_line_columns[2])
alignment_length = float(m8_line_columns[3])
e_value = float(m8_line_columns[10])
# These have been filtered out before the creation of m8_f and hit_f
assert alignment_length > 0
assert -0.25 < percent_identity < 100.25
assert e_value == e_value
if e_value_type != 'log10':
# e_value could be 0 when large contigs are mapped
if e_value <= MIN_NORMAL_POSITIVE_DOUBLE:
e_value = MIN_NORMAL_POSITIVE_DOUBLE
e_value = math.log10(e_value)
# Retrieve the taxon lineage and mark meaningless calls with fake
# taxids.
hit_taxids_all_levels = lineage_map.get(
hit_taxid, lineage.NULL_LINEAGE)
cleaned_hit_taxids_all_levels = lineage.validate_taxid_lineage(
hit_taxids_all_levels, hit_taxid, hit_level)
assert num_ranks == len(cleaned_hit_taxids_all_levels)
if should_keep(cleaned_hit_taxids_all_levels):
# Aggregate each level and collect statistics
agg_key = tuple(cleaned_hit_taxids_all_levels)
while agg_key:
agg_bucket = aggregation.get(agg_key)
if not agg_bucket:
agg_bucket = {
'nonunique_count': 0,
'unique_count': 0,
'sum_percent_identity': 0.0,
'sum_alignment_length': 0.0,
'sum_e_value': 0.0
}
aggregation[agg_key] = agg_bucket
agg_bucket['nonunique_count'] += get_read_cluster_size(
cdhit_cluster_sizes, read_id)
agg_bucket['unique_count'] += 1
agg_bucket['sum_percent_identity'] += percent_identity
agg_bucket['sum_alignment_length'] += alignment_length
agg_bucket['sum_e_value'] += e_value
# Chomp off the lowest rank as we aggregate up the tree
agg_key = agg_key[1:]
hit_line = hit_f.readline()
m8_line = m8_f.readline()
# Produce the final output
taxon_counts_attributes = []
with log.log_context("generate_taxon_count_json_from_m8",
{"substep": "loop_2"}):
for agg_key, agg_bucket in aggregation.items():
unique_count = agg_bucket['unique_count']
nonunique_count = agg_bucket['nonunique_count']
tax_level = num_ranks - len(agg_key) + 1
# TODO: Extend taxonomic ranks as indicated on the commented out lines.
taxon_counts_attributes.append({
"tax_id":
agg_key[0],
"tax_level":
tax_level,
# 'species_taxid' : agg_key[tax_level - 1] if tax_level == 1 else "-100",
'genus_taxid':
agg_key[2 - tax_level] if tax_level <= 2 else "-200",
'family_taxid':
agg_key[3 - tax_level] if tax_level <= 3 else "-300",
# 'order_taxid' : agg_key[4 - tax_level] if tax_level <= 4 else "-400",
# 'class_taxid' : agg_key[5 - tax_level] if tax_level <= 5 else "-500",
# 'phyllum_taxid' : agg_key[6 - tax_level] if tax_level <= 6 else "-600",
# 'kingdom_taxid' : agg_key[7 - tax_level] if tax_level <= 7 else "-700",
# 'domain_taxid' : agg_key[8 - tax_level] if tax_level <= 8 else "-800",
"count": # this field will be consumed by the webapp
nonunique_count if READ_COUNTING_MODE == ReadCountingMode.COUNT_ALL else unique_count,
"nonunique_count":
nonunique_count,
"unique_count":
unique_count,
"dcr":
nonunique_count / unique_count,
"percent_identity":
agg_bucket['sum_percent_identity'] / unique_count,
"alignment_length":
agg_bucket['sum_alignment_length'] / unique_count,
"e_value":
agg_bucket['sum_e_value'] / unique_count,
"count_type":
count_type
})
output_dict = {
"pipeline_output": {
"taxon_counts_attributes": taxon_counts_attributes
}
}
with log.log_context("generate_taxon_count_json_from_m8", {
"substep": "json_dump",
"output_json_file": output_json_file
}):
with open(output_json_file, 'w') as outf:
json.dump(output_dict, outf)
outf.flush()
def count_reads(self):
self.should_count_reads = True
self.counts_dict[self.name] = reads_in_group(
file_group=self.output_files_local()[0:2],
cluster_sizes=load_cdhit_cluster_sizes(self.input_cluster_sizes_path()),
cluster_key=lambda x: x)