def load_refjson(self, refjson_fname):
try:
self.refjson = RefJsonParser(refjson_fname)
except ValueError:
self.cfg.exit_user_error("ERROR: Invalid json file format!")
#validate input json format
(valid, err) = self.refjson.validate()
if not valid:
self.cfg.log.error(
"ERROR: Parsing reference JSON file failed:\n%s", err)
self.cfg.exit_user_error()
self.rate = self.refjson.get_rate()
self.node_height = self.refjson.get_node_height()
self.origin_taxonomy = self.refjson.get_origin_taxonomy()
self.tax_tree = self.refjson.get_tax_tree()
self.cfg.compress_patterns = self.refjson.get_pattern_compression()
self.bid_taxonomy_map = self.refjson.get_branch_tax_map()
if not self.bid_taxonomy_map:
# old file format (before 1.6), need to rebuild this map from scratch
th = TaxTreeHelper(self.cfg, self.origin_taxonomy)
th.set_mf_rooted_tree(self.tax_tree)
th.set_bf_unrooted_tree(self.refjson.get_reftree())
self.bid_taxonomy_map = th.get_bid_taxonomy_map()
self.write_bid_tax_map(self.bid_taxonomy_map, final=False)
reftree_str = self.refjson.get_raxml_readable_tree()
self.reftree = Tree(reftree_str)
self.reftree_size = len(self.reftree.get_leaves())
# IMPORTANT: set EPA heuristic rate based on tree size!
self.cfg.resolve_auto_settings(self.reftree_size)
# If we're loading the pre-optimized model, we MUST set the same rate het. mode as in the ref file
if self.cfg.epa_load_optmod:
self.cfg.raxml_model = self.refjson.get_ratehet_model()
self.classify_helper = TaxClassifyHelper(self.cfg,
self.bid_taxonomy_map,
self.rate, self.node_height)
self.taxtree_helper = TaxTreeHelper(self.cfg, self.origin_taxonomy,
self.tax_tree)
tax_code_name = self.refjson.get_taxcode()
self.tax_code = TaxCode(tax_code_name)
self.taxonomy = Taxonomy(prefix=EpacConfig.REF_SEQ_PREFIX,
tax_map=self.origin_taxonomy)
self.tax_common_ranks = self.taxonomy.get_common_ranks()
# print "Common ranks: ", self.tax_common_ranks
self.mislabels_cnt = [0] * TaxCode.UNI_TAX_LEVELS
self.rank_mislabels_cnt = [0] * TaxCode.UNI_TAX_LEVELS
def test_zoo_taxcode(self):
self.taxcode = TaxCode("ZOO")
tests = []
ranks = "Animalia;Chordata;Mammalia;Rodentia;Muridae;Apodemus;agrarius"
expected_levels = "kingdom;phylum;class;order;family;genus;species"
tests += [(ranks, expected_levels)]
ranks = "Animalia;Chordata;Mammalia;Primates;Hominidae;Homininae;Hominini;Hominina;H**o;sapiens"
expected_levels = "kingdom;phylum;class;order;family;subfamily;tribe;subtribe;genus;species"
tests += [(ranks, expected_levels)]
self.check_ranks(tests)
def test_bac_taxcode(self):
self.taxcode = TaxCode("BAC")
tests = []
ranks = "Bacteria;Firmicutes;Clostridia;Clostridiales;Clostridiaceae;Clostridium;Clostridium rectum"
expected_levels = "kingdom;phylum;class;order;family;genus;species"
tests += [(ranks, expected_levels)]
ranks = "Bacteria;Actinobacteria;Actinobacteria;Actinobacteridae;Actinomycetales;Micrococcineae;Micrococcaceae;Acaricomes;Acaricomes phytoseiuli"
expected_levels = "kingdom;phylum;class;subclass;order;suborder;family;genus;species"
tests += [(ranks, expected_levels)]
ranks = "k__Bacteria; p__Acidobacteria; c__[Chloracidobacteria]; o__[Chloracidobacterales]; f__[Chloracidobacteraceae]; g__Candidatus Chloracidobacterium; s__"
expected_levels = "kingdom;phylum;class;order;family;genus;species"
tests += [(ranks, expected_levels)]
self.check_ranks(tests)
def load_refjson(self, refjson_fname):
try:
self.refjson = RefJsonParser(refjson_fname)
except ValueError:
self.cfg.exit_user_error("ERROR: Invalid json file format!")
#validate input json format
(valid, err) = self.refjson.validate()
if not valid:
self.cfg.log.error("ERROR: Parsing reference JSON file failed:\n%s", err)
self.cfg.exit_user_error()
self.rate = self.refjson.get_rate()
self.node_height = self.refjson.get_node_height()
self.origin_taxonomy = self.refjson.get_origin_taxonomy()
self.tax_tree = self.refjson.get_tax_tree()
self.cfg.compress_patterns = self.refjson.get_pattern_compression()
self.bid_taxonomy_map = self.refjson.get_branch_tax_map()
if not self.bid_taxonomy_map:
# old file format (before 1.6), need to rebuild this map from scratch
th = TaxTreeHelper(self.cfg, self.origin_taxonomy)
th.set_mf_rooted_tree(self.tax_tree)
th.set_bf_unrooted_tree(self.refjson.get_reftree())
self.bid_taxonomy_map = th.get_bid_taxonomy_map()
self.write_bid_tax_map(self.bid_taxonomy_map, final=False)
reftree_str = self.refjson.get_raxml_readable_tree()
self.reftree = Tree(reftree_str)
self.reftree_size = len(self.reftree.get_leaves())
# IMPORTANT: set EPA heuristic rate based on tree size!
self.cfg.resolve_auto_settings(self.reftree_size)
# If we're loading the pre-optimized model, we MUST set the same rate het. mode as in the ref file
if self.cfg.epa_load_optmod:
self.cfg.raxml_model = self.refjson.get_ratehet_model()
self.classify_helper = TaxClassifyHelper(self.cfg, self.bid_taxonomy_map, self.rate, self.node_height)
self.taxtree_helper = TaxTreeHelper(self.cfg, self.origin_taxonomy, self.tax_tree)
tax_code_name = self.refjson.get_taxcode()
self.tax_code = TaxCode(tax_code_name)
self.taxonomy = Taxonomy(prefix=EpacConfig.REF_SEQ_PREFIX, tax_map=self.origin_taxonomy)
self.tax_common_ranks = self.taxonomy.get_common_ranks()
# print "Common ranks: ", self.tax_common_ranks
self.mislabels_cnt = [0] * TaxCode.UNI_TAX_LEVELS
self.rank_mislabels_cnt = [0] * TaxCode.UNI_TAX_LEVELS
def test_bot_taxcode(self):
self.taxcode = TaxCode("BOT")
tests = []
ranks = "Apocynoideae;Nerieae;Neriinae;Adenium;Adenium_swazicum"
expected_levels = "subfamily;tribe;subtribe;genus;species"
tests += [(ranks, expected_levels)]
ranks = "Apocynoideae;Echiteae;Parsonsiinae;Parsonsia;Parsonsia_heterophylla"
expected_levels = "subfamily;tribe;subtribe;genus;species"
tests += [(ranks, expected_levels)]
# ranks = "Eukaryota;Opisthokonta;Nucletmycea;Fungi;Dikarya;Ascomycota;Saccharomycotina;Saccharomycetes;Saccharomycetales;Metschnikowiaceae;Clavispora;Clavispora lusitaniae ATCC 42720"
ranks = "Eukaryota;Fungi;Dikarya;Ascomycota;Saccharomycotina;Saccharomycetes;Saccharomycetales;Metschnikowiaceae;Clavispora;Clavispora lusitaniae ATCC 42720"
expected_levels = "domain;kingdom;subkingdom;phylum;subphylum;class;order;family;genus;species"
tests += [(ranks, expected_levels)]
self.check_ranks(tests)
def run_leave_subtree_out_test(self):
job_name = self.cfg.subst_name("l1out_rank_%NAME%")
# if self.jplace_fname:
# jp = EpaJsonParser(self.jplace_fname)
# else:
#create file with subtrees
rank_parent, rank_tips = self.get_parent_tip_ranks(self.tax_tree)
subtree_list = list(rank_tips.items())
if len(subtree_list) == 0:
return 0
subtree_list_file = self.cfg.tmp_fname("treelist_%NAME%.txt")
with open(subtree_list_file, "w") as fout:
for rank_name, tips in subtree_list:
fout.write("%s\n" % " ".join(tips))
jp_list = self.raxml.run_epa(job_name,
self.refalign_fname,
self.reftree_fname,
self.optmod_fname,
mode="l1o_subtree",
subtree_fname=subtree_list_file)
subtree_count = 0
for jp in jp_list:
placements = jp.get_placement()
for place in placements:
ranks, lws = self.classify_seq(place)
tax_path = subtree_list[subtree_count][0]
orig_ranks = Taxonomy.split_rank_uid(tax_path)
rank_level = Taxonomy.lowest_assigned_rank_level(orig_ranks)
rank_prefix = TaxCode.guess_rank_level_name(
orig_ranks, rank_level)[0]
rank_name = orig_ranks[rank_level]
if not rank_name.startswith(rank_prefix):
rank_name = rank_prefix + rank_name
parent_ranks = rank_parent[tax_path]
# print orig_ranks, "\n", parent_ranks, "\n", ranks, "\n"
mis_rec = self.check_rank_tax_labels(rank_name, parent_ranks,
ranks, lws)
if mis_rec:
self.misrank_conf_map[tax_path] = mis_rec['conf']
subtree_count += 1
return subtree_count
class LeaveOneTest:
def __init__(self, config):
self.cfg = config
self.mis_fname = self.cfg.out_fname("%NAME%.mis")
self.premis_fname = self.cfg.out_fname("%NAME%.premis")
self.misrank_fname = self.cfg.out_fname("%NAME%.misrank")
self.stats_fname = self.cfg.out_fname("%NAME%.stats")
if os.path.isfile(self.mis_fname):
print "\nERROR: Output file already exists: %s" % self.mis_fname
print "Please specify a different job name using -n or remove old output files."
self.cfg.exit_user_error()
self.tmp_refaln = config.tmp_fname("%NAME%.refaln")
self.reftree_lbl_fname = config.tmp_fname("%NAME%_lbl.tre")
self.reftree_tax_fname = config.tmp_fname("%NAME%_tax.tre")
self.optmod_fname = self.cfg.tmp_fname("%NAME%.opt")
self.reftree_fname = self.cfg.tmp_fname("ref_%NAME%.tre")
self.mislabels = []
self.mislabels_cnt = []
self.rank_mislabels = []
self.rank_mislabels_cnt = []
self.misrank_conf_map = {}
def write_bid_tax_map(self, bid_tax_map, final):
if self.cfg.debug:
fname_suffix = "final" if final else "l1out"
bid_fname = self.cfg.tmp_fname("%NAME%_" + "bid_tax_map_%s.txt" % fname_suffix)
with open(bid_fname, "w") as outf:
for bid, bid_rec in bid_tax_map.iteritems():
outf.write("%s\t%s\t%d\t%f\n" % (bid, bid_rec[0], bid_rec[1], bid_rec[2]));
def write_assignments(self, assign_map, final):
if self.cfg.debug:
fname_suffix = "final" if final else "l1out"
assign_fname = self.cfg.tmp_fname("%NAME%_" + "taxassign_%s.txt" % fname_suffix)
with open(assign_fname, "w") as outf:
for seq_name in assign_map.iterkeys():
ranks, lws = assign_map[seq_name]
outf.write("%s\t%s\t%s\n" % (seq_name, ";".join(ranks), ";".join(["%.3f" % l for l in lws])))
def load_refjson(self, refjson_fname):
try:
self.refjson = RefJsonParser(refjson_fname)
except ValueError:
self.cfg.exit_user_error("ERROR: Invalid json file format!")
#validate input json format
(valid, err) = self.refjson.validate()
if not valid:
self.cfg.log.error("ERROR: Parsing reference JSON file failed:\n%s", err)
self.cfg.exit_user_error()
self.rate = self.refjson.get_rate()
self.node_height = self.refjson.get_node_height()
self.origin_taxonomy = self.refjson.get_origin_taxonomy()
self.tax_tree = self.refjson.get_tax_tree()
self.cfg.compress_patterns = self.refjson.get_pattern_compression()
self.bid_taxonomy_map = self.refjson.get_branch_tax_map()
if not self.bid_taxonomy_map:
# old file format (before 1.6), need to rebuild this map from scratch
th = TaxTreeHelper(self.cfg, self.origin_taxonomy)
th.set_mf_rooted_tree(self.tax_tree)
th.set_bf_unrooted_tree(self.refjson.get_reftree())
self.bid_taxonomy_map = th.get_bid_taxonomy_map()
self.write_bid_tax_map(self.bid_taxonomy_map, final=False)
reftree_str = self.refjson.get_raxml_readable_tree()
self.reftree = Tree(reftree_str)
self.reftree_size = len(self.reftree.get_leaves())
# IMPORTANT: set EPA heuristic rate based on tree size!
self.cfg.resolve_auto_settings(self.reftree_size)
# If we're loading the pre-optimized model, we MUST set the same rate het. mode as in the ref file
if self.cfg.epa_load_optmod:
self.cfg.raxml_model = self.refjson.get_ratehet_model()
self.classify_helper = TaxClassifyHelper(self.cfg, self.bid_taxonomy_map, self.rate, self.node_height)
self.taxtree_helper = TaxTreeHelper(self.cfg, self.origin_taxonomy, self.tax_tree)
tax_code_name = self.refjson.get_taxcode()
self.tax_code = TaxCode(tax_code_name)
self.taxonomy = Taxonomy(prefix=EpacConfig.REF_SEQ_PREFIX, tax_map=self.origin_taxonomy)
self.tax_common_ranks = self.taxonomy.get_common_ranks()
# print "Common ranks: ", self.tax_common_ranks
self.mislabels_cnt = [0] * TaxCode.UNI_TAX_LEVELS
self.rank_mislabels_cnt = [0] * TaxCode.UNI_TAX_LEVELS
def run_epa_trainer(self):
epa_trainer.run_trainer(self.cfg)
if not os.path.isfile(self.cfg.refjson_fname):
self.cfg.log.error("\nBuilding reference tree failed, see error messages above.")
self.cfg.exit_fatal_error()
def classify_seq(self, placement):
edges = placement["p"]
if len(edges) > 0:
return self.classify_helper.classify_seq(edges)
else:
print "ERROR: no placements! something is definitely wrong!"
def check_seq_tax_labels(self, seq_name, orig_ranks, ranks, lws):
mis_rec = None
num_common_ranks = len(self.tax_common_ranks)
orig_rank_level = Taxonomy.lowest_assigned_rank_level(orig_ranks)
new_rank_level = Taxonomy.lowest_assigned_rank_level(ranks)
#if new_rank_level < 0 or (new_rank_level < num_common_ranks and orig_rank_level >= num_common_ranks):
# if new_rank_level < 0:
if len(ranks) == 0:
mis_rec = {}
mis_rec['name'] = seq_name
mis_rec['orig_level'] = -1
mis_rec['real_level'] = 0
mis_rec['level_name'] = "[NotIngroup]"
mis_rec['inv_level'] = -1 * mis_rec['real_level'] # just for sorting
mis_rec['orig_ranks'] = orig_ranks
mis_rec['ranks'] = []
mis_rec['lws'] = [1.0]
mis_rec['conf'] = mis_rec['lws'][0]
else:
mislabel_lvl = -1
min_len = min(len(orig_ranks),len(ranks))
for rank_lvl in range(min_len):
if ranks[rank_lvl] != Taxonomy.EMPTY_RANK and ranks[rank_lvl] != orig_ranks[rank_lvl]:
mislabel_lvl = rank_lvl
break
if mislabel_lvl >= 0:
real_lvl = self.tax_code.guess_rank_level(orig_ranks, mislabel_lvl)
mis_rec = {}
mis_rec['name'] = seq_name
mis_rec['orig_level'] = mislabel_lvl
mis_rec['real_level'] = real_lvl
mis_rec['level_name'] = self.tax_code.rank_level_name(real_lvl)[0]
mis_rec['inv_level'] = -1 * mis_rec['real_level'] # just for sorting
mis_rec['orig_ranks'] = orig_ranks
mis_rec['ranks'] = ranks
mis_rec['lws'] = lws
mis_rec['conf'] = lws[mislabel_lvl]
if mis_rec:
self.mislabels.append(mis_rec)
return mis_rec
def filter_mislabels(self):
filtered_mis = []
for i in range(len(self.mislabels)):
if self.mislabels[i]['conf'] >= self.cfg.conf_cutoff:
filtered_mis.append(self.mislabels[i])
self.mislabels = filtered_mis
def check_rank_tax_labels(self, rank_name, orig_ranks, ranks, lws):
mislabel_lvl = -1
min_len = min(len(orig_ranks),len(ranks))
for rank_lvl in range(min_len):
if ranks[rank_lvl] != Taxonomy.EMPTY_RANK and ranks[rank_lvl] != orig_ranks[rank_lvl]:
mislabel_lvl = rank_lvl
break
if mislabel_lvl >= 0:
real_lvl = self.tax_code.guess_rank_level(orig_ranks, mislabel_lvl)
mis_rec = {}
mis_rec['name'] = rank_name
mis_rec['orig_level'] = mislabel_lvl
mis_rec['real_level'] = real_lvl
mis_rec['level_name'] = self.tax_code.rank_level_name(real_lvl)[0]
mis_rec['inv_level'] = -1 * real_lvl # just for sorting
mis_rec['orig_ranks'] = orig_ranks
mis_rec['ranks'] = ranks
mis_rec['lws'] = lws
mis_rec['conf'] = lws[mislabel_lvl]
self.rank_mislabels.append(mis_rec)
return mis_rec
else:
return None
def mis_rec_to_string_old(self, mis_rec):
lvl = mis_rec['orig_level']
output = mis_rec['name'] + "\t"
output += "%s\t%s\t%s\t%.3f\n" % (mis_rec['level_name'],
mis_rec['orig_ranks'][lvl], mis_rec['ranks'][lvl], mis_rec['lws'][lvl])
output += ";".join(mis_rec['orig_ranks']) + "\n"
output += ";".join(mis_rec['ranks']) + "\n"
output += "\t".join(["%.3f" % conf for conf in mis_rec['lws']]) + "\n"
return output
def mis_rec_to_string(self, mis_rec):
lvl = mis_rec['orig_level']
uncorr_name = EpacConfig.strip_ref_prefix(self.refjson.get_uncorr_seqid(mis_rec['name']))
uncorr_orig_ranks = self.refjson.get_uncorr_ranks(mis_rec['orig_ranks'])
uncorr_ranks = self.refjson.get_uncorr_ranks(mis_rec['ranks'])
output = uncorr_name + "\t"
if lvl >= 0:
output += "%s\t%s\t%s\t%.3f\t" % (mis_rec['level_name'],
uncorr_orig_ranks[lvl], uncorr_ranks[lvl], mis_rec['lws'][lvl])
else:
output += "%s\t%s\t%s\t%.3f\t" % (mis_rec['level_name'],
"NA", "NA", mis_rec['lws'][0])
output += Taxonomy.lineage_str(uncorr_orig_ranks) + "\t"
output += Taxonomy.lineage_str(uncorr_ranks) + "\t"
output += ";".join(["%.3f" % conf for conf in mis_rec['lws']])
if 'rank_conf' in mis_rec:
output += "\t%.3f" % mis_rec['rank_conf']
return output
def sort_mislabels(self):
self.mislabels = sorted(self.mislabels, key=itemgetter('inv_level', 'conf', 'name'), reverse=True)
for mis_rec in self.mislabels:
real_lvl = mis_rec["real_level"]
self.mislabels_cnt[real_lvl] += 1
if self.cfg.ranktest:
self.rank_mislabels = sorted(self.rank_mislabels, key=itemgetter('inv_level', 'conf', 'name'), reverse=True)
for mis_rec in self.rank_mislabels:
real_lvl = mis_rec["real_level"]
self.rank_mislabels_cnt[real_lvl] += 1
def write_stats(self, toFile=False):
self.cfg.log.info("Mislabeled sequences by rank:")
seq_sum = 0
rank_sum = 0
stats = []
for i in range(len(self.mislabels_cnt)):
if i > 0:
rname = self.tax_code.rank_level_name(i)[0].ljust(12)
else:
rname = "[NotIngroup]"
if self.mislabels_cnt[i] > 0:
seq_sum += self.mislabels_cnt[i]
# output = "%s:\t%d" % (rname, seq_sum)
output = "%s:\t%d" % (rname, self.mislabels_cnt[i])
if self.cfg.ranktest:
rank_sum += self.rank_mislabels_cnt[i]
output += "\t%d" % rank_sum
self.cfg.log.info(output)
stats.append(output)
if toFile:
with open(self.stats_fname, "w") as fo_stat:
for line in stats:
fo_stat.write(line + "\n")
def write_mislabels(self, final=True):
if final:
out_fname = self.mis_fname
else:
out_fname = self.premis_fname
with open(out_fname, "w") as fo_all:
fields = ["SeqID", "MislabeledLevel", "OriginalLabel", "ProposedLabel", "Confidence", "OriginalTaxonomyPath", "ProposedTaxonomyPath", "PerRankConfidence"]
if self.cfg.ranktest:
fields += ["HigherRankMisplacedConfidence"]
header = ";" + "\t".join(fields) + "\n"
fo_all.write(header)
if self.cfg.verbose and len(self.mislabels) > 0 and final:
print "Mislabeled sequences:\n"
print header
for mis_rec in self.mislabels:
output = self.mis_rec_to_string(mis_rec) + "\n"
fo_all.write(output)
if self.cfg.verbose and final:
print(output)
if not final:
return
if self.cfg.ranktest:
with open(self.misrank_fname, "w") as fo_all:
fields = ["RankID", "MislabeledLevel", "OriginalLabel", "ProposedLabel", "Confidence", "OriginalTaxonomyPath", "ProposedTaxonomyPath", "PerRankConfidence"]
header = ";" + "\t".join(fields) + "\n"
fo_all.write(header)
if self.cfg.verbose and len(self.rank_mislabels) > 0:
print "\nMislabeled higher ranks:\n"
print header
for mis_rec in self.rank_mislabels:
output = self.mis_rec_to_string(mis_rec) + "\n"
fo_all.write(output)
if self.cfg.verbose:
print(output)
self.write_stats()
def run_leave_subtree_out_test(self):
job_name = self.cfg.subst_name("l1out_rank_%NAME%")
# if self.jplace_fname:
# jp = EpaJsonParser(self.jplace_fname)
# else:
#create file with subtrees
rank_tips = {}
rank_parent = {}
for node in self.tax_tree.traverse("postorder"):
if node.is_leaf() or node.is_root():
continue
tax_path = node.name
ranks = Taxonomy.split_rank_uid(tax_path)
rank_lvl = Taxonomy.lowest_assigned_rank_level(ranks)
if rank_lvl < 2:
continue
parent_ranks = Taxonomy.split_rank_uid(node.up.name)
parent_lvl = Taxonomy.lowest_assigned_rank_level(parent_ranks)
if parent_lvl < 1:
continue
rank_seqs = node.get_leaf_names()
rank_size = len(rank_seqs)
if rank_size < 2 or rank_size > self.reftree_size-4:
continue
# print rank_lvl, "\t", tax_path, "\t", rank_seqs, "\n"
rank_tips[tax_path] = node.get_leaf_names()
rank_parent[tax_path] = parent_ranks
subtree_list = rank_tips.items()
if len(subtree_list) == 0:
return 0
subtree_list_file = self.cfg.tmp_fname("treelist_%NAME%.txt")
with open(subtree_list_file, "w") as fout:
for rank_name, tips in subtree_list:
fout.write("%s\n" % " ".join(tips))
jp_list = self.raxml.run_epa(job_name, self.refalign_fname, self.reftree_fname, self.optmod_fname,
mode="l1o_subtree", subtree_fname=subtree_list_file)
subtree_count = 0
for jp in jp_list:
placements = jp.get_placement()
for place in placements:
ranks, lws = self.classify_seq(place)
tax_path = subtree_list[subtree_count][0]
orig_ranks = Taxonomy.split_rank_uid(tax_path)
rank_level = Taxonomy.lowest_assigned_rank_level(orig_ranks)
rank_prefix = self.guess_rank_level_name(orig_ranks, rank_level)[0]
rank_name = orig_ranks[rank_level]
if not rank_name.startswith(rank_prefix):
rank_name = rank_prefix + rank_name
parent_ranks = rank_parent[tax_path]
# print orig_ranks, "\n", parent_ranks, "\n", ranks, "\n"
mis_rec = self.check_rank_tax_labels(rank_name, parent_ranks, ranks, lws)
if mis_rec:
self.misrank_conf_map[tax_path] = mis_rec['conf']
subtree_count += 1
return subtree_count
def run_leave_seq_out_test(self):
job_name = self.cfg.subst_name("l1out_seq_%NAME%")
placements = []
if self.cfg.jplace_fname:
if os.path.isdir(self.cfg.jplace_fname):
jplace_fmask = os.path.join(self.cfg.jplace_fname, '*.jplace')
else:
jplace_fmask = self.cfg.jplace_fname
jplace_fname_list = glob.glob(jplace_fmask)
for jplace_fname in jplace_fname_list:
jp = EpaJsonParser(jplace_fname)
placements += jp.get_placement()
config.log.debug("Loaded %d placements from %s\n", len(placements), jplace_fmask)
else:
jp = self.raxml.run_epa(job_name, self.refalign_fname, self.reftree_fname, self.optmod_fname, mode="l1o_seq")
placements = jp.get_placement()
if self.cfg.output_interim_files:
out_jplace_fname = self.cfg.out_fname("%NAME%.l1out_seq.jplace")
self.raxml.copy_epa_jplace(job_name, out_jplace_fname, move=True, mode="l1o_seq")
seq_count = 0
l1out_ass = {}
for place in placements:
seq_name = place["n"][0]
# get original taxonomic label
# orig_ranks = self.get_orig_ranks(seq_name)
orig_ranks = self.taxtree_helper.get_seq_ranks_from_tree(seq_name)
# get EPA tax label
ranks, lws = self.classify_seq(place)
l1out_ass[seq_name] = (ranks, lws)
# check if they match
mis_rec = self.check_seq_tax_labels(seq_name, orig_ranks, ranks, lws)
# cross-check with higher rank mislabels
if self.cfg.ranktest and mis_rec:
rank_conf = 0
for lvl in range(2,len(orig_ranks)):
tax_path = Taxonomy.get_rank_uid(orig_ranks, lvl)
if tax_path in self.misrank_conf_map:
rank_conf = max(rank_conf, self.misrank_conf_map[tax_path])
mis_rec['rank_conf'] = rank_conf
seq_count += 1
self.write_assignments(l1out_ass, final=False)
return seq_count
def run_final_epa_test(self):
self.reftree_outgroup = self.refjson.get_outgroup()
tmp_reftree = self.reftree.copy(method="newick")
name2refnode = {}
for leaf in tmp_reftree.iter_leaves():
name2refnode[leaf.name] = leaf
tmp_taxtree = self.tax_tree.copy(method="newick")
name2taxnode = {}
for leaf in tmp_taxtree.iter_leaves():
name2taxnode[leaf.name] = leaf
for mis_rec in self.mislabels:
rname = mis_rec['name']
# rname = EpacConfig.REF_SEQ_PREFIX + name
if rname in name2refnode:
name2refnode[rname].delete()
else:
print "Node not found in the reference tree: %s" % rname
if rname in name2taxnode:
name2taxnode[rname].delete()
else:
print "Node not found in the taxonomic tree: %s" % rname
# remove unifurcation at the root
if len(tmp_reftree.children) == 1:
tmp_reftree = tmp_reftree.children[0]
self.mislabels = []
th = TaxTreeHelper(self.cfg, self.origin_taxonomy)
th.set_mf_rooted_tree(tmp_taxtree)
epa_result = self.run_epa_once(tmp_reftree)
reftree_epalbl_str = epa_result.get_std_newick_tree()
placements = epa_result.get_placement()
# update branchid-taxonomy mapping to account for possible changes in branch numbering
reftree_tax = Tree(reftree_epalbl_str)
th.set_bf_unrooted_tree(reftree_tax)
bid_tax_map = th.get_bid_taxonomy_map()
self.write_bid_tax_map(bid_tax_map, final=True)
cl = TaxClassifyHelper(self.cfg, bid_tax_map, self.rate, self.node_height)
# newtax_fname = self.cfg.subst_name("newtax_%NAME%.tre")
# th.get_tax_tree().write(outfile=newtax_fname, format=3)
final_ass = {}
for place in placements:
seq_name = place["n"][0]
# get original taxonomic label
orig_ranks = self.taxtree_helper.get_seq_ranks_from_tree(seq_name)
# EXPERIMENTAL FEATURE - disabled for now!
# It could happen that certain ranks were present in the "original" reference tree, but
# are completely missing in the pruned tree (e.g., all seqs of a species were considered "suspicious"
# after the leave-one-out test and thus pruned)
# In this case, EPA has no chance to infer full original taxonomic annotation (=species) since the corresponding clade
# is now missing. To account for this fact, we amend the original taxonomic annotation and set ranks missing from
# pruned tree to "Undefined".
# orig_ranks = th.strip_missing_ranks(orig_ranks)
# print orig_ranks
# get EPA tax label
ranks, lws = cl.classify_seq(place["p"])
final_ass[seq_name] = (ranks, lws)
#print seq_name, ": ", orig_ranks, "--->", ranks
# check if they match
mis_rec = self.check_seq_tax_labels(seq_name, orig_ranks, ranks, lws)
self.write_assignments(final_ass, final=True)
def run_epa_once(self, reftree):
reftree_fname = self.cfg.tmp_fname("final_ref_%NAME%.tre")
job_name = self.cfg.subst_name("final_epa_%NAME%")
reftree.write(outfile=reftree_fname)
# IMPORTANT: don't load the model, since it's invalid for the pruned true !!!
optmod_fname=""
epa_result = self.raxml.run_epa(job_name, self.refalign_fname, reftree_fname, optmod_fname)
if self.cfg.output_interim_files:
out_jplace_fname = self.cfg.out_fname("%NAME%.final_epa.jplace")
self.raxml.copy_epa_jplace(job_name, out_jplace_fname, move=True)
return epa_result
def run_test(self):
self.raxml = RaxmlWrapper(self.cfg)
# config.log.info("Number of sequences in the reference: %d\n", self.reftree_size)
self.refjson.get_raxml_readable_tree(self.reftree_fname)
self.refalign_fname = self.refjson.get_alignment(self.tmp_refaln)
self.refjson.get_binary_model(self.optmod_fname)
if self.cfg.ranktest:
config.log.info("Running the leave-one-rank-out test...\n")
subtree_count = self.run_leave_subtree_out_test()
config.log.info("Running the leave-one-sequence-out test...\n")
self.run_leave_seq_out_test()
if len(self.mislabels) > 0:
config.log.info("Leave-one-out test identified %d suspicious sequences; running final EPA test to check them...\n", len(self.mislabels))
if self.cfg.debug:
self.write_mislabels(final=False)
self.run_final_epa_test()
self.filter_mislabels()
self.sort_mislabels()
self.write_mislabels()
config.log.info("\nTotal mislabels: %d / %.2f %%", len(self.mislabels), (float(len(self.mislabels)) / self.reftree_size * 100))
class LeaveOneTest:
def __init__(self, config):
self.cfg = config
self.mis_fname = self.cfg.out_fname("%NAME%.mis")
self.premis_fname = self.cfg.out_fname("%NAME%.premis")
self.misrank_fname = self.cfg.out_fname("%NAME%.misrank")
self.stats_fname = self.cfg.out_fname("%NAME%.stats")
if os.path.isfile(self.mis_fname):
print("\nERROR: Output file already exists: %s" % self.mis_fname)
print(
"Please specify a different job name using -n or remove old output files."
)
self.cfg.exit_user_error()
self.tmp_refaln = config.tmp_fname("%NAME%.refaln")
self.reftree_lbl_fname = config.tmp_fname("%NAME%_lbl.tre")
self.reftree_tax_fname = config.tmp_fname("%NAME%_tax.tre")
self.optmod_fname = self.cfg.tmp_fname("%NAME%.opt")
self.reftree_fname = self.cfg.tmp_fname("ref_%NAME%.tre")
self.mislabels = []
self.mislabels_cnt = []
self.rank_mislabels = []
self.rank_mislabels_cnt = []
self.misrank_conf_map = {}
def write_bid_tax_map(self, bid_tax_map, final):
if self.cfg.debug:
fname_suffix = "final" if final else "l1out"
bid_fname = self.cfg.tmp_fname("%NAME%_" +
"bid_tax_map_%s.txt" % fname_suffix)
with open(bid_fname, "w") as outf:
for bid, bid_rec in bid_tax_map.items():
outf.write("%s\t%s\t%d\t%f\n" %
(bid, bid_rec[0], bid_rec[1], bid_rec[2]))
def write_assignments(self, assign_map, final):
if self.cfg.debug:
fname_suffix = "final" if final else "l1out"
assign_fname = self.cfg.tmp_fname("%NAME%_" + "taxassign_%s.txt" %
fname_suffix)
with open(assign_fname, "w") as outf:
for seq_name in assign_map.keys():
ranks, lws = assign_map[seq_name]
outf.write("%s\t%s\t%s\n" %
(seq_name, ";".join(ranks), ";".join(
["%.3f" % l for l in lws])))
def load_refjson(self, refjson_fname):
try:
self.refjson = RefJsonParser(refjson_fname)
except ValueError:
self.cfg.exit_user_error("ERROR: Invalid json file format!")
#validate input json format
(valid, err) = self.refjson.validate()
if not valid:
self.cfg.log.error(
"ERROR: Parsing reference JSON file failed:\n%s", err)
self.cfg.exit_user_error()
self.rate = self.refjson.get_rate()
self.node_height = self.refjson.get_node_height()
self.origin_taxonomy = self.refjson.get_origin_taxonomy()
self.tax_tree = self.refjson.get_tax_tree()
self.cfg.compress_patterns = self.refjson.get_pattern_compression()
self.bid_taxonomy_map = self.refjson.get_branch_tax_map()
if not self.bid_taxonomy_map:
# old file format (before 1.6), need to rebuild this map from scratch
th = TaxTreeHelper(self.cfg, self.origin_taxonomy)
th.set_mf_rooted_tree(self.tax_tree)
th.set_bf_unrooted_tree(self.refjson.get_reftree())
self.bid_taxonomy_map = th.get_bid_taxonomy_map()
self.write_bid_tax_map(self.bid_taxonomy_map, final=False)
reftree_str = self.refjson.get_raxml_readable_tree()
self.reftree = Tree(reftree_str)
self.reftree_size = len(self.reftree.get_leaves())
# IMPORTANT: set EPA heuristic rate based on tree size!
self.cfg.resolve_auto_settings(self.reftree_size)
# If we're loading the pre-optimized model, we MUST set the same rate het. mode as in the ref file
if self.cfg.epa_load_optmod:
self.cfg.raxml_model = self.refjson.get_ratehet_model()
self.classify_helper = TaxClassifyHelper(self.cfg,
self.bid_taxonomy_map,
self.rate, self.node_height)
self.taxtree_helper = TaxTreeHelper(self.cfg, self.origin_taxonomy,
self.tax_tree)
tax_code_name = self.refjson.get_taxcode()
self.tax_code = TaxCode(tax_code_name)
self.taxonomy = Taxonomy(prefix=EpacConfig.REF_SEQ_PREFIX,
tax_map=self.origin_taxonomy)
self.tax_common_ranks = self.taxonomy.get_common_ranks()
# print "Common ranks: ", self.tax_common_ranks
self.mislabels_cnt = [0] * TaxCode.UNI_TAX_LEVELS
self.rank_mislabels_cnt = [0] * TaxCode.UNI_TAX_LEVELS
def run_epa_trainer(self):
epa_trainer.run_trainer(self.cfg)
if not os.path.isfile(self.cfg.refjson_fname):
self.cfg.log.error(
"\nBuilding reference tree failed, see error messages above.")
self.cfg.exit_fatal_error()
def classify_seq(self, placement):
edges = placement["p"]
if len(edges) > 0:
return self.classify_helper.classify_seq(edges)
else:
print("ERROR: no placements! something is definitely wrong!")
def check_seq_tax_labels(self, seq_name, orig_ranks, ranks, lws):
mis_rec = None
num_common_ranks = len(self.tax_common_ranks)
orig_rank_level = Taxonomy.lowest_assigned_rank_level(orig_ranks)
new_rank_level = Taxonomy.lowest_assigned_rank_level(ranks)
#if new_rank_level < 0 or (new_rank_level < num_common_ranks and orig_rank_level >= num_common_ranks):
# if new_rank_level < 0:
if len(ranks) == 0:
mis_rec = {}
mis_rec['name'] = seq_name
mis_rec['orig_level'] = -1
mis_rec['real_level'] = 0
mis_rec['level_name'] = "[NotIngroup]"
mis_rec['inv_level'] = -1 * mis_rec[
'real_level'] # just for sorting
mis_rec['orig_ranks'] = orig_ranks
mis_rec['ranks'] = []
mis_rec['lws'] = [1.0]
mis_rec['conf'] = mis_rec['lws'][0]
else:
mislabel_lvl = -1
min_len = min(len(orig_ranks), len(ranks))
for rank_lvl in range(min_len):
if ranks[rank_lvl] != Taxonomy.EMPTY_RANK and ranks[
rank_lvl] != orig_ranks[rank_lvl]:
mislabel_lvl = rank_lvl
break
if mislabel_lvl >= 0:
real_lvl = self.tax_code.guess_rank_level(
orig_ranks, mislabel_lvl)
mis_rec = {}
mis_rec['name'] = seq_name
mis_rec['orig_level'] = mislabel_lvl
mis_rec['real_level'] = real_lvl
mis_rec['level_name'] = self.tax_code.rank_level_name(
real_lvl)[0]
mis_rec['inv_level'] = -1 * mis_rec[
'real_level'] # just for sorting
mis_rec['orig_ranks'] = orig_ranks
mis_rec['ranks'] = ranks
mis_rec['lws'] = lws
mis_rec['conf'] = lws[mislabel_lvl]
if mis_rec:
self.mislabels.append(mis_rec)
return mis_rec
def filter_mislabels(self):
filtered_mis = []
for i in range(len(self.mislabels)):
if self.mislabels[i]['conf'] >= self.cfg.conf_cutoff:
filtered_mis.append(self.mislabels[i])
self.mislabels = filtered_mis
def check_rank_tax_labels(self, rank_name, orig_ranks, ranks, lws):
mislabel_lvl = -1
min_len = min(len(orig_ranks), len(ranks))
for rank_lvl in range(min_len):
if ranks[rank_lvl] != Taxonomy.EMPTY_RANK and ranks[
rank_lvl] != orig_ranks[rank_lvl]:
mislabel_lvl = rank_lvl
break
if mislabel_lvl >= 0:
real_lvl = self.tax_code.guess_rank_level(orig_ranks, mislabel_lvl)
mis_rec = {}
mis_rec['name'] = rank_name
mis_rec['orig_level'] = mislabel_lvl
mis_rec['real_level'] = real_lvl
mis_rec['level_name'] = self.tax_code.rank_level_name(real_lvl)[0]
mis_rec['inv_level'] = -1 * real_lvl # just for sorting
mis_rec['orig_ranks'] = orig_ranks
mis_rec['ranks'] = ranks
mis_rec['lws'] = lws
mis_rec['conf'] = lws[mislabel_lvl]
self.rank_mislabels.append(mis_rec)
return mis_rec
else:
return None
def mis_rec_to_string_old(self, mis_rec):
lvl = mis_rec['orig_level']
output = mis_rec['name'] + "\t"
output += "%s\t%s\t%s\t%.3f\n" % (
mis_rec['level_name'], mis_rec['orig_ranks'][lvl],
mis_rec['ranks'][lvl], mis_rec['lws'][lvl])
output += ";".join(mis_rec['orig_ranks']) + "\n"
output += ";".join(mis_rec['ranks']) + "\n"
output += "\t".join(["%.3f" % conf for conf in mis_rec['lws']]) + "\n"
return output
def mis_rec_to_string(self, mis_rec):
lvl = mis_rec['orig_level']
uncorr_name = EpacConfig.strip_ref_prefix(
self.refjson.get_uncorr_seqid(mis_rec['name']))
uncorr_orig_ranks = self.refjson.get_uncorr_ranks(
mis_rec['orig_ranks'])
uncorr_ranks = self.refjson.get_uncorr_ranks(mis_rec['ranks'])
output = uncorr_name + "\t"
if lvl >= 0:
output += "%s\t%s\t%s\t%.3f\t" % (
mis_rec['level_name'], uncorr_orig_ranks[lvl],
uncorr_ranks[lvl], mis_rec['lws'][lvl])
else:
output += "%s\t%s\t%s\t%.3f\t" % (mis_rec['level_name'], "NA",
"NA", mis_rec['lws'][0])
output += Taxonomy.lineage_str(uncorr_orig_ranks) + "\t"
output += Taxonomy.lineage_str(uncorr_ranks) + "\t"
output += ";".join(["%.3f" % conf for conf in mis_rec['lws']])
if 'rank_conf' in mis_rec:
output += "\t%.3f" % mis_rec['rank_conf']
return output
def sort_mislabels(self):
self.mislabels = sorted(self.mislabels,
key=itemgetter('inv_level', 'conf', 'name'),
reverse=True)
for mis_rec in self.mislabels:
real_lvl = mis_rec["real_level"]
self.mislabels_cnt[real_lvl] += 1
if self.cfg.ranktest:
self.rank_mislabels = sorted(self.rank_mislabels,
key=itemgetter(
'inv_level', 'conf', 'name'),
reverse=True)
for mis_rec in self.rank_mislabels:
real_lvl = mis_rec["real_level"]
self.rank_mislabels_cnt[real_lvl] += 1
def write_stats(self, toFile=False):
self.cfg.log.info("Mislabeled sequences by rank:")
seq_sum = 0
rank_sum = 0
stats = []
for i in range(len(self.mislabels_cnt)):
if i > 0:
rname = self.tax_code.rank_level_name(i)[0].ljust(12)
else:
rname = "[NotIngroup]"
if self.mislabels_cnt[i] > 0:
seq_sum += self.mislabels_cnt[i]
# output = "%s:\t%d" % (rname, seq_sum)
output = "%s:\t%d" % (rname, self.mislabels_cnt[i])
if self.cfg.ranktest:
rank_sum += self.rank_mislabels_cnt[i]
output += "\t%d" % rank_sum
self.cfg.log.info(output)
stats.append(output)
if toFile:
with open(self.stats_fname, "w") as fo_stat:
for line in stats:
fo_stat.write(line + "\n")
def write_mislabels_header(self, fo, final, fields):
header = ";" + "\t".join(fields) + "\n"
# write to file
if final:
for line in DISCLAIMER.split("\n"):
fo.write(";%s\n" % line)
fo.write(";\n")
fo.write(header)
# print to console
if final and self.cfg.verbose and len(self.rank_mislabels) > 0:
print(DISCLAIMER, "\n")
print("Mislabeled sequences:\n")
print(header)
def write_rank_mislabels(self, final=True):
if not self.cfg.ranktest:
return
with open(self.misrank_fname, "w") as fo_all:
fields = [
"RankID", "MislabeledLevel", "OriginalLabel", "ProposedLabel",
"Confidence", "OriginalTaxonomyPath", "ProposedTaxonomyPath",
"PerRankConfidence"
]
self.write_mislabels_header(fo_all, final, fields)
for mis_rec in self.rank_mislabels:
output = self.mis_rec_to_string(mis_rec) + "\n"
fo_all.write(output)
if self.cfg.verbose:
print(output)
def write_mislabels(self, final=True):
if final:
out_fname = self.mis_fname
else:
out_fname = self.premis_fname
with open(out_fname, "w") as fo_all:
fields = [
"SeqID", "MislabeledLevel", "OriginalLabel", "ProposedLabel",
"Confidence", "OriginalTaxonomyPath", "ProposedTaxonomyPath",
"PerRankConfidence"
]
if self.cfg.ranktest:
fields += ["HigherRankMisplacedConfidence"]
self.write_mislabels_header(fo_all, final, fields)
for mis_rec in self.mislabels:
output = self.mis_rec_to_string(mis_rec) + "\n"
fo_all.write(output)
if self.cfg.verbose and final:
print(output)
if final:
self.write_rank_mislabels()
self.write_stats()
def get_parent_tip_ranks(self, tax_tree):
rank_tips = {}
rank_parent = {}
for node in tax_tree.traverse("postorder"):
if node.is_leaf() or node.is_root():
continue
tax_path = node.name
ranks = Taxonomy.split_rank_uid(tax_path)
rank_lvl = Taxonomy.lowest_assigned_rank_level(ranks)
if rank_lvl < 2:
continue
parent_ranks = Taxonomy.split_rank_uid(node.up.name)
parent_lvl = Taxonomy.lowest_assigned_rank_level(parent_ranks)
if parent_lvl < 1:
continue
rank_seqs = node.get_leaf_names()
rank_size = len(rank_seqs)
if rank_size < 2 or rank_size > self.reftree_size - 4:
continue
# print rank_lvl, "\t", tax_path, "\t", rank_seqs, "\n"
rank_tips[tax_path] = node.get_leaf_names()
rank_parent[tax_path] = parent_ranks
return rank_parent, rank_tips
def run_leave_subtree_out_test(self):
job_name = self.cfg.subst_name("l1out_rank_%NAME%")
# if self.jplace_fname:
# jp = EpaJsonParser(self.jplace_fname)
# else:
#create file with subtrees
rank_parent, rank_tips = self.get_parent_tip_ranks(self.tax_tree)
subtree_list = list(rank_tips.items())
if len(subtree_list) == 0:
return 0
subtree_list_file = self.cfg.tmp_fname("treelist_%NAME%.txt")
with open(subtree_list_file, "w") as fout:
for rank_name, tips in subtree_list:
fout.write("%s\n" % " ".join(tips))
jp_list = self.raxml.run_epa(job_name,
self.refalign_fname,
self.reftree_fname,
self.optmod_fname,
mode="l1o_subtree",
subtree_fname=subtree_list_file)
subtree_count = 0
for jp in jp_list:
placements = jp.get_placement()
for place in placements:
ranks, lws = self.classify_seq(place)
tax_path = subtree_list[subtree_count][0]
orig_ranks = Taxonomy.split_rank_uid(tax_path)
rank_level = Taxonomy.lowest_assigned_rank_level(orig_ranks)
rank_prefix = self.tax_code.guess_rank_level_name(
orig_ranks, rank_level)[0]
rank_name = orig_ranks[rank_level]
if not rank_name.startswith(rank_prefix):
rank_name = rank_prefix + rank_name
parent_ranks = rank_parent[tax_path]
# print orig_ranks, "\n", parent_ranks, "\n", ranks, "\n"
mis_rec = self.check_rank_tax_labels(rank_name, parent_ranks,
ranks, lws)
if mis_rec:
self.misrank_conf_map[tax_path] = mis_rec['conf']
subtree_count += 1
return subtree_count
def run_leave_seq_out_test(self):
job_name = self.cfg.subst_name("l1out_seq_%NAME%")
placements = []
if self.cfg.jplace_fname:
if os.path.isdir(self.cfg.jplace_fname):
jplace_fmask = os.path.join(self.cfg.jplace_fname, '*.jplace')
else:
jplace_fmask = self.cfg.jplace_fname
jplace_fname_list = glob.glob(jplace_fmask)
for jplace_fname in jplace_fname_list:
jp = EpaJsonParser(jplace_fname)
placements += jp.get_placement()
config.log.debug("Loaded %d placements from %s\n", len(placements),
jplace_fmask)
else:
jp = self.raxml.run_epa(job_name,
self.refalign_fname,
self.reftree_fname,
self.optmod_fname,
mode="l1o_seq")
placements = jp.get_placement()
if self.cfg.output_interim_files:
out_jplace_fname = self.cfg.out_fname(
"%NAME%.l1out_seq.jplace")
self.raxml.copy_epa_jplace(job_name,
out_jplace_fname,
move=True,
mode="l1o_seq")
seq_count = 0
l1out_ass = {}
for place in placements:
seq_name = place["n"][0]
# get original taxonomic label
# orig_ranks = self.get_orig_ranks(seq_name)
orig_ranks = self.taxtree_helper.get_seq_ranks_from_tree(seq_name)
# get EPA tax label
ranks, lws = self.classify_seq(place)
l1out_ass[seq_name] = (ranks, lws)
# check if they match
mis_rec = self.check_seq_tax_labels(seq_name, orig_ranks, ranks,
lws)
# cross-check with higher rank mislabels
if self.cfg.ranktest and mis_rec:
rank_conf = 0
for lvl in range(2, len(orig_ranks)):
tax_path = Taxonomy.get_rank_uid(orig_ranks, lvl)
if tax_path in self.misrank_conf_map:
rank_conf = max(rank_conf,
self.misrank_conf_map[tax_path])
mis_rec['rank_conf'] = rank_conf
seq_count += 1
self.write_assignments(l1out_ass, final=False)
return seq_count
def prune_mislabels_from_tree(self, src_tree, tree_name):
pruned_tree = src_tree.copy(method="newick")
name2node = {}
for leaf in pruned_tree.iter_leaves():
name2node[leaf.name] = leaf
for mis_rec in self.mislabels:
rname = mis_rec['name']
# rname = EpacConfig.REF_SEQ_PREFIX + name
if rname in name2node:
name2node[rname].delete()
else:
config.log.debug("Node not found in the %s tree: %s" %
(tree_name, rname))
return pruned_tree
def run_final_epa_test(self):
self.reftree_outgroup = self.refjson.get_outgroup()
pruned_reftree = self.prune_mislabels_from_tree(
self.reftree, "reference")
pruned_taxtree = self.prune_mislabels_from_tree(
self.reftree, "taxonomic")
# remove unifurcation at the root
if len(pruned_reftree.children) == 1:
pruned_reftree = pruned_reftree.children[0]
self.mislabels = []
th = TaxTreeHelper(self.cfg, self.origin_taxonomy)
th.set_mf_rooted_tree(pruned_taxtree)
reftree_epalbl_str = None
if self.cfg.final_jplace_fname:
if os.path.isdir(self.cfg.final_jplace_fname):
jplace_fmask = os.path.join(self.cfg.final_jplace_fname,
'*.jplace')
else:
jplace_fmask = self.cfg.final_jplace_fname
jplace_fname_list = glob.glob(jplace_fmask)
placements = []
for jplace_fname in jplace_fname_list:
jp = EpaJsonParser(jplace_fname)
placements += jp.get_placement()
if not reftree_epalbl_str:
reftree_epalbl_str = jp.get_std_newick_tree()
config.log.debug("Loaded %d final epa placements from %s\n",
len(placements), jplace_fmask)
else:
epa_result = self.run_epa_once(pruned_reftree)
reftree_epalbl_str = epa_result.get_std_newick_tree()
placements = epa_result.get_placement()
# update branchid-taxonomy mapping to account for possible changes in branch numbering
reftree_tax = Tree(reftree_epalbl_str)
th.set_bf_unrooted_tree(reftree_tax)
bid_tax_map = th.get_bid_taxonomy_map()
self.write_bid_tax_map(bid_tax_map, final=True)
cl = TaxClassifyHelper(self.cfg, bid_tax_map, self.rate,
self.node_height)
# newtax_fname = self.cfg.subst_name("newtax_%NAME%.tre")
# th.get_tax_tree().write(outfile=newtax_fname, format=3)
final_ass = {}
for place in placements:
seq_name = place["n"][0]
# get original taxonomic label
orig_ranks = self.taxtree_helper.get_seq_ranks_from_tree(seq_name)
# EXPERIMENTAL FEATURE - disabled for now!
# It could happen that certain ranks were present in the "original" reference tree, but
# are completely missing in the pruned tree (e.g., all seqs of a species were considered "suspicious"
# after the leave-one-out test and thus pruned)
# In this case, EPA has no chance to infer full original taxonomic annotation (=species) since the corresponding clade
# is now missing. To account for this fact, we amend the original taxonomic annotation and set ranks missing from
# pruned tree to "Undefined".
# orig_ranks = th.strip_missing_ranks(orig_ranks)
# print orig_ranks
# get EPA tax label
ranks, lws = cl.classify_seq(place["p"])
final_ass[seq_name] = (ranks, lws)
#print seq_name, ": ", orig_ranks, "--->", ranks
# check if they match
mis_rec = self.check_seq_tax_labels(seq_name, orig_ranks, ranks,
lws)
self.write_assignments(final_ass, final=True)
def run_epa_once(self, reftree):
reftree_fname = self.cfg.tmp_fname("final_ref_%NAME%.tre")
job_name = self.cfg.subst_name("final_epa_%NAME%")
reftree.write(outfile=reftree_fname)
# IMPORTANT: don't load the model, since it's invalid for the pruned true !!!
optmod_fname = ""
epa_result = self.raxml.run_epa(job_name, self.refalign_fname,
reftree_fname, optmod_fname)
if self.cfg.output_interim_files:
out_jplace_fname = self.cfg.out_fname("%NAME%.final_epa.jplace")
self.raxml.copy_epa_jplace(job_name, out_jplace_fname, move=True)
return epa_result
def run_test(self):
self.raxml = RaxmlWrapper(self.cfg)
# config.log.info("Number of sequences in the reference: %d\n", self.reftree_size)
self.refjson.get_raxml_readable_tree(self.reftree_fname)
self.refalign_fname = self.refjson.get_alignment(self.tmp_refaln)
self.refjson.get_binary_model(self.optmod_fname)
if self.cfg.ranktest:
config.log.info("Running the leave-one-rank-out test...\n")
subtree_count = self.run_leave_subtree_out_test()
config.log.info("Running the leave-one-sequence-out test...\n")
self.run_leave_seq_out_test()
if len(self.mislabels) > 0:
config.log.info(
"Leave-one-out test identified %d suspicious sequences; running final EPA test to check them...\n",
len(self.mislabels))
if self.cfg.debug:
self.write_mislabels(final=False)
self.run_final_epa_test()
self.filter_mislabels()
self.sort_mislabels()
self.write_mislabels()
config.log.info("\nTotal mislabels: %d / %.2f %%", len(self.mislabels),
(float(len(self.mislabels)) / self.reftree_size * 100))