def load_refjson(self, refjson_fname):
try:
self.refjson = RefJsonParser(refjson_fname)
except ValueError:
self.cfg.exit_user_error("ERROR: Invalid json file format!")
#validate input json format
(valid, err) = self.refjson.validate()
if not valid:
self.cfg.log.error(
"ERROR: Parsing reference JSON file failed:\n%s", err)
self.cfg.exit_user_error()
self.rate = self.refjson.get_rate()
self.node_height = self.refjson.get_node_height()
self.origin_taxonomy = self.refjson.get_origin_taxonomy()
self.tax_tree = self.refjson.get_tax_tree()
self.cfg.compress_patterns = self.refjson.get_pattern_compression()
self.bid_taxonomy_map = self.refjson.get_branch_tax_map()
if not self.bid_taxonomy_map:
# old file format (before 1.6), need to rebuild this map from scratch
th = TaxTreeHelper(self.cfg, self.origin_taxonomy)
th.set_mf_rooted_tree(self.tax_tree)
th.set_bf_unrooted_tree(self.refjson.get_reftree())
self.bid_taxonomy_map = th.get_bid_taxonomy_map()
self.write_bid_tax_map(self.bid_taxonomy_map, final=False)
reftree_str = self.refjson.get_raxml_readable_tree()
self.reftree = Tree(reftree_str)
self.reftree_size = len(self.reftree.get_leaves())
# IMPORTANT: set EPA heuristic rate based on tree size!
self.cfg.resolve_auto_settings(self.reftree_size)
# If we're loading the pre-optimized model, we MUST set the same rate het. mode as in the ref file
if self.cfg.epa_load_optmod:
self.cfg.raxml_model = self.refjson.get_ratehet_model()
self.classify_helper = TaxClassifyHelper(self.cfg,
self.bid_taxonomy_map,
self.rate, self.node_height)
self.taxtree_helper = TaxTreeHelper(self.cfg, self.origin_taxonomy,
self.tax_tree)
tax_code_name = self.refjson.get_taxcode()
self.tax_code = TaxCode(tax_code_name)
self.taxonomy = Taxonomy(prefix=EpacConfig.REF_SEQ_PREFIX,
tax_map=self.origin_taxonomy)
self.tax_common_ranks = self.taxonomy.get_common_ranks()
# print "Common ranks: ", self.tax_common_ranks
self.mislabels_cnt = [0] * TaxCode.UNI_TAX_LEVELS
self.rank_mislabels_cnt = [0] * TaxCode.UNI_TAX_LEVELS
def setUp(self):
self.testfile_dir = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "testfiles")
self.tax_fname = os.path.join(self.testfile_dir, "test_clean.tax")
cfg = EpacClassifierConfig()
map_fname = os.path.join(self.testfile_dir, "bid_tax_map2.txt")
self.bid_tax_map = {}
with open(map_fname) as inf:
for line in inf:
bid, rank_id, rdiff, brlen = line.strip().split("\t")
self.bid_tax_map[bid] = (rank_id, int(rdiff), float(brlen))
self.classify_helper = TaxClassifyHelper(cfg, self.bid_tax_map)
def __init__(self, config, args):
self.cfg = config
self.jplace_fname = args.jplace_fname
self.ignore_refalign = args.ignore_refalign
self.tmp_refaln = config.tmp_fname("%NAME%.refaln")
#here is the final alignment file for running EPA
self.epa_alignment = config.tmp_fname("%NAME%.afa")
self.hmmprofile = config.tmp_fname("%NAME%.hmmprofile")
self.tmpquery = config.tmp_fname("%NAME%.tmpquery")
self.noalign = config.tmp_fname("%NAME%.noalign")
self.seqs = None
assign_fname = args.output_name + ".assignment.txt"
self.out_assign_fname = os.path.join(args.output_dir, assign_fname)
jplace_fname = args.output_name + ".jplace"
self.out_jplace_fname = os.path.join(args.output_dir, jplace_fname)
try:
self.refjson = RefJsonParser(config.refjson_fname)
except ValueError:
self.cfg.exit_user_error("Invalid json file format: %s" %
config.refjson_fname)
#validate input json format
self.refjson.validate()
self.reftree = self.refjson.get_reftree()
self.rate = self.refjson.get_rate()
self.node_height = self.refjson.get_node_height()
self.cfg.compress_patterns = self.refjson.get_pattern_compression()
self.bid_taxonomy_map = self.refjson.get_branch_tax_map()
if not self.bid_taxonomy_map:
# old file format (before 1.6), need to rebuild this map from scratch
th = TaxTreeHelper(self.cfg, self.refjson.get_origin_taxonomy())
th.set_mf_rooted_tree(self.refjson.get_tax_tree())
th.set_bf_unrooted_tree(self.refjson.get_reftree())
self.bid_taxonomy_map = th.get_bid_taxonomy_map()
self.cfg.log.info("Loaded reference tree with %d taxa\n" %
len(self.reftree.get_leaves()))
self.classify_helper = TaxClassifyHelper(self.cfg,
self.bid_taxonomy_map,
self.rate, self.node_height)
def run_final_epa_test(self):
self.reftree_outgroup = self.refjson.get_outgroup()
pruned_reftree = self.prune_mislabels_from_tree(
self.reftree, "reference")
pruned_taxtree = self.prune_mislabels_from_tree(
self.reftree, "taxonomic")
# remove unifurcation at the root
if len(pruned_reftree.children) == 1:
pruned_reftree = pruned_reftree.children[0]
self.mislabels = []
th = TaxTreeHelper(self.cfg, self.origin_taxonomy)
th.set_mf_rooted_tree(pruned_taxtree)
reftree_epalbl_str = None
if self.cfg.final_jplace_fname:
if os.path.isdir(self.cfg.final_jplace_fname):
jplace_fmask = os.path.join(self.cfg.final_jplace_fname,
'*.jplace')
else:
jplace_fmask = self.cfg.final_jplace_fname
jplace_fname_list = glob.glob(jplace_fmask)
placements = []
for jplace_fname in jplace_fname_list:
jp = EpaJsonParser(jplace_fname)
placements += jp.get_placement()
if not reftree_epalbl_str:
reftree_epalbl_str = jp.get_std_newick_tree()
config.log.debug("Loaded %d final epa placements from %s\n",
len(placements), jplace_fmask)
else:
epa_result = self.run_epa_once(pruned_reftree)
reftree_epalbl_str = epa_result.get_std_newick_tree()
placements = epa_result.get_placement()
# update branchid-taxonomy mapping to account for possible changes in branch numbering
reftree_tax = Tree(reftree_epalbl_str)
th.set_bf_unrooted_tree(reftree_tax)
bid_tax_map = th.get_bid_taxonomy_map()
self.write_bid_tax_map(bid_tax_map, final=True)
cl = TaxClassifyHelper(self.cfg, bid_tax_map, self.rate,
self.node_height)
# newtax_fname = self.cfg.subst_name("newtax_%NAME%.tre")
# th.get_tax_tree().write(outfile=newtax_fname, format=3)
final_ass = {}
for place in placements:
seq_name = place["n"][0]
# get original taxonomic label
orig_ranks = self.taxtree_helper.get_seq_ranks_from_tree(seq_name)
# EXPERIMENTAL FEATURE - disabled for now!
# It could happen that certain ranks were present in the "original" reference tree, but
# are completely missing in the pruned tree (e.g., all seqs of a species were considered "suspicious"
# after the leave-one-out test and thus pruned)
# In this case, EPA has no chance to infer full original taxonomic annotation (=species) since the corresponding clade
# is now missing. To account for this fact, we amend the original taxonomic annotation and set ranks missing from
# pruned tree to "Undefined".
# orig_ranks = th.strip_missing_ranks(orig_ranks)
# print orig_ranks
# get EPA tax label
ranks, lws = cl.classify_seq(place["p"])
final_ass[seq_name] = (ranks, lws)
#print seq_name, ": ", orig_ranks, "--->", ranks
# check if they match
mis_rec = self.check_seq_tax_labels(seq_name, orig_ranks, ranks,
lws)
self.write_assignments(final_ass, final=True)
def run_final_epa_test(self):
self.reftree_outgroup = self.refjson.get_outgroup()
tmp_reftree = self.reftree.copy(method="newick")
name2refnode = {}
for leaf in tmp_reftree.iter_leaves():
name2refnode[leaf.name] = leaf
tmp_taxtree = self.tax_tree.copy(method="newick")
name2taxnode = {}
for leaf in tmp_taxtree.iter_leaves():
name2taxnode[leaf.name] = leaf
for mis_rec in self.mislabels:
rname = mis_rec['name']
# rname = EpacConfig.REF_SEQ_PREFIX + name
if rname in name2refnode:
name2refnode[rname].delete()
else:
print "Node not found in the reference tree: %s" % rname
if rname in name2taxnode:
name2taxnode[rname].delete()
else:
print "Node not found in the taxonomic tree: %s" % rname
# remove unifurcation at the root
if len(tmp_reftree.children) == 1:
tmp_reftree = tmp_reftree.children[0]
self.mislabels = []
th = TaxTreeHelper(self.cfg, self.origin_taxonomy)
th.set_mf_rooted_tree(tmp_taxtree)
epa_result = self.run_epa_once(tmp_reftree)
reftree_epalbl_str = epa_result.get_std_newick_tree()
placements = epa_result.get_placement()
# update branchid-taxonomy mapping to account for possible changes in branch numbering
reftree_tax = Tree(reftree_epalbl_str)
th.set_bf_unrooted_tree(reftree_tax)
bid_tax_map = th.get_bid_taxonomy_map()
self.write_bid_tax_map(bid_tax_map, final=True)
cl = TaxClassifyHelper(self.cfg, bid_tax_map, self.rate,
self.node_height)
# newtax_fname = self.cfg.subst_name("newtax_%NAME%.tre")
# th.get_tax_tree().write(outfile=newtax_fname, format=3)
final_ass = {}
for place in placements:
seq_name = place["n"][0]
# get original taxonomic label
orig_ranks = self.taxtree_helper.get_seq_ranks_from_tree(seq_name)
# EXPERIMENTAL FEATURE - disabled for now!
# It could happen that certain ranks were present in the "original" reference tree, but
# are completely missing in the pruned tree (e.g., all seqs of a species were considered "suspicious"
# after the leave-one-out test and thus pruned)
# In this case, EPA has no chance to infer full original taxonomic annotation (=species) since the corresponding clade
# is now missing. To account for this fact, we amend the original taxonomic annotation and set ranks missing from
# pruned tree to "Undefined".
# orig_ranks = th.strip_missing_ranks(orig_ranks)
# print orig_ranks
# get EPA tax label
ranks, lws = cl.classify_seq(place["p"])
final_ass[seq_name] = (ranks, lws)
#print seq_name, ": ", orig_ranks, "--->", ranks
# check if they match
mis_rec = self.check_seq_tax_labels(seq_name, orig_ranks, ranks,
lws)
self.write_assignments(final_ass, final=True)
