def annotate_node(t, final_task):
cladeid2node = {}
# Annotate cladeid in the whole tree
for n in t.traverse():
if n.is_leaf():
n.add_feature("realname", db.get_seq_name(n.name))
#n.name = n.realname
if hasattr(n, "cladeid"):
cladeid2node[n.cladeid] = n
alltasks = GLOBALS[final_task.configid]["_nodeinfo"][final_task.nodeid]["tasks"]
npr_iter = get_iternumber(final_task.threadid)
n = cladeid2node[t.cladeid]
n.add_features(size=final_task.size)
for task in alltasks:
params = ["%s %s" %(k,v) for k,v in six.iteritems(task.args)
if not k.startswith("_")]
params = " ".join(params)
if task.ttype == "msf":
n.add_features(msf_outseqs=task.out_seqs,
msf_file=task.multiseq_file)
elif task.ttype == "acleaner":
n.add_features(clean_alg_mean_ident=task.mean_ident,
clean_alg_std_ident=task.std_ident,
clean_alg_max_ident=task.max_ident,
clean_alg_min_ident=task.min_ident,
clean_alg_type=task.tname,
clean_alg_cmd=params,
clean_alg_path=task.clean_alg_fasta_file)
elif task.ttype == "alg":
n.add_features(alg_mean_ident=task.mean_ident,
alg_std_ident=task.std_ident,
alg_max_ident=task.max_ident,
alg_min_ident=task.min_ident,
alg_type=task.tname,
alg_cmd=params,
alg_path=task.alg_fasta_file)
elif task.ttype == "tree":
n.add_features(tree_model=task.model,
tree_seqtype=task.seqtype,
tree_type=task.tname,
tree_cmd=params,
tree_path=task.tree_file,
tree_constrain=task.constrain_tree,
npr_iter=npr_iter)
elif task.ttype == "mchooser":
n.add_features(modeltester_models=task.models,
modeltester_type=task.tname,
modeltester_params=params,
modeltester_bestmodel=task.best_model,
)
elif task.ttype == "treemerger":
n.add_features(treemerger_type=task.tname,
treemerger_rf="RF=%s [%s]" %(task.rf[0], task.rf[1]),
treemerger_out_match_dist = task.outgroup_match_dist,
treemerger_out_match = task.outgroup_match,
)
def annotate_node(t, final_task):
cladeid2node = {}
# Annotate cladeid in the whole tree
for n in t.traverse():
if n.is_leaf():
n.add_feature("realname", db.get_seq_name(n.name))
#n.name = n.realname
if hasattr(n, "cladeid"):
cladeid2node[n.cladeid] = n
alltasks = GLOBALS[final_task.configid]["_nodeinfo"][
final_task.nodeid]["tasks"]
npr_iter = get_iternumber(final_task.threadid)
n = cladeid2node[t.cladeid]
n.add_features(size=final_task.size)
for task in alltasks:
params = [
"%s %s" % (k, v) for k, v in six.iteritems(task.args)
if not k.startswith("_")
]
params = " ".join(params)
if task.ttype == "tree":
n.add_features(tree_model=task.model,
tree_seqtype=task.seqtype,
tree_type=task.tname,
tree_cmd=params,
tree_file=rpath(task.tree_file),
tree_constrain=task.constrain_tree,
npr_iter=npr_iter)
elif task.ttype == "treemerger":
n.add_features(treemerger_type=task.tname,
treemerger_rf="RF=%s [%s]" %
(task.rf[0], task.rf[1]),
treemerger_out_match_dist=task.outgroup_match_dist,
treemerger_out_match=task.outgroup_match)
elif task.ttype == "concat_alg":
n.add_features(concatalg_cogs="%d" % task.used_cogs,
alg_path=task.alg_fasta_file)
def annotate_node(t, final_task):
cladeid2node = {}
# Annotate cladeid in the whole tree
for n in t.traverse():
if n.is_leaf():
n.add_feature("realname", db.get_seq_name(n.name))
#n.name = n.realname
if hasattr(n, "cladeid"):
cladeid2node[n.cladeid] = n
alltasks = GLOBALS[final_task.configid]["_nodeinfo"][final_task.nodeid]["tasks"]
npr_iter = get_iternumber(final_task.threadid)
n = cladeid2node[t.cladeid]
n.add_features(size=final_task.size)
for task in alltasks:
params = ["%s %s" %(k,v) for k,v in six.iteritems(task.args)
if not k.startswith("_")]
params = " ".join(params)
if task.ttype == "tree":
n.add_features(tree_model=task.model,
tree_seqtype=task.seqtype,
tree_type=task.tname,
tree_cmd=params,
tree_file=rpath(task.tree_file),
tree_constrain=task.constrain_tree,
npr_iter=npr_iter)
elif task.ttype == "treemerger":
n.add_features(treemerger_type=task.tname,
treemerger_rf="RF=%s [%s]" %(task.rf[0], task.rf[1]),
treemerger_out_match_dist = task.outgroup_match_dist,
treemerger_out_match = task.outgroup_match)
elif task.ttype == "concat_alg":
n.add_features(concatalg_cogs="%d"%task.used_cogs,
alg_path=task.alg_fasta_file)
def process_task(task, wkname, npr_conf, nodeid2info):
cogconf, cogclass = npr_conf.cog_selector
concatconf, concatclass = npr_conf.alg_concatenator
treebuilderconf, treebuilderclass = npr_conf.tree_builder
splitterconf, splitterclass = npr_conf.tree_splitter
threadid, nodeid, seqtype, ttype = (task.threadid, task.nodeid,
task.seqtype, task.ttype)
cladeid, targets, outgroups = db.get_node_info(threadid, nodeid)
if not treebuilderclass or task.size < 4:
# Allows to dump algs in workflows with no tree tasks or if tree
# inference does not make sense given the number of sequences. DummyTree
# will produce a fake fully collapsed newick tree.
treebuilderclass = DummyTree
if outgroups and len(outgroups) > 1:
constrain_id = nodeid
else:
constrain_id = None
node_info = nodeid2info[nodeid]
conf = GLOBALS[task.configid]
new_tasks = []
if ttype == "cog_selector":
# Generates a md5 id based on the genetree configuration workflow used
# for the concat alg task. If something changes, concat alg will change
# and the associated tree will be rebuilt
config_blocks = set([wkname])
for key, value in six.iteritems(conf[wkname]):
if isinstance(value, list) or isinstance(value, tuple) \
or isinstance(value, set):
for elem in value:
config_blocks.add(elem[1:]) if isinstance(
elem, str) and elem.startswith("@") else None
elif isinstance(value, str):
config_blocks.add(value[1:]) if value.startswith("@") else None
config_checksum = md5(''.join([
"[%s]\n%s" % (x, dict_string(conf[x]))
for x in sorted(config_blocks)
]))
# THIS PART HAS BEEN MOVED TO COG_SELECTOR TASK
# Check that current selection of cogs will cover all target and
# outgroup species
#cog_hard_limit = int(conf[concatconf]["_max_cogs"])
#sp_repr = defaultdict(int)
#for co in task.raw_cogs[:cog_hard_limit]:
# for sp, seq in co:
# sp_repr[sp] += 1
#missing_sp = (targets | outgroups) - set(sp_repr.keys())
#if missing_sp:
# raise TaskError("missing species under current cog selection: %s" %missing_sp)
#else:
# log.log(28, "Analysis of current COG selection:")
# for sp, ncogs in sorted(sp_repr.items(), key=lambda x:x[1]):
# log.log(28, " % 30s species present in % 6d COGs" %(sp, ncogs))
# register concat alignment task. NodeId associated to concat_alg tasks
# and all its children jobs should take into account cog information and
# not only species and outgroups included.
concat_job = concatclass(task.cogs, seqtype, conf, concatconf,
config_checksum)
db.add_node(threadid, concat_job.nodeid, cladeid, targets, outgroups)
# Register Tree constrains
constrain_tree = "(%s, (%s));" % (','.join(
sorted(outgroups)), ','.join(sorted(targets)))
_outs = "\n".join([">%s\n0" % name for name in sorted(outgroups)])
_tars = "\n".join([">%s\n1" % name for name in sorted(targets)])
constrain_alg = '\n'.join([_outs, _tars])
db.add_task_data(concat_job.nodeid, DATATYPES.constrain_tree,
constrain_tree)
db.add_task_data(concat_job.nodeid, DATATYPES.constrain_alg,
constrain_alg)
db.dataconn.commit() # since the creation of some Task objects
# may require this info, I need to commit
# right now.
concat_job.size = task.size
new_tasks.append(concat_job)
elif ttype == "concat_alg":
# register tree for concat alignment, using constraint tree if
# necessary
alg_id = db.get_dataid(task.taskid, DATATYPES.concat_alg_phylip)
try:
parts_id = db.get_dataid(task.taskid, DATATYPES.model_partitions)
except ValueError:
parts_id = None
nodeid2info[nodeid]["size"] = task.size
nodeid2info[nodeid]["target_seqs"] = targets
nodeid2info[nodeid]["out_seqs"] = outgroups
tree_task = treebuilderclass(nodeid,
alg_id,
constrain_id,
None,
seqtype,
conf,
treebuilderconf,
parts_id=parts_id)
tree_task.size = task.size
new_tasks.append(tree_task)
elif ttype == "tree":
merger_task = splitterclass(nodeid, seqtype, task.tree_file, conf,
splitterconf)
merger_task.size = task.size
new_tasks.append(merger_task)
elif ttype == "treemerger":
# Lets merge with main tree
if not task.task_tree:
task.finish()
log.log(24, "Saving task tree...")
annotate_node(task.task_tree, task)
db.update_node(nid=task.nodeid,
runid=task.threadid,
newick=db.encode(task.task_tree))
db.commit()
if not isinstance(treebuilderclass,
DummyTree) and npr_conf.max_iters > 1:
current_iter = get_iternumber(threadid)
if npr_conf.max_iters and current_iter >= npr_conf.max_iters:
log.warning("Maximum number of iterations reached!")
else:
# Add new nodes
source_seqtype = "aa" if "aa" in GLOBALS["seqtypes"] else "nt"
ttree, mtree = task.task_tree, task.main_tree
log.log(26, "Processing tree: %s seqs, %s outgroups",
len(targets), len(outgroups))
target_cladeids = None
if tobool(conf[splitterconf].get("_find_ncbi_targets", False)):
tcopy = mtree.copy()
ncbi.connect_database()
tax2name, tax2track = ncbi.annotate_tree_with_taxa(
tcopy, None)
#tax2name, tax2track = ncbi.annotate_tree_with_taxa(tcopy, "fake") # for testing sptree example
n2content = tcopy.get_cached_content()
broken_branches, broken_clades, broken_clade_sizes, tax2name = ncbi.get_broken_branches(
tcopy, n2content)
log.log(
28, 'restricting NPR to broken clades: ' + colorify(
', '.join(
["%s" % tax2name[x]
for x in broken_clades]), "wr"))
target_cladeids = set()
for branch in broken_branches:
print(
branch.get_ascii(attributes=['spname', 'taxid'],
compact=True))
print([
"%s" % tax2name[x] for x in broken_branches[branch]
])
target_cladeids.add(branch.cladeid)
for node, seqs, outs, wkname in get_next_npr_node(
task.configid, ttree, task.out_seqs, mtree, None,
npr_conf,
target_cladeids): # None is to avoid alg checks
log.log(24, "Adding new node: %s seqs, %s outgroups",
len(seqs), len(outs))
new_task_node = cogclass(seqs, outs, source_seqtype, conf,
cogconf)
new_task_node.target_wkname = wkname
new_tasks.append(new_task_node)
db.add_node(threadid, new_task_node.nodeid,
new_task_node.cladeid, new_task_node.targets,
new_task_node.outgroups)
return new_tasks
def process_task(task, wkname, npr_conf, nodeid2info):
cogconf, cogclass = npr_conf.cog_selector
concatconf, concatclass = npr_conf.alg_concatenator
treebuilderconf, treebuilderclass = npr_conf.tree_builder
splitterconf, splitterclass = npr_conf.tree_splitter
threadid, nodeid, seqtype, ttype = (task.threadid, task.nodeid,
task.seqtype, task.ttype)
cladeid, targets, outgroups = db.get_node_info(threadid, nodeid)
if not treebuilderclass or task.size < 4:
# Allows to dump algs in workflows with no tree tasks or if tree
# inference does not make sense given the number of sequences. DummyTree
# will produce a fake fully collapsed newick tree.
treebuilderclass = DummyTree
if outgroups and len(outgroups) > 1:
constrain_id = nodeid
else:
constrain_id = None
node_info = nodeid2info[nodeid]
conf = GLOBALS[task.configid]
new_tasks = []
if ttype == "cog_selector":
# Generates a md5 id based on the genetree configuration workflow used
# for the concat alg task. If something changes, concat alg will change
# and the associated tree will be rebuilt
config_blocks = set([wkname])
for key, value in six.iteritems(conf[wkname]):
if isinstance(value, list) or isinstance(value, tuple) \
or isinstance(value, set):
for elem in value:
config_blocks.add(elem[1:]) if isinstance(elem, str) and elem.startswith("@") else None
elif isinstance(value, str):
config_blocks.add(value[1:]) if value.startswith("@") else None
config_checksum = md5(''.join(["[%s]\n%s" %(x, dict_string(conf[x]))
for x in sorted(config_blocks)]))
# THIS PART HAS BEEN MOVED TO COG_SELECTOR TASK
# Check that current selection of cogs will cover all target and
# outgroup species
#cog_hard_limit = int(conf[concatconf]["_max_cogs"])
#sp_repr = defaultdict(int)
#for co in task.raw_cogs[:cog_hard_limit]:
# for sp, seq in co:
# sp_repr[sp] += 1
#missing_sp = (targets | outgroups) - set(sp_repr.keys())
#if missing_sp:
# raise TaskError("missing species under current cog selection: %s" %missing_sp)
#else:
# log.log(28, "Analysis of current COG selection:")
# for sp, ncogs in sorted(sp_repr.items(), key=lambda x:x[1]):
# log.log(28, " % 30s species present in % 6d COGs" %(sp, ncogs))
# register concat alignment task. NodeId associated to concat_alg tasks
# and all its children jobs should take into account cog information and
# not only species and outgroups included.
concat_job = concatclass(task.cogs, seqtype, conf, concatconf,
config_checksum)
db.add_node(threadid,
concat_job.nodeid, cladeid,
targets, outgroups)
# Register Tree constrains
constrain_tree = "(%s, (%s));" %(','.join(sorted(outgroups)),
','.join(sorted(targets)))
_outs = "\n".join([">%s\n0" %name for name in sorted(outgroups)])
_tars = "\n".join([">%s\n1" %name for name in sorted(targets)])
constrain_alg = '\n'.join([_outs, _tars])
db.add_task_data(concat_job.nodeid, DATATYPES.constrain_tree, constrain_tree)
db.add_task_data(concat_job.nodeid, DATATYPES.constrain_alg, constrain_alg)
db.dataconn.commit() # since the creation of some Task objects
# may require this info, I need to commit
# right now.
concat_job.size = task.size
new_tasks.append(concat_job)
elif ttype == "concat_alg":
# register tree for concat alignment, using constraint tree if
# necessary
alg_id = db.get_dataid(task.taskid, DATATYPES.concat_alg_phylip)
try:
parts_id = db.get_dataid(task.taskid, DATATYPES.model_partitions)
except ValueError:
parts_id = None
nodeid2info[nodeid]["size"] = task.size
nodeid2info[nodeid]["target_seqs"] = targets
nodeid2info[nodeid]["out_seqs"] = outgroups
tree_task = treebuilderclass(nodeid, alg_id,
constrain_id, None,
seqtype, conf, treebuilderconf,
parts_id=parts_id)
tree_task.size = task.size
new_tasks.append(tree_task)
elif ttype == "tree":
merger_task = splitterclass(nodeid, seqtype, task.tree_file, conf, splitterconf)
merger_task.size = task.size
new_tasks.append(merger_task)
elif ttype == "treemerger":
# Lets merge with main tree
if not task.task_tree:
task.finish()
log.log(24, "Saving task tree...")
annotate_node(task.task_tree, task)
db.update_node(nid=task.nodeid, runid=task.threadid,
newick=db.encode(task.task_tree))
db.commit()
if not isinstance(treebuilderclass, DummyTree) and npr_conf.max_iters > 1:
current_iter = get_iternumber(threadid)
if npr_conf.max_iters and current_iter >= npr_conf.max_iters:
log.warning("Maximum number of iterations reached!")
else:
# Add new nodes
source_seqtype = "aa" if "aa" in GLOBALS["seqtypes"] else "nt"
ttree, mtree = task.task_tree, task.main_tree
log.log(26, "Processing tree: %s seqs, %s outgroups",
len(targets), len(outgroups))
target_cladeids = None
if tobool(conf[splitterconf].get("_find_ncbi_targets", False)):
tcopy = mtree.copy()
ncbi.connect_database()
tax2name, tax2track = ncbi.annotate_tree_with_taxa(tcopy, None)
#tax2name, tax2track = ncbi.annotate_tree_with_taxa(tcopy, "fake") # for testing sptree example
n2content = tcopy.get_cached_content()
broken_branches, broken_clades, broken_clade_sizes, tax2name = ncbi.get_broken_branches(tcopy, n2content)
log.log(28, 'restricting NPR to broken clades: '+
colorify(', '.join(["%s"%tax2name[x] for x in broken_clades]), "wr"))
target_cladeids = set()
for branch in broken_branches:
print(branch.get_ascii(attributes=['spname', 'taxid'], compact=True))
print(["%s"%tax2name[x] for x in broken_branches[branch]])
target_cladeids.add(branch.cladeid)
for node, seqs, outs, wkname in get_next_npr_node(task.configid, ttree,
task.out_seqs, mtree, None,
npr_conf, target_cladeids): # None is to avoid alg checks
log.log(24, "Adding new node: %s seqs, %s outgroups",
len(seqs), len(outs))
new_task_node = cogclass(seqs, outs,
source_seqtype, conf, cogconf)
new_task_node.target_wkname = wkname
new_tasks.append(new_task_node)
db.add_node(threadid,
new_task_node.nodeid, new_task_node.cladeid,
new_task_node.targets,
new_task_node.outgroups)
return new_tasks
def annotate_node(t, final_task):
cladeid2node = {}
# Annotate cladeid in the whole tree
for n in t.traverse():
if n.is_leaf():
n.add_feature("realname", db.get_seq_name(n.name))
#n.name = n.realname
if hasattr(n, "cladeid"):
cladeid2node[n.cladeid] = n
alltasks = GLOBALS[final_task.configid]["_nodeinfo"][
final_task.nodeid]["tasks"]
npr_iter = get_iternumber(final_task.threadid)
n = cladeid2node[t.cladeid]
n.add_features(size=final_task.size)
for task in alltasks:
params = [
"%s %s" % (k, v) for k, v in six.iteritems(task.args)
if not k.startswith("_")
]
params = " ".join(params)
if task.ttype == "msf":
n.add_features(msf_outseqs=task.out_seqs,
msf_file=task.multiseq_file)
elif task.ttype == "acleaner":
n.add_features(clean_alg_mean_ident=task.mean_ident,
clean_alg_std_ident=task.std_ident,
clean_alg_max_ident=task.max_ident,
clean_alg_min_ident=task.min_ident,
clean_alg_type=task.tname,
clean_alg_cmd=params,
clean_alg_path=task.clean_alg_fasta_file)
elif task.ttype == "alg":
n.add_features(alg_mean_ident=task.mean_ident,
alg_std_ident=task.std_ident,
alg_max_ident=task.max_ident,
alg_min_ident=task.min_ident,
alg_type=task.tname,
alg_cmd=params,
alg_path=task.alg_fasta_file)
elif task.ttype == "tree":
n.add_features(tree_model=task.model,
tree_seqtype=task.seqtype,
tree_type=task.tname,
tree_cmd=params,
tree_path=task.tree_file,
tree_constrain=task.constrain_tree,
npr_iter=npr_iter)
elif task.ttype == "mchooser":
n.add_features(
modeltester_models=task.models,
modeltester_type=task.tname,
modeltester_params=params,
modeltester_bestmodel=task.best_model,
)
elif task.ttype == "treemerger":
n.add_features(
treemerger_type=task.tname,
treemerger_rf="RF=%s [%s]" % (task.rf[0], task.rf[1]),
treemerger_out_match_dist=task.outgroup_match_dist,
treemerger_out_match=task.outgroup_match,
)
def process_task(task, wkname, npr_conf, nodeid2info):
alignerconf, alignerclass = npr_conf.aligner
cleanerconf, cleanerclass = npr_conf.alg_cleaner
mtesterconf, mtesterclass = npr_conf.model_tester
treebuilderconf, treebuilderclass = npr_conf.tree_builder
if not treebuilderclass:
# Allows to dump algs in workflows with no tree tasks
treebuilderclass = DummyTree
splitterconf, splitterclass = npr_conf.tree_splitter
conf = GLOBALS[task.configid]
seqtype = task.seqtype
nodeid = task.nodeid
ttype = task.ttype
taskid = task.taskid
threadid = task.threadid
node_info = nodeid2info[nodeid]
size = task.size #node_info.get("size", 0)
target_seqs = node_info.get("target_seqs", [])
out_seqs = node_info.get("out_seqs", [])
if not treebuilderclass or size < 4:
# Allows to dump algs in workflows with no tree tasks or if tree
# inference does not make sense given the number of sequences. DummyTree
# will produce a fake fully collapsed newick tree.
treebuilderclass = DummyTree
# If more than one outgroup are used, enable the use of constrain
if out_seqs and len(out_seqs) > 1:
constrain_id = nodeid
else:
constrain_id = None
new_tasks = []
if ttype == "msf":
# Register Tree constrains
constrain_tree = "(%s, (%s));" % (','.join(sorted(
task.out_seqs)), ','.join(sorted(task.target_seqs)))
_outs = "\n".join([">%s\n0" % name for name in sorted(task.out_seqs)])
_tars = "\n".join(
[">%s\n1" % name for name in sorted(task.target_seqs)])
constrain_alg = '\n'.join([_outs, _tars])
db.add_task_data(nodeid, DATATYPES.constrain_tree, constrain_tree)
db.add_task_data(nodeid, DATATYPES.constrain_alg, constrain_alg)
db.dataconn.commit() # since the creation of some Task
# objects may require this info, I need
# to commit right now.
# Register node
db.add_node(task.threadid, task.nodeid, task.cladeid, task.target_seqs,
task.out_seqs)
nodeid2info[nodeid]["size"] = task.size
nodeid2info[nodeid]["target_seqs"] = task.target_seqs
nodeid2info[nodeid]["out_seqs"] = task.out_seqs
alg_task = alignerclass(nodeid, task.multiseq_file, seqtype, conf,
alignerconf)
alg_task.size = task.size
new_tasks.append(alg_task)
elif ttype == "alg" or ttype == "acleaner":
if ttype == "alg":
nodeid2info[nodeid]["alg_path"] = task.alg_fasta_file
elif ttype == "acleaner":
nodeid2info[nodeid]["alg_clean_path"] = task.clean_alg_fasta_file
alg_fasta_file = getattr(task, "clean_alg_fasta_file",
task.alg_fasta_file)
alg_phylip_file = getattr(task, "clean_alg_phylip_file",
task.alg_phylip_file)
# Calculate alignment stats
# cons_mean, cons_std = get_trimal_conservation(task.alg_fasta_file,
# conf["app"]["trimal"])
#
# max_identity = get_trimal_identity(task.alg_fasta_file,
# conf["app"]["trimal"])
# log.info("Conservation: %0.2f +-%0.2f", cons_mean, cons_std)
# log.info("Max. Identity: %0.2f", max_identity)
#import time
#t1 = time.time()
#mx, mn, mean, std = get_identity(task.alg_fasta_file)
#print time.time()-t1
#log.log(26, "Identity: max=%0.2f min=%0.2f mean=%0.2f +- %0.2f",
# mx, mn, mean, std)
#t1 = time.time()
if seqtype == "aa" and npr_conf.switch_aa_similarity < 1:
try:
alg_stats = db.get_task_data(taskid, DATATYPES.alg_stats)
except Exception as e:
alg_stats = {}
if ttype == "alg":
algfile = pjoin(GLOBALS["input_dir"], task.alg_phylip_file)
dataid = DATATYPES.alg_phylip
elif ttype == "acleaner":
algfile = pjoin(GLOBALS["input_dir"],
task.clean_alg_phylip_file)
dataid = DATATYPES.clean_alg_phylip
if "i_mean" not in alg_stats:
log.log(24, "Calculating alignment stats...")
# dump data if necesary
algfile = pjoin(GLOBALS["input_dir"], task.alg_phylip_file)
if not pexist(algfile):
# dump phylip alg
open(algfile,
"w").write(db.get_data(db.get_dataid(taskid, dataid)))
mx, mn, mean, std = get_statal_identity(
algfile, conf["app"]["statal"])
alg_stats = {
"i_max": mx,
"i_mean": mean,
"i_min": mn,
"i_std": std
}
db.add_task_data(taskid, DATATYPES.alg_stats, alg_stats)
log.log(22, "Alignment stats (sequence similarity):")
log.log(
22,
" max: %(i_max)0.2f, min:%(i_min)0.2f, avg:%(i_mean)0.2f+-%(i_std)0.2f"
% (alg_stats))
else:
alg_stats = {"i_max": -1, "i_mean": -1, "i_min": -1, "i_std": -1}
#print time.time()-t1
#log.log(24, "Identity: max=%0.2f min=%0.2f mean=%0.2f +- %0.2f",
# mx, mn, mean, std)
task.max_ident = alg_stats["i_max"]
task.min_ident = alg_stats["i_min"]
task.mean_ident = alg_stats["i_mean"]
task.std_ident = alg_stats["i_std"]
next_task = None
if ttype == "alg" and cleanerclass:
next_task = cleanerclass(nodeid, seqtype, alg_fasta_file,
alg_phylip_file, conf, cleanerconf)
else:
# Converts aa alignment into nt if necessary
if seqtype == "aa" and \
"nt" in GLOBALS["seqtypes"] and \
task.mean_ident >= npr_conf.switch_aa_similarity:
log.log(28, "@@2:Switching to codon alignment!@@1: amino-acid sequence similarity: %0.2f >= %0.2f" %\
(task.mean_ident, npr_conf.switch_aa_similarity))
alg_fasta_file = "%s.%s" % (taskid, DATATYPES.alg_nt_fasta)
alg_phylip_file = "%s.%s" % (taskid, DATATYPES.alg_nt_phylip)
try:
alg_fasta_file = db.get_dataid(taskid,
DATATYPES.alg_nt_fasta)
alg_fasta_file = db.get_dataid(taskid,
DATATYPES.alg_nt_phylip)
except ValueError:
log.log(22, "Calculating codon alignment...")
source_alg = pjoin(GLOBALS["input_dir"],
task.alg_fasta_file)
if ttype == "alg":
kept_columns = []
elif ttype == "acleaner":
# if original alignment was trimmed, use it as reference
# but make the nt alignment only on the kept columns
kept_columns = db.get_task_data(
taskid, DATATYPES.kept_alg_columns)
if not pexist(source_alg):
open(source_alg, "w").write(
db.get_task_data(taskid, DATATYPES.alg_fasta))
nt_alg = switch_to_codon(source_alg,
kept_columns=kept_columns)
db.add_task_data(taskid, DATATYPES.alg_nt_fasta,
nt_alg.write())
db.add_task_data(taskid, DATATYPES.alg_nt_phylip,
nt_alg.write(format='iphylip_relaxed'))
npr_conf = IterConfig(conf, wkname, task.size, "nt")
seqtype = "nt"
if mtesterclass:
next_task = mtesterclass(nodeid, alg_fasta_file,
alg_phylip_file, constrain_id, conf,
mtesterconf)
elif treebuilderclass:
next_task = treebuilderclass(nodeid, alg_phylip_file,
constrain_id, None, seqtype, conf,
treebuilderconf)
if next_task:
next_task.size = task.size
new_tasks.append(next_task)
elif ttype == "mchooser":
if treebuilderclass:
alg_fasta_file = task.alg_fasta_file
alg_phylip_file = task.alg_phylip_file
model = task.best_model
tree_task = treebuilderclass(nodeid, alg_phylip_file, constrain_id,
model, seqtype, conf, treebuilderconf)
tree_task.size = task.size
new_tasks.append(tree_task)
elif ttype == "tree":
treemerge_task = splitterclass(nodeid, seqtype, task.tree_file, conf,
splitterconf)
#if conf["tree_splitter"]["_outgroup_size"]:
# treemerge_task = TreeSplitterWithOutgroups(nodeid, seqtype, task.tree_file, main_tree, conf)
#else:
# treemerge_task = TreeSplitter(nodeid, seqtype, task.tree_file, main_tree, conf)
treemerge_task.size = task.size
new_tasks.append(treemerge_task)
elif ttype == "treemerger":
if not task.task_tree:
task.finish()
log.log(24, "Saving task tree...")
annotate_node(task.task_tree, task)
db.update_node(nid=task.nodeid,
runid=task.threadid,
newick=db.encode(task.task_tree))
db.commit()
if not isinstance(treebuilderclass,
DummyTree) and npr_conf.max_iters > 1:
current_iter = get_iternumber(threadid)
if npr_conf.max_iters and current_iter >= npr_conf.max_iters:
log.warning("Maximum number of iterations reached!")
else:
# Add new nodes
source_seqtype = "aa" if "aa" in GLOBALS["seqtypes"] else "nt"
ttree, mtree = task.task_tree, task.main_tree
log.log(26, "Processing tree: %s seqs, %s outgroups",
len(target_seqs), len(out_seqs))
alg_path = node_info.get("clean_alg_path",
node_info["alg_path"])
for node, seqs, outs, wkname in get_next_npr_node(
threadid, ttree, task.out_seqs, mtree, alg_path,
npr_conf):
log.log(24, "Registering new node: %s seqs, %s outgroups",
len(seqs), len(outs))
new_task_node = Msf(seqs, outs, seqtype=source_seqtype)
new_task_node.target_wkname = wkname
new_tasks.append(new_task_node)
return new_tasks
def process_task(task, wkname, npr_conf, nodeid2info):
alignerconf, alignerclass = npr_conf.aligner
cleanerconf, cleanerclass = npr_conf.alg_cleaner
mtesterconf, mtesterclass = npr_conf.model_tester
treebuilderconf, treebuilderclass = npr_conf.tree_builder
if not treebuilderclass:
# Allows to dump algs in workflows with no tree tasks
treebuilderclass = DummyTree
splitterconf, splitterclass = npr_conf.tree_splitter
conf = GLOBALS[task.configid]
seqtype = task.seqtype
nodeid = task.nodeid
ttype = task.ttype
taskid = task.taskid
threadid = task.threadid
node_info = nodeid2info[nodeid]
size = task.size#node_info.get("size", 0)
target_seqs = node_info.get("target_seqs", [])
out_seqs = node_info.get("out_seqs", [])
if not treebuilderclass or size < 4:
# Allows to dump algs in workflows with no tree tasks or if tree
# inference does not make sense given the number of sequences. DummyTree
# will produce a fake fully collapsed newick tree.
treebuilderclass = DummyTree
# If more than one outgroup are used, enable the use of constrain
if out_seqs and len(out_seqs) > 1:
constrain_id = nodeid
else:
constrain_id = None
new_tasks = []
if ttype == "msf":
# Register Tree constrains
constrain_tree = "(%s, (%s));" %(','.join(sorted(task.out_seqs)),
','.join(sorted(task.target_seqs)))
_outs = "\n".join([">%s\n0" %name for name in sorted(task.out_seqs)])
_tars = "\n".join([">%s\n1" %name for name in sorted(task.target_seqs)])
constrain_alg = '\n'.join([_outs, _tars])
db.add_task_data(nodeid, DATATYPES.constrain_tree, constrain_tree)
db.add_task_data(nodeid, DATATYPES.constrain_alg, constrain_alg)
db.dataconn.commit() # since the creation of some Task
# objects may require this info, I need
# to commit right now.
# Register node
db.add_node(task.threadid,
task.nodeid, task.cladeid,
task.target_seqs,
task.out_seqs)
nodeid2info[nodeid]["size"] = task.size
nodeid2info[nodeid]["target_seqs"] = task.target_seqs
nodeid2info[nodeid]["out_seqs"] = task.out_seqs
alg_task = alignerclass(nodeid, task.multiseq_file,
seqtype, conf, alignerconf)
alg_task.size = task.size
new_tasks.append(alg_task)
elif ttype == "alg" or ttype == "acleaner":
if ttype == "alg":
nodeid2info[nodeid]["alg_path"] = task.alg_fasta_file
elif ttype == "acleaner":
nodeid2info[nodeid]["alg_clean_path"] = task.clean_alg_fasta_file
alg_fasta_file = getattr(task, "clean_alg_fasta_file",
task.alg_fasta_file)
alg_phylip_file = getattr(task, "clean_alg_phylip_file",
task.alg_phylip_file)
# Calculate alignment stats
# cons_mean, cons_std = get_trimal_conservation(task.alg_fasta_file,
# conf["app"]["trimal"])
#
# max_identity = get_trimal_identity(task.alg_fasta_file,
# conf["app"]["trimal"])
# log.info("Conservation: %0.2f +-%0.2f", cons_mean, cons_std)
# log.info("Max. Identity: %0.2f", max_identity)
#import time
#t1 = time.time()
#mx, mn, mean, std = get_identity(task.alg_fasta_file)
#print time.time()-t1
#log.log(26, "Identity: max=%0.2f min=%0.2f mean=%0.2f +- %0.2f",
# mx, mn, mean, std)
#t1 = time.time()
if seqtype == "aa" and npr_conf.switch_aa_similarity < 1:
try:
alg_stats = db.get_task_data(taskid, DATATYPES.alg_stats)
except Exception as e:
alg_stats = {}
if ttype == "alg":
algfile = pjoin(GLOBALS["input_dir"], task.alg_phylip_file)
dataid = DATATYPES.alg_phylip
elif ttype == "acleaner":
algfile = pjoin(GLOBALS["input_dir"], task.clean_alg_phylip_file)
dataid = DATATYPES.clean_alg_phylip
if "i_mean" not in alg_stats:
log.log(24, "Calculating alignment stats...")
# dump data if necesary
algfile = pjoin(GLOBALS["input_dir"], task.alg_phylip_file)
if not pexist(algfile):
# dump phylip alg
open(algfile, "w").write(db.get_data(db.get_dataid(taskid, dataid)))
mx, mn, mean, std = get_statal_identity(algfile,
conf["app"]["statal"])
alg_stats = {"i_max":mx, "i_mean":mean, "i_min":mn, "i_std":std}
db.add_task_data(taskid, DATATYPES.alg_stats, alg_stats)
log.log(22, "Alignment stats (sequence similarity):")
log.log(22, " max: %(i_max)0.2f, min:%(i_min)0.2f, avg:%(i_mean)0.2f+-%(i_std)0.2f" %
(alg_stats))
else:
alg_stats = {"i_max":-1, "i_mean":-1, "i_min":-1, "i_std":-1}
#print time.time()-t1
#log.log(24, "Identity: max=%0.2f min=%0.2f mean=%0.2f +- %0.2f",
# mx, mn, mean, std)
task.max_ident = alg_stats["i_max"]
task.min_ident = alg_stats["i_min"]
task.mean_ident = alg_stats["i_mean"]
task.std_ident = alg_stats["i_std"]
next_task = None
if ttype == "alg" and cleanerclass:
next_task = cleanerclass(nodeid, seqtype, alg_fasta_file,
alg_phylip_file,
conf, cleanerconf)
else:
# Converts aa alignment into nt if necessary
if seqtype == "aa" and \
"nt" in GLOBALS["seqtypes"] and \
task.mean_ident >= npr_conf.switch_aa_similarity:
log.log(28, "@@2:Switching to codon alignment!@@1: amino-acid sequence similarity: %0.2f >= %0.2f" %\
(task.mean_ident, npr_conf.switch_aa_similarity))
alg_fasta_file = "%s.%s" %(taskid, DATATYPES.alg_nt_fasta)
alg_phylip_file = "%s.%s" %(taskid, DATATYPES.alg_nt_phylip)
try:
alg_fasta_file = db.get_dataid(taskid, DATATYPES.alg_nt_fasta)
alg_fasta_file = db.get_dataid(taskid, DATATYPES.alg_nt_phylip)
except ValueError:
log.log(22, "Calculating codon alignment...")
source_alg = pjoin(GLOBALS["input_dir"], task.alg_fasta_file)
if ttype == "alg":
kept_columns = []
elif ttype == "acleaner":
# if original alignment was trimmed, use it as reference
# but make the nt alignment only on the kept columns
kept_columns = db.get_task_data(taskid, DATATYPES.kept_alg_columns)
if not pexist(source_alg):
open(source_alg, "w").write(db.get_task_data(taskid, DATATYPES.alg_fasta))
nt_alg = switch_to_codon(source_alg, kept_columns=kept_columns)
db.add_task_data(taskid, DATATYPES.alg_nt_fasta, nt_alg.write())
db.add_task_data(taskid, DATATYPES.alg_nt_phylip, nt_alg.write(format='iphylip_relaxed'))
npr_conf = IterConfig(conf, wkname, task.size, "nt")
seqtype = "nt"
if mtesterclass:
next_task = mtesterclass(nodeid, alg_fasta_file,
alg_phylip_file,
constrain_id,
conf, mtesterconf)
elif treebuilderclass:
next_task = treebuilderclass(nodeid, alg_phylip_file,
constrain_id,
None, seqtype,
conf, treebuilderconf)
if next_task:
next_task.size = task.size
new_tasks.append(next_task)
elif ttype == "mchooser":
if treebuilderclass:
alg_fasta_file = task.alg_fasta_file
alg_phylip_file = task.alg_phylip_file
model = task.best_model
tree_task = treebuilderclass(nodeid, alg_phylip_file,
constrain_id,
model, seqtype,
conf, treebuilderconf)
tree_task.size = task.size
new_tasks.append(tree_task)
elif ttype == "tree":
treemerge_task = splitterclass(nodeid, seqtype,
task.tree_file, conf, splitterconf)
#if conf["tree_splitter"]["_outgroup_size"]:
# treemerge_task = TreeSplitterWithOutgroups(nodeid, seqtype, task.tree_file, main_tree, conf)
#else:
# treemerge_task = TreeSplitter(nodeid, seqtype, task.tree_file, main_tree, conf)
treemerge_task.size = task.size
new_tasks.append(treemerge_task)
elif ttype == "treemerger":
if not task.task_tree:
task.finish()
log.log(24, "Saving task tree...")
annotate_node(task.task_tree, task)
db.update_node(nid=task.nodeid,
runid=task.threadid,
newick=db.encode(task.task_tree))
db.commit()
if not isinstance(treebuilderclass, DummyTree) and npr_conf.max_iters > 1:
current_iter = get_iternumber(threadid)
if npr_conf.max_iters and current_iter >= npr_conf.max_iters:
log.warning("Maximum number of iterations reached!")
else:
# Add new nodes
source_seqtype = "aa" if "aa" in GLOBALS["seqtypes"] else "nt"
ttree, mtree = task.task_tree, task.main_tree
log.log(26, "Processing tree: %s seqs, %s outgroups",
len(target_seqs), len(out_seqs))
alg_path = node_info.get("clean_alg_path", node_info["alg_path"])
for node, seqs, outs, wkname in get_next_npr_node(threadid, ttree,
task.out_seqs, mtree,
alg_path, npr_conf):
log.log(24, "Registering new node: %s seqs, %s outgroups",
len(seqs), len(outs))
new_task_node = Msf(seqs, outs, seqtype=source_seqtype)
new_task_node.target_wkname = wkname
new_tasks.append(new_task_node)
return new_tasks