def finish(self): lks = [] if self.lk_mode == "phyml": for job in self.jobs: if job.jobcat != "bionj": continue phyml_job = job tree_file = pjoin(phyml_job.jobdir, self.alg_phylip_file+"_phyml_tree.txt") stats_file = pjoin(phyml_job.jobdir, self.alg_phylip_file+"_phyml_stats.txt") tree = PhyloTree(tree_file) m = re.search('Log-likelihood:\s+(-?\d+\.\d+)', open(stats_file).read()) lk = float(m.groups()[0]) tree.add_feature("lk", lk) tree.add_feature("model", phyml_job.args["--model"]) lks.append([float(tree.lk), tree.model, tree]) elif self.lk_mode == "raxml": for job in self.jobs: if job.jobcat != "raxml": continue raxml_job = job lk = open(pjoin(raxml_job.jobdir, "RAxML_log.%s" %raxml_job.args["-n"])).readline().split()[1] tree = PhyloTree(raxml_job.args["-t"]) tree.add_feature("lk", lk) tree.add_feature("model", raxml_job.model) lks.append([float(tree.lk), tree.model, tree]) # sort lks in ASC order lks.sort() # choose the model with higher likelihood, the lastone in the list best_model = lks[-1][1] best_tree = lks[-1][2] log.log(22, "%s model selected from the following lk values:\n%s" %(best_model, '\n'.join(map(str, lks)))) ModelTesterTask.store_data(self, best_model, lks)
def load_jobs(self): conf = self.conf for m in self.models: args = self.args.copy() args["--model"] = m bionj_job = Job(conf["app"]["phyml"], args, parent_ids=[self.nodeid]) bionj_job.jobname += "-bionj-" + m bionj_job.jobcat = "bionj" bionj_job.add_input_file(self.alg_phylip_file, bionj_job.jobdir) self.jobs.append(bionj_job) if self.lk_mode == "raxml": raxml_args = { "-f": "e", "-s": pjoin(bionj_job.jobdir, self.alg_phylip_file), "-m": "PROTGAMMA%s" % m, "-n": self.alg_phylip_file + "." + m, "-t": pjoin(bionj_job.jobdir, self.alg_phylip_file + "_phyml_tree.txt") } raxml_job = Job(conf["app"]["raxml"], raxml_args, parent_ids=[bionj_job.jobid]) raxml_job.jobname += "-lk-optimize" raxml_job.dependencies.add(bionj_job) raxml_job.model = m raxml_job.jobcat = "raxml" self.jobs.append(raxml_job)
def load_jobs(self): conf = self.conf for m in self.models: args = self.args.copy() args["--model"] = m bionj_job = Job(conf["app"]["phyml"], args, parent_ids=[self.nodeid]) bionj_job.jobname += "-bionj-" + m bionj_job.jobcat = "bionj" bionj_job.add_input_file(self.alg_phylip_file, bionj_job.jobdir) self.jobs.append(bionj_job) if self.lk_mode == "raxml": raxml_args = { "-f": "e", "-s": pjoin(bionj_job.jobdir, self.alg_phylip_file), "-m": "PROTGAMMA%s" % m, "-n": self.alg_phylip_file+"."+m, "-t": pjoin(bionj_job.jobdir, self.alg_phylip_file+"_phyml_tree.txt") } raxml_job = Job(conf["app"]["raxml"], raxml_args, parent_ids=[bionj_job.jobid]) raxml_job.jobname += "-lk-optimize" raxml_job.dependencies.add(bionj_job) raxml_job.model = m raxml_job.jobcat = "raxml" self.jobs.append(raxml_job)
def zdecode(x): if x.startswith("__DBDIR__:"): data_id = x.split(':', 1)[1] data = six.moves.cPickle.load(open(pjoin(GLOBALS['db_dir'], data_id+".pkl"))) else: data = six.moves.cPickle.loads(zlib.decompress(base64.decodestring(x))) return data
def finish(self): # Once executed, alignment is converted into relaxed # interleaved phylip format. Both files, fasta and phylip, # remain accessible. # Set Task specific attributes main_job = self.jobs[0] fasta_path = pjoin(main_job.jobdir, "clean.alg.fasta") alg = SeqGroup(fasta_path) if len(alg) != self.size: log.warning( "Trimming was to aggressive and it tried" " to remove one or more sequences." " Alignment trimming will be disabled for this dataset.") self.clean_alg_fasta_file = db.register_task_data( self.taskid, DATATYPES.clean_alg_fasta, self.alg_fasta_file) self.clean_alg_phylip_file = db.register_task_data( self.taskid, DATATYPES.clean_alg_phylip, self.alg_phylip_file) else: for line in open(self.jobs[0].stdout_file): line = line.strip() if line.startswith("#ColumnsMap"): kept_columns = list( map(int, line.split("\t")[1].split(","))) fasta = alg.write(format="fasta") phylip = alg.write(format="iphylip_relaxed") AlgCleanerTask.store_data(self, fasta, phylip, kept_columns)
def load_jobs(self): appname = self.conf[self.confname]["_app"] args = self.args.copy() args["-in"] = pjoin(GLOBALS["input_dir"], self.alg_fasta_file) args["-out"] = "clean.alg.fasta" job = Job(self.conf["app"][appname], args, parent_ids=[self.nodeid]) job.add_input_file(self.alg_fasta_file) self.jobs.append(job)
def zdecode(x): if x.startswith("__DBDIR__:"): data_id = x.split(':', 1)[1] data = six.moves.cPickle.load( open(pjoin(GLOBALS['db_dir'], data_id + ".pkl"))) else: data = six.moves.cPickle.loads(zlib.decompress(base64.decodestring(x))) return data
def zencode(x, data_id): pdata = six.moves.cPickle.dumps(x) if sys.getsizeof(pdata) > MAX_SQLITE_SIZE: # using protocol 2 fails because of the integer overflow python bug # i.e. http://bugs.python.org/issue13555 six.moves.cPickle.dump(x, open(pjoin(GLOBALS['db_dir'], data_id+".pkl"), "wb")) return "__DBDIR__:%s" %data_id else: return base64.encodestring(zlib.compress(pdata))
def load_jobs(self): # Only one Muscle job is necessary to run this task appname = self.conf[self.confname]["_app"] args = OrderedDict(self.args) args["-in"] = pjoin(GLOBALS["input_dir"], self.multiseq_file) args["-out"] = "alg.fasta" job = Job(self.conf["app"][appname], args, parent_ids=[self.nodeid]) job.add_input_file(self.multiseq_file) self.jobs.append(job)
def load_jobs(self): # Only one Muscle job is necessary to run this task appname = self.conf[self.confname]["_app"] args = OrderedDict(self.args) args[''] = "%s %s" % (pjoin(GLOBALS["input_dir"], self.multiseq_file), "alg.fasta") job = Job(self.conf["app"][appname], args, parent_ids=[self.nodeid]) job.add_input_file(self.multiseq_file) self.jobs.append(job)
def load_jobs(self): appname = self.conf[self.confname]["_app"] # Only one Muscle job is necessary to run this task args = OrderedDict(self.args) args["-i"] = pjoin(GLOBALS["input_dir"], self.multiseq_file) args["-o"] = "alg.fasta" job = Job(self.conf["app"][appname], args, parent_ids=[self.nodeid]) job.cores = self.conf["threading"].get(appname, 1) job.add_input_file(self.multiseq_file) self.jobs.append(job)
def zencode(x, data_id): pdata = six.moves.cPickle.dumps(x) if sys.getsizeof(pdata) > MAX_SQLITE_SIZE: # using protocol 2 fails because of the integer overflow python bug # i.e. http://bugs.python.org/issue13555 six.moves.cPickle.dump( x, open(pjoin(GLOBALS['db_dir'], data_id + ".pkl"), "wb")) return "__DBDIR__:%s" % data_id else: return base64.encodestring(zlib.compress(pdata))
def load_jobs(self): appname = self.conf[self.confname]["_app"] args = OrderedDict(self.args) # Mafft redirects resulting alg to std.output. The order of # arguments is important, input file must be the last # one. args[""] = pjoin(GLOBALS["input_dir"], self.multiseq_file) job = Job(self.conf["app"][appname], args, parent_ids=[self.nodeid]) job.add_input_file(self.multiseq_file) job.cores = self.conf["threading"][appname] self.jobs.append(job)
def load_jobs(self): args = self.args.copy() args["-outfile"] = "mcoffee.fasta" alg_paths = [pjoin(GLOBALS["input_dir"], algid) for algid in self.all_alg_files] args["-aln"] = ' '.join(alg_paths) job = Job(self.conf["app"]["tcoffee"], args, parent_ids=self.parent_ids) for key in self.all_alg_files: job.add_input_file(key) self.jobs.append(job)
def app_wrapper(func, args): global NCURSES base_dir = GLOBALS.get("scratch_dir", GLOBALS["basedir"]) lock_file = pjoin(base_dir, "alive") if not args.enable_ui: NCURSES = False if not pexist(lock_file) or args.clearall: open(lock_file, "w").write(time.ctime()) else: clear_env() print('\nThe same process seems to be running. Use --clearall or remove the lock file "alive" within the output dir', file=sys.stderr) sys.exit(-1) try: if NCURSES: curses.wrapper(main, func, args) else: main(None, func, args) except ConfigError as e: if GLOBALS.get('_background_scheduler', None): GLOBALS['_background_scheduler'].terminate() print("\nConfiguration Error:", e, file=sys.stderr) clear_env() sys.exit(-1) except DataError as e: if GLOBALS.get('_background_scheduler', None): GLOBALS['_background_scheduler'].terminate() print("\nData Error:", e, file=sys.stderr) clear_env() sys.exit(-1) except KeyboardInterrupt: # Control-C is also grabbed by the back_launcher, so it is no necessary # to terminate from here print("\nProgram was interrupted.", file=sys.stderr) if args.monitor: print(("VERY IMPORTANT !!!: Note that launched" " jobs will keep running as you provided the --monitor flag"), file=sys.stderr) clear_env() sys.exit(-1) except: if GLOBALS.get('_background_scheduler', None): GLOBALS['_background_scheduler'].terminate() clear_env() raise else: if GLOBALS.get('_background_scheduler', None): GLOBALS['_background_scheduler'].terminate() clear_env()
def load_jobs(self): args = self.args.copy() try: del args["-wag"] except KeyError: pass if self.constrain_tree: args["-constraints"] = pjoin(GLOBALS["input_dir"], self.constrain_tree) args[pjoin(GLOBALS["input_dir"], self.alg_phylip_file)] = "" appname = self.conf[self.confname]["_app"] job = Job(self.conf["app"][appname], args, parent_ids=[self.nodeid]) job.cores = self.conf["threading"][appname] if self.constrain_tree: job.add_input_file(self.constrain_tree) job.add_input_file(self.alg_phylip_file) self.jobs.append(job)
def finish(self): # Once executed, alignment is converted into relaxed # interleaved phylip format. alg = SeqGroup(os.path.join(self.jobs[0].jobdir, "mcoffee.fasta")) fasta = alg.write(format="fasta") phylip = alg.write(format="iphylip_relaxed") alg_list_string = '\n'.join([pjoin(GLOBALS["input_dir"], aname) for aname in self.all_alg_files]) db.add_task_data(self.taskid, DATATYPES.alg_list, alg_list_string) AlgTask.store_data(self, fasta, phylip)
def finish(self): lks = [] if self.lk_mode == "phyml": for job in self.jobs: if job.jobcat != "bionj": continue phyml_job = job tree_file = pjoin(phyml_job.jobdir, self.alg_phylip_file + "_phyml_tree.txt") stats_file = pjoin(phyml_job.jobdir, self.alg_phylip_file + "_phyml_stats.txt") tree = PhyloTree(tree_file) m = re.search('Log-likelihood:\s+(-?\d+\.\d+)', open(stats_file).read()) lk = float(m.groups()[0]) tree.add_feature("lk", lk) tree.add_feature("model", phyml_job.args["--model"]) lks.append([float(tree.lk), tree.model, tree]) elif self.lk_mode == "raxml": for job in self.jobs: if job.jobcat != "raxml": continue raxml_job = job lk = open( pjoin(raxml_job.jobdir, "RAxML_log.%s" % raxml_job.args["-n"])).readline().split()[1] tree = PhyloTree(raxml_job.args["-t"]) tree.add_feature("lk", lk) tree.add_feature("model", raxml_job.model) lks.append([float(tree.lk), tree.model, tree]) # sort lks in ASC order lks.sort() # choose the model with higher likelihood, the lastone in the list best_model = lks[-1][1] best_tree = lks[-1][2] log.log( 22, "%s model selected from the following lk values:\n%s" % (best_model, '\n'.join(map(str, lks)))) ModelTesterTask.store_data(self, best_model, lks)
def clear_env(): try: terminate_job_launcher() except: pass base_dir = GLOBALS["basedir"] lock_file = pjoin(base_dir, "alive") try: os.remove(lock_file) except Exception: print("could not remove lock file %s" %lock_file, file=sys.stderr) clear_tempdir()
def finish(self): if self.conf[self.confname]["_alg_trimming"]: # If trimming happened after mcoffee, let's save the # resulting output trim_job = self.jobs[-1] alg = SeqGroup(pjoin(trim_job.jobdir, trim_job.alg_fasta_file)) fasta = alg.write(format="fasta") phylip = alg.write(format="iphylip_relaxed") AlgTask.store_data(self, fasta, phylip) else: # If no post trimming, output is just what Mcoffee # produced, so we can recycle its data ids. mc_task = self.jobs[-1] fasta_id = db.get_dataid(mc_task.taskid, DATATYPES.alg_fasta) phylip_id = db.get_dataid(mc_task.taskid, DATATYPES.alg_phylip) db.register_task_data(self.taskid, DATATYPES.alg_fasta, fasta_id) db.register_task_data(self.taskid, DATATYPES.alg_phylip, phylip_id)
def dump_tree_debug(msg, taskdir, mtree, ttree, target_seqs, out_seqs): try: if out_seqs is None: out_seqs = set() if target_seqs is None: target_seqs = set() if ttree: for n in ttree.get_leaves(): if n.name in out_seqs: n.name = n.name + " *__OUTGROUP__*" if mtree: for n in mtree.get_leaves(): if n.name in out_seqs: n.name = n.name + " *__OUTGROUP__*" if n.name in target_seqs: n.name = n.name + " [ TARGET ]" OUT = open(pjoin(taskdir, "__debug__"), "w") print(msg, file=OUT) print("MainTree:", mtree, file=OUT) print("TaskTree:", ttree, file=OUT) print("Expected outgroups:", out_seqs, file=OUT) OUT.close() except Exception as e: print(e)
def finish(self): # Once executed, alignment is converted into relaxed # interleaved phylip format. Both files, fasta and phylip, # remain accessible. # Set Task specific attributes main_job = self.jobs[0] fasta_path = pjoin(main_job.jobdir, "clean.alg.fasta") alg = SeqGroup(fasta_path) if len(alg) != self.size: log.warning("Trimming was to aggressive and it tried" " to remove one or more sequences." " Alignment trimming will be disabled for this dataset." ) self.clean_alg_fasta_file = db.register_task_data(self.taskid, DATATYPES.clean_alg_fasta, self.alg_fasta_file) self.clean_alg_phylip_file = db.register_task_data(self.taskid, DATATYPES.clean_alg_phylip, self.alg_phylip_file) else: for line in open(self.jobs[0].stdout_file): line = line.strip() if line.startswith("#ColumnsMap"): kept_columns = list(map(int, line.split("\t")[1].split(","))) fasta = alg.write(format="fasta") phylip = alg.write(format="iphylip_relaxed") AlgCleanerTask.store_data(self, fasta, phylip, kept_columns)
def __init__(self, bin, args, jobname=None, parent_ids=None): # Used at execution time self.status = None # How to run the app self.bin = bin # command line arguments self.args = args # Default number of cores used by the job. If more than 1, # this attribute should be changed self.cores = 1 self.exec_type = "insitu" self.jobname = jobname # generates the unique job identifier based on the params of # the app. Some params include path names that can prevent # recycling the job, so a clean it. clean = lambda x: basename(x) if GLOBALS["basedir"] in x or GLOBALS["tasks_dir"] in x else x parsed_id_string = ["%s %s" %(clean(str(pair[0])), clean(str(pair[1]))) for pair in six.iteritems(self.args)] #print '\n'.join(map(str, self.args.items())) self.jobid = md5(','.join(sorted([md5(e) for e in parsed_id_string]))) # self.jobid = md5(','.join(sorted([md5(str(pair)) for pair in # self.args.iteritems()]))) if parent_ids: self.jobid = md5(','.join(sorted(parent_ids+[self.jobid]))) if not self.jobname: self.jobname = re.sub("[^0-9a-zA-Z]", "-", basename(self.bin)) self.ifdone_cmd = "" self.iffail_cmd = "" self.set_jobdir(pjoin(GLOBALS["tasks_dir"], self.jobid)) self.input_files = {} self.dependencies = set()
def schedule(workflow_task_processor, pending_tasks, schedule_time, execution, debug, norender): # Adjust debug mode if debug == "all": log.setLevel(10) pending_tasks = set(pending_tasks) ## =================================== ## INITIALIZE BASIC VARS execution, run_detached = execution thread2tasks = defaultdict(list) for task in pending_tasks: thread2tasks[task.configid].append(task) expected_threads = set(thread2tasks.keys()) past_threads = {} thread_errors = defaultdict(list) ## END OF VARS AND SHORTCUTS ## =================================== cores_total = GLOBALS["_max_cores"] if cores_total > 0: job_queue = Queue() back_launcher = Process(target=background_job_launcher, args=(job_queue, run_detached, GLOBALS["launch_time"], cores_total)) back_launcher.start() else: job_queue = None back_launcher = None GLOBALS["_background_scheduler"] = back_launcher GLOBALS["_job_queue"] = job_queue # Captures Ctrl-C for debuging DEBUG #signal.signal(signal.SIGINT, control_c) last_report_time = None BUG = set() try: # Enters into task scheduling while pending_tasks: wtime = schedule_time # ask SGE for running jobs if execution == "sge": #sgeid2jobs = db.get_sge_tasks() #qstat_jobs = sge.qstat() pass else: qstat_jobs = None # Show summary of pending tasks per thread thread2tasks = defaultdict(list) for task in pending_tasks: thread2tasks[task.configid].append(task) set_logindent(0) log.log(28, "@@13: Updating tasks status:@@1: (%s)" % (ctime())) info_lines = [] for tid, tlist in six.iteritems(thread2tasks): threadname = GLOBALS[tid]["_name"] sizelist = ["%s" %getattr(_ts, "size", "?") for _ts in tlist] info = "Thread @@13:%s@@1:: pending tasks: @@8:%s@@1: of sizes: %s" %( threadname, len(tlist), ', '.join(sizelist)) info_lines.append(info) for line in info_lines: log.log(28, line) if GLOBALS["email"] and last_report_time is None: last_report_time = time() send_mail(GLOBALS["email"], "Your NPR process has started", '\n'.join(info_lines)) ## ================================ ## CHECK AND UPDATE CURRENT TASKS checked_tasks = set() check_start_time = time() to_add_tasks = set() GLOBALS["cached_status"] = {} for task in sorted(pending_tasks, sort_tasks): # Avoids endless periods without new job submissions elapsed_time = time() - check_start_time #if not back_launcher and pending_tasks and \ # elapsed_time > schedule_time * 2: # log.log(26, "@@8:Interrupting task checks to schedule new jobs@@1:") # db.commit() # wtime = launch_jobs(sorted(pending_tasks, sort_tasks), # execution, run_detached) # check_start_time = time() # Enter debuging mode if necessary if debug and log.level > 10 and task.taskid.startswith(debug): log.setLevel(10) log.debug("ENTERING IN DEBUGGING MODE") thread2tasks[task.configid].append(task) # Update tasks and job statuses if task.taskid not in checked_tasks: try: show_task_info(task) task.status = task.get_status(qstat_jobs) db.dataconn.commit() if back_launcher and task.status not in set("DE"): for j, cmd in task.iter_waiting_jobs(): j.status = "Q" GLOBALS["cached_status"][j.jobid] = "Q" if j.jobid not in BUG: if not os.path.exists(j.jobdir): os.makedirs(j.jobdir) for ifile, outpath in six.iteritems(j.input_files): try: _tid, _did = ifile.split(".") _did = int(_did) except (IndexError, ValueError): dataid = ifile else: dataid = db.get_dataid(_tid, _did) if not outpath: outfile = pjoin(GLOBALS["input_dir"], ifile) else: outfile = pjoin(outpath, ifile) if not os.path.exists(outfile): open(outfile, "w").write(db.get_data(dataid)) log.log(24, " @@8:Queueing @@1: %s from %s" %(j, task)) if execution: job_queue.put([j.jobid, j.cores, cmd, j.status_file]) BUG.add(j.jobid) update_task_states_recursively(task) db.commit() checked_tasks.add(task.taskid) except TaskError as e: log.error("Errors found in %s" %task) import traceback traceback.print_exc() if GLOBALS["email"]: threadname = GLOBALS[task.configid]["_name"] send_mail(GLOBALS["email"], "Errors found in %s!" %threadname, '\n'.join(map(str, [task, e.value, e.msg]))) pending_tasks.discard(task) thread_errors[task.configid].append([task, e.value, e.msg]) continue else: # Set temporary Queued state to avoids launching # jobs from clones task.status = "Q" if log.level < 24: show_task_info(task) if task.status == "D": #db.commit() show_task_info(task) logindent(3) # Log commands of every task if 'cmd_log_file' not in GLOBALS[task.configid]: GLOBALS[task.configid]['cmd_log_file'] = pjoin(GLOBALS[task.configid]["_outpath"], "cmd.log") O = open(GLOBALS[task.configid]['cmd_log_file'], "w") O.close() cmd_lines = get_cmd_log(task) CMD_LOG = open(GLOBALS[task.configid]['cmd_log_file'], "a") print(task, file=CMD_LOG) for c in cmd_lines: print(' '+'\t'.join(map(str, c)), file=CMD_LOG) CMD_LOG.close() # try: #wkname = GLOBALS[task.configid]['_name'] create_tasks = workflow_task_processor(task, task.target_wkname) except TaskError as e: log.error("Errors found in %s" %task) pending_tasks.discard(task) thread_errors[task.configid].append([task, e.value, e.msg]) continue else: logindent(-3) to_add_tasks.update(create_tasks) pending_tasks.discard(task) elif task.status == "E": log.error("task contains errors: %s " %task) log.error("Errors found in %s") pending_tasks.discard(task) thread_errors[task.configid].append([task, None, "Found (E) task status"]) #db.commit() #if not back_launcher: # wtime = launch_jobs(sorted(pending_tasks, sort_tasks), # execution, run_detached) # Update global task list with recently added jobs to be check # during next cycle pending_tasks.update(to_add_tasks) ## END CHECK AND UPDATE CURRENT TASKS ## ================================ if wtime: set_logindent(0) log.log(28, "@@13:Waiting %s seconds@@1:" %wtime) sleep(wtime) else: sleep(schedule_time) # Dump / show ended threads error_lines = [] for configid, etasks in six.iteritems(thread_errors): error_lines.append("Thread @@10:%s@@1: contains errors:" %\ (GLOBALS[configid]["_name"])) for error in etasks: error_lines.append(" ** %s" %error[0]) e_obj = error[1] if error[1] else error[0] error_path = e_obj.jobdir if isjob(e_obj) else e_obj.taskid if e_obj is not error[0]: error_lines.append(" -> %s" %e_obj) error_lines.append(" -> %s" %error_path) error_lines.append(" -> %s" %error[2]) for eline in error_lines: log.error(eline) pending_threads = set([ts.configid for ts in pending_tasks]) finished_threads = expected_threads - (pending_threads | set(thread_errors.keys())) just_finished_lines = [] finished_lines = [] for configid in finished_threads: # configid is the the same as threadid in master tasks final_tree_file = pjoin(GLOBALS[configid]["_outpath"], GLOBALS["inputname"] + ".final_tree") threadname = GLOBALS[configid]["_name"] if configid in past_threads: log.log(28, "Done thread @@12:%s@@1: in %d iteration(s)", threadname, past_threads[configid]) finished_lines.append("Finished %s in %d iteration(s)" %( threadname, past_threads[configid])) else: log.log(28, "Assembling final tree...") main_tree, treeiters = assembly_tree(configid) past_threads[configid] = treeiters - 1 log.log(28, "Done thread @@12:%s@@1: in %d iteration(s)", threadname, past_threads[configid]) log.log(28, "Writing final tree for @@13:%s@@1:\n %s\n %s", threadname, final_tree_file+".nw", final_tree_file+".nwx (newick extended)") main_tree.write(outfile=final_tree_file+".nw") main_tree.write(outfile=final_tree_file+ ".nwx", features=[], format_root_node=True) if hasattr(main_tree, "alg_path"): log.log(28, "Writing root node alignment @@13:%s@@1:\n %s", threadname, final_tree_file+".fa") alg = SeqGroup(get_stored_data(main_tree.alg_path)) OUT = open(final_tree_file+".fa", "w") for name, seq, comments in alg: realname = db.get_seq_name(name) print(">%s\n%s" %(realname, seq), file=OUT) OUT.close() if hasattr(main_tree, "clean_alg_path"): log.log(28, "Writing root node trimmed alignment @@13:%s@@1:\n %s", threadname, final_tree_file+".trimmed.fa") alg = SeqGroup(get_stored_data(main_tree.clean_alg_path)) OUT = open(final_tree_file+".trimmed.fa", "w") for name, seq, comments in alg: realname = db.get_seq_name(name) print(">%s\n%s" %(realname, seq), file=OUT) OUT.close() if norender == False: log.log(28, "Generating tree image for @@13:%s@@1:\n %s", threadname, final_tree_file+".png") for lf in main_tree: lf.add_feature("sequence", alg.get_seq(lf.safename)) try: from ete3.tools.phylobuild_lib.visualize import draw_tree draw_tree(main_tree, GLOBALS[configid], final_tree_file+".png") except Exception as e: log.warning('@@8:something went wrong when generating the tree image. Try manually :(@@1:') if DEBUG: import traceback, sys traceback.print_exc(file=sys.stdout) just_finished_lines.append("Finished %s in %d iteration(s)" %( threadname, past_threads[configid])) if GLOBALS["email"]: if not pending_tasks: all_lines = finished_lines + just_finished_lines + error_lines send_mail(GLOBALS["email"], "Your NPR process has ended", '\n'.join(all_lines)) elif GLOBALS["email_report_time"] and time() - last_report_time >= \ GLOBALS["email_report_time"]: all_lines = info_lines + error_lines + just_finished_lines send_mail(GLOBALS["email"], "Your NPR report", '\n'.join(all_lines)) last_report_time = time() elif just_finished_lines: send_mail(GLOBALS["email"], "Finished threads!", '\n'.join(just_finished_lines)) log.log(26, "") except: raise if thread_errors: log.error("Done with ERRORS") else: log.log(28, "Done") return thread_errors
def load_jobs(self): args = OrderedDict(self.args) args["-s"] = pjoin(GLOBALS["input_dir"], self.alg_phylip_file) args["-m"] = self.model_string args["-n"] = self.alg_phylip_file if self.constrain_tree: log.log(24, "Using constrain tree %s" %self.constrain_tree) args["-g"] = pjoin(GLOBALS["input_dir"], self.constrain_tree) if self.partitions_file: log.log(24, "Using alg partitions %s" %self.partitions_file) args['-q'] = pjoin(GLOBALS["input_dir"], self.partitions_file) tree_job = Job(self.raxml_bin, args, parent_ids=[self.nodeid]) tree_job.jobname += "-"+self.model_string tree_job.cores = self.threads # Register input files necessary to run the job tree_job.add_input_file(self.alg_phylip_file) if self.constrain_tree: tree_job.add_input_file(self.constrain_tree) if self.partitions_file: tree_job.add_input_file(self.partitions_file) self.jobs.append(tree_job) self.out_tree_file = os.path.join(tree_job.jobdir, "RAxML_bestTree." + self.alg_phylip_file) if self.bootstrap == "alrt": alrt_args = tree_job.args.copy() if self.constrain_tree: del alrt_args["-g"] if self.partitions_file: alrt_args["-q"] = args['-q'] alrt_args["-f"] = "J" alrt_args["-t"] = self.out_tree_file alrt_job = Job(self.raxml_bin, alrt_args, parent_ids=[tree_job.jobid]) alrt_job.jobname += "-alrt" alrt_job.dependencies.add(tree_job) alrt_job.cores = self.threads # Register necessary input files alrt_job.add_input_file(self.alg_phylip_file) if self.partitions_file: alrt_job.add_input_file(self.partitions_file) self.jobs.append(alrt_job) self.alrt_job = alrt_job elif self.bootstrap == "alrt_phyml": alrt_args = { "-o": "n", "-i": self.alg_phylip_file, "--bootstrap": "-2", "-d": self.seqtype, "-u": self.out_tree_file, "--model": self.model, "--quiet": "", "--no_memory_check": "", } #if self.constrain_tree: # alrt_args["--constraint_tree"] = self.constrain_tree alrt_job = Job(self.conf["app"]["phyml"], alrt_args, parent_ids=[tree_job.jobid]) alrt_job.add_input_file(self.alg_phylip_file, alrt_job.jobdir) alrt_job.jobname += "-alrt" alrt_job.dependencies.add(tree_job) alrt_job.add_input_file(self.alg_phylip_file) self.jobs.append(alrt_job) self.alrt_job = alrt_job else: # Bootstrap calculation boot_args = tree_job.args.copy() boot_args["-n"] = "bootstraps."+boot_args["-n"] boot_args["-N"] = int(self.bootstrap) boot_args["-b"] = 31416 boot_job = Job(self.raxml_bin, boot_args, parent_ids=[tree_job.jobid]) boot_job.jobname += "-%d-bootstraps" %(boot_args['-N']) boot_job.dependencies.add(tree_job) boot_job.cores = self.threads # Register necessary input files boot_job.add_input_file(self.alg_phylip_file) if self.constrain_tree: boot_job.add_input_file(self.constrain_tree) if self.partitions_file: boot_job.add_input_file(self.partitions_file) self.jobs.append(boot_job) # Bootstrap drawing on top of best tree bootd_args = tree_job.args.copy() if self.constrain_tree: del bootd_args["-g"] if self.partitions_file: del bootd_args["-q"] bootd_args["-n"] = "bootstrapped."+ tree_job.args["-n"] bootd_args["-f"] = "b" bootd_args["-t"] = self.out_tree_file bootd_args["-z"] = pjoin(boot_job.jobdir, "RAxML_bootstrap." + boot_job.args["-n"]) bootd_job = Job(self.raxml_bin, bootd_args, parent_ids=[tree_job.jobid]) bootd_job.jobname += "-bootstrapped" bootd_job.dependencies.add(boot_job) bootd_job.cores = self.threads self.jobs.append(bootd_job) self.boot_job = boot_job self.bootd_job = bootd_job
def process_task(task, wkname, npr_conf, nodeid2info): alignerconf, alignerclass = npr_conf.aligner cleanerconf, cleanerclass = npr_conf.alg_cleaner mtesterconf, mtesterclass = npr_conf.model_tester treebuilderconf, treebuilderclass = npr_conf.tree_builder if not treebuilderclass: # Allows to dump algs in workflows with no tree tasks treebuilderclass = DummyTree splitterconf, splitterclass = npr_conf.tree_splitter conf = GLOBALS[task.configid] seqtype = task.seqtype nodeid = task.nodeid ttype = task.ttype taskid = task.taskid threadid = task.threadid node_info = nodeid2info[nodeid] size = task.size#node_info.get("size", 0) target_seqs = node_info.get("target_seqs", []) out_seqs = node_info.get("out_seqs", []) if not treebuilderclass or size < 4: # Allows to dump algs in workflows with no tree tasks or if tree # inference does not make sense given the number of sequences. DummyTree # will produce a fake fully collapsed newick tree. treebuilderclass = DummyTree # If more than one outgroup are used, enable the use of constrain if out_seqs and len(out_seqs) > 1: constrain_id = nodeid else: constrain_id = None new_tasks = [] if ttype == "msf": # Register Tree constrains constrain_tree = "(%s, (%s));" %(','.join(sorted(task.out_seqs)), ','.join(sorted(task.target_seqs))) _outs = "\n".join([">%s\n0" %name for name in sorted(task.out_seqs)]) _tars = "\n".join([">%s\n1" %name for name in sorted(task.target_seqs)]) constrain_alg = '\n'.join([_outs, _tars]) db.add_task_data(nodeid, DATATYPES.constrain_tree, constrain_tree) db.add_task_data(nodeid, DATATYPES.constrain_alg, constrain_alg) db.dataconn.commit() # since the creation of some Task # objects may require this info, I need # to commit right now. # Register node db.add_node(task.threadid, task.nodeid, task.cladeid, task.target_seqs, task.out_seqs) nodeid2info[nodeid]["size"] = task.size nodeid2info[nodeid]["target_seqs"] = task.target_seqs nodeid2info[nodeid]["out_seqs"] = task.out_seqs alg_task = alignerclass(nodeid, task.multiseq_file, seqtype, conf, alignerconf) alg_task.size = task.size new_tasks.append(alg_task) elif ttype == "alg" or ttype == "acleaner": if ttype == "alg": nodeid2info[nodeid]["alg_path"] = task.alg_fasta_file elif ttype == "acleaner": nodeid2info[nodeid]["alg_clean_path"] = task.clean_alg_fasta_file alg_fasta_file = getattr(task, "clean_alg_fasta_file", task.alg_fasta_file) alg_phylip_file = getattr(task, "clean_alg_phylip_file", task.alg_phylip_file) # Calculate alignment stats # cons_mean, cons_std = get_trimal_conservation(task.alg_fasta_file, # conf["app"]["trimal"]) # # max_identity = get_trimal_identity(task.alg_fasta_file, # conf["app"]["trimal"]) # log.info("Conservation: %0.2f +-%0.2f", cons_mean, cons_std) # log.info("Max. Identity: %0.2f", max_identity) #import time #t1 = time.time() #mx, mn, mean, std = get_identity(task.alg_fasta_file) #print time.time()-t1 #log.log(26, "Identity: max=%0.2f min=%0.2f mean=%0.2f +- %0.2f", # mx, mn, mean, std) #t1 = time.time() if seqtype == "aa" and npr_conf.switch_aa_similarity < 1: try: alg_stats = db.get_task_data(taskid, DATATYPES.alg_stats) except Exception as e: alg_stats = {} if ttype == "alg": algfile = pjoin(GLOBALS["input_dir"], task.alg_phylip_file) dataid = DATATYPES.alg_phylip elif ttype == "acleaner": algfile = pjoin(GLOBALS["input_dir"], task.clean_alg_phylip_file) dataid = DATATYPES.clean_alg_phylip if "i_mean" not in alg_stats: log.log(24, "Calculating alignment stats...") # dump data if necesary algfile = pjoin(GLOBALS["input_dir"], task.alg_phylip_file) if not pexist(algfile): # dump phylip alg open(algfile, "w").write(db.get_data(db.get_dataid(taskid, dataid))) mx, mn, mean, std = get_statal_identity(algfile, conf["app"]["statal"]) alg_stats = {"i_max":mx, "i_mean":mean, "i_min":mn, "i_std":std} db.add_task_data(taskid, DATATYPES.alg_stats, alg_stats) log.log(22, "Alignment stats (sequence similarity):") log.log(22, " max: %(i_max)0.2f, min:%(i_min)0.2f, avg:%(i_mean)0.2f+-%(i_std)0.2f" % (alg_stats)) else: alg_stats = {"i_max":-1, "i_mean":-1, "i_min":-1, "i_std":-1} #print time.time()-t1 #log.log(24, "Identity: max=%0.2f min=%0.2f mean=%0.2f +- %0.2f", # mx, mn, mean, std) task.max_ident = alg_stats["i_max"] task.min_ident = alg_stats["i_min"] task.mean_ident = alg_stats["i_mean"] task.std_ident = alg_stats["i_std"] next_task = None if ttype == "alg" and cleanerclass: next_task = cleanerclass(nodeid, seqtype, alg_fasta_file, alg_phylip_file, conf, cleanerconf) else: # Converts aa alignment into nt if necessary if seqtype == "aa" and \ "nt" in GLOBALS["seqtypes"] and \ task.mean_ident >= npr_conf.switch_aa_similarity: log.log(28, "@@2:Switching to codon alignment!@@1: amino-acid sequence similarity: %0.2f >= %0.2f" %\ (task.mean_ident, npr_conf.switch_aa_similarity)) alg_fasta_file = "%s.%s" %(taskid, DATATYPES.alg_nt_fasta) alg_phylip_file = "%s.%s" %(taskid, DATATYPES.alg_nt_phylip) try: alg_fasta_file = db.get_dataid(taskid, DATATYPES.alg_nt_fasta) alg_fasta_file = db.get_dataid(taskid, DATATYPES.alg_nt_phylip) except ValueError: log.log(22, "Calculating codon alignment...") source_alg = pjoin(GLOBALS["input_dir"], task.alg_fasta_file) if ttype == "alg": kept_columns = [] elif ttype == "acleaner": # if original alignment was trimmed, use it as reference # but make the nt alignment only on the kept columns kept_columns = db.get_task_data(taskid, DATATYPES.kept_alg_columns) if not pexist(source_alg): open(source_alg, "w").write(db.get_task_data(taskid, DATATYPES.alg_fasta)) nt_alg = switch_to_codon(source_alg, kept_columns=kept_columns) db.add_task_data(taskid, DATATYPES.alg_nt_fasta, nt_alg.write()) db.add_task_data(taskid, DATATYPES.alg_nt_phylip, nt_alg.write(format='iphylip_relaxed')) npr_conf = IterConfig(conf, wkname, task.size, "nt") seqtype = "nt" if mtesterclass: next_task = mtesterclass(nodeid, alg_fasta_file, alg_phylip_file, constrain_id, conf, mtesterconf) elif treebuilderclass: next_task = treebuilderclass(nodeid, alg_phylip_file, constrain_id, None, seqtype, conf, treebuilderconf) if next_task: next_task.size = task.size new_tasks.append(next_task) elif ttype == "mchooser": if treebuilderclass: alg_fasta_file = task.alg_fasta_file alg_phylip_file = task.alg_phylip_file model = task.best_model tree_task = treebuilderclass(nodeid, alg_phylip_file, constrain_id, model, seqtype, conf, treebuilderconf) tree_task.size = task.size new_tasks.append(tree_task) elif ttype == "tree": treemerge_task = splitterclass(nodeid, seqtype, task.tree_file, conf, splitterconf) #if conf["tree_splitter"]["_outgroup_size"]: # treemerge_task = TreeSplitterWithOutgroups(nodeid, seqtype, task.tree_file, main_tree, conf) #else: # treemerge_task = TreeSplitter(nodeid, seqtype, task.tree_file, main_tree, conf) treemerge_task.size = task.size new_tasks.append(treemerge_task) elif ttype == "treemerger": if not task.task_tree: task.finish() log.log(24, "Saving task tree...") annotate_node(task.task_tree, task) db.update_node(nid=task.nodeid, runid=task.threadid, newick=db.encode(task.task_tree)) db.commit() if not isinstance(treebuilderclass, DummyTree) and npr_conf.max_iters > 1: current_iter = get_iternumber(threadid) if npr_conf.max_iters and current_iter >= npr_conf.max_iters: log.warning("Maximum number of iterations reached!") else: # Add new nodes source_seqtype = "aa" if "aa" in GLOBALS["seqtypes"] else "nt" ttree, mtree = task.task_tree, task.main_tree log.log(26, "Processing tree: %s seqs, %s outgroups", len(target_seqs), len(out_seqs)) alg_path = node_info.get("clean_alg_path", node_info["alg_path"]) for node, seqs, outs, wkname in get_next_npr_node(threadid, ttree, task.out_seqs, mtree, alg_path, npr_conf): log.log(24, "Registering new node: %s seqs, %s outgroups", len(seqs), len(outs)) new_task_node = Msf(seqs, outs, seqtype=source_seqtype) new_task_node.target_wkname = wkname new_tasks.append(new_task_node) return new_tasks
def load_jobs(self): readal_bin = self.conf["app"]["readal"] trimal_bin = self.conf["app"]["trimal"] input_dir = GLOBALS["input_dir"] multiseq_file = pjoin(input_dir, self.multiseq_file) multiseq_file_r = pjoin(input_dir, self.multiseq_file+"_reversed") first = seq_reverser_job(multiseq_file, multiseq_file_r, [self.nodeid], readal_bin) #print self.multiseq_file first.add_input_file(self.multiseq_file) self.jobs.append(first) all_alg_names = [] mcoffee_parents = [] for aligner_name in self.conf[self.confname]["_aligners"]: aligner_name = aligner_name[1:] _classname = APP2CLASS[self.conf[aligner_name]["_app"]] _module = __import__(CLASS2MODULE[_classname], globals(), locals(), [], -1) _aligner = getattr(_module, _classname) # Normal alg task1 = _aligner(self.nodeid, self.multiseq_file, self.seqtype, self.conf, aligner_name) task1.size = self.size self.jobs.append(task1) all_alg_names.append(task1.alg_fasta_file) # Alg of the reverse task2 = _aligner(self.nodeid, self.multiseq_file+"_reversed", self.seqtype, self.conf, aligner_name) task2.size = self.size task2.dependencies.add(first) self.jobs.append(task2) # Restore reverse alg reverse_out = pjoin(input_dir, task2.alg_fasta_file) task3 = seq_reverser_job(reverse_out, reverse_out+"_restored", [task2.taskid], readal_bin) task3.dependencies.add(task2) task3.add_input_file(task2.alg_fasta_file) all_alg_names.append(reverse_out+"_restored") self.jobs.append(task3) mcoffee_parents.extend([task1.taskid, task2.taskid]) # Combine signal from all algs using Mcoffee mcoffee_task = MCoffee(self.nodeid, self.seqtype, all_alg_names, self.conf, self.confname, parent_ids=mcoffee_parents) # reversed algs are not actually saved into db, but it should # be present since the reverser job is always executed mcoffee_task.dependencies.update(list(self.jobs)) self.jobs.append(mcoffee_task) if self.conf[self.confname]["_alg_trimming"]: trimming_cutoff = 1.0 / len(all_alg_names) targs = {} targs["-forceselect"] = pjoin(input_dir, mcoffee_task.alg_fasta_file) targs["-compareset"] = pjoin(input_dir, mcoffee_task.alg_list_file) targs["-out"] = "mcoffee.trimmed.fasta" targs["-fasta"] = "" targs["-ct"] = trimming_cutoff trim_job = Job(trimal_bin, targs, parent_ids=[mcoffee_task.taskid]) trim_job.jobname = "McoffeeTrimming" trim_job.dependencies.add(mcoffee_task) trim_job.alg_fasta_file = targs["-out"] for key in all_alg_names: trim_job.add_input_file(key) trim_job.add_input_file(mcoffee_task.alg_fasta_file) trim_job.add_input_file(mcoffee_task.alg_list_file) self.jobs.append(trim_job)
def schedule(workflow_task_processor, pending_tasks, schedule_time, execution, debug, norender): # Adjust debug mode if debug == "all": log.setLevel(10) pending_tasks = set(pending_tasks) ## =================================== ## INITIALIZE BASIC VARS execution, run_detached = execution thread2tasks = defaultdict(list) for task in pending_tasks: thread2tasks[task.configid].append(task) expected_threads = set(thread2tasks.keys()) past_threads = {} thread_errors = defaultdict(list) ## END OF VARS AND SHORTCUTS ## =================================== cores_total = GLOBALS["_max_cores"] if cores_total > 0: job_queue = Queue() back_launcher = Process(target=background_job_launcher, args=(job_queue, run_detached, GLOBALS["launch_time"], cores_total)) back_launcher.start() else: job_queue = None back_launcher = None GLOBALS["_background_scheduler"] = back_launcher GLOBALS["_job_queue"] = job_queue # Captures Ctrl-C for debuging DEBUG #signal.signal(signal.SIGINT, control_c) last_report_time = None BUG = set() try: # Enters into task scheduling while pending_tasks: wtime = schedule_time # ask SGE for running jobs if execution == "sge": #sgeid2jobs = db.get_sge_tasks() #qstat_jobs = sge.qstat() pass else: qstat_jobs = None # Show summary of pending tasks per thread thread2tasks = defaultdict(list) for task in pending_tasks: thread2tasks[task.configid].append(task) set_logindent(0) log.log(28, "@@13: Updating tasks status:@@1: (%s)" % (ctime())) info_lines = [] for tid, tlist in six.iteritems(thread2tasks): threadname = GLOBALS[tid]["_name"] sizelist = ["%s" % getattr(_ts, "size", "?") for _ts in tlist] info = "Thread @@13:%s@@1:: pending tasks: @@8:%s@@1: of sizes: %s" % ( threadname, len(tlist), ', '.join(sizelist)) info_lines.append(info) for line in info_lines: log.log(28, line) if GLOBALS["email"] and last_report_time is None: last_report_time = time() send_mail(GLOBALS["email"], "Your NPR process has started", '\n'.join(info_lines)) ## ================================ ## CHECK AND UPDATE CURRENT TASKS checked_tasks = set() check_start_time = time() to_add_tasks = set() GLOBALS["cached_status"] = {} for task in sorted(pending_tasks, sort_tasks): # Avoids endless periods without new job submissions elapsed_time = time() - check_start_time #if not back_launcher and pending_tasks and \ # elapsed_time > schedule_time * 2: # log.log(26, "@@8:Interrupting task checks to schedule new jobs@@1:") # db.commit() # wtime = launch_jobs(sorted(pending_tasks, sort_tasks), # execution, run_detached) # check_start_time = time() # Enter debuging mode if necessary if debug and log.level > 10 and task.taskid.startswith(debug): log.setLevel(10) log.debug("ENTERING IN DEBUGGING MODE") thread2tasks[task.configid].append(task) # Update tasks and job statuses if task.taskid not in checked_tasks: try: show_task_info(task) task.status = task.get_status(qstat_jobs) db.dataconn.commit() if back_launcher and task.status not in set("DE"): for j, cmd in task.iter_waiting_jobs(): j.status = "Q" GLOBALS["cached_status"][j.jobid] = "Q" if j.jobid not in BUG: if not os.path.exists(j.jobdir): os.makedirs(j.jobdir) for ifile, outpath in six.iteritems( j.input_files): try: _tid, _did = ifile.split(".") _did = int(_did) except (IndexError, ValueError): dataid = ifile else: dataid = db.get_dataid(_tid, _did) if not outpath: outfile = pjoin( GLOBALS["input_dir"], ifile) else: outfile = pjoin(outpath, ifile) if not os.path.exists(outfile): open(outfile, "w").write( db.get_data(dataid)) log.log( 24, " @@8:Queueing @@1: %s from %s" % (j, task)) if execution: job_queue.put([ j.jobid, j.cores, cmd, j.status_file ]) BUG.add(j.jobid) update_task_states_recursively(task) db.commit() checked_tasks.add(task.taskid) except TaskError as e: log.error("Errors found in %s" % task) import traceback traceback.print_exc() if GLOBALS["email"]: threadname = GLOBALS[task.configid]["_name"] send_mail( GLOBALS["email"], "Errors found in %s!" % threadname, '\n'.join(map(str, [task, e.value, e.msg]))) pending_tasks.discard(task) thread_errors[task.configid].append( [task, e.value, e.msg]) continue else: # Set temporary Queued state to avoids launching # jobs from clones task.status = "Q" if log.level < 24: show_task_info(task) if task.status == "D": #db.commit() show_task_info(task) logindent(3) # Log commands of every task if 'cmd_log_file' not in GLOBALS[task.configid]: GLOBALS[task.configid]['cmd_log_file'] = pjoin( GLOBALS[task.configid]["_outpath"], "cmd.log") O = open(GLOBALS[task.configid]['cmd_log_file'], "w") O.close() cmd_lines = get_cmd_log(task) CMD_LOG = open(GLOBALS[task.configid]['cmd_log_file'], "a") print(task, file=CMD_LOG) for c in cmd_lines: print(' ' + '\t'.join(map(str, c)), file=CMD_LOG) CMD_LOG.close() # try: #wkname = GLOBALS[task.configid]['_name'] create_tasks = workflow_task_processor( task, task.target_wkname) except TaskError as e: log.error("Errors found in %s" % task) pending_tasks.discard(task) thread_errors[task.configid].append( [task, e.value, e.msg]) continue else: logindent(-3) to_add_tasks.update(create_tasks) pending_tasks.discard(task) elif task.status == "E": log.error("task contains errors: %s " % task) log.error("Errors found in %s") pending_tasks.discard(task) thread_errors[task.configid].append( [task, None, "Found (E) task status"]) #db.commit() #if not back_launcher: # wtime = launch_jobs(sorted(pending_tasks, sort_tasks), # execution, run_detached) # Update global task list with recently added jobs to be check # during next cycle pending_tasks.update(to_add_tasks) ## END CHECK AND UPDATE CURRENT TASKS ## ================================ if wtime: set_logindent(0) log.log(28, "@@13:Waiting %s seconds@@1:" % wtime) sleep(wtime) else: sleep(schedule_time) # Dump / show ended threads error_lines = [] for configid, etasks in six.iteritems(thread_errors): error_lines.append("Thread @@10:%s@@1: contains errors:" %\ (GLOBALS[configid]["_name"])) for error in etasks: error_lines.append(" ** %s" % error[0]) e_obj = error[1] if error[1] else error[0] error_path = e_obj.jobdir if isjob(e_obj) else e_obj.taskid if e_obj is not error[0]: error_lines.append(" -> %s" % e_obj) error_lines.append(" -> %s" % error_path) error_lines.append(" -> %s" % error[2]) for eline in error_lines: log.error(eline) pending_threads = set([ts.configid for ts in pending_tasks]) finished_threads = expected_threads - (pending_threads | set(thread_errors.keys())) just_finished_lines = [] finished_lines = [] for configid in finished_threads: # configid is the the same as threadid in master tasks final_tree_file = pjoin(GLOBALS[configid]["_outpath"], GLOBALS["inputname"] + ".final_tree") threadname = GLOBALS[configid]["_name"] if configid in past_threads: log.log(28, "Done thread @@12:%s@@1: in %d iteration(s)", threadname, past_threads[configid]) finished_lines.append("Finished %s in %d iteration(s)" % (threadname, past_threads[configid])) else: log.log(28, "Assembling final tree...") main_tree, treeiters = assembly_tree(configid) past_threads[configid] = treeiters - 1 log.log(28, "Done thread @@12:%s@@1: in %d iteration(s)", threadname, past_threads[configid]) log.log( 28, "Writing final tree for @@13:%s@@1:\n %s\n %s", threadname, final_tree_file + ".nw", final_tree_file + ".nwx (newick extended)") main_tree.write(outfile=final_tree_file + ".nw") main_tree.write(outfile=final_tree_file + ".nwx", features=[], format_root_node=True) if hasattr(main_tree, "alg_path"): log.log( 28, "Writing root node alignment @@13:%s@@1:\n %s", threadname, final_tree_file + ".fa") alg = SeqGroup(get_stored_data(main_tree.alg_path)) OUT = open(final_tree_file + ".fa", "w") for name, seq, comments in alg: realname = db.get_seq_name(name) print(">%s\n%s" % (realname, seq), file=OUT) OUT.close() if hasattr(main_tree, "clean_alg_path"): log.log( 28, "Writing root node trimmed alignment @@13:%s@@1:\n %s", threadname, final_tree_file + ".trimmed.fa") alg = SeqGroup( get_stored_data(main_tree.clean_alg_path)) OUT = open(final_tree_file + ".trimmed.fa", "w") for name, seq, comments in alg: realname = db.get_seq_name(name) print(">%s\n%s" % (realname, seq), file=OUT) OUT.close() if norender == False: log.log( 28, "Generating tree image for @@13:%s@@1:\n %s", threadname, final_tree_file + ".png") for lf in main_tree: lf.add_feature("sequence", alg.get_seq(lf.safename)) try: from ete3.tools.phylobuild_lib.visualize import draw_tree draw_tree(main_tree, GLOBALS[configid], final_tree_file + ".png") except Exception as e: log.warning( '@@8:something went wrong when generating the tree image. Try manually :(@@1:' ) if DEBUG: import traceback, sys traceback.print_exc(file=sys.stdout) just_finished_lines.append( "Finished %s in %d iteration(s)" % (threadname, past_threads[configid])) if GLOBALS["email"]: if not pending_tasks: all_lines = finished_lines + just_finished_lines + error_lines send_mail(GLOBALS["email"], "Your NPR process has ended", '\n'.join(all_lines)) elif GLOBALS["email_report_time"] and time() - last_report_time >= \ GLOBALS["email_report_time"]: all_lines = info_lines + error_lines + just_finished_lines send_mail(GLOBALS["email"], "Your NPR report", '\n'.join(all_lines)) last_report_time = time() elif just_finished_lines: send_mail(GLOBALS["email"], "Finished threads!", '\n'.join(just_finished_lines)) log.log(26, "") except: raise if thread_errors: log.error("Done with ERRORS") else: log.log(28, "Done") return thread_errors
def load_jobs(self): args = OrderedDict(self.args) args["-s"] = pjoin(GLOBALS["input_dir"], self.alg_phylip_file) args["-m"] = self.model_string args["-n"] = self.alg_phylip_file if self.constrain_tree: log.log(24, "Using constrain tree %s" % self.constrain_tree) args["-g"] = pjoin(GLOBALS["input_dir"], self.constrain_tree) if self.partitions_file: log.log(24, "Using alg partitions %s" % self.partitions_file) args['-q'] = pjoin(GLOBALS["input_dir"], self.partitions_file) tree_job = Job(self.raxml_bin, args, parent_ids=[self.nodeid]) tree_job.jobname += "-" + self.model_string tree_job.cores = self.threads # Register input files necessary to run the job tree_job.add_input_file(self.alg_phylip_file) if self.constrain_tree: tree_job.add_input_file(self.constrain_tree) if self.partitions_file: tree_job.add_input_file(self.partitions_file) self.jobs.append(tree_job) self.out_tree_file = os.path.join( tree_job.jobdir, "RAxML_bestTree." + self.alg_phylip_file) if self.bootstrap == "alrt": alrt_args = tree_job.args.copy() if self.constrain_tree: del alrt_args["-g"] if self.partitions_file: alrt_args["-q"] = args['-q'] alrt_args["-f"] = "J" alrt_args["-t"] = self.out_tree_file alrt_job = Job(self.raxml_bin, alrt_args, parent_ids=[tree_job.jobid]) alrt_job.jobname += "-alrt" alrt_job.dependencies.add(tree_job) alrt_job.cores = self.threads # Register necessary input files alrt_job.add_input_file(self.alg_phylip_file) if self.partitions_file: alrt_job.add_input_file(self.partitions_file) self.jobs.append(alrt_job) self.alrt_job = alrt_job elif self.bootstrap == "alrt_phyml": alrt_args = { "-o": "n", "-i": self.alg_phylip_file, "--bootstrap": "-2", "-d": self.seqtype, "-u": self.out_tree_file, "--model": self.model, "--quiet": "", "--no_memory_check": "", } #if self.constrain_tree: # alrt_args["--constraint_tree"] = self.constrain_tree alrt_job = Job(self.conf["app"]["phyml"], alrt_args, parent_ids=[tree_job.jobid]) alrt_job.add_input_file(self.alg_phylip_file, alrt_job.jobdir) alrt_job.jobname += "-alrt" alrt_job.dependencies.add(tree_job) alrt_job.add_input_file(self.alg_phylip_file) self.jobs.append(alrt_job) self.alrt_job = alrt_job else: # Bootstrap calculation boot_args = tree_job.args.copy() boot_args["-n"] = "bootstraps." + boot_args["-n"] boot_args["-N"] = int(self.bootstrap) boot_args["-b"] = 31416 boot_job = Job(self.raxml_bin, boot_args, parent_ids=[tree_job.jobid]) boot_job.jobname += "-%d-bootstraps" % (boot_args['-N']) boot_job.dependencies.add(tree_job) boot_job.cores = self.threads # Register necessary input files boot_job.add_input_file(self.alg_phylip_file) if self.constrain_tree: boot_job.add_input_file(self.constrain_tree) if self.partitions_file: boot_job.add_input_file(self.partitions_file) self.jobs.append(boot_job) # Bootstrap drawing on top of best tree bootd_args = tree_job.args.copy() if self.constrain_tree: del bootd_args["-g"] if self.partitions_file: del bootd_args["-q"] bootd_args["-n"] = "bootstrapped." + tree_job.args["-n"] bootd_args["-f"] = "b" bootd_args["-t"] = self.out_tree_file bootd_args["-z"] = pjoin(boot_job.jobdir, "RAxML_bootstrap." + boot_job.args["-n"]) bootd_job = Job(self.raxml_bin, bootd_args, parent_ids=[tree_job.jobid]) bootd_job.jobname += "-bootstrapped" bootd_job.dependencies.add(boot_job) bootd_job.cores = self.threads self.jobs.append(bootd_job) self.boot_job = boot_job self.bootd_job = bootd_job
def process_task(task, wkname, npr_conf, nodeid2info): alignerconf, alignerclass = npr_conf.aligner cleanerconf, cleanerclass = npr_conf.alg_cleaner mtesterconf, mtesterclass = npr_conf.model_tester treebuilderconf, treebuilderclass = npr_conf.tree_builder if not treebuilderclass: # Allows to dump algs in workflows with no tree tasks treebuilderclass = DummyTree splitterconf, splitterclass = npr_conf.tree_splitter conf = GLOBALS[task.configid] seqtype = task.seqtype nodeid = task.nodeid ttype = task.ttype taskid = task.taskid threadid = task.threadid node_info = nodeid2info[nodeid] size = task.size #node_info.get("size", 0) target_seqs = node_info.get("target_seqs", []) out_seqs = node_info.get("out_seqs", []) if not treebuilderclass or size < 4: # Allows to dump algs in workflows with no tree tasks or if tree # inference does not make sense given the number of sequences. DummyTree # will produce a fake fully collapsed newick tree. treebuilderclass = DummyTree # If more than one outgroup are used, enable the use of constrain if out_seqs and len(out_seqs) > 1: constrain_id = nodeid else: constrain_id = None new_tasks = [] if ttype == "msf": # Register Tree constrains constrain_tree = "(%s, (%s));" % (','.join(sorted( task.out_seqs)), ','.join(sorted(task.target_seqs))) _outs = "\n".join([">%s\n0" % name for name in sorted(task.out_seqs)]) _tars = "\n".join( [">%s\n1" % name for name in sorted(task.target_seqs)]) constrain_alg = '\n'.join([_outs, _tars]) db.add_task_data(nodeid, DATATYPES.constrain_tree, constrain_tree) db.add_task_data(nodeid, DATATYPES.constrain_alg, constrain_alg) db.dataconn.commit() # since the creation of some Task # objects may require this info, I need # to commit right now. # Register node db.add_node(task.threadid, task.nodeid, task.cladeid, task.target_seqs, task.out_seqs) nodeid2info[nodeid]["size"] = task.size nodeid2info[nodeid]["target_seqs"] = task.target_seqs nodeid2info[nodeid]["out_seqs"] = task.out_seqs alg_task = alignerclass(nodeid, task.multiseq_file, seqtype, conf, alignerconf) alg_task.size = task.size new_tasks.append(alg_task) elif ttype == "alg" or ttype == "acleaner": if ttype == "alg": nodeid2info[nodeid]["alg_path"] = task.alg_fasta_file elif ttype == "acleaner": nodeid2info[nodeid]["alg_clean_path"] = task.clean_alg_fasta_file alg_fasta_file = getattr(task, "clean_alg_fasta_file", task.alg_fasta_file) alg_phylip_file = getattr(task, "clean_alg_phylip_file", task.alg_phylip_file) # Calculate alignment stats # cons_mean, cons_std = get_trimal_conservation(task.alg_fasta_file, # conf["app"]["trimal"]) # # max_identity = get_trimal_identity(task.alg_fasta_file, # conf["app"]["trimal"]) # log.info("Conservation: %0.2f +-%0.2f", cons_mean, cons_std) # log.info("Max. Identity: %0.2f", max_identity) #import time #t1 = time.time() #mx, mn, mean, std = get_identity(task.alg_fasta_file) #print time.time()-t1 #log.log(26, "Identity: max=%0.2f min=%0.2f mean=%0.2f +- %0.2f", # mx, mn, mean, std) #t1 = time.time() if seqtype == "aa" and npr_conf.switch_aa_similarity < 1: try: alg_stats = db.get_task_data(taskid, DATATYPES.alg_stats) except Exception as e: alg_stats = {} if ttype == "alg": algfile = pjoin(GLOBALS["input_dir"], task.alg_phylip_file) dataid = DATATYPES.alg_phylip elif ttype == "acleaner": algfile = pjoin(GLOBALS["input_dir"], task.clean_alg_phylip_file) dataid = DATATYPES.clean_alg_phylip if "i_mean" not in alg_stats: log.log(24, "Calculating alignment stats...") # dump data if necesary algfile = pjoin(GLOBALS["input_dir"], task.alg_phylip_file) if not pexist(algfile): # dump phylip alg open(algfile, "w").write(db.get_data(db.get_dataid(taskid, dataid))) mx, mn, mean, std = get_statal_identity( algfile, conf["app"]["statal"]) alg_stats = { "i_max": mx, "i_mean": mean, "i_min": mn, "i_std": std } db.add_task_data(taskid, DATATYPES.alg_stats, alg_stats) log.log(22, "Alignment stats (sequence similarity):") log.log( 22, " max: %(i_max)0.2f, min:%(i_min)0.2f, avg:%(i_mean)0.2f+-%(i_std)0.2f" % (alg_stats)) else: alg_stats = {"i_max": -1, "i_mean": -1, "i_min": -1, "i_std": -1} #print time.time()-t1 #log.log(24, "Identity: max=%0.2f min=%0.2f mean=%0.2f +- %0.2f", # mx, mn, mean, std) task.max_ident = alg_stats["i_max"] task.min_ident = alg_stats["i_min"] task.mean_ident = alg_stats["i_mean"] task.std_ident = alg_stats["i_std"] next_task = None if ttype == "alg" and cleanerclass: next_task = cleanerclass(nodeid, seqtype, alg_fasta_file, alg_phylip_file, conf, cleanerconf) else: # Converts aa alignment into nt if necessary if seqtype == "aa" and \ "nt" in GLOBALS["seqtypes"] and \ task.mean_ident >= npr_conf.switch_aa_similarity: log.log(28, "@@2:Switching to codon alignment!@@1: amino-acid sequence similarity: %0.2f >= %0.2f" %\ (task.mean_ident, npr_conf.switch_aa_similarity)) alg_fasta_file = "%s.%s" % (taskid, DATATYPES.alg_nt_fasta) alg_phylip_file = "%s.%s" % (taskid, DATATYPES.alg_nt_phylip) try: alg_fasta_file = db.get_dataid(taskid, DATATYPES.alg_nt_fasta) alg_fasta_file = db.get_dataid(taskid, DATATYPES.alg_nt_phylip) except ValueError: log.log(22, "Calculating codon alignment...") source_alg = pjoin(GLOBALS["input_dir"], task.alg_fasta_file) if ttype == "alg": kept_columns = [] elif ttype == "acleaner": # if original alignment was trimmed, use it as reference # but make the nt alignment only on the kept columns kept_columns = db.get_task_data( taskid, DATATYPES.kept_alg_columns) if not pexist(source_alg): open(source_alg, "w").write( db.get_task_data(taskid, DATATYPES.alg_fasta)) nt_alg = switch_to_codon(source_alg, kept_columns=kept_columns) db.add_task_data(taskid, DATATYPES.alg_nt_fasta, nt_alg.write()) db.add_task_data(taskid, DATATYPES.alg_nt_phylip, nt_alg.write(format='iphylip_relaxed')) npr_conf = IterConfig(conf, wkname, task.size, "nt") seqtype = "nt" if mtesterclass: next_task = mtesterclass(nodeid, alg_fasta_file, alg_phylip_file, constrain_id, conf, mtesterconf) elif treebuilderclass: next_task = treebuilderclass(nodeid, alg_phylip_file, constrain_id, None, seqtype, conf, treebuilderconf) if next_task: next_task.size = task.size new_tasks.append(next_task) elif ttype == "mchooser": if treebuilderclass: alg_fasta_file = task.alg_fasta_file alg_phylip_file = task.alg_phylip_file model = task.best_model tree_task = treebuilderclass(nodeid, alg_phylip_file, constrain_id, model, seqtype, conf, treebuilderconf) tree_task.size = task.size new_tasks.append(tree_task) elif ttype == "tree": treemerge_task = splitterclass(nodeid, seqtype, task.tree_file, conf, splitterconf) #if conf["tree_splitter"]["_outgroup_size"]: # treemerge_task = TreeSplitterWithOutgroups(nodeid, seqtype, task.tree_file, main_tree, conf) #else: # treemerge_task = TreeSplitter(nodeid, seqtype, task.tree_file, main_tree, conf) treemerge_task.size = task.size new_tasks.append(treemerge_task) elif ttype == "treemerger": if not task.task_tree: task.finish() log.log(24, "Saving task tree...") annotate_node(task.task_tree, task) db.update_node(nid=task.nodeid, runid=task.threadid, newick=db.encode(task.task_tree)) db.commit() if not isinstance(treebuilderclass, DummyTree) and npr_conf.max_iters > 1: current_iter = get_iternumber(threadid) if npr_conf.max_iters and current_iter >= npr_conf.max_iters: log.warning("Maximum number of iterations reached!") else: # Add new nodes source_seqtype = "aa" if "aa" in GLOBALS["seqtypes"] else "nt" ttree, mtree = task.task_tree, task.main_tree log.log(26, "Processing tree: %s seqs, %s outgroups", len(target_seqs), len(out_seqs)) alg_path = node_info.get("clean_alg_path", node_info["alg_path"]) for node, seqs, outs, wkname in get_next_npr_node( threadid, ttree, task.out_seqs, mtree, alg_path, npr_conf): log.log(24, "Registering new node: %s seqs, %s outgroups", len(seqs), len(outs)) new_task_node = Msf(seqs, outs, seqtype=source_seqtype) new_task_node.target_wkname = wkname new_tasks.append(new_task_node) return new_tasks