def SubmitSuqJob(suq_basedir, datapath, outpath, priority, scriptfile): #{{{ myfunc.WriteFile("Entering SubmitSuqJob()\n", g_params['debugfile'], "a") rmsg = "" cmd = [ suq_exec, "-b", suq_basedir, "run", "-d", outpath, "-p", "%d" % (priority), scriptfile ] cmdline = " ".join(cmd) myfunc.WriteFile("cmdline: %s\n\n" % (cmdline), g_params['debugfile'], "a") MAX_TRY = 5 cnttry = 0 isSubmitSuccess = False while cnttry < MAX_TRY: try: myfunc.WriteFile( "run cmd: cnttry = %d, MAX_TRY=%d\n" % (cnttry, MAX_TRY), g_params['debugfile'], "a") rmsg = subprocess.check_output(cmd, stderr=subprocess.STDOUT) isSubmitSuccess = True break except subprocess.CalledProcessError, e: print e print rmsg myfunc.WriteFile( str(e) + "\n" + rmsg + "\n", g_params['debugfile'], "a") pass cnttry += 1 time.sleep(0.05 + cnttry * 0.03)
def SubmitJobToQueue( jobid, datapath, outpath, numseq, numseq_this_user, email, #{{{ host_ip, base_www_url): myfunc.WriteFile("Entering SubmitJobToQueue()\n", g_params['debugfile'], "a") fafile = "%s/query.fa" % (datapath) if numseq == -1: numseq = myfunc.CountFastaSeq(fafile) if numseq_this_user == -1: numseq_this_user = numseq name_software = "boctopus2" runjob = "%s %s/run_job.py" % (python_exec, rundir) scriptfile = "%s/runjob;%s;%s;%s;%s;%d.sh" % ( outpath, name_software, jobid, host_ip, email, numseq) code_str_list = [] code_str_list.append("#!/bin/bash") cmdline = "%s %s -outpath %s -tmpdir %s -jobid %s " % ( runjob, fafile, outpath, datapath, jobid) if email != "": cmdline += "-email \"%s\" " % (email) if base_www_url != "": cmdline += "-baseurl \"%s\" " % (base_www_url) if g_params['isForceRun']: cmdline += "-force " code_str_list.append(cmdline) code = "\n".join(code_str_list) msg = "Write scriptfile %s" % (scriptfile) myfunc.WriteFile(msg + "\n", g_params['debugfile'], "a") myfunc.WriteFile(code, scriptfile) os.chmod(scriptfile, 0755) myfunc.WriteFile("Getting priority" + "\n", g_params['debugfile'], "a") priority = myfunc.GetSuqPriority(numseq_this_user) priority = 10 # quick fix debug 2017-09-18 if email in vip_user_list: priority = 999999999.0 myfunc.WriteFile("priority=%d\n" % (priority), g_params['debugfile'], "a") st1 = SubmitSuqJob(suq_basedir, datapath, outpath, priority, scriptfile) return st1
def WriteDateTimeTagFile(outfile, runjob_logfile, runjob_errfile): # {{{ if not os.path.exists(outfile): date_str = time.strftime(FORMAT_DATETIME) try: myfunc.WriteFile(date_str, outfile) msg = "Write tag file %s succeeded" % (outfile) myfunc.WriteFile("[%s] %s\n" % (date_str, msg), runjob_logfile, "a", True) except Exception as e: msg = "Failed to write to file %s with message: \"%s\"" % (outfile, str(e)) myfunc.WriteFile("[%s] %s\n" % (date_str, msg), runjob_errfile, "a", True)
def RunJob(infile, outpath, tmpdir, email, jobid, g_params): #{{{ blastdb = "/data3/data/blastdb/swissprot" rootname = os.path.basename(os.path.splitext(infile)[0]) starttagfile = "%s/runjob.start" % (outpath) runjob_errfile = "%s/runjob.err" % (outpath) runjob_logfile = "%s/runjob.log" % (outpath) finishtagfile = "%s/runjob.finish" % (outpath) tmp_outfile = "%s/query.result" % (tmpdir) resultpathname = jobid outpath_result = "%s/%s" % (outpath, resultpathname) outfile = "%s/query.result" % (outpath_result) tarball = "%s.tar.gz" % (resultpathname) tarball_fullpath = "%s.tar.gz" % (outpath_result) isOK = True try: os.makedirs(outpath_result) isOK = True except OSError: msg = "Failed to create folder %s" % (outpath_result) myfunc.WriteFile(msg + "\n", runjob_errfile, "a") isOK = False if isOK: g_params['runjob_log'].append("tmpdir = %s" % (tmpdir)) #cmd = [script_getseqlen, infile, "-o", tmp_outfile , "-printid"] datetime = time.strftime("%Y-%m-%d %H:%M:%S") rt_msg = myfunc.WriteFile(datetime, starttagfile) if rt_msg: g_params['runjob_err'].append(rt_msg) cmd = [ blastall, "-i", infile, "-p", "blastp", "-o", tmp_outfile, "-d", blastdb ] g_params['runjob_log'].append(" ".join(cmd)) try: myfunc.check_output(cmd) except subprocess.CalledProcessError, e: g_params['runjob_err'].append(str(e)) suqoutfilelist = glob.glob("%s/*.sh.*.out" % (tmpdir)) if len(suqoutfilelist) > 0: suqoutfile = suqoutfilelist[0] g_params['runjob_err'].append(myfunc.ReadFile(suqoutfile)) if os.path.exists(tmp_outfile): cmd = ["cp", "-f", tmp_outfile, outfile] try: myfunc.check_output(cmd) except subprocess.CalledProcessError, e: g_params['runjob_err'].append(str(e))
def CleanServerFile(logfile, errfile): #{{{ """Clean old files on the server""" # clean tmp files msg = "CleanServerFile..." date_str = time.strftime(FORMAT_DATETIME) myfunc.WriteFile("[%s] %s\n" % (date_str, msg), logfile, "a", True) cmd = ["bash", "%s/clean_server_file.sh" % (rundir)] RunCmd(cmd, logfile, errfile)
def RunCmd(cmd, runjob_logfile, runjob_errfile, verbose=False): # {{{ """Input cmd in list Run the command and also output message to logs """ begin_time = time.time() isCmdSuccess = False cmdline = " ".join(cmd) date_str = time.strftime(FORMAT_DATETIME) rmsg = "" try: rmsg = subprocess.check_output(cmd) if verbose: msg = "workflow: %s" % (cmdline) myfunc.WriteFile("[%s] %s\n" % (date_str, msg), runjob_logfile, "a", True) isCmdSuccess = True except subprocess.CalledProcessError, e: msg = "cmdline: %s\nFailed with message \"%s\"" % (cmdline, str(e)) myfunc.WriteFile("[%s] %s\n" % (date_str, msg), runjob_errfile, "a", True) isCmdSuccess = False pass
def GetNumSameUserInQueue(suq_ls_content, basename_scriptfile, email, host_ip): #{{{ myfunc.WriteFile("Entering GetNumSameUserInQueue()\n", g_params['debugfile'], "a") num_same_user_in_queue = 0 if email == "" and host_ip == "": num_same_user_in_queue = 0 else: lines = suq_ls_content.split("\n") if email != "" and host_ip != "": for line in lines: if line.find(email) != -1 or line.find(host_ip) != -1: num_same_user_in_queue += 1 elif email != "": for line in lines: if line.find(email) != -1: num_same_user_in_queue += 1 elif host_ip != "": for line in lines: if line.find(host_ip) != -1: num_same_user_in_queue += 1 return num_same_user_in_queue
myfunc.WriteFile( "run cmd: cnttry = %d, MAX_TRY=%d\n" % (cnttry, MAX_TRY), g_params['debugfile'], "a") rmsg = subprocess.check_output(cmd, stderr=subprocess.STDOUT) isSubmitSuccess = True break except subprocess.CalledProcessError, e: print e print rmsg myfunc.WriteFile( str(e) + "\n" + rmsg + "\n", g_params['debugfile'], "a") pass cnttry += 1 time.sleep(0.05 + cnttry * 0.03) if isSubmitSuccess: myfunc.WriteFile("Leaving SubmitSuqJob() with success\n\n", g_params['debugfile'], "a") return 0 else: myfunc.WriteFile("Leaving SubmitSuqJob() with error\n\n", g_params['debugfile'], "a") return 1 #}}} def main(g_params): #{{{ argv = sys.argv numArgv = len(argv) if numArgv < 2: PrintHelp() return 1
def RunJob(infile, outpath, tmpdir, email, jobid, g_params):#{{{ rootname = os.path.basename(os.path.splitext(infile)[0]) starttagfile = "%s/runjob.start"%(outpath) runjob_errfile = "%s/runjob.err"%(outpath) runjob_logfile = "%s/runjob.log"%(outpath) finishtagfile = "%s/runjob.finish"%(outpath) rmsg = "" resultpathname = jobid outpath_result = "%s/%s"%(outpath, resultpathname) tarball = "%s.tar.gz"%(resultpathname) zipfile = "%s.zip"%(resultpathname) tarball_fullpath = "%s.tar.gz"%(outpath_result) zipfile_fullpath = "%s.zip"%(outpath_result) outfile = "%s/%s/Topcons/topcons.top"%(outpath_result, "seq_%d"%(0)) resultfile_text = "%s/%s"%(outpath_result, "query.result.txt") tmp_outpath_result = "%s/%s"%(tmpdir, resultpathname) isOK = True try: os.makedirs(tmp_outpath_result) isOK = True except OSError: msg = "Failed to create folder %s"%(tmp_outpath_result) myfunc.WriteFile(msg+"\n", runjob_errfile, "a") isOK = False print "isOK =", isOK if isOK: tmp_mapfile = "%s/seqid_index_map.txt"%(tmp_outpath_result) maplist = [] maplist_simple = [] hdl = myfunc.ReadFastaByBlock(infile, method_seqid=0, method_seq=0) if hdl.failure: isOK = False else: recordList = hdl.readseq() cnt = 0 while recordList != None: for rd in recordList: maplist.append("%s\t%d\t%s\t%s"%("seq_%d"%cnt, len(rd.seq), rd.description, rd.seq)) maplist_simple.append("%s\t%d\t%s"%("seq_%d"%cnt, len(rd.seq), rd.description)) cnt += 1 recordList = hdl.readseq() hdl.close() myfunc.WriteFile("\n".join(maplist_simple), tmp_mapfile) if isOK: # g_params['runjob_log'].append("tmpdir = %s"%(tmpdir)) #cmd = [script_getseqlen, infile, "-o", tmp_outfile , "-printid"] datetime = time.strftime("%Y-%m-%d %H:%M:%S") rt_msg = myfunc.WriteFile(datetime, starttagfile) if rt_msg: g_params['runjob_err'].append(rt_msg) cmd = [runscript, infile, tmp_outpath_result, blastdir, blastdb ] g_params['runjob_log'].append(" ".join(cmd)) begin_time = time.time() try: rmsg = subprocess.check_output(cmd) except subprocess.CalledProcessError, e: g_params['runjob_err'].append(str(e)+"\n") g_params['runjob_err'].append(rmsg + "\n") suqoutfilelist = glob.glob("%s/*.sh.*.out"%(tmpdir)) if len(suqoutfilelist)>0: suqoutfile = suqoutfilelist[0] g_params['runjob_err'].append(myfunc.ReadFile(suqoutfile)) end_time = time.time() runtime_in_sec = end_time - begin_time if os.path.exists(tmp_outpath_result): cmd = ["cp","-rf", tmp_outpath_result, outpath] try: subprocess.check_output(cmd) except subprocess.CalledProcessError, e: g_params['runjob_err'].append(str(e)) if len(g_params['runjob_log']) > 0 : rt_msg = myfunc.WriteFile("\n".join(g_params['runjob_log']), runjob_logfile, "a") if rt_msg: g_params['runjob_err'].append(rt_msg) datetime = time.strftime("%Y-%m-%d %H:%M:%S") if os.path.exists(outfile): rt_msg = myfunc.WriteFile(datetime, finishtagfile) if rt_msg: g_params['runjob_err'].append(rt_msg) # now write the text output to a single file WriteTextResultFile(resultfile_text, maplist, runtime_in_sec) # now making zip instead (for windows users) pwd = os.getcwd() os.chdir(outpath) # cmd = ["tar", "-czf", tarball, resultpathname] cmd = ["zip", "-rq", zipfile, resultpathname] try: subprocess.check_output(cmd) except subprocess.CalledProcessError, e: g_params['runjob_err'].append(str(e))
# cmd = ["tar", "-czf", tarball, resultpathname] cmd = ["zip", "-rq", zipfile, resultpathname] try: subprocess.check_output(cmd) except subprocess.CalledProcessError, e: g_params['runjob_err'].append(str(e)) os.chdir(pwd) isSuccess = False if (os.path.exists(finishtagfile) and os.path.exists(zipfile_fullpath)): isSuccess = True else: isSuccess = False failtagfile = "%s/runjob.failed"%(outpath) datetime = time.strftime("%Y-%m-%d %H:%M:%S") rt_msg = myfunc.WriteFile(datetime, failtagfile) if rt_msg: g_params['runjob_err'].append(rt_msg) # send the result to email if myfunc.IsValidEmailAddress(email): from_email = "*****@*****.**" to_email = email subject = "Your result for TOPCONS2 JOBID=%s"%(jobid) if isSuccess: bodytext = """ Your result is ready at %s/pred/result/%s Thanks for using TOPCONS2 """%(g_params['base_www_url'], jobid)
def RunJob(infile, outpath, tmpdir, email, jobid, g_params): #{{{ all_begin_time = time.time() rootname = os.path.basename(os.path.splitext(infile)[0]) starttagfile = "%s/runjob.start" % (outpath) runjob_errfile = "%s/runjob.err" % (outpath) runjob_logfile = "%s/runjob.log" % (outpath) finishtagfile = "%s/runjob.finish" % (outpath) rmsg = "" resultpathname = jobid outpath_result = "%s/%s" % (outpath, resultpathname) tarball = "%s.tar.gz" % (resultpathname) zipfile = "%s.zip" % (resultpathname) tarball_fullpath = "%s.tar.gz" % (outpath_result) zipfile_fullpath = "%s.zip" % (outpath_result) outfile = "%s/%s/Topcons/topcons.top" % (outpath_result, "seq_%d" % (0)) resultfile_text = "%s/%s" % (outpath_result, "query.result.txt") mapfile = "%s/seqid_index_map.txt" % (outpath_result) finished_seq_file = "%s/finished_seqs.txt" % (outpath_result) tmp_outpath_result = "%s/%s" % (tmpdir, resultpathname) isOK = True try: os.makedirs(tmp_outpath_result) isOK = True except OSError: msg = "Failed to create folder %s" % (tmp_outpath_result) myfunc.WriteFile(msg + "\n", runjob_errfile, "a") isOK = False pass try: os.makedirs(outpath_result) isOK = True except OSError: msg = "Failed to create folder %s" % (outpath_result) myfunc.WriteFile(msg + "\n", runjob_errfile, "a") isOK = False pass if isOK: try: open(finished_seq_file, 'w').close() except: pass #first getting result from caches # ================================== maplist = [] maplist_simple = [] toRunDict = {} hdl = myfunc.ReadFastaByBlock(infile, method_seqid=0, method_seq=0) if hdl.failure: isOK = False else: datetime = time.strftime("%Y-%m-%d %H:%M:%S") rt_msg = myfunc.WriteFile(datetime, starttagfile) recordList = hdl.readseq() cnt = 0 origpath = os.getcwd() while recordList != None: for rd in recordList: isSkip = False # temp outpath for the sequence is always seq_0, and I feed # only one seq a time to the workflow tmp_outpath_this_seq = "%s/%s" % (tmp_outpath_result, "seq_%d" % 0) outpath_this_seq = "%s/%s" % (outpath_result, "seq_%d" % cnt) subfoldername_this_seq = "seq_%d" % (cnt) if os.path.exists(tmp_outpath_this_seq): try: shutil.rmtree(tmp_outpath_this_seq) except OSError: pass maplist.append( "%s\t%d\t%s\t%s" % ("seq_%d" % cnt, len(rd.seq), rd.description, rd.seq)) maplist_simple.append( "%s\t%d\t%s" % ("seq_%d" % cnt, len(rd.seq), rd.description)) if not g_params['isForceRun']: md5_key = hashlib.md5(rd.seq).hexdigest() subfoldername = md5_key[:2] md5_link = "%s/%s/%s" % (path_md5cache, subfoldername, md5_key) if os.path.exists(md5_link): # create a symlink to the cache rela_path = os.path.relpath( md5_link, outpath_result) #relative path os.chdir(outpath_result) os.symlink(rela_path, subfoldername_this_seq) if os.path.exists(outpath_this_seq): runtime = 0.0 #in seconds topfile = "%s/%s/topcons.top" % ( outpath_this_seq, "Topcons") top = myfunc.ReadFile(topfile).strip() numTM = myfunc.CountTM(top) posSP = myfunc.GetSPPosition(top) if len(posSP) > 0: isHasSP = True else: isHasSP = False info_finish = [ "seq_%d" % cnt, str(len(rd.seq)), str(numTM), str(isHasSP), "cached", str(runtime), rd.description ] myfunc.WriteFile("\t".join(info_finish) + "\n", finished_seq_file, "a", isFlush=True) isSkip = True if not isSkip: # first try to delete the outfolder if exists if os.path.exists(outpath_this_seq): try: shutil.rmtree(outpath_this_seq) except OSError: pass origIndex = cnt numTM = 0 toRunDict[origIndex] = [rd.seq, numTM, rd.description ] #init value for numTM is 0 cnt += 1 recordList = hdl.readseq() hdl.close() myfunc.WriteFile("\n".join(maplist_simple) + "\n", mapfile) # run scampi single to estimate the number of TM helices and then run # the query sequences in the descending order of numTM torun_all_seqfile = "%s/%s" % (tmp_outpath_result, "query.torun.fa") dumplist = [] for key in toRunDict: top = toRunDict[key][0] dumplist.append(">%s\n%s" % (str(key), top)) myfunc.WriteFile("\n".join(dumplist) + "\n", torun_all_seqfile, "w") del dumplist topfile_scampiseq = "%s/%s" % (tmp_outpath_result, "query.torun.fa.topo") if os.path.exists(torun_all_seqfile): # run scampi to estimate the number of TM helices cmd = [ script_scampi, torun_all_seqfile, "-outpath", tmp_outpath_result ] try: rmsg = subprocess.check_output(cmd) except subprocess.CalledProcessError, e: g_params['runjob_err'].append(str(e) + "\n") pass if os.path.exists(topfile_scampiseq): (idlist_scampi, annolist_scampi, toplist_scampi) = myfunc.ReadFasta(topfile_scampiseq) for jj in xrange(len(idlist_scampi)): numTM = myfunc.CountTM(toplist_scampi[jj]) try: toRunDict[int(idlist_scampi[jj])][1] = numTM except (KeyError, ValueError, TypeError): pass sortedlist = sorted(toRunDict.items(), key=lambda x: x[1][1], reverse=True) #format of sortedlist [(origIndex: [seq, numTM, description]), ...] # submit sequences one by one to the workflow according to orders in # sortedlist for item in sortedlist: # g_params['runjob_log'].append("tmpdir = %s"%(tmpdir)) #cmd = [script_getseqlen, infile, "-o", tmp_outfile , "-printid"] origIndex = item[0] seq = item[1][0] description = item[1][2] outpath_this_seq = "%s/%s" % (outpath_result, "seq_%d" % origIndex) tmp_outpath_this_seq = "%s/%s" % (tmp_outpath_result, "seq_%d" % (0)) if os.path.exists(tmp_outpath_this_seq): try: shutil.rmtree(tmp_outpath_this_seq) except OSError: pass seqfile_this_seq = "%s/%s" % (tmp_outpath_result, "query_%d.fa" % (origIndex)) seqcontent = ">%d\n%s\n" % (origIndex, seq) myfunc.WriteFile(seqcontent, seqfile_this_seq, "w") if not os.path.exists(seqfile_this_seq): g_params['runjob_err'].append( "failed to generate seq index %d" % (origIndex)) continue cmd = [ runscript, seqfile_this_seq, tmp_outpath_result, blastdir, blastdb ] g_params['runjob_log'].append(" ".join(cmd)) begin_time = time.time() try: rmsg = subprocess.check_output(cmd) g_params['runjob_log'].append("workflow:\n" + rmsg + "\n") except subprocess.CalledProcessError, e: g_params['runjob_err'].append(str(e) + "\n") g_params['runjob_err'].append(rmsg + "\n") pass #suqoutfilelist = glob.glob("%s/*.sh.*.out"%(tmpdir)) #if len(suqoutfilelist)>0: # suqoutfile = suqoutfilelist[0] #g_params['runjob_err'].append(myfunc.ReadFile(suqoutfile)) end_time = time.time() runtime_in_sec = end_time - begin_time if os.path.exists(tmp_outpath_this_seq): cmd = ["mv", "-f", tmp_outpath_this_seq, outpath_this_seq] isCmdSuccess = False try: subprocess.check_output(cmd) isCmdSuccess = True except subprocess.CalledProcessError, e: msg = "Failed to run prediction for sequence No. %d\n" % ( origIndex) g_params['runjob_err'].append(msg) g_params['runjob_err'].append(str(e) + "\n") pass timefile = "%s/time.txt" % (tmp_outpath_result) targetfile = "%s/time.txt" % (outpath_this_seq) if os.path.exists(timefile) and os.path.exists( outpath_this_seq): try: shutil.move(timefile, targetfile) except: g_params['runjob_err'].append( "Failed to move %s/time.txt" % (tmp_outpath_result) + "\n") pass if isCmdSuccess: runtime = runtime_in_sec #in seconds topfile = "%s/%s/topcons.top" % (outpath_this_seq, "Topcons") top = myfunc.ReadFile(topfile).strip() numTM = myfunc.CountTM(top) posSP = myfunc.GetSPPosition(top) if len(posSP) > 0: isHasSP = True else: isHasSP = False info_finish = [ "seq_%d" % origIndex, str(len(seq)), str(numTM), str(isHasSP), "newrun", str(runtime), description ] myfunc.WriteFile("\t".join(info_finish) + "\n", finished_seq_file, "a", isFlush=True) # now write the text output for this seq info_this_seq = "%s\t%d\t%s\t%s" % ( "seq_%d" % origIndex, len(seq), description, seq) resultfile_text_this_seq = "%s/%s" % (outpath_this_seq, "query.result.txt") myfunc.WriteTOPCONSTextResultFile(resultfile_text_this_seq, outpath_result, [info_this_seq], runtime_in_sec, g_params['base_www_url']) # create or update the md5 cache # create cache only on the front-end if g_params['base_www_url'].find("topcons.net") != -1: md5_key = hashlib.md5(seq).hexdigest() subfoldername = md5_key[:2] md5_subfolder = "%s/%s" % (path_md5cache, subfoldername) md5_link = "%s/%s/%s" % (path_md5cache, subfoldername, md5_key) if os.path.exists(md5_link): try: os.unlink(md5_link) except: pass subfolder_md5 = "%s/%s" % (path_md5cache, subfoldername) if not os.path.exists(subfolder_md5): try: os.makedirs(subfolder_md5) except: pass rela_path = os.path.relpath( outpath_this_seq, md5_subfolder) #relative path try: os.chdir(md5_subfolder) os.symlink(rela_path, md5_key) except: pass
except: pass rela_path = os.path.relpath( outpath_this_seq, md5_subfolder) #relative path try: os.chdir(md5_subfolder) os.symlink(rela_path, md5_key) except: pass all_end_time = time.time() all_runtime_in_sec = all_end_time - all_begin_time if len(g_params['runjob_log']) > 0: rt_msg = myfunc.WriteFile("\n".join(g_params['runjob_log']) + "\n", runjob_logfile, "a") if rt_msg: g_params['runjob_err'].append(rt_msg) datetime = time.strftime("%Y-%m-%d %H:%M:%S") if os.path.exists(finished_seq_file): rt_msg = myfunc.WriteFile(datetime, finishtagfile) if rt_msg: g_params['runjob_err'].append(rt_msg) # now write the text output to a single file statfile = "%s/%s" % (outpath_result, "stat.txt") myfunc.WriteTOPCONSTextResultFile(resultfile_text, outpath_result, maplist, all_runtime_in_sec,
def start_boctopus(infile, blastpath, modHome, hmmfilename, ws_cytosolic, ws_extracellular, ws_lipidfacing, ws_porefacing, \ fakedbpath, dbpath, blastpgppath, hhsearchpath, hhblitspath, rpath): print "boctopus2 will start with ", infile # f = open(infile, "r")#{{{ DELETED # lines = f.readlines() # f.close() # # pname = [] # seqname = [] # tempseq = "" # for line in lines: # line = line.strip() # # if line.startswith(">"): # pname.append(line[1:]) # if len(tempseq) > 0: # seqname.append(tempseq) # tempseq = "" # else: # tempseq += line # # if len(tempseq) > 0: # seqname.append(tempseq) # # print pname # print seqname # # if len(pname) != len(seqname): # print "number of pnames and seqs not the same." # else:#}}} # rewrite sequence reading part (seqidlist, seqannolist, seqlist) = myfunc.ReadFasta(infile) if len(seqidlist) <= 0: print >> sys.stderr, "No valid sequences read from file '%s'"%(infile) return 1 #for i in range(0, len(pname)): for i in xrange(len(seqidlist)): seqid = seqidlist[i] seq = seqlist[i] seqanno = seqannolist[i] print "processing ", i , seqanno subtmpdir = "%s/seq_%d"%(tmpdir, i) if os.path.exists(subtmpdir): shutil.rmtree(subtmpdir) os.makedirs(subtmpdir) singleseqfile = "%s/query.fa"%(subtmpdir) myfunc.WriteFile(">%s\n%s\n"%(seqanno, seq), singleseqfile, mode="w", isFlush=True) if not os.path.exists(singleseqfile): print >> sys.stderr, "Failed to write to singleseqfile %s"%(singleseqfile) continue command = "python "+ "%s/boctopus_startHMM.py "%(rundir) + singleseqfile + " " + blastpath + " " + modHome + " " + hmmfilename + " " + ws_cytosolic + " " + ws_extracellular + " " + ws_lipidfacing + " " + ws_porefacing + " " + rpath+ " " +fakedbpath+\ " " + dbpath+ " " + blastpgppath+ " " + hhsearchpath + " " + hhblitspath print command os.system(command) outpath_this_seq = "%s/seq_%d"%(outpath, i) if not os.path.exists(outpath_this_seq): os.makedirs(outpath_this_seq) filepair_to_copy = [ ("%s/query.fa"%subtmpdir, "%s/query.fa"%outpath_this_seq), ("%s/output/query_ioIOS.prf.txt_svm_topo.png"%subtmpdir, "%s/query.predict.png"%(outpath_this_seq)), ("%s/output/query_topologies.txt"%(subtmpdir), "%s/query_topologies.txt"%outpath_this_seq), ("%s/svmoutput/query_ioIOS.prf.txt"%subtmpdir, "%s/profile.txt"%outpath_this_seq), ("%s/pssm/query.filtered.pssmvals"%subtmpdir, "%s/pssm.txt"%(outpath_this_seq)) ] for tup in filepair_to_copy: shutil.move(tup[0], tup[1]) return
def RunJob(modelfile, seqfile, outpath, tmpdir, email, jobid, g_params): #{{{ all_begin_time = time.time() rootname = os.path.basename(os.path.splitext(modelfile)[0]) starttagfile = "%s/runjob.start" % (outpath) runjob_errfile = "%s/runjob.err" % (outpath) runjob_logfile = "%s/runjob.log" % (outpath) finishtagfile = "%s/runjob.finish" % (outpath) rmsg = "" query_parafile = "%s/query.para.txt" % (outpath) query_para = {} content = myfunc.ReadFile(query_parafile) if content != "": query_para = json.loads(content) resultpathname = jobid outpath_result = "%s/%s" % (outpath, resultpathname) tarball = "%s.tar.gz" % (resultpathname) zipfile = "%s.zip" % (resultpathname) tarball_fullpath = "%s.tar.gz" % (outpath_result) zipfile_fullpath = "%s.zip" % (outpath_result) mapfile = "%s/seqid_index_map.txt" % (outpath_result) finished_model_file = "%s/finished_models.txt" % (outpath_result) timefile = "%s/time.txt" % (outpath_result) tmp_outpath_result = "%s/%s" % (tmpdir, resultpathname) isOK = True if os.path.exists(tmp_outpath_result): shutil.rmtree(tmp_outpath_result) try: os.makedirs(tmp_outpath_result) isOK = True except OSError: msg = "Failed to create folder %s" % (tmp_outpath_result) myfunc.WriteFile(msg + "\n", runjob_errfile, "a") isOK = False pass if os.path.exists(outpath_result): shutil.rmtree(outpath_result) try: os.makedirs(outpath_result) isOK = True except OSError: msg = "Failed to create folder %s" % (outpath_result) myfunc.WriteFile(msg + "\n", runjob_errfile, "a") isOK = False pass if isOK: try: open(finished_model_file, 'w').close() except: pass #first getting result from caches # cache profiles for sequences, but do not cache predictions for models webserver_common.WriteDateTimeTagFile(starttagfile, runjob_logfile, runjob_errfile) # ================================== numModel = 0 modelFileList = [] if seqfile != "": # if the fasta sequence is supplied, all models should be using this sequence subfoldername_profile = "profile_%d" % (0) outpath_profile = "%s/%s" % (outpath_result, subfoldername_profile) CreateProfile(seqfile, outpath_profile, outpath_result, tmp_outpath_result, timefile, runjob_errfile) # run proq3 for models modelList = myfunc.ReadPDBModel(modelfile) numModel = len(modelList) for ii in range(len(modelList)): model = modelList[ii] tmp_model_file = "%s/query_%d.pdb" % (tmp_outpath_result, ii) myfunc.WriteFile(model + "\n", tmp_model_file) profilename = "%s/%s" % (outpath_profile, "query.fasta") subfoldername_this_model = "model_%d" % (ii) outpath_this_model = "%s/%s" % (outpath_result, subfoldername_this_model) modelinfo = ScoreModel(query_para, tmp_model_file, outpath_this_model, profilename, outpath_result, tmp_outpath_result, timefile, runjob_errfile) myfunc.WriteFile("\t".join(modelinfo) + "\n", finished_model_file, "a") modelFileList.append( "%s/%s" % (outpath_this_model, "query_%d.pdb" % (ii))) else: # no seqfile supplied, sequences are obtained from the model file modelList = myfunc.ReadPDBModel(modelfile) numModel = len(modelList) for ii in range(len(modelList)): model = modelList[ii] tmp_model_file = "%s/query_%d.pdb" % (tmp_outpath_result, ii) myfunc.WriteFile(model + "\n", tmp_model_file) subfoldername_this_model = "model_%d" % (ii) tmp_outpath_this_model = "%s/%s" % (tmp_outpath_result, subfoldername_this_model) if not os.path.exists(tmp_outpath_this_model): os.makedirs(tmp_outpath_this_model) tmp_seqfile = "%s/query.fasta" % (tmp_outpath_this_model) cmd = [pdb2aa_script, tmp_model_file] g_params['runjob_log'].append(" ".join(cmd)) try: rmsg = subprocess.check_output(cmd) g_params['runjob_log'].append( "extracting sequence from modelfile:\n" + rmsg + "\n") except subprocess.CalledProcessError as e: g_params['runjob_err'].append(str(e) + "\n") g_params['runjob_err'].append(rmsg + "\n") if rmsg != "": myfunc.WriteFile(">seq\n" + rmsg.strip(), tmp_seqfile) subfoldername_profile = "profile_%d" % (ii) outpath_profile = "%s/%s" % (outpath_result, subfoldername_profile) CreateProfile(tmp_seqfile, outpath_profile, outpath_result, tmp_outpath_result, timefile, runjob_errfile) outpath_this_model = "%s/%s" % (outpath_result, subfoldername_this_model) profilename = "%s/%s" % (outpath_profile, "query.fasta") modelinfo = ScoreModel(query_para, tmp_model_file, outpath_this_model, profilename, outpath_result, tmp_outpath_result, timefile, runjob_errfile) myfunc.WriteFile("\t".join(modelinfo) + "\n", finished_model_file, "a") modelFileList.append( "%s/%s" % (outpath_this_model, "query_%d.pdb" % (ii))) all_end_time = time.time() all_runtime_in_sec = all_end_time - all_begin_time if len(g_params['runjob_log']) > 0: rt_msg = myfunc.WriteFile("\n".join(g_params['runjob_log']) + "\n", runjob_logfile, "a") if rt_msg: g_params['runjob_err'].append(rt_msg) webserver_common.WriteDateTimeTagFile(finishtagfile, runjob_logfile, runjob_errfile) # now write the text output to a single file #statfile = "%s/%s"%(outpath_result, "stat.txt") statfile = "" dumped_resultfile = "%s/%s" % (outpath_result, "query.proq3.txt") proq3opt = GetProQ3Option(query_para) webserver_common.WriteProQ3TextResultFile(dumped_resultfile, query_para, modelFileList, all_runtime_in_sec, g_params['base_www_url'], proq3opt, statfile=statfile) # now making zip instead (for windows users) # note that zip rq will zip the real data for symbolic links os.chdir(outpath) # cmd = ["tar", "-czf", tarball, resultpathname] cmd = ["zip", "-rq", zipfile, resultpathname] try: subprocess.check_output(cmd) except subprocess.CalledProcessError as e: g_params['runjob_err'].append(str(e)) pass isSuccess = False if (os.path.exists(finishtagfile) and os.path.exists(zipfile_fullpath)): isSuccess = True flist = glob.glob("%s/*.out" % (tmpdir)) if len(flist) > 0: outfile_runscript = flist[0] else: outfile_runscript = "" if os.path.exists(outfile_runscript): shutil.move(outfile_runscript, outpath) # delete the tmpdir if succeeded shutil.rmtree(tmpdir) #DEBUG, keep tmpdir else: isSuccess = False failedtagfile = "%s/runjob.failed" % (outpath) webserver_common.WriteDateTimeTagFile(failedtagfile, runjob_logfile, runjob_errfile) # send the result to email # do not sendmail at the cloud VM if (webserver_common.IsFrontEndNode(g_params['base_www_url']) and myfunc.IsValidEmailAddress(email)): from_email = "*****@*****.**" to_email = email subject = "Your result for ProQ3 JOBID=%s" % (jobid) if isSuccess: bodytext = """ Your result is ready at %s/pred/result/%s Thanks for using ProQ3 """ % (g_params['base_www_url'], jobid) else: bodytext = """ We are sorry that your job with jobid %s is failed. Please contact %s if you have any questions. Attached below is the error message: %s """ % (jobid, contact_email, "\n".join(g_params['runjob_err'])) g_params['runjob_log'].append("Sendmail %s -> %s, %s" % (from_email, to_email, subject)) #debug rtValue = myfunc.Sendmail(from_email, to_email, subject, bodytext) if rtValue != 0: g_params['runjob_err'].append( "Sendmail to {} failed with status {}".format( to_email, rtValue)) if len(g_params['runjob_err']) > 0: rt_msg = myfunc.WriteFile("\n".join(g_params['runjob_err']) + "\n", runjob_errfile, "w") return 1 return 0
def loginfo(msg, outfile): # {{{ """Write loginfo to outfile, appending current time""" date_str = time.strftime(FORMAT_DATETIME) myfunc.WriteFile("[%s] %s\n" % (date_str, msg), outfile, "a", True)
def CreateProfile(seqfile, outpath_profile, outpath_result, tmp_outpath_result, timefile, runjob_errfile): #{{{ (seqid, seqanno, seq) = myfunc.ReadSingleFasta(seqfile) subfoldername_profile = os.path.basename(outpath_profile) tmp_outpath_profile = "%s/%s" % (tmp_outpath_result, subfoldername_profile) isSkip = False rmsg = "" if not g_params['isForceRun']: md5_key = hashlib.md5(seq).hexdigest() subfoldername = md5_key[:2] md5_link = "%s/%s/%s" % (path_md5cache, subfoldername, md5_key) if os.path.exists(md5_link): # create a symlink to the cache rela_path = os.path.relpath(md5_link, outpath_result) #relative path os.chdir(outpath_result) os.symlink(rela_path, subfoldername_profile) isSkip = True if not isSkip: # build profiles if not os.path.exists(tmp_outpath_profile): try: os.makedirs(tmp_outpath_profile) except OSError: msg = "Failed to create folder %s" % (tmp_outpath_profile) myfunc.WriteFile(msg + "\n", runjob_errfile, "a") return 1 cmd = [ runscript, "-fasta", seqfile, "-outpath", tmp_outpath_profile, "-only-build-profile" ] g_params['runjob_log'].append(" ".join(cmd)) begin_time = time.time() cmdline = " ".join(cmd) #os.system("%s >> %s 2>&1"%(cmdline, runjob_errfile)) #DEBUG try: rmsg = subprocess.check_output(cmd) g_params['runjob_log'].append("profile_building:\n" + rmsg + "\n") except subprocess.CalledProcessError, e: g_params['runjob_err'].append(str(e) + "\n") g_params['runjob_err'].append("cmdline: " + cmdline + "\n") g_params['runjob_err'].append("profile_building:\n" + rmsg + "\n") pass end_time = time.time() runtime_in_sec = end_time - begin_time msg = "%s\t%f\n" % (subfoldername_profile, runtime_in_sec) myfunc.WriteFile(msg, timefile, "a") if os.path.exists(tmp_outpath_profile): md5_key = hashlib.md5(seq).hexdigest() md5_subfoldername = md5_key[:2] subfolder_profile_cache = "%s/%s" % (path_profile_cache, md5_subfoldername) outpath_profile_cache = "%s/%s" % (subfolder_profile_cache, md5_key) if os.path.exists(outpath_profile_cache): shutil.rmtree(outpath_profile_cache) if not os.path.exists(subfolder_profile_cache): os.makedirs(subfolder_profile_cache) cmd = ["mv", "-f", tmp_outpath_profile, outpath_profile_cache] isCmdSuccess = False try: subprocess.check_output(cmd) isCmdSuccess = True except subprocess.CalledProcessError, e: msg = "Failed to run get profile for the target sequence %s" % ( seq) g_params['runjob_err'].append(msg) g_params['runjob_err'].append(str(e) + "\n") pass if isCmdSuccess and webserver_common.IsFrontEndNode( g_params['base_www_url']): # make zip folder for the cached profile cwd = os.getcwd() os.chdir(subfolder_profile_cache) cmd = ["zip", "-rq", "%s.zip" % (md5_key), md5_key] try: subprocess.check_output(cmd) except subprocess.CalledProcessError, e: g_params['runjob_err'].append(str(e)) pass os.chdir(cwd) # create soft link for profile and for md5 # first create a soft link for outpath_profile to outpath_profile_cache rela_path = os.path.relpath(outpath_profile_cache, outpath_result) #relative path try: os.chdir(outpath_result) os.symlink(rela_path, subfoldername_profile) except: pass # then create a soft link for md5 to outpath_proifle_cache md5_subfolder = "%s/%s" % (path_md5cache, md5_subfoldername) md5_link = "%s/%s/%s" % (path_md5cache, md5_subfoldername, md5_key) if os.path.exists(md5_link): try: os.unlink(md5_link) except: pass if not os.path.exists(md5_subfolder): try: os.makedirs(md5_subfolder) except: pass rela_path = os.path.relpath(outpath_profile_cache, md5_subfolder) #relative path try: os.chdir(md5_subfolder) os.symlink(rela_path, md5_key) except: pass
def RunJob(modelfile, seqfile, outpath, tmpdir, email, jobid, g_params): #{{{ all_begin_time = time.time() rootname = os.path.basename(os.path.splitext(modelfile)[0]) starttagfile = "%s/runjob.start" % (outpath) runjob_errfile = "%s/runjob.err" % (outpath) runjob_logfile = "%s/runjob.log" % (outpath) finishtagfile = "%s/runjob.finish" % (outpath) rmsg = "" query_parafile = "%s/query.para.txt" % (outpath) query_para = {} content = myfunc.ReadFile(query_parafile) if content != "": query_para = json.loads(content) resultpathname = jobid outpath_result = "%s/%s" % (outpath, resultpathname) tarball = "%s.tar.gz" % (resultpathname) zipfile = "%s.zip" % (resultpathname) tarball_fullpath = "%s.tar.gz" % (outpath_result) zipfile_fullpath = "%s.zip" % (outpath_result) mapfile = "%s/seqid_index_map.txt" % (outpath_result) finished_model_file = "%s/finished_models.txt" % (outpath_result) timefile = "%s/time.txt" % (outpath_result) tmp_outpath_result = "%s/%s" % (tmpdir, resultpathname) isOK = True if os.path.exists(tmp_outpath_result): shutil.rmtree(tmp_outpath_result) try: os.makedirs(tmp_outpath_result) isOK = True except OSError: msg = "Failed to create folder %s" % (tmp_outpath_result) myfunc.WriteFile(msg + "\n", runjob_errfile, "a") isOK = False pass if os.path.exists(outpath_result): shutil.rmtree(outpath_result) try: os.makedirs(outpath_result) isOK = True except OSError: msg = "Failed to create folder %s" % (outpath_result) myfunc.WriteFile(msg + "\n", runjob_errfile, "a") isOK = False pass if isOK: try: open(finished_model_file, 'w').close() except: pass #first getting result from caches # cache profiles for sequences, but do not cache predictions for models webserver_common.WriteDateTimeTagFile(starttagfile, runjob_logfile, runjob_errfile) # ================================== numModel = 0 modelFileList = [] if seqfile != "": # if the fasta sequence is supplied, all models should be using this sequence subfoldername_profile = "profile_%d" % (0) outpath_profile = "%s/%s" % (outpath_result, subfoldername_profile) CreateProfile(seqfile, outpath_profile, outpath_result, tmp_outpath_result, timefile, runjob_errfile) # run proq3 for models modelList = myfunc.ReadPDBModel(modelfile) numModel = len(modelList) for ii in xrange(len(modelList)): model = modelList[ii] tmp_model_file = "%s/query_%d.pdb" % (tmp_outpath_result, ii) myfunc.WriteFile(model + "\n", tmp_model_file) profilename = "%s/%s" % (outpath_profile, "query.fasta") subfoldername_this_model = "model_%d" % (ii) outpath_this_model = "%s/%s" % (outpath_result, subfoldername_this_model) modelinfo = ScoreModel(query_para, tmp_model_file, outpath_this_model, profilename, outpath_result, tmp_outpath_result, timefile, runjob_errfile) myfunc.WriteFile("\t".join(modelinfo) + "\n", finished_model_file, "a") modelFileList.append( "%s/%s" % (outpath_this_model, "query_%d.pdb" % (ii))) else: # no seqfile supplied, sequences are obtained from the model file modelList = myfunc.ReadPDBModel(modelfile) numModel = len(modelList) for ii in xrange(len(modelList)): model = modelList[ii] tmp_model_file = "%s/query_%d.pdb" % (tmp_outpath_result, ii) myfunc.WriteFile(model + "\n", tmp_model_file) subfoldername_this_model = "model_%d" % (ii) tmp_outpath_this_model = "%s/%s" % (tmp_outpath_result, subfoldername_this_model) if not os.path.exists(tmp_outpath_this_model): os.makedirs(tmp_outpath_this_model) tmp_seqfile = "%s/query.fasta" % (tmp_outpath_this_model) cmd = [pdb2aa_script, tmp_model_file] g_params['runjob_log'].append(" ".join(cmd)) try: rmsg = subprocess.check_output(cmd) g_params['runjob_log'].append( "extracting sequence from modelfile:\n" + rmsg + "\n") except subprocess.CalledProcessError, e: g_params['runjob_err'].append(str(e) + "\n") g_params['runjob_err'].append(rmsg + "\n") if rmsg != "": myfunc.WriteFile(">seq\n" + rmsg.strip(), tmp_seqfile) subfoldername_profile = "profile_%d" % (ii) outpath_profile = "%s/%s" % (outpath_result, subfoldername_profile) CreateProfile(tmp_seqfile, outpath_profile, outpath_result, tmp_outpath_result, timefile, runjob_errfile) outpath_this_model = "%s/%s" % (outpath_result, subfoldername_this_model) profilename = "%s/%s" % (outpath_profile, "query.fasta") modelinfo = ScoreModel(query_para, tmp_model_file, outpath_this_model, profilename, outpath_result, tmp_outpath_result, timefile, runjob_errfile) myfunc.WriteFile("\t".join(modelinfo) + "\n", finished_model_file, "a") modelFileList.append( "%s/%s" % (outpath_this_model, "query_%d.pdb" % (ii))) all_end_time = time.time() all_runtime_in_sec = all_end_time - all_begin_time if len(g_params['runjob_log']) > 0: rt_msg = myfunc.WriteFile("\n".join(g_params['runjob_log']) + "\n", runjob_logfile, "a") if rt_msg: g_params['runjob_err'].append(rt_msg) webserver_common.WriteDateTimeTagFile(finishtagfile, runjob_logfile, runjob_errfile) # now write the text output to a single file #statfile = "%s/%s"%(outpath_result, "stat.txt") statfile = "" dumped_resultfile = "%s/%s" % (outpath_result, "query.proq3.txt") proq3opt = GetProQ3Option(query_para) webserver_common.WriteProQ3TextResultFile(dumped_resultfile, query_para, modelFileList, all_runtime_in_sec, g_params['base_www_url'], proq3opt, statfile=statfile) # now making zip instead (for windows users) # note that zip rq will zip the real data for symbolic links os.chdir(outpath) # cmd = ["tar", "-czf", tarball, resultpathname] cmd = ["zip", "-rq", zipfile, resultpathname] try: subprocess.check_output(cmd) except subprocess.CalledProcessError, e: g_params['runjob_err'].append(str(e)) pass
] + proq3opt g_params['runjob_log'].append(" ".join(cmd)) cmdline = " ".join(cmd) begin_time = time.time() try: rmsg = subprocess.check_output(cmd) g_params['runjob_log'].append("model scoring:\n" + rmsg + "\n") except subprocess.CalledProcessError, e: g_params['runjob_err'].append(str(e) + "\n") g_params['runjob_err'].append("cmdline: " + cmdline + "\n") g_params['runjob_err'].append("model scoring:\n" + rmsg + "\n") pass end_time = time.time() runtime_in_sec = end_time - begin_time msg = "%s\t%f\n" % (subfoldername_this_model, runtime_in_sec) myfunc.WriteFile(msg, timefile, "a") if os.path.exists(tmp_outpath_this_model): cmd = ["mv", "-f", tmp_outpath_this_model, outpath_this_model] isCmdSuccess = False try: subprocess.check_output(cmd) isCmdSuccess = True except subprocess.CalledProcessError, e: msg = "Failed to move result from %s to %s." % ( tmp_outpath_this_model, outpath_this_model) g_params['runjob_err'].append(msg) g_params['runjob_err'].append(str(e) + "\n") pass modelfile = "%s/query_%d.pdb" % (outpath_this_model, modelidx) globalscorefile = "%s.proq3.%s.global" % (modelfile, method_quality) if not os.path.exists(globalscorefile):
def ScoreModel( query_para, model_file, outpath_this_model, profilename, outpath_result, #{{{ tmp_outpath_result, timefile, runjob_errfile): subfoldername_this_model = os.path.basename(outpath_this_model) modelidx = int(subfoldername_this_model.split("model_")[1]) try: method_quality = query_para['method_quality'] except KeyError: method_quality = 'sscore' rmsg = "" tmp_outpath_this_model = "%s/%s" % (tmp_outpath_result, subfoldername_this_model) proq3opt = GetProQ3Option(query_para) cmd = [ runscript, "-profile", profilename, "-outpath", tmp_outpath_this_model, model_file ] + proq3opt g_params['runjob_log'].append(" ".join(cmd)) cmdline = " ".join(cmd) begin_time = time.time() try: rmsg = subprocess.check_output(cmd) g_params['runjob_log'].append("model scoring:\n" + rmsg + "\n") except subprocess.CalledProcessError as e: g_params['runjob_err'].append(str(e) + "\n") g_params['runjob_err'].append("cmdline: " + cmdline + "\n") g_params['runjob_err'].append("model scoring:\n" + rmsg + "\n") pass end_time = time.time() runtime_in_sec = end_time - begin_time msg = "%s\t%f\n" % (subfoldername_this_model, runtime_in_sec) myfunc.WriteFile(msg, timefile, "a") if os.path.exists(tmp_outpath_this_model): cmd = ["mv", "-f", tmp_outpath_this_model, outpath_this_model] isCmdSuccess = False try: subprocess.check_output(cmd) isCmdSuccess = True except subprocess.CalledProcessError as e: msg = "Failed to move result from %s to %s." % ( tmp_outpath_this_model, outpath_this_model) g_params['runjob_err'].append(msg) g_params['runjob_err'].append(str(e) + "\n") pass modelfile = "%s/query_%d.pdb" % (outpath_this_model, modelidx) globalscorefile = "%s.proq3.%s.global" % (modelfile, method_quality) if not os.path.exists(globalscorefile): globalscorefile = "%s.proq3.global" % (modelfile) (globalscore, itemList) = webserver_common.ReadProQ3GlobalScore(globalscorefile) modelseqfile = "%s/query_%d.pdb.fasta" % (outpath_this_model, modelidx) modellength = myfunc.GetSingleFastaLength(modelseqfile) modelinfo = [ subfoldername_this_model, str(modellength), str(runtime_in_sec) ] if globalscore: for i in range(len(itemList)): modelinfo.append(str(globalscore[itemList[i]])) return modelinfo
def SubmitJobToQueue( jobid, datapath, outpath, nummodel, nummodel_this_user, email, #{{{ host_ip, base_www_url): myfunc.WriteFile("Entering SubmitJobToQueue()\n", g_params['debugfile'], "a") modelfile = "%s/query.pdb" % (datapath) seqfile = "%s/query.fa" % (datapath) if nummodel == -1: nummodel = myfunc.ReadFile(modelfile).count("\nENDMDL") if nummodel == 0: nummodel = 1 if nummodel_this_user == -1: nummodel_this_user = nummodel query_parafile = "%s/query.para.txt" % (outpath) query_para = {} content = myfunc.ReadFile(query_parafile) para_str = content if content != "": query_para = json.loads(content) try: name_software = query_para['name_software'] except KeyError: name_software = "proq3" runjob = "%s %s/run_job.py" % (python_exec, rundir) scriptfile = "%s/runjob,%s,%s,%s,%s,%d.sh" % ( outpath, name_software, jobid, host_ip, email, nummodel) code_str_list = [] code_str_list.append("#!/bin/bash") code_str_list.append("source %s/bin/activate" % (virt_env_path)) cmdline = "%s %s -outpath %s -tmpdir %s -jobid %s " % ( runjob, modelfile, outpath, datapath, jobid) if email != "": cmdline += "-email \"%s\" " % (email) if os.path.exists(seqfile): cmdline += "-fasta \"%s\" " % (seqfile) if base_www_url != "": cmdline += "-baseurl \"%s\" " % (base_www_url) if g_params['isForceRun']: cmdline += "-force " code_str_list.append(cmdline) code = "\n".join(code_str_list) msg = "Write scriptfile %s" % (scriptfile) myfunc.WriteFile(msg + "\n", g_params['debugfile'], "a") myfunc.WriteFile(code, scriptfile) os.chmod(scriptfile, 0755) myfunc.WriteFile("Getting priority" + "\n", g_params['debugfile'], "a") priority = myfunc.GetSuqPriority(nummodel_this_user) if email in vip_user_list: priority = 999999999.0 myfunc.WriteFile("priority=%d\n" % (priority), g_params['debugfile'], "a") st1 = SubmitSuqJob(suq_basedir, datapath, outpath, priority, scriptfile) return st1