Exemple #1
0
def SubmitQueryToLocalQueue(query, tmpdir, rstdir, isOnlyGetCache=False):  #{{{
    scriptfile = "%s/app/submit_job_to_queue.py" % (SITE_ROOT)
    rstdir = "%s/%s" % (path_result, query['jobid'])
    debugfile = "%s/debug.log" % (rstdir)  #this log only for debugging
    runjob_logfile = "%s/runjob.log" % (rstdir)
    runjob_errfile = "%s/runjob.err" % (rstdir)
    failedtagfile = "%s/%s" % (rstdir, "runjob.failed")
    rmsg = ""

    cmd = [
        python_exec, scriptfile, "-nseq",
        "%d" % query['numseq'], "-nseq-this-user",
        "%d" % query['numseq_this_user'], "-jobid", query['jobid'], "-outpath",
        rstdir, "-datapath", tmpdir, "-baseurl", query['base_www_url']
    ]
    if query['email'] != "":
        cmd += ["-email", query['email']]
    if query['client_ip'] != "":
        cmd += ["-host", query['client_ip']]
    if query['isForceRun']:
        cmd += ["-force"]
    if isOnlyGetCache:
        cmd += ["-only-get-cache"]

    (isSuccess, t_runtime) = webcom.RunCmd(cmd, runjob_logfile, runjob_errfile)
    if not isSuccess:
        webcom.WriteDateTimeTagFile(failedtagfile, runjob_logfile,
                                    runjob_errfile)
        return 1
    else:
        return 0
Exemple #2
0
def RunJob(infile, outpath, tmpdir, email, jobid, g_params):  #{{{
    all_begin_time = time.time()

    rootname = os.path.basename(os.path.splitext(infile)[0])
    starttagfile = "%s/runjob.start" % (outpath)
    runjob_errfile = "%s/runjob.err" % (outpath)
    runjob_logfile = "%s/runjob.log" % (outpath)
    app_logfile = "%s/app.log" % (outpath)
    finishtagfile = "%s/runjob.finish" % (outpath)
    rmsg = ""

    resultpathname = jobid

    outpath_result = "%s/%s" % (outpath, resultpathname)
    tmp_outpath_result = "%s/%s" % (tmpdir, resultpathname)

    tarball = "%s.tar.gz" % (resultpathname)
    zipfile = "%s.zip" % (resultpathname)
    tarball_fullpath = "%s.tar.gz" % (outpath_result)
    zipfile_fullpath = "%s.zip" % (outpath_result)
    resultfile_text = "%s/%s" % (outpath_result, "query.top")
    mapfile = "%s/seqid_index_map.txt" % (outpath_result)
    finished_seq_file = "%s/finished_seqs.txt" % (outpath_result)
    finished_idx_file = "%s/finished_seqindex.txt" % (outpath)

    for folder in [outpath_result, tmp_outpath_result]:
        try:
            os.makedirs(folder)
        except OSError:
            msg = "Failed to create folder %s" % (folder)
            myfunc.WriteFile(msg + "\n", gen_errfile, "a")
            return 1

    try:
        open(finished_seq_file, 'w').close()
    except:
        pass
#first getting result from caches
# ==================================

    maplist = []
    maplist_simple = []
    toRunDict = {}
    hdl = myfunc.ReadFastaByBlock(infile, method_seqid=0, method_seq=0)
    if hdl.failure:
        isOK = False
    else:
        webcom.WriteDateTimeTagFile(starttagfile, runjob_logfile,
                                    runjob_errfile)

        recordList = hdl.readseq()
        cnt = 0
        origpath = os.getcwd()
        while recordList != None:
            for rd in recordList:
                isSkip = False
                # temp outpath for the sequence is always seq_0, and I feed
                # only one seq a time to the workflow
                tmp_outpath_this_seq = "%s/%s" % (tmp_outpath_result,
                                                  "seq_%d" % 0)
                outpath_this_seq = "%s/%s" % (outpath_result, "seq_%d" % cnt)
                subfoldername_this_seq = "seq_%d" % (cnt)
                if os.path.exists(tmp_outpath_this_seq):
                    try:
                        shutil.rmtree(tmp_outpath_this_seq)
                    except OSError:
                        pass

                maplist.append(
                    "%s\t%d\t%s\t%s" %
                    ("seq_%d" % cnt, len(rd.seq), rd.description, rd.seq))
                maplist_simple.append(
                    "%s\t%d\t%s" %
                    ("seq_%d" % cnt, len(rd.seq), rd.description))
                if not g_params['isForceRun']:
                    md5_key = hashlib.md5(rd.seq.encode('utf-8')).hexdigest()
                    subfoldername = md5_key[:2]
                    cachedir = "%s/%s/%s" % (path_cache, subfoldername,
                                             md5_key)
                    zipfile_cache = cachedir + ".zip"
                    if os.path.exists(cachedir) or os.path.exists(
                            zipfile_cache):
                        if os.path.exists(cachedir):
                            try:
                                shutil.copytree(cachedir, outpath_this_seq)
                            except Exception as e:
                                msg = "Failed to copytree  %s -> %s" % (
                                    cachedir, outpath_this_seq)
                                date_str = time.strftime(FORMAT_DATETIME)
                                myfunc.WriteFile(
                                    "[%s] %s with errmsg=%s\n" %
                                    (date_str, msg, str(e)), runjob_errfile,
                                    "a")
                        elif os.path.exists(zipfile_cache):
                            cmd = [
                                "unzip", zipfile_cache, "-d", outpath_result
                            ]
                            webcom.RunCmd(cmd, runjob_logfile, runjob_errfile)
                            shutil.move("%s/%s" % (outpath_result, md5_key),
                                        outpath_this_seq)

                        checkfile = "%s/query.predict.png" % (outpath_this_seq)
                        fafile_this_seq = '%s/seq.fa' % (outpath_this_seq)

                        if os.path.exists(outpath_this_seq) and os.path.exists(
                                checkfile):
                            info_finish = webcom.GetInfoFinish_Boctopus2(
                                outpath_this_seq,
                                cnt,
                                len(rd.seq),
                                rd.description,
                                source_result="cached",
                                runtime=0.0)
                            myfunc.WriteFile("\t".join(info_finish) + "\n",
                                             finished_seq_file,
                                             "a",
                                             isFlush=True)
                            myfunc.WriteFile("%d\n" % (cnt),
                                             finished_idx_file,
                                             "a",
                                             isFlush=True)
                            isSkip = True

                if not isSkip:
                    # first try to delete the outfolder if exists
                    if os.path.exists(outpath_this_seq):
                        try:
                            shutil.rmtree(outpath_this_seq)
                        except OSError:
                            pass
                    origIndex = cnt
                    numTM = 0
                    toRunDict[origIndex] = [rd.seq, numTM, rd.description
                                            ]  #init value for numTM is 0

                cnt += 1
            recordList = hdl.readseq()
        hdl.close()
    myfunc.WriteFile("\n".join(maplist_simple) + "\n", mapfile)

    if not g_params['isOnlyGetCache']:
        torun_all_seqfile = "%s/%s" % (tmp_outpath_result, "query.torun.fa")
        dumplist = []
        for key in toRunDict:
            top = toRunDict[key][0]
            dumplist.append(">%s\n%s" % (str(key), top))
        myfunc.WriteFile("\n".join(dumplist) + "\n", torun_all_seqfile, "w")
        del dumplist

        sortedlist = sorted(list(toRunDict.items()),
                            key=lambda x: x[1][1],
                            reverse=True)
        #format of sortedlist [(origIndex: [seq, numTM, description]), ...]

        # submit sequences one by one to the workflow according to orders in
        # sortedlist

        for item in sortedlist:
            origIndex = item[0]
            seq = item[1][0]
            description = item[1][2]

            subfoldername_this_seq = "seq_%d" % (origIndex)
            outpath_this_seq = "%s/%s" % (outpath_result,
                                          subfoldername_this_seq)
            tmp_outpath_this_seq = "%s/%s" % (tmp_outpath_result, "seq_%d" %
                                              (0))
            if os.path.exists(tmp_outpath_this_seq):
                try:
                    shutil.rmtree(tmp_outpath_this_seq)
                except OSError:
                    pass

            seqfile_this_seq = "%s/%s" % (tmp_outpath_result, "query_%d.fa" %
                                          (origIndex))
            seqcontent = ">query_%d\n%s\n" % (origIndex, seq)
            myfunc.WriteFile(seqcontent, seqfile_this_seq, "w")

            if not os.path.exists(seqfile_this_seq):
                msg = "Failed to generate seq file for index %d" % (origIndex)
                date_str = time.strftime(FORMAT_DATETIME)
                myfunc.WriteFile("[%s] %s\n" % (date_str, msg), runjob_errfile,
                                 "a", True)
                continue

            cmd = [runscript, seqfile_this_seq, tmp_outpath_result]
            (t_success,
             runtime_in_sec) = webcom.RunCmd(cmd, runjob_logfile,
                                             runjob_errfile, True)

            aaseqfile = "%s/seq.fa" % (tmp_outpath_this_seq)
            if not os.path.exists(aaseqfile):
                seqcontent = ">%s\n%s\n" % (description, seq)
                myfunc.WriteFile(seqcontent, aaseqfile, "w")

            if os.path.exists(tmp_outpath_this_seq):
                cmd = ["mv", "-f", tmp_outpath_this_seq, outpath_this_seq]
                (isCmdSuccess,
                 t_runtime) = webcom.RunCmd(cmd, runjob_logfile,
                                            runjob_errfile)
                timefile = "%s/time.txt" % (tmp_outpath_result)
                targetfile = "%s/time.txt" % (outpath_this_seq)
                if os.path.exists(timefile) and os.path.exists(
                        outpath_this_seq):
                    try:
                        shutil.move(timefile, targetfile)
                    except:
                        msg = "Failed to move %s/time.txt" % (
                            tmp_outpath_result)
                        date_str = time.strftime(FORMAT_DATETIME)
                        myfunc.WriteFile("[%s] %s\n" % (date_str, msg),
                                         runjob_errfile, "a", True)
                        pass

                if isCmdSuccess:
                    runtime = runtime_in_sec  #in seconds
                    info_finish = webcom.GetInfoFinish_Boctopus2(
                        outpath_this_seq,
                        origIndex,
                        len(seq),
                        description,
                        source_result="newrun",
                        runtime=runtime)
                    myfunc.WriteFile("\t".join(info_finish) + "\n",
                                     finished_seq_file,
                                     "a",
                                     isFlush=True)
                    # now write the text output for this seq

                    info_this_seq = "%s\t%d\t%s\t%s" % (
                        "seq_%d" % origIndex, len(seq), description, seq)
                    resultfile_text_this_seq = "%s/%s" % (outpath_this_seq,
                                                          "query.result.txt")
                    webcom.WriteBoctopusTextResultFile(
                        resultfile_text_this_seq, outpath_result,
                        [info_this_seq], runtime_in_sec,
                        g_params['base_www_url'])
                    # create or update the md5 cache
                    # create cache only on the front-end
                    figurefile = "%s/plot/query_0.png" % (outpath_this_seq)
                    # Note: do not create cache is figure file does not exist
                    if webcom.IsFrontEndNode(g_params['base_www_url']
                                             ) and os.path.exists(figurefile):
                        md5_key = hashlib.md5(seq.encode('utf-8')).hexdigest()
                        subfoldername = md5_key[:2]
                        md5_subfolder = "%s/%s" % (path_cache, subfoldername)
                        cachedir = "%s/%s/%s" % (path_cache, subfoldername,
                                                 md5_key)
                        if os.path.exists(cachedir):
                            try:
                                shutil.rmtree(cachedir)
                            except:
                                msg = "Failed to shutil.rmtree(%s)" % (
                                    cachedir)
                                date_str = time.strftime(FORMAT_DATETIME)
                                myfunc.WriteFile("[%s] %s\n" % (date_str, msg),
                                                 runjob_errfile, "a", True)
                                pass

                        if not os.path.exists(md5_subfolder):
                            try:
                                os.makedirs(md5_subfolder)
                            except:
                                pass

                        if os.path.exists(md5_subfolder
                                          ) and not os.path.exists(cachedir):
                            cmd = ["mv", "-f", outpath_this_seq, cachedir]
                            webcom.RunCmd(cmd, runjob_logfile, runjob_errfile)

                        if not os.path.exists(
                                outpath_this_seq) and os.path.exists(cachedir):
                            rela_path = os.path.relpath(
                                cachedir, outpath_result)  #relative path
                            try:
                                os.chdir(outpath_result)
                                os.symlink(rela_path, subfoldername_this_seq)
                            except:
                                pass

    all_end_time = time.time()
    all_runtime_in_sec = all_end_time - all_begin_time

    if not g_params['isOnlyGetCache'] or len(toRunDict) == 0:
        # now write the text output to a single file
        statfile = "%s/%s" % (outpath_result, "stat.txt")
        webcom.WriteBoctopusTextResultFile(resultfile_text,
                                           outpath_result,
                                           maplist,
                                           all_runtime_in_sec,
                                           g_params['base_www_url'],
                                           statfile=statfile)

        # now making zip instead (for windows users)
        # note that zip rq will zip the real data for symbolic links
        os.chdir(outpath)
        cmd = ["zip", "-rq", zipfile, resultpathname]
        webcom.RunCmd(cmd, runjob_logfile, runjob_errfile)

        # write finish tag file
        if os.path.exists(finished_seq_file):
            webcom.WriteDateTimeTagFile(finishtagfile, runjob_logfile,
                                        runjob_errfile)

        isSuccess = False
        if (os.path.exists(finishtagfile)
                and os.path.exists(zipfile_fullpath)):
            isSuccess = True
        else:
            isSuccess = False
            failedtagfile = "%s/runjob.failed" % (outpath)
            webcom.WriteDateTimeTagFile(failedtagfile, runjob_logfile,
                                        runjob_errfile)

# send the result to email
# do not sendmail at the cloud VM
        if webcom.IsFrontEndNode(g_params['base_www_url']
                                 ) and myfunc.IsValidEmailAddress(email):
            if isSuccess:
                finish_status = "success"
            else:
                finish_status = "failed"
            webcom.SendEmail_on_finish(
                jobid,
                g_params['base_www_url'],
                finish_status,
                name_server="BOCTOPUS2",
                from_email="[email protected])",
                to_email=email,
                contact_email=contact_email,
                logfile=runjob_logfile,
                errfile=runjob_errfile)

    if os.path.exists(runjob_errfile) and os.path.getsize(runjob_errfile) > 1:
        return 1
    else:
        date_str = time.strftime(FORMAT_DATETIME)
        try:
            shutil.rmtree(tmpdir)
            msg = "rmtree(%s)" % (tmpdir)
            myfunc.WriteFile("[%s] %s\n" % (date_str, msg), runjob_logfile,
                             "a", True)
        except Exception as e:
            msg = "Failed to rmtree(%s)" % (tmpdir)
            myfunc.WriteFile("[%s] %s\n" % (date_str, msg), runjob_errfile,
                             "a", True)
            pass
        return 0
Exemple #3
0
def RunJob_msa(infile, outpath, tmpdir, email, jobid, g_params):  #{{{
    all_begin_time = time.time()

    rootname = os.path.basename(os.path.splitext(infile)[0])
    runjob_errfile = "%s/runjob.err" % (outpath)
    runjob_logfile = "%s/runjob.log" % (outpath)
    starttagfile = "%s/runjob.start" % (outpath)
    finishtagfile = "%s/runjob.finish" % (outpath)
    failtagfile = "%s/runjob.failed" % (outpath)
    rmsg = ""
    qdinit_start_tagfile = "%s/runjob.qdinit.start" % (outpath)

    # if the daemon starts to process the job before the run_job.py running
    # in the local queue, skip it
    if os.path.exists(qdinit_start_tagfile):
        return 0

    resultpathname = jobid

    outpath_result = "%s/%s" % (outpath, resultpathname)
    tmp_outpath_result = "%s/%s" % (tmpdir, resultpathname)

    tarball = "%s.tar.gz" % (resultpathname)
    zipfile = "%s.zip" % (resultpathname)
    tarball_fullpath = "%s.tar.gz" % (outpath_result)
    zipfile_fullpath = "%s.zip" % (outpath_result)
    resultfile_text = "%s/%s" % (outpath_result, "query.result.txt")
    mapfile = "%s/seqid_index_map.txt" % (outpath_result)
    finished_seq_file = "%s/finished_seqs.txt" % (outpath_result)
    finished_idx_file = "%s/finished_seqindex.txt" % (outpath)

    for folder in [outpath_result, tmp_outpath_result]:
        try:
            os.makedirs(folder)
        except OSError:
            msg = "Failed to create folder %s" % (folder)
            myfunc.WriteFile(msg + "\n", gen_errfile, "a")
            return 1

    try:
        open(finished_seq_file, 'w').close()
    except:
        pass
#first getting result from caches
# ==================================
    maplist = []
    toRunDict = {}
    hdl = myfunc.ReadFastaByBlock(infile, method_seqid=0, method_seq=0)
    if hdl.failure:
        isOK = False
    else:
        webcom.WriteDateTimeTagFile(starttagfile, runjob_logfile,
                                    runjob_errfile)
        cnt = 0
        origpath = os.getcwd()
        con = sqlite3.connect(db_cache_SCAMPI2MSA)
        with con:
            cur = con.cursor()
            cur.execute("""
                CREATE TABLE IF NOT EXISTS %s
                (
                    md5 VARCHAR(100),
                    seq VARCHAR(30000),
                    top VARCHAR(30000),
                    PRIMARY KEY (md5)
                )""" % (dbmsa_tablename))
            recordList = hdl.readseq()
            while recordList != None:
                for rd in recordList:
                    isSkip = False
                    if not g_params['isForceRun']:
                        md5_key = hashlib.md5(
                            rd.seq.encode('utf-8')).hexdigest()
                        cmd = "SELECT md5, seq, top FROM %s WHERE md5 =  \"%s\"" % (
                            dbmsa_tablename, md5_key)
                        cur.execute(cmd)
                        rows = cur.fetchall()
                        for row in rows:
                            top = row[2]
                            numTM = myfunc.CountTM(top)
                            # info_finish has 8 items
                            info_finish = [
                                "seq_%d" % cnt,
                                str(len(rd.seq)),
                                str(numTM), "cached",
                                str(0.0), rd.description, rd.seq, top
                            ]
                            myfunc.WriteFile("\t".join(info_finish) + "\n",
                                             finished_seq_file,
                                             "a",
                                             isFlush=True)
                            myfunc.WriteFile("%d\n" % (cnt),
                                             finished_idx_file,
                                             "a",
                                             isFlush=True)
                            isSkip = True

                    if not isSkip:
                        # first try to delete the outfolder if exists
                        origIndex = cnt
                        numTM = 0
                        toRunDict[origIndex] = [rd.seq, numTM, rd.description
                                                ]  #init value for numTM is 0
                    cnt += 1
                recordList = hdl.readseq()
            hdl.close()

    if not g_params['isOnlyGetCache']:
        torun_all_seqfile = "%s/%s" % (tmp_outpath_result, "query.torun.fa")
        dumplist = []
        for key in toRunDict:
            top = toRunDict[key][0]
            dumplist.append(">%s\n%s" % (str(key), top))
        myfunc.WriteFile("\n".join(dumplist) + "\n", torun_all_seqfile, "w")
        del dumplist
        sortedlist = sorted(list(toRunDict.items()),
                            key=lambda x: x[1][1],
                            reverse=True)
        #format of sortedlist [(origIndex: [seq, numTM, description]), ...]
        # submit sequences one by one to the workflow according to orders in
        # sortedlist
        for item in sortedlist:
            origIndex = item[0]
            seq = item[1][0]
            description = item[1][2]

            outpath_this_seq = "%s/%s" % (outpath_result, "seq_%d" % origIndex)
            tmp_outpath_this_seq = "%s/%s" % (tmp_outpath_result, "seq_%d" %
                                              (0))
            if os.path.exists(tmp_outpath_this_seq):
                try:
                    shutil.rmtree(tmp_outpath_this_seq)
                except OSError:
                    pass
            try:
                os.makedirs(tmp_outpath_this_seq)
            except OSError:
                g_params['runjob_err'].append(
                    "Failed to create the tmp_outpath_this_seq %s" %
                    (tmp_outpath_this_seq))
                continue

            seqfile_this_seq = "%s/%s" % (tmp_outpath_result, "query_%d.fa" %
                                          (origIndex))
            seqcontent = ">%d\n%s\n" % (origIndex, seq)
            myfunc.WriteFile(seqcontent, seqfile_this_seq, "w")

            if not os.path.exists(seqfile_this_seq):
                g_params['runjob_err'].append(
                    "failed to generate seq index %d" % (origIndex))
                continue

            if not os.path.exists("%s/seq.fa" % (tmp_outpath_this_seq)):
                try:
                    shutil.copyfile(seqfile_this_seq,
                                    "%s/seq.fa" % (tmp_outpath_this_seq))
                except OSError:
                    pass

            numCPU = 4
            outtopfile = "%s/query.top" % (tmp_outpath_this_seq)
            cmd = [
                runscript_msa, seqfile_this_seq, outtopfile, blastdir, blastdb
            ]
            (t_success, runtime_in_sec) = webcom.RunCmd(cmd,
                                                        runjob_logfile,
                                                        runjob_errfile,
                                                        verbose=True)

            if os.path.exists(tmp_outpath_this_seq):
                cmd = ["mv", "-f", tmp_outpath_this_seq, outpath_this_seq]
                (isCmdSuccess, t_runtime) = webcom.RunCmd(cmd,
                                                          runjob_logfile,
                                                          runjob_errfile,
                                                          verbose=True)

                if isCmdSuccess:
                    runtime = runtime_in_sec  #in seconds
                    predfile = "%s/query.top" % (outpath_this_seq)
                    (seqid, seqanno, top) = myfunc.ReadSingleFasta(predfile)
                    numTM = myfunc.CountTM(top)
                    # info_finish has 8 items
                    info_finish = [
                        "seq_%d" % origIndex,
                        str(len(seq)),
                        str(numTM), "newrun",
                        str(runtime), description, seq, top
                    ]
                    myfunc.WriteFile("\t".join(info_finish) + "\n",
                                     finished_seq_file,
                                     "a",
                                     isFlush=True)

    all_end_time = time.time()
    all_runtime_in_sec = all_end_time - all_begin_time

    if len(g_params['runjob_log']) > 0:
        rt_msg = myfunc.WriteFile("\n".join(g_params['runjob_log']) + "\n",
                                  runjob_logfile, "a")
        if rt_msg:
            g_params['runjob_err'].append(rt_msg)

    if not g_params['isOnlyGetCache'] or len(toRunDict) == 0:
        if os.path.exists(finished_seq_file):
            webcom.WriteDateTimeTagFile(finishtagfile, runjob_logfile,
                                        runjob_errfile)

# now write the text output to a single file
        dumped_resultfile = "%s/%s" % (outpath_result, "query.top")
        statfile = "%s/%s" % (outpath_result, "stat.txt")
        webcom.WriteSCAMPI2MSATextResultFile(dumped_resultfile,
                                             outpath_result,
                                             maplist,
                                             all_runtime_in_sec,
                                             g_params['base_www_url'],
                                             statfile=statfile)

        # now making zip instead (for windows users)
        pwd = os.getcwd()
        os.chdir(outpath)
        cmd = ["zip", "-rq", zipfile, resultpathname]
        webcom.RunCmd(cmd, runjob_logfile, runjob_errfile)
        os.chdir(pwd)

        isSuccess = False
        if (os.path.exists(finishtagfile)
                and os.path.exists(zipfile_fullpath)):
            isSuccess = True
            # delete the tmpdir if succeeded
            shutil.rmtree(tmpdir)  #DEBUG, keep tmpdir
        else:
            isSuccess = False
            webcom.WriteDateTimeTagFile(failtagfile, runjob_logfile,
                                        runjob_errfile)

        finish_status = ""  #["success", "failed", "partly_failed"]
        if isSuccess:
            finish_status = "success"
        else:
            finish_status = "failed"

# send the result to email
# do not sendmail at the cloud VM
        if webcom.IsFrontEndNode(g_params['base_www_url']
                                 ) and myfunc.IsValidEmailAddress(email):
            webcom.SendEmail_on_finish(jobid,
                                       g_params['base_www_url'],
                                       finish_status,
                                       name_server="SCAMPI2-msa",
                                       from_email="*****@*****.**",
                                       to_email=email,
                                       contact_email=contact_email,
                                       logfile=runjob_logfile,
                                       errfile=runjob_errfile)
    return 0
def RunJob(infile, outpath, tmpdir, email, jobid, g_params):  #{{{
    all_begin_time = time.time()

    rootname = os.path.basename(os.path.splitext(infile)[0])
    starttagfile = "%s/runjob.start" % (outpath)
    runjob_errfile = "%s/runjob.err" % (outpath)
    runjob_logfile = "%s/runjob.log" % (outpath)
    app_logfile = "%s/app.log" % (outpath)
    finishtagfile = "%s/runjob.finish" % (outpath)
    failedtagfile = "%s/runjob.failed" % (outpath)
    query_parafile = "%s/query.para.txt" % (outpath)

    query_para = ""
    content = myfunc.ReadFile(query_parafile)
    if content != "":
        query_para = json.loads(content)

    rmsg = ""

    resultpathname = jobid

    outpath_result = "%s/%s" % (outpath, resultpathname)
    tmp_outpath_result = "%s/%s" % (tmpdir, resultpathname)

    tarball = "%s.tar.gz" % (resultpathname)
    zipfile = "%s.zip" % (resultpathname)
    tarball_fullpath = "%s.tar.gz" % (outpath_result)
    zipfile_fullpath = "%s.zip" % (outpath_result)
    resultfile_text = "%s/%s" % (outpath_result, "query.result.txt")
    mapfile = "%s/seqid_index_map.txt" % (outpath_result)
    finished_seq_file = "%s/finished_seqs.txt" % (outpath_result)

    for folder in [outpath_result, tmp_outpath_result]:
        try:
            os.makedirs(folder)
        except OSError:
            msg = "Failed to create folder %s" % (folder)
            myfunc.WriteFile(msg + "\n", gen_errfile, "a")
            return 1
    try:
        open(finished_seq_file, 'w').close()
    except:
        pass
#first getting result from caches
# ==================================

    maplist = []
    maplist_simple = []
    toRunDict = {}
    hdl = myfunc.ReadFastaByBlock(infile, method_seqid=0, method_seq=0)
    if hdl.failure:
        isOK = False
    else:
        webcom.WriteDateTimeTagFile(starttagfile, runjob_logfile,
                                    runjob_errfile)
        recordList = hdl.readseq()
        cnt = 0
        origpath = os.getcwd()
        while recordList != None:
            for rd in recordList:
                isSkip = False
                # temp outpath for the sequence is always seq_0, and I feed
                # only one seq a time to the workflow
                tmp_outpath_this_seq = "%s/%s" % (tmp_outpath_result,
                                                  "seq_%d" % 0)
                outpath_this_seq = "%s/%s" % (outpath_result, "seq_%d" % cnt)
                subfoldername_this_seq = "seq_%d" % (cnt)
                if os.path.exists(tmp_outpath_this_seq):
                    try:
                        shutil.rmtree(tmp_outpath_this_seq)
                    except OSError:
                        pass

                maplist.append(
                    "%s\t%d\t%s\t%s" %
                    ("seq_%d" % cnt, len(rd.seq), rd.description, rd.seq))
                maplist_simple.append(
                    "%s\t%d\t%s" %
                    ("seq_%d" % cnt, len(rd.seq), rd.description))
                if not g_params['isForceRun']:
                    md5_key = hashlib.md5(
                        (rd.seq +
                         str(query_para)).encode('utf-8')).hexdigest()
                    subfoldername = md5_key[:2]
                    cachedir = "%s/%s/%s" % (path_cache, subfoldername,
                                             md5_key)
                    zipfile_cache = cachedir + ".zip"

                    if os.path.exists(cachedir) or os.path.exists(
                            zipfile_cache):
                        if os.path.exists(cachedir):
                            try:
                                shutil.copytree(cachedir, outpath_this_seq)
                            except Exception as e:
                                msg = "Failed to copytree  %s -> %s" % (
                                    cachedir, outpath_this_seq)
                                date_str = time.strftime(FORMAT_DATETIME)
                                myfunc.WriteFile(
                                    "[%s] %s with errmsg=%s\n" %
                                    (date_str, msg, str(e)), runjob_errfile,
                                    "a")
                        elif os.path.exists(zipfile_cache):
                            cmd = [
                                "unzip", zipfile_cache, "-d", outpath_result
                            ]
                            webcom.RunCmd(cmd, runjob_logfile, runjob_errfile)
                            shutil.move("%s/%s" % (outpath_result, md5_key),
                                        outpath_this_seq)

                        if os.path.exists(outpath_this_seq):
                            info_finish = webcom.GetInfoFinish_PRODRES(
                                outpath_this_seq,
                                cnt,
                                len(rd.seq),
                                rd.description,
                                source_result="cached",
                                runtime=0.0)
                            myfunc.WriteFile("\t".join(info_finish) + "\n",
                                             finished_seq_file,
                                             "a",
                                             isFlush=True)
                            isSkip = True

                if not isSkip:
                    # first try to delete the outfolder if exists
                    if os.path.exists(outpath_this_seq):
                        try:
                            shutil.rmtree(outpath_this_seq)
                        except OSError:
                            pass
                    origIndex = cnt
                    numTM = 0
                    toRunDict[origIndex] = [rd.seq, numTM, rd.description
                                            ]  #init value for numTM is 0

                cnt += 1
            recordList = hdl.readseq()
        hdl.close()
    myfunc.WriteFile("\n".join(maplist_simple) + "\n", mapfile)

    if not g_params['isOnlyGetCache']:
        torun_all_seqfile = "%s/%s" % (tmp_outpath_result, "query.torun.fa")
        dumplist = []
        for key in toRunDict:
            top = toRunDict[key][0]
            dumplist.append(">%s\n%s" % (str(key), top))
        myfunc.WriteFile("\n".join(dumplist) + "\n", torun_all_seqfile, "w")
        del dumplist

        sortedlist = sorted(list(toRunDict.items()),
                            key=lambda x: x[1][1],
                            reverse=True)
        #format of sortedlist [(origIndex: [seq, numTM, description]), ...]

        # submit sequences one by one to the workflow according to orders in
        # sortedlist

        for item in sortedlist:
            origIndex = item[0]
            seq = item[1][0]
            description = item[1][2]

            subfoldername_this_seq = "seq_%d" % (origIndex)
            outpath_this_seq = "%s/%s" % (outpath_result,
                                          subfoldername_this_seq)
            tmp_outpath_this_seq = "%s/%s" % (tmp_outpath_result, "seq_%d" %
                                              (0))
            if os.path.exists(tmp_outpath_this_seq):
                try:
                    shutil.rmtree(tmp_outpath_this_seq)
                except OSError:
                    pass

            seqfile_this_seq = "%s/%s" % (tmp_outpath_result, "query_%d.fa" %
                                          (origIndex))
            seqcontent = ">query_%d\n%s\n" % (origIndex, seq)
            myfunc.WriteFile(seqcontent, seqfile_this_seq, "w")

            if not os.path.exists(seqfile_this_seq):
                msg = "failed to generate seq index %d" % (origIndex)
                date_str = time.strftime(g_params['FORMAT_DATETIME'])
                myfunc.WriteFile("[%s] %s\n" % (date_str, msg), runjob_errfile,
                                 "a", True)
                continue

            cmd = [
                "python", runscript, "--input", seqfile_this_seq, "--output",
                tmp_outpath_this_seq, "--pfam-dir", path_pfamdatabase,
                "--pfamscan-script", path_pfamscanscript,
                "--fallback-db-fasta", blastdb
            ]

            if 'second_method' in query_para and query_para[
                    'second_method'] != "":
                cmd += ['--second-search', query_para['second_method']]

            if 'pfamscan_evalue' in query_para and query_para[
                    'pfamscan_evalue'] != "":
                cmd += ['--pfamscan_e-val', query_para['pfamscan_evalue']]
            elif 'pfamscan_bitscore' in query_para and query_para[
                    'pfamscan_bitscore'] != "":
                cmd += ['--pfamscan_bitscore', query_para['pfamscan_bitscore']]

            if 'pfamscan_clanoverlap' in query_para:
                if query_para['pfamscan_clanoverlap'] == False:
                    cmd += ['--pfamscan_clan-overlap', 'no']
                else:
                    cmd += ['--pfamscan_clan-overlap', 'yes']

            if 'jackhmmer_iteration' in query_para and query_para[
                    'jackhmmer_iteration'] != "":
                cmd += [
                    '--jackhmmer_max_iter', query_para['jackhmmer_iteration']
                ]

            if 'jackhmmer_threshold_type' in query_para and query_para[
                    'jackhmmer_threshold_type'] != "":
                cmd += [
                    '--jackhmmer-threshold-type',
                    query_para['jackhmmer_threshold_type']
                ]

            if 'jackhmmer_evalue' in query_para and query_para[
                    'jackhmmer_evalue'] != "":
                cmd += ['--jackhmmer_e-val', query_para['jackhmmer_evalue']]
            elif 'jackhmmer_bitscore' in query_para and query_para[
                    'jackhmmer_bitscore'] != "":
                cmd += [
                    '--jackhmmer_bit-score', query_para['jackhmmer_bitscore']
                ]

            if 'psiblast_iteration' in query_para and query_para[
                    'psiblast_iteration'] != "":
                cmd += ['--psiblast_iter', query_para['psiblast_iteration']]
            if 'psiblast_outfmt' in query_para and query_para[
                    'psiblast_outfmt'] != "":
                cmd += ['--psiblast_outfmt', query_para['psiblast_outfmt']]

            (t_success,
             runtime_in_sec) = webcom.RunCmd(cmd, runjob_logfile,
                                             runjob_errfile, True)

            aaseqfile = "%s/seq.fa" % (tmp_outpath_this_seq + os.sep +
                                       "query_0")
            if not os.path.exists(aaseqfile):
                seqcontent = ">%s\n%s\n" % (description, seq)
                myfunc.WriteFile(seqcontent, aaseqfile, "w")

            if os.path.exists(tmp_outpath_this_seq):
                cmd = [
                    "mv", "-f", tmp_outpath_this_seq + os.sep + "query_0",
                    outpath_this_seq
                ]
                isCmdSuccess = False
                (isCmdSuccess,
                 t_runtime) = webcom.RunCmd(cmd, runjob_logfile,
                                            runjob_errfile, True)

                if not 'isKeepTempFile' in query_para or query_para[
                        'isKeepTempFile'] == False:
                    try:
                        temp_result_folder = "%s/temp" % (outpath_this_seq)
                        shutil.rmtree(temp_result_folder)
                    except:
                        msg = "Failed to delete the folder %s" % (
                            temp_result_folder)
                        date_str = time.strftime(g_params['FORMAT_DATETIME'])
                        myfunc.WriteFile("[%s] %s\n" % (date_str, msg),
                                         runjob_errfile, "a", True)

                    flist = [
                        "%s/outputs/%s" % (outpath_this_seq, "Alignment.txt"),
                        "%s/outputs/%s" % (outpath_this_seq, "tableOut.txt"),
                        "%s/outputs/%s" % (outpath_this_seq, "fullOut.txt")
                    ]
                    for f in flist:
                        if os.path.exists(f):
                            try:
                                os.remove(f)
                            except:
                                msg = "Failed to delete the file %s" % (f)
                                date_str = time.strftime(
                                    g_params['FORMAT_DATETIME'])
                                myfunc.WriteFile("[%s] %s\n" % (date_str, msg),
                                                 runjob_errfile, "a", True)

                if isCmdSuccess:
                    timefile = "%s/time.txt" % (outpath_this_seq)
                    runtime = webcom.ReadRuntimeFromFile(timefile,
                                                         default_runtime=0.0)
                    info_finish = webcom.GetInfoFinish_PRODRES(
                        outpath_this_seq,
                        origIndex,
                        len(seq),
                        description,
                        source_result="newrun",
                        runtime=runtime)
                    myfunc.WriteFile("\t".join(info_finish) + "\n",
                                     finished_seq_file,
                                     "a",
                                     isFlush=True)
                    # now write the text output for this seq

                    info_this_seq = "%s\t%d\t%s\t%s" % (
                        "seq_%d" % origIndex, len(seq), description, seq)
                    resultfile_text_this_seq = "%s/%s" % (outpath_this_seq,
                                                          "query.result.txt")
                    #webcom.WriteSubconsTextResultFile(resultfile_text_this_seq,
                    #        outpath_result, [info_this_seq], runtime_in_sec, g_params['base_www_url'])
                    # create or update the md5 cache
                    # create cache only on the front-end
                    if webcom.IsFrontEndNode(g_params['base_www_url']):
                        md5_key = hashlib.md5(
                            (seq +
                             str(query_para)).encode('utf-8')).hexdigest()
                        subfoldername = md5_key[:2]
                        md5_subfolder = "%s/%s" % (path_cache, subfoldername)
                        cachedir = "%s/%s/%s" % (path_cache, subfoldername,
                                                 md5_key)

                        # copy the zipped folder to the cache path
                        origpath = os.getcwd()
                        os.chdir(outpath_result)
                        shutil.copytree("seq_%d" % (origIndex), md5_key)
                        cmd = ["zip", "-rq", "%s.zip" % (md5_key), md5_key]
                        webcom.RunCmd(cmd, runjob_logfile, runjob_logfile)
                        if not os.path.exists(md5_subfolder):
                            os.makedirs(md5_subfolder)
                        shutil.move("%s.zip" % (md5_key),
                                    "%s.zip" % (cachedir))
                        shutil.rmtree(
                            md5_key
                        )  # delete the temp folder named as md5 hash
                        os.chdir(origpath)

                        # Add the finished date to the database
                        date_str = time.strftime(FORMAT_DATETIME)
                        webcom.InsertFinishDateToDB(date_str, md5_key, seq,
                                                    finished_date_db)

    all_end_time = time.time()
    all_runtime_in_sec = all_end_time - all_begin_time

    if not g_params['isOnlyGetCache'] or len(toRunDict) == 0:
        # now write the text output to a single file
        statfile = "%s/%s" % (outpath_result, "stat.txt")
        #webcom.WriteSubconsTextResultFile(resultfile_text, outpath_result, maplist,
        #        all_runtime_in_sec, g_params['base_www_url'], statfile=statfile)

        # now making zip instead (for windows users)
        # note that zip rq will zip the real data for symbolic links
        os.chdir(outpath)
        #             cmd = ["tar", "-czf", tarball, resultpathname]
        cmd = ["zip", "-rq", zipfile, resultpathname]
        webcom.RunCmd(cmd, runjob_logfile, runjob_errfile)

        # write finish tag file
        if os.path.exists(finished_seq_file):
            webcom.WriteDateTimeTagFile(finishtagfile, runjob_logfile,
                                        runjob_errfile)

        isSuccess = False
        if (os.path.exists(finishtagfile)
                and os.path.exists(zipfile_fullpath)):
            isSuccess = True
        else:
            isSuccess = False
            webcom.WriteDateTimeTagFile(failedtagfile, runjob_logfile,
                                        runjob_errfile)

# send the result to email
# do not sendmail at the cloud VM
        if webcom.IsFrontEndNode(g_params['base_www_url']
                                 ) and myfunc.IsValidEmailAddress(email):
            if isSuccess:
                finish_status = "success"
            else:
                finish_status = "failed"
            webcom.SendEmail_on_finish(
                jobid,
                g_params['base_www_url'],
                finish_status,
                name_server="PRODRES",
                from_email="*****@*****.**",
                to_email=email,
                contact_email=contact_email,
                logfile=runjob_logfile,
                errfile=runjob_errfile)

    if os.path.exists(runjob_errfile) and os.path.getsize(runjob_errfile) > 1:
        return 1
    else:
        try:
            shutil.rmtree(tmpdir)
            msg = "rmtree(%s)" % (tmpdir)
            webcom.loginfo("rmtree(%s)" % (tmpdir), runjob_logfile)
        except Exception as e:
            msg = "Failed to rmtree(%s)" % (tmpdir)
            webcom.loginfo(
                "Failed to rmtree(%s) with error message: %s" %
                (tmpdir, str(e)), runjob_errfile)
        return 0
Exemple #5
0
def RunJob_single(infile, outpath, tmpdir, email, jobid, g_params):  #{{{
    all_begin_time = time.time()

    rootname = os.path.basename(os.path.splitext(infile)[0])
    starttagfile = "%s/runjob.start" % (outpath)
    finishtagfile = "%s/runjob.finish" % (outpath)
    failtagfile = "%s/runjob.failed" % (outpath)
    runjob_errfile = "%s/runjob.err" % (outpath)
    runjob_logfile = "%s/runjob.log" % (outpath)
    rmsg = ""

    resultpathname = jobid

    outpath_result = "%s/%s" % (outpath, resultpathname)
    tmp_outpath_result = "%s/%s" % (tmpdir, resultpathname)

    tarball = "%s.tar.gz" % (resultpathname)
    zipfile = "%s.zip" % (resultpathname)
    tarball_fullpath = "%s.tar.gz" % (outpath_result)
    zipfile_fullpath = "%s.zip" % (outpath_result)
    resultfile_text = "%s/%s" % (outpath_result, "query.result.txt")
    outfile = "%s/%s" % (outpath_result, "query.top")

    isOK = True
    try:
        os.makedirs(tmp_outpath_result)
        isOK = True
    except OSError:
        msg = "Failed to create folder %s" % (tmp_outpath_result)
        myfunc.WriteFile(msg + "\n", runjob_errfile, "a")
        isOK = False
        pass
    tmp_outfile = "%s/%s" % (tmp_outpath_result, "query.top")

    try:
        os.makedirs(outpath_result)
        isOK = True
    except OSError:
        msg = "Failed to create folder %s" % (outpath_result)
        myfunc.WriteFile(msg + "\n", runjob_errfile, "a")
        isOK = False
        pass

    if isOK:
        webcom.WriteDateTimeTagFile(starttagfile, runjob_logfile,
                                    runjob_errfile)

        cmd = [runscript_single, infile, tmp_outfile]
        (t_success, runtime_in_sec) = webcom.RunCmd(cmd,
                                                    runjob_logfile,
                                                    runjob_errfile,
                                                    verbose=True)

        if os.path.exists(tmp_outfile):
            cmd = ["mv", "-f", tmp_outfile, outfile]
            (isCmdSuccess, t_runtime) = webcom.RunCmd(cmd, runjob_logfile,
                                                      runjob_errfile)
            if isCmdSuccess:
                runtime = runtime_in_sec  #in seconds

        if len(g_params['runjob_log']) > 0:
            rt_msg = myfunc.WriteFile("\n".join(g_params['runjob_log']) + "\n",
                                      runjob_logfile, "a")
            if rt_msg:
                g_params['runjob_err'].append(rt_msg)

        if os.path.exists(outfile):
            webcom.WriteDateTimeTagFile(finishtagfile, runjob_logfile,
                                        runjob_errfile)

    isSuccess = False
    if (os.path.exists(finishtagfile) and os.path.exists(outfile)):
        isSuccess = True
        finish_status = "success"
    else:
        isSuccess = False
        finish_status = "failed"
        webcom.WriteDateTimeTagFile(failtagfile, runjob_logfile,
                                    runjob_errfile)

# send the result to email
# do not sendmail at the cloud VM
    if webcom.IsFrontEndNode(
            g_params['base_www_url']) and myfunc.IsValidEmailAddress(email):
        webcom.SendEmail_on_finish(jobid,
                                   g_params['base_www_url'],
                                   finish_status,
                                   name_server="SCAMPI2-single",
                                   from_email="*****@*****.**",
                                   to_email=email,
                                   contact_email=contact_email,
                                   logfile=runjob_logfile,
                                   errfile=runjob_errfile)
    return 0