def download(request): #{{{
info = {}
webcom.set_basic_config(request, info, g_params)
for key in ["db_prodres", "db_subcons"]:
zipfile = os.path.join(path_static, "download", f"{key}.zip")
md5file = f"{zipfile}.md5"
if os.path.exists(zipfile):
filesize = os.path.getsize(os.path.realpath(zipfile))
filesize_humanreadable = myfunc.Size_byte2human(filesize)
info[f"size_{key}"] = filesize_humanreadable
info[f"zipfile_{key}"] = os.path.basename(zipfile)
else:
info[f"zipfile_{key}"] = ""
info[f"size_{key}"] = ""
if os.path.exists(md5file):
try:
md5_key = myfunc.ReadFile(md5file).strip().split()[0]
info[f"md5_key_{key}"] = md5_key
except (OSError, IndexError):
info[f"md5_key_{key}"] = ""
else:
info[f"md5_key_{key}"] = ""
info['jobcounter'] = webcom.GetJobCounter(info)
return render(request, 'pred/download.html', info)
def get_results_eachseq(request, jobid="1", seqindex="1"): #{{{
resultdict = {}
webcom.set_basic_config(request, resultdict, g_params)
rstdir = "%s/%s" % (path_result, jobid)
outpathname = jobid
jobinfofile = "%s/jobinfo" % (rstdir)
jobinfo = myfunc.ReadFile(jobinfofile).strip()
jobinfolist = jobinfo.split("\t")
if len(jobinfolist) >= 8:
submit_date_str = jobinfolist[0]
numseq = int(jobinfolist[3])
jobname = jobinfolist[5]
email = jobinfolist[6]
method_submission = jobinfolist[7]
else:
submit_date_str = ""
numseq = 1
jobname = ""
email = ""
method_submission = "web"
status = ""
resultdict['jobid'] = jobid
resultdict['subdirname'] = seqindex
resultdict['jobname'] = jobname
resultdict['outpathname'] = os.path.basename(outpathname)
resultdict['BASEURL'] = g_params['BASEURL']
resultdict['status'] = status
resultdict['numseq'] = numseq
base_www_url = "http://" + request.META['HTTP_HOST']
resultfile = "%s/%s/%s/%s" % (rstdir, outpathname, seqindex,
"query.result.txt")
htmlfigure_file = "%s/%s/%s/plot/%s" % (rstdir, outpathname, seqindex,
"query_0.html")
if os.path.exists(htmlfigure_file):
resultdict['htmlfigure'] = "%s/%s/%s/%s/plot/%s" % (
"result", jobid, jobid, seqindex,
os.path.basename(htmlfigure_file))
else:
resultdict['htmlfigure'] = ""
if os.path.exists(rstdir):
resultdict['isResultFolderExist'] = True
else:
resultdict['isResultFolderExist'] = False
if os.path.exists(resultfile):
resultdict['resultfile'] = os.path.basename(resultfile)
else:
resultdict['resultfile'] = ""
resultdict['jobcounter'] = webcom.GetJobCounter(resultdict)
return render(request, 'pred/get_results_eachseq.html', resultdict)
def get_results(request, jobid="1"):#{{{
resultdict = {}
webcom.set_basic_config(request, resultdict, g_params)
rstdir = "%s/%s"%(path_result, jobid)
outpathname = jobid
resultfile = "%s/%s/%s/%s"%(rstdir, jobid, outpathname, "query.result.txt")
tarball = "%s/%s.tar.gz"%(rstdir, outpathname)
zipfile = "%s/%s.zip"%(rstdir, outpathname)
starttagfile = "%s/%s"%(rstdir, "runjob.start")
finishtagfile = "%s/%s"%(rstdir, "runjob.finish")
failtagfile = "%s/%s"%(rstdir, "runjob.failed")
errfile = "%s/%s"%(rstdir, "runjob.err")
query_seqfile = "%s/%s"%(rstdir, "query.fa")
raw_query_seqfile = "%s/%s"%(rstdir, "query.raw.fa")
seqid_index_mapfile = "%s/%s/%s"%(rstdir,jobid, "seqid_index_map.txt")
finished_seq_file = "%s/%s/finished_seqs.txt"%(rstdir, jobid)
statfile = "%s/%s/stat.txt"%(rstdir, jobid)
method_submission = "web"
jobinfofile = "%s/jobinfo"%(rstdir)
jobinfo = myfunc.ReadFile(jobinfofile).strip()
jobinfolist = jobinfo.split("\t")
if len(jobinfolist) >= 8:
submit_date_str = jobinfolist[0]
numseq = int(jobinfolist[3])
jobname = jobinfolist[5]
email = jobinfolist[6]
method_submission = jobinfolist[7]
else:
submit_date_str = ""
numseq = 1
jobname = ""
email = ""
method_submission = "web"
isValidSubmitDate = True
try:
submit_date = webcom.datetime_str_to_time(submit_date_str)
except ValueError:
isValidSubmitDate = False
current_time = datetime.now(timezone(TZ))
resultdict['isResultFolderExist'] = True
resultdict['errinfo'] = ""
if os.path.exists(errfile):
resultdict['errinfo'] = myfunc.ReadFile(errfile)
status = ""
queuetime = ""
runtime = ""
if not os.path.exists(rstdir):
resultdict['isResultFolderExist'] = False
resultdict['isFinished'] = False
resultdict['isFailed'] = True
resultdict['isStarted'] = False
elif os.path.exists(failtagfile):
resultdict['isFinished'] = False
resultdict['isFailed'] = True
resultdict['isStarted'] = True
status = "Failed"
start_date_str = ""
if os.path.exists(starttagfile):
start_date_str = myfunc.ReadFile(starttagfile).strip()
isValidStartDate = True
isValidFailedDate = True
try:
start_date = webcom.datetime_str_to_time(start_date_str)
except ValueError:
isValidStartDate = False
failed_date_str = myfunc.ReadFile(failtagfile).strip()
try:
failed_date = webcom.datetime_str_to_time(failed_date_str)
except ValueError:
isValidFailedDate = False
if isValidSubmitDate and isValidStartDate:
queuetime = myfunc.date_diff(submit_date, start_date)
if isValidStartDate and isValidFailedDate:
runtime = myfunc.date_diff(start_date, failed_date)
else:
resultdict['isFailed'] = False
if os.path.exists(finishtagfile):
resultdict['isFinished'] = True
resultdict['isStarted'] = True
status = "Finished"
isValidStartDate = True
isValidFinishDate = True
if os.path.exists(starttagfile):
start_date_str = myfunc.ReadFile(starttagfile).strip()
else:
start_date_str = ""
try:
start_date = webcom.datetime_str_to_time(start_date_str)
except ValueError:
isValidStartDate = False
finish_date_str = myfunc.ReadFile(finishtagfile).strip()
try:
finish_date = webcom.datetime_str_to_time(finish_date_str)
except ValueError:
isValidFinishDate = False
if isValidSubmitDate and isValidStartDate:
queuetime = myfunc.date_diff(submit_date, start_date)
if isValidStartDate and isValidFinishDate:
runtime = myfunc.date_diff(start_date, finish_date)
else:
resultdict['isFinished'] = False
if os.path.exists(starttagfile):
isValidStartDate = True
start_date_str = ""
if os.path.exists(starttagfile):
start_date_str = myfunc.ReadFile(starttagfile).strip()
try:
start_date = webcom.datetime_str_to_time(start_date_str)
except ValueError:
isValidStartDate = False
resultdict['isStarted'] = True
status = "Running"
if isValidSubmitDate and isValidStartDate:
queuetime = myfunc.date_diff(submit_date, start_date)
if isValidStartDate:
runtime = myfunc.date_diff(start_date, current_time)
else:
resultdict['isStarted'] = False
status = "Wait"
if isValidSubmitDate:
queuetime = myfunc.date_diff(submit_date, current_time)
color_status = webcom.SetColorStatus(status)
file_seq_warning = "%s/%s/%s/%s"%(SITE_ROOT, "static/result", jobid, "query.warn.txt")
seqwarninfo = ""
if os.path.exists(file_seq_warning):
seqwarninfo = myfunc.ReadFile(file_seq_warning)
seqwarninfo = seqwarninfo.strip()
resultdict['file_seq_warning'] = os.path.basename(file_seq_warning)
resultdict['seqwarninfo'] = seqwarninfo
resultdict['jobid'] = jobid
resultdict['subdirname'] = "seq_0"
resultdict['jobname'] = jobname
resultdict['outpathname'] = os.path.basename(outpathname)
resultdict['resultfile'] = os.path.basename(resultfile)
resultdict['tarball'] = os.path.basename(tarball)
resultdict['zipfile'] = os.path.basename(zipfile)
resultdict['submit_date'] = submit_date_str
resultdict['queuetime'] = queuetime
resultdict['runtime'] = runtime
resultdict['BASEURL'] = g_params['BASEURL']
resultdict['status'] = status
resultdict['color_status'] = color_status
resultdict['numseq'] = numseq
resultdict['query_seqfile'] = os.path.basename(query_seqfile)
resultdict['raw_query_seqfile'] = os.path.basename(raw_query_seqfile)
base_www_url = "http://" + request.META['HTTP_HOST']
# note that here one must add http:// in front of the url
resultdict['url_result'] = "%s/pred/result/%s"%(base_www_url, jobid)
sum_run_time = 0.0
average_run_time = float(g_params['AVERAGE_RUNTIME_PER_SEQ_IN_SEC']) # default average_run_time
num_finished = 0
cntnewrun = 0
cntcached = 0
newrun_table_list = [] # this is used for calculating the remaining time
# get seqid_index_map
if os.path.exists(finished_seq_file):
resultdict['index_table_header'] = ["No.", "ID", "Length",
"Variant", "Prediction", "Prediction score", "Severity", "Severity score", "RunTime(s)", "Source"]
index_table_content_list = []
indexmap_content = myfunc.ReadFile(finished_seq_file).split("\n")
cnt = 0
for line in indexmap_content:
strs = line.split("\t")
if len(strs)>=7:
subfolder = strs[0]
length_str = strs[1]
source = strs[4]
try:
runtime_in_sec_str = "%.1f"%(float(strs[5]))
if source == "newrun":
sum_run_time += float(strs[5])
cntnewrun += 1
elif source == "cached":
cntcached += 1
except:
runtime_in_sec_str = ""
desp = strs[6]
#Attempt to read the output prediction file in the subfolder
output_pred_file = "%s/%s/%s/output_predictions"%(rstdir, jobid, subfolder)
resultdict['output_pred_file'] = None
if os.path.exists(output_pred_file):
resultdict['output_pred_file'] = os.path.basename(output_pred_file)
prediction_lines = myfunc.ReadFile(output_pred_file).split("\n")
for pred_line in prediction_lines:
if pred_line == "": continue
print(pred_line)
print((pred_line.split('\t')))
pred_identifier = pred_line.split('\t')[0]
pred_variant = pred_line.split('\t')[1]
pred_class = pred_line.split('\t')[2]
pred_score = pred_line.split('\t')[3]
pred_severity = pred_line.split('\t')[4]
pred_sev_score = pred_line.split('\t')[5]
rank = "%d"%(cnt+1)
if cnt < g_params['MAX_ROWS_TO_SHOW_IN_TABLE']:
content_line = [rank, pred_identifier, length_str, pred_variant, pred_class,
pred_score, pred_severity, pred_sev_score, runtime_in_sec_str, source]
index_table_content_list.append(content_line)
cnt += 1
#See if the entropy csv exists in result directory
resultdict['entropy_data_file'] = None
output_entropy_csv = "%s/%s/%s/entropy_data.csv"%(rstdir, jobid, subfolder)
if os.path.exists(output_entropy_csv):
resultdict['entropy_data_file'] = os.path.basename(output_entropy_csv)
if source == "newrun":
newrun_table_list.append([rank, subfolder])
if cntnewrun > 0:
average_run_time = sum_run_time / float(cntnewrun)
resultdict['index_table_content_list'] = index_table_content_list
resultdict['indexfiletype'] = "finishedfile"
resultdict['num_finished'] = cnt
num_finished = cnt
resultdict['percent_finished'] = "%.1f"%(float(cnt)/numseq*100)
else:
resultdict['index_table_header'] = []
resultdict['index_table_content_list'] = []
resultdict['indexfiletype'] = "finishedfile"
resultdict['num_finished'] = 0
resultdict['percent_finished'] = "%.1f"%(0.0)
num_remain = numseq - num_finished
time_remain_in_sec = num_remain * average_run_time # set default value
resultdict['num_row_result_table'] = len(resultdict['index_table_content_list'])
# calculate the remaining time based on the average_runtime of the last x
# number of newrun sequences
avg_newrun_time = webcom.GetAverageNewRunTime(finished_seq_file, window=10)
if cntnewrun > 0 and avg_newrun_time >= 0:
time_remain_in_sec = int(avg_newrun_time*num_remain+0.5)
time_remain = myfunc.second_to_human(time_remain_in_sec)
resultdict['time_remain'] = time_remain
qdinittagfile = "%s/runjob.qdinit"%(rstdir)
if os.path.exists(rstdir):
resultdict['isResultFolderExist'] = True
else:
resultdict['isResultFolderExist'] = False
if numseq <= 1:
if method_submission == "web":
resultdict['refresh_interval'] = 2
else:
resultdict['refresh_interval'] = 5
else:
#resultdict['refresh_interval'] = numseq * 2
if os.path.exists(qdinittagfile):
addtime = int(math.sqrt(max(0,min(num_remain, num_finished))))+1
resultdict['refresh_interval'] = average_run_time + addtime
else:
resultdict['refresh_interval'] = 2
# get stat info
if os.path.exists(statfile):#{{{
content = myfunc.ReadFile(statfile)
lines = content.split("\n")
for line in lines:
strs = line.split()
if len(strs) >= 2:
resultdict[strs[0]] = strs[1]
percent = "%.1f"%(int(strs[1])/float(numseq)*100)
newkey = strs[0].replace('num_', 'per_')
resultdict[newkey] = percent
#}}}
resultdict['MAX_ROWS_TO_SHOW_IN_TABLE'] = g_params['MAX_ROWS_TO_SHOW_IN_TABLE']
resultdict['jobcounter'] = webcom.GetJobCounter(resultdict)
return render(request, 'pred/get_results.html', resultdict)
def download(request):#{{{
info = {}
webcom.set_basic_config(request, info, g_params)
info['jobcounter'] = webcom.GetJobCounter(info)
return render(request, 'pred/download.html', info)
def get_example(request):#{{{
info = {}
webcom.set_basic_config(request, info, g_params)
info['jobcounter'] = webcom.GetJobCounter(info)
return render(request, 'pred/example.html', info)
def login(request):#{{{
#logout(request)
info = {}
webcom.set_basic_config(request, info, g_params)
info['jobcounter'] = webcom.GetJobCounter(info)
return render(request, 'pred/login.html', info)
def submit_seq(request):#{{{
info = {}
webcom.set_basic_config(request, info, g_params)
# if this is a POST request we need to process the form data
if request.method == 'POST':
# create a form instance and populate it with data from the request:
form = SubmissionForm(request.POST)
# check whether it's valid:
if form.is_valid():
# process the data in form.cleaned_data as required
# redirect to a new URL:
jobname = request.POST['jobname']
email = request.POST['email']
rawseq = request.POST['rawseq'] + "\n" # force add a new line
variants = request.POST['variants'] #also POST variants from form
Nfix = ""
Cfix = ""
fix_str = ""
isForceRun = False
try:
Nfix = request.POST['Nfix']
except:
pass
try:
Cfix = request.POST['Cfix']
except:
pass
try:
fix_str = request.POST['fix_str']
except:
pass
if 'forcerun' in request.POST:
isForceRun = True
try:
seqfile = request.FILES['seqfile']
except KeyError as MultiValueDictKeyError:
seqfile = ""
date_str = time.strftime(g_params['FORMAT_DATETIME'])
query = {}
query['rawseq'] = rawseq
query['variants'] = variants
query['seqfile'] = seqfile
query['email'] = email
query['jobname'] = jobname
query['date'] = date_str
query['client_ip'] = info['client_ip']
query['errinfo'] = ""
query['method_submission'] = "web"
query['Nfix'] = Nfix
query['Cfix'] = Cfix
query['fix_str'] = fix_str
query['isForceRun'] = isForceRun
query['username'] = info['username']
query['STATIC_URL'] = settings.STATIC_URL
is_valid = webcom.ValidateQuery(request, query, g_params)
if is_valid:
jobid = RunQuery(request, query)
# type of method_submission can be web or wsdl
#date, jobid, IP, numseq, size, jobname, email, method_submission
log_record = "%s\t%s\t%s\t%s\t%d\t%s\t%s\t%s\n"%(query['date'], jobid,
query['client_ip'], query['numseq'],
len(query['rawseq']),query['jobname'], query['email'],
query['method_submission'])
main_logfile_query = "%s/%s/%s"%(SITE_ROOT, "static/log", "submitted_seq.log")
myfunc.WriteFile(log_record, main_logfile_query, "a")
divided_logfile_query = "%s/%s/%s"%(SITE_ROOT,
"static/log/divided", "%s_submitted_seq.log"%(info['client_ip']))
divided_logfile_finished_jobid = "%s/%s/%s"%(SITE_ROOT,
"static/log/divided", "%s_finished_job.log"%(info['client_ip']))
if query['client_ip'] != "":
myfunc.WriteFile(log_record, divided_logfile_query, "a")
file_seq_warning = "%s/%s/%s/%s"%(SITE_ROOT, "static/result", jobid, "query.warn.txt")
query['file_seq_warning'] = os.path.basename(file_seq_warning)
if query['warninfo'] != "":
myfunc.WriteFile(query['warninfo'], file_seq_warning, "a")
query['jobid'] = jobid
query['raw_query_seqfile'] = "query.raw.fa"
query['BASEURL'] = g_params['BASEURL']
# start the qd_fe if not, in the background
# cmd = [qd_fe_scriptfile]
base_www_url = "http://" + request.META['HTTP_HOST']
# run the daemon only at the frontend
if webcom.IsFrontEndNode(base_www_url):
cmd = "nohup %s %s &"%(python_exec, qd_fe_scriptfile)
os.system(cmd)
if query['numseq'] < 0: #go to result page anyway
info['jobcounter'] = webcom.GetJobCounter(info)
return render(request, 'pred/thanks.html', info)
else:
return get_results(request, jobid)
else:
info['jobcounter'] = webcom.GetJobCounter(info)
return render(request, 'pred/badquery.html', info)
# if a GET (or any other method) we'll create a blank form
else:
form = SubmissionForm()
jobcounter = webcom.GetJobCounter(info)
info['form'] = form
info['jobcounter'] = jobcounter
return render(request, 'pred/submit_seq.html', info)
def findjob(request):#{{{
info = {}
errmsg = ""
webcom.set_basic_config(request, info, g_params)
all_logfile_query = "%s/%s/%s"%(SITE_ROOT, "static/log", "submitted_seq.log")
info['header'] = ["No.", "JobID","JobName", "NumSeq", "Email", "Submit date"]
matched_list = []
num_matched = 0
is_form_submitted = False
info['jobid'] = ""
info['jobname'] = ""
if g_params['DEBUG']:
myfunc.WriteFile("request.method=%s\n"%(str(request.method)), gen_logfile, "a", True)
if request.method == 'GET':
form = SubmissionForm_findjob(request.GET)
if request.GET.get('do'):
is_form_submitted = True
if g_params['DEBUG']: myfunc.WriteFile("Enter POST\n", gen_logfile, "a", True)
if form.is_valid():
if g_params['DEBUG']: myfunc.WriteFile("form.is_valid == True\n", gen_logfile, "a", True)
st_jobid = request.GET.get('jobid')
st_jobname = request.GET.get('jobname')
matched_jobidlist = []
if not (st_jobid or st_jobname):
errmsg = "Error! Neither Job ID nor Job Name is set."
else:
alljob_dict = myfunc.ReadSubmittedLogFile(all_logfile_query)
all_jobidList = list(alljob_dict.keys())
all_jobnameList = [alljob_dict[x][1] for x in all_jobidList]
if st_jobid:
if st_jobid.startswith("rst_") and len(st_jobid) >= 5:
for jobid in all_jobidList:
if jobid.find(st_jobid) != -1:
matched_jobidlist.append(jobid)
else:
errmsg = "Error! Searching text for Job ID must be started with 'rst_'\
and contains at least one char after 'rst_'"
else:
matched_jobidlist = all_jobidList
if st_jobname:
newli = []
for jobid in matched_jobidlist:
jobname = alljob_dict[jobid][1]
if jobname.find(st_jobname) != -1:
newli.append(jobid)
matched_jobidlist = newli
num_matched = len(matched_jobidlist)
for i in range(num_matched):
jobid = matched_jobidlist[i]
li = alljob_dict[jobid]
submit_date_str = li[0]
jobname = li[1]
email = li[3]
numseq_str = li[4]
rstdir = "%s/%s"%(path_result, jobid)
if os.path.exists(rstdir):
matched_list.append([i+1, jobid, jobname, numseq_str, email, submit_date_str])
else:
#errmsg = "Error! Neither Job ID nor Job Name is set."
form = SubmissionForm_findjob()
num_matched = len(matched_list)
info['errmsg'] = errmsg
info['form'] = form
try:
info['jobid'] = st_jobid
except:
pass
try:
info['jobname'] = st_jobname
except:
pass
info['num_matched'] = num_matched
info['content'] = matched_list
info['BASEURL'] = g_params['BASEURL']
info['is_form_submitted'] = is_form_submitted
info['jobcounter'] = webcom.GetJobCounter(info)
return render(request, 'pred/findjob.html', info)
def get_results(request, jobid="1"): #{{{
resultdict = {}
webcom.set_basic_config(request, resultdict, g_params)
#img1 = "%s/%s/%s/%s"%(SITE_ROOT, "result", jobid, "PconsC2.s400.jpg")
#url_img1 = serve(request, os.path.basename(img1), os.path.dirname(img1))
rstdir = "%s/%s" % (path_result, jobid)
outpathname = jobid
resultfile = "%s/%s/%s/%s" % (rstdir, jobid, outpathname,
"query.result.txt")
tarball = "%s/%s.tar.gz" % (rstdir, outpathname)
zipfile = "%s/%s.zip" % (rstdir, outpathname)
starttagfile = "%s/%s" % (rstdir, "runjob.start")
finishtagfile = "%s/%s" % (rstdir, "runjob.finish")
failtagfile = "%s/%s" % (rstdir, "runjob.failed")
errfile = "%s/%s" % (rstdir, "runjob.err")
query_seqfile = "%s/%s" % (rstdir, "query.fa")
raw_query_seqfile = "%s/%s" % (rstdir, "query.raw.fa")
seqid_index_mapfile = "%s/%s/%s" % (rstdir, jobid, "seqid_index_map.txt")
finished_seq_file = "%s/%s/finished_seqs.txt" % (rstdir, jobid)
statfile = "%s/%s/stat.txt" % (rstdir, jobid)
method_submission = "web"
jobinfofile = "%s/jobinfo" % (rstdir)
jobinfo = myfunc.ReadFile(jobinfofile).strip()
jobinfolist = jobinfo.split("\t")
if len(jobinfolist) >= 8:
submit_date_str = jobinfolist[0]
numseq = int(jobinfolist[3])
jobname = jobinfolist[5]
email = jobinfolist[6]
method_submission = jobinfolist[7]
else:
submit_date_str = ""
numseq = 1
jobname = ""
email = ""
method_submission = "web"
isValidSubmitDate = True
try:
submit_date = webcom.datetime_str_to_time(submit_date_str)
except ValueError:
isValidSubmitDate = False
current_time = datetime.now(timezone(TZ))
resultdict['isResultFolderExist'] = True
resultdict['errinfo'] = ""
if os.path.exists(errfile):
resultdict['errinfo'] = myfunc.ReadFile(errfile)
status = ""
queuetime = ""
runtime = ""
queuetime_in_sec = 0
runtime_in_sec = 0
if not os.path.exists(rstdir):
resultdict['isResultFolderExist'] = False
resultdict['isFinished'] = False
resultdict['isFailed'] = True
resultdict['isStarted'] = False
elif os.path.exists(failtagfile):
resultdict['isFinished'] = False
resultdict['isFailed'] = True
resultdict['isStarted'] = True
status = "Failed"
start_date_str = ""
if os.path.exists(starttagfile):
start_date_str = myfunc.ReadFile(starttagfile).strip()
isValidStartDate = True
isValidFailedDate = True
try:
start_date = webcom.datetime_str_to_time(start_date_str)
except ValueError:
isValidStartDate = False
failed_date_str = myfunc.ReadFile(failtagfile).strip()
try:
failed_date = webcom.datetime_str_to_time(failed_date_str)
except ValueError:
isValidFailedDate = False
if isValidSubmitDate and isValidStartDate:
queuetime = myfunc.date_diff(submit_date, start_date)
queuetime_in_sec = (start_date - submit_date).total_seconds()
if isValidStartDate and isValidFailedDate:
runtime = myfunc.date_diff(start_date, failed_date)
runtime_in_sec = (failed_date - start_date).total_seconds()
else:
resultdict['isFailed'] = False
if os.path.exists(finishtagfile):
resultdict['isFinished'] = True
resultdict['isStarted'] = True
status = "Finished"
isValidStartDate = True
isValidFinishDate = True
if os.path.exists(starttagfile):
start_date_str = myfunc.ReadFile(starttagfile).strip()
else:
start_date_str = ""
try:
start_date = webcom.datetime_str_to_time(start_date_str)
except ValueError:
isValidStartDate = False
finish_date_str = myfunc.ReadFile(finishtagfile).strip()
try:
finish_date = webcom.datetime_str_to_time(finish_date_str)
except ValueError:
isValidFinishDate = False
if isValidSubmitDate and isValidStartDate:
queuetime = myfunc.date_diff(submit_date, start_date)
queuetime_in_sec = (start_date - submit_date).total_seconds()
if isValidStartDate and isValidFinishDate:
runtime = myfunc.date_diff(start_date, finish_date)
runtime_in_sec = (finish_date - start_date).total_seconds()
else:
resultdict['isFinished'] = False
if os.path.exists(starttagfile):
isValidStartDate = True
start_date_str = ""
if os.path.exists(starttagfile):
start_date_str = myfunc.ReadFile(starttagfile).strip()
try:
start_date = webcom.datetime_str_to_time(start_date_str)
except ValueError:
isValidStartDate = False
resultdict['isStarted'] = True
status = "Running"
if isValidSubmitDate and isValidStartDate:
queuetime = myfunc.date_diff(submit_date, start_date)
queuetime_in_sec = (start_date -
submit_date).total_seconds()
if isValidStartDate:
runtime = myfunc.date_diff(start_date, current_time)
runtime_in_sec = (current_time -
start_date).total_seconds()
else:
resultdict['isStarted'] = False
status = "Wait"
if isValidSubmitDate:
queuetime = myfunc.date_diff(submit_date, current_time)
queuetime_in_sec = (current_time -
submit_date).total_seconds()
color_status = webcom.SetColorStatus(status)
file_seq_warning = "%s/%s/%s/%s" % (SITE_ROOT, "static/result", jobid,
"query.warn.txt")
seqwarninfo = ""
if os.path.exists(file_seq_warning):
seqwarninfo = myfunc.ReadFile(file_seq_warning)
seqwarninfo = seqwarninfo.strip()
resultdict['file_seq_warning'] = os.path.basename(file_seq_warning)
resultdict['seqwarninfo'] = seqwarninfo
resultdict['jobid'] = jobid
resultdict['subdirname'] = "seq_0"
resultdict['jobname'] = jobname
resultdict['outpathname'] = os.path.basename(outpathname)
resultdict['resultfile'] = os.path.basename(resultfile)
resultdict['tarball'] = os.path.basename(tarball)
resultdict['zipfile'] = os.path.basename(zipfile)
resultdict['submit_date'] = submit_date_str
resultdict['queuetime'] = queuetime
resultdict['runtime'] = runtime
resultdict['BASEURL'] = g_params['BASEURL']
resultdict['status'] = status
resultdict['color_status'] = color_status
resultdict['numseq'] = numseq
resultdict['query_seqfile'] = os.path.basename(query_seqfile)
resultdict['raw_query_seqfile'] = os.path.basename(raw_query_seqfile)
base_www_url = "http://" + request.META['HTTP_HOST']
# note that here one must add http:// in front of the url
resultdict['url_result'] = "%s/pred/result/%s" % (base_www_url, jobid)
sum_run_time = 0.0
average_run_time = float(
g_params['AVERAGE_RUNTIME_PER_SEQ_IN_SEC']) # default average_run_time
num_finished = 0
cntnewrun = 0
cntcached = 0
newrun_table_list = [] # this is used for calculating the remaining time
# get seqid_index_map
if os.path.exists(finished_seq_file):
resultdict['index_table_header'] = [
"No.", "Length", "LOC_DEF", "LOC_DEF_SCORE", "RunTime(s)",
"SequenceName", "Source", "FinishDate"
]
index_table_content_list = []
indexmap_content = myfunc.ReadFile(finished_seq_file).split("\n")
cnt = 0
for line in indexmap_content:
strs = line.split("\t")
if len(strs) >= 7:
subfolder = strs[0]
length_str = strs[1]
loc_def_str = strs[2]
loc_def_score_str = strs[3]
source = strs[4]
try:
finishdate = strs[7]
except IndexError:
finishdate = "N/A"
try:
runtime_in_sec_str = "%.1f" % (float(strs[5]))
if source == "newrun":
sum_run_time += float(strs[5])
cntnewrun += 1
elif source == "cached":
cntcached += 1
except:
runtime_in_sec_str = ""
desp = strs[6]
rank = "%d" % (cnt + 1)
if cnt < g_params['MAX_ROWS_TO_SHOW_IN_TABLE']:
index_table_content_list.append([
rank, length_str, loc_def_str, loc_def_score_str,
runtime_in_sec_str, desp[:30], subfolder, source,
finishdate
])
if source == "newrun":
newrun_table_list.append([rank, subfolder])
cnt += 1
if cntnewrun > 0:
average_run_time = sum_run_time / float(cntnewrun)
resultdict['index_table_content_list'] = index_table_content_list
resultdict['indexfiletype'] = "finishedfile"
resultdict['num_finished'] = cnt
num_finished = cnt
resultdict['percent_finished'] = "%.1f" % (float(cnt) / numseq * 100)
else:
resultdict['index_table_header'] = []
resultdict['index_table_content_list'] = []
resultdict['indexfiletype'] = "finishedfile"
resultdict['num_finished'] = 0
resultdict['percent_finished'] = "%.1f" % (0.0)
num_remain = numseq - num_finished
time_remain_in_sec = num_remain * average_run_time # set default value
# re-define runtime as the sum of all real running time
if sum_run_time > 0.0:
resultdict['runtime'] = myfunc.second_to_human(int(sum_run_time + 0.5))
resultdict['num_row_result_table'] = len(
resultdict['index_table_content_list'])
# calculate the remaining time based on the average_runtime of the last x
# number of newrun sequences
avg_newrun_time = webcom.GetAverageNewRunTime(finished_seq_file, window=10)
if cntnewrun > 0 and avg_newrun_time >= 0:
time_remain_in_sec = int(avg_newrun_time * num_remain + 0.5)
time_remain = myfunc.second_to_human(int(time_remain_in_sec + 0.5))
resultdict['time_remain'] = time_remain
qdinittagfile = "%s/runjob.qdinit" % (rstdir)
if os.path.exists(rstdir):
resultdict['isResultFolderExist'] = True
else:
resultdict['isResultFolderExist'] = False
if numseq <= 1:
resultdict['refresh_interval'] = webcom.GetRefreshInterval(
queuetime_in_sec, runtime_in_sec, method_submission)
else:
if os.path.exists(qdinittagfile):
addtime = int(math.sqrt(max(0, min(num_remain, num_finished)))) + 1
resultdict['refresh_interval'] = average_run_time + addtime
else:
resultdict['refresh_interval'] = webcom.GetRefreshInterval(
queuetime_in_sec, runtime_in_sec, method_submission)
# get stat info
if os.path.exists(statfile): #{{{
content = myfunc.ReadFile(statfile)
lines = content.split("\n")
for line in lines:
strs = line.split()
if len(strs) >= 2:
resultdict[strs[0]] = strs[1]
percent = "%.1f" % (int(strs[1]) / float(numseq) * 100)
newkey = strs[0].replace('num_', 'per_')
resultdict[newkey] = percent
#}}}
resultdict['MAX_ROWS_TO_SHOW_IN_TABLE'] = g_params[
'MAX_ROWS_TO_SHOW_IN_TABLE']
resultdict['jobcounter'] = webcom.GetJobCounter(resultdict)
return render(request, 'pred/get_results.html', resultdict)
def get_results(request, jobid="1"): #{{{
resultdict = {}
webcom.set_basic_config(request, resultdict, g_params)
#img1 = "%s/%s/%s/%s"%(SITE_ROOT, "result", jobid, "PconsC2.s400.jpg")
#url_img1 = serve(request, os.path.basename(img1), os.path.dirname(img1))
rstdir = "%s/%s" % (path_result, jobid)
outpathname = jobid
resultfile = "%s/%s/%s/%s" % (rstdir, jobid, outpathname,
"query.result.txt")
tarball = "%s/%s.tar.gz" % (rstdir, outpathname)
zipfile = "%s/%s.zip" % (rstdir, outpathname)
starttagfile = "%s/%s" % (rstdir, "runjob.start")
finishtagfile = "%s/%s" % (rstdir, "runjob.finish")
failtagfile = "%s/%s" % (rstdir, "runjob.failed")
runjob_errfile = "%s/%s" % (rstdir, "runjob.err")
query_seqfile = "%s/%s" % (rstdir, "query.fa")
raw_query_seqfile = "%s/%s" % (rstdir, "query.raw.fa")
seqid_index_mapfile = "%s/%s/%s" % (rstdir, jobid, "seqid_index_map.txt")
finished_seq_file = "%s/%s/finished_seqs.txt" % (rstdir, jobid)
statfile = "%s/%s/stat.txt" % (rstdir, jobid)
method_submission = "web"
finished_seq_file = "%s/%s/finished_seqs.txt" % (rstdir, jobid)
part_predfile = "%s/%s/query.part.top" % (rstdir, jobid)
jobinfofile = "%s/jobinfo" % (rstdir)
jobinfo = myfunc.ReadFile(jobinfofile).strip()
jobinfolist = jobinfo.split("\t")
app_type = "SCAMPI-single"
if len(jobinfolist) >= 8:
submit_date_str = jobinfolist[0]
numseq = int(jobinfolist[3])
jobname = jobinfolist[5]
email = jobinfolist[6]
method_submission = jobinfolist[7]
try:
app_type = jobinfolist[8]
except:
pass
else:
submit_date_str = ""
numseq = 1
jobname = ""
email = ""
method_submission = "web"
isValidSubmitDate = True
try:
submit_date = webcom.datetime_str_to_time(submit_date_str)
except ValueError:
isValidSubmitDate = False
current_time = datetime.now(timezone(TZ))
resultdict['isResultFolderExist'] = True
resultdict['errinfo'] = myfunc.ReadFile(runjob_errfile)
status = ""
queuetime = ""
runtime = ""
if not os.path.exists(rstdir):
resultdict['isResultFolderExist'] = False
resultdict['isFinished'] = False
resultdict['isFailed'] = True
resultdict['isStarted'] = False
elif os.path.exists(failtagfile):
resultdict['isFinished'] = False
resultdict['isFailed'] = True
resultdict['isStarted'] = True
status = "Failed"
start_date_str = myfunc.ReadFile(starttagfile).strip()
isValidStartDate = True
isValidFailedDate = True
try:
start_date = webcom.datetime_str_to_time(start_date_str)
except ValueError:
isValidStartDate = False
failed_date_str = myfunc.ReadFile(failtagfile).strip()
try:
failed_date = webcom.datetime_str_to_time(failed_date_str)
except ValueError:
isValidFailedDate = False
if isValidSubmitDate and isValidStartDate:
queuetime = myfunc.date_diff(submit_date, start_date)
if isValidStartDate and isValidFailedDate:
runtime = myfunc.date_diff(start_date, failed_date)
else:
resultdict['isFailed'] = False
if os.path.exists(finishtagfile):
resultdict['isFinished'] = True
resultdict['isStarted'] = True
status = "Finished"
isValidStartDate = True
isValidFinishDate = True
start_date_str = myfunc.ReadFile(starttagfile).strip()
try:
start_date = webcom.datetime_str_to_time(start_date_str)
except ValueError:
isValidStartDate = False
finish_date_str = myfunc.ReadFile(finishtagfile).strip()
try:
finish_date = webcom.datetime_str_to_time(finish_date_str)
except ValueError:
isValidFinishDate = False
if isValidSubmitDate and isValidStartDate:
queuetime = myfunc.date_diff(submit_date, start_date)
if isValidStartDate and isValidFinishDate:
runtime = myfunc.date_diff(start_date, finish_date)
else:
resultdict['isFinished'] = False
if os.path.exists(starttagfile):
isValidStartDate = True
start_date_str = myfunc.ReadFile(starttagfile).strip()
try:
start_date = webcom.datetime_str_to_time(start_date_str)
except ValueError:
isValidStartDate = False
resultdict['isStarted'] = True
status = "Running"
if isValidSubmitDate and isValidStartDate:
queuetime = myfunc.date_diff(submit_date, start_date)
if isValidStartDate:
runtime = myfunc.date_diff(start_date, current_time)
else:
resultdict['isStarted'] = False
status = "Wait"
if isValidSubmitDate:
queuetime = myfunc.date_diff(submit_date, current_time)
color_status = webcom.SetColorStatus(status)
file_seq_warning = "%s/%s/%s/%s" % (SITE_ROOT, "static/result", jobid,
"query.warn.txt")
seqwarninfo = ""
if os.path.exists(file_seq_warning):
seqwarninfo = myfunc.ReadFile(file_seq_warning).strip()
resultdict['file_seq_warning'] = os.path.basename(file_seq_warning)
resultdict['seqwarninfo'] = seqwarninfo
resultdict['app_type'] = app_type
resultdict['jobid'] = jobid
resultdict['jobname'] = jobname
resultdict['outpathname'] = os.path.basename(outpathname)
resultdict['resultfile'] = os.path.basename(resultfile)
resultdict['tarball'] = os.path.basename(tarball)
resultdict['zipfile'] = os.path.basename(zipfile)
resultdict['submit_date'] = submit_date_str
resultdict['queuetime'] = queuetime
resultdict['runtime'] = runtime
resultdict['status'] = status
resultdict['color_status'] = color_status
resultdict['numseq'] = numseq
resultdict['query_seqfile'] = os.path.basename(query_seqfile)
resultdict['raw_query_seqfile'] = os.path.basename(raw_query_seqfile)
base_www_url = "http://" + request.META['HTTP_HOST']
# note that here one must add http:// in front of the url
resultdict['url_result'] = "%s/pred/result/%s" % (base_www_url, jobid)
num_finished = 0
if os.path.exists(finished_seq_file):
lines = myfunc.ReadFile(finished_seq_file).split("\n")
lines = [_f for _f in lines if _f]
num_finished = len(lines)
sum_run_time = 0.0
average_run_time_single = 0.1 # default average_run_time
average_run_time_msa = 300 # default average_run_time
num_finished = 0
cntnewrun = 0
cntcached = 0
topcontentList = []
# get seqid_index_map
if os.path.exists(finished_seq_file):
resultdict['index_table_header'] = [
"No.", "Length", "numTM", "RunTime(s)", "SequenceName", "Source"
]
index_table_content_list = []
indexmap_content = myfunc.ReadFile(finished_seq_file).split("\n")
cnt = 0
added_idx_set = set([])
for line in indexmap_content:
strs = line.split("\t")
if len(strs) >= 8:
subfolder = strs[0]
if not subfolder in added_idx_set:
length_str = strs[1]
numTM_str = strs[2]
source = strs[3]
try:
runtime_in_sec_str = "%.1f" % (float(strs[4]))
if source == "newrun":
sum_run_time += float(strs[4])
cntnewrun += 1
elif source == "cached":
cntcached += 1
except:
runtime_in_sec_str = ""
desp = strs[5]
top = strs[7]
rank = "%d" % (cnt + 1)
index_table_content_list.append([
rank, length_str, numTM_str, runtime_in_sec_str,
desp[:30], source
])
cnt += 1
added_idx_set.add(subfolder)
topcontentList.append(">%s\n%s" % (desp, top))
if cntnewrun > 0:
average_run_time_msa = sum_run_time / cntnewrun
resultdict['index_table_content_list'] = index_table_content_list
resultdict['indexfiletype'] = "finishedfile"
resultdict['num_finished'] = cnt
num_finished = cnt
resultdict['percent_finished'] = "%.1f" % (float(cnt) / numseq * 100)
else:
resultdict['index_table_header'] = []
resultdict['index_table_content_list'] = []
resultdict['indexfiletype'] = "finishedfile"
resultdict['num_finished'] = 0
resultdict['percent_finished'] = "%.1f" % (0.0)
num_remain = numseq - num_finished
myfunc.WriteFile("\n".join(topcontentList), part_predfile, "w")
time_remain_in_sec = numseq * 120 # set default value
if os.path.exists(starttagfile):
start_date_str = myfunc.ReadFile(starttagfile).strip()
isValidStartDate = False
try:
start_date_epoch = webcom.datetime_str_to_epoch(start_date_str)
isValidStartDate = True
except:
pass
if isValidStartDate:
time_now = time.time()
runtime_total_in_sec = float(time_now) - float(start_date_epoch)
cnt_torun = numseq - cntcached #
if cntnewrun <= 0:
time_remain_in_sec = cnt_torun * 120
else:
time_remain_in_sec = int(runtime_total_in_sec /
float(cntnewrun) * cnt_torun + 0.5)
time_remain = myfunc.second_to_human(time_remain_in_sec)
resultdict['time_remain'] = time_remain
qdinittagfile = "%s/runjob.qdinit" % (rstdir)
if numseq <= 1:
if method_submission == "web":
if app_type == "SCAMPI-single":
resultdict['refresh_interval'] = 1
else:
resultdict['refresh_interval'] = 5.0
else:
if app_type == "SCAMPI-single":
resultdict['refresh_interval'] = 1.0
else:
resultdict['refresh_interval'] = 5.0
else:
#resultdict['refresh_interval'] = numseq * 2
addtime = int(math.sqrt(max(0, min(num_remain, num_finished)))) + 1
if app_type == "SCAMPI-single":
resultdict['refresh_interval'] = max(
1, num_remain * average_run_time_single)
else:
if not os.path.exists(qdinittagfile):
resultdict['refresh_interval'] = 2
else:
if num_finished == 0:
resultdict['refresh_interval'] = 5
else:
resultdict['refresh_interval'] = 10 + addtime
# get stat info
if os.path.exists(statfile): #{{{
content = myfunc.ReadFile(statfile)
lines = content.split("\n")
for line in lines:
strs = line.split()
if len(strs) >= 2:
resultdict[strs[0]] = strs[1]
percent = "%.1f" % (int(strs[1]) / float(numseq) * 100)
newkey = strs[0].replace('num_', 'per_')
resultdict[newkey] = percent
#}}}
topfile = "%s/%s/query.top" % (rstdir, jobid)
TM_listfile = "%s/%s/query.TM_list.txt" % (rstdir, jobid)
nonTM_listfile = "%s/%s/query.nonTM_list.txt" % (rstdir, jobid)
str_TMlist = []
str_nonTMlist = []
lenseq_list = []
num_TMPro = 0
if os.path.exists(topfile):
(tmpidlist, tmpannolist, tmptoplist) = myfunc.ReadFasta(topfile)
cnt_TMPro = 0
for ii in range(len(tmpidlist)):
top = tmptoplist[ii]
lenseq_list.append(len(top))
if top.find('M') != -1:
cnt_TMPro += 1
str_TMlist.append(tmpannolist[ii])
else:
str_nonTMlist.append(tmpannolist[ii])
num_TMPro = cnt_TMPro
if not os.path.exists(TM_listfile) or os.path.getsize(TM_listfile) < 1:
myfunc.WriteFile("\n".join(str_TMlist), TM_listfile, "w")
if not os.path.exists(
nonTM_listfile) or os.path.getsize(nonTM_listfile) < 1:
myfunc.WriteFile("\n".join(str_nonTMlist), nonTM_listfile, "w")
avg_lenseq = myfunc.FloatDivision(sum(lenseq_list), len(lenseq_list))
resultdict['avg_lenseq'] = int(avg_lenseq + 0.5)
resultdict['app_type'] = app_type
resultdict['num_TMPro'] = num_TMPro
resultdict['per_TMPro'] = "%.1f" % (
myfunc.FloatDivision(num_TMPro, numseq) * 100)
resultdict['num_nonTMPro'] = numseq - num_TMPro
resultdict['per_nonTMPro'] = "%.1f" % (
100.0 - myfunc.FloatDivision(num_TMPro, numseq) * 100)
resultdict['num_finished'] = num_finished
resultdict['percent_finished'] = "%.1f" % (float(num_finished) / numseq *
100)
resultdict['jobcounter'] = webcom.GetJobCounter(resultdict)
return render(request, 'pred/get_results.html', resultdict)
