def job_wait(path, secs):
## WAITS UNTIL A JOB ENDS SUCCESSFULLY (IF ERROR -> ABORTS) BASED ON THE LOG FILES CREATED BY THE RESOURCE MANAGER
## Input:
## path: String with the path to the log files corresponding to the jobs that will be checked (i.e. single job: '/some_path/job1.log', job array: '/some_path/job1.log|/some_path/job2.log')
## secs: Number of seconds between subsequent checks
## Returns:
## 1: Once all the jobs have finished
## If an error is detected -> exit()
if "|" in path:
jobarray = path.split("|")
else:
jobarray = [path]
for path in jobarray:
while 1:
rt = manager.job_status(path)
if rt == 1:
break
elif rt == -1:
exit("Error on: " + path)
else:
time.sleep(secs)
return 1
def job_wait(path, secs):
## WAITS UNTIL A JOB ENDS SUCCESSFULLY (IF ERROR -> ABORTS) BASED ON THE LOG FILES CREATED BY THE RESOURCE MANAGER
## Input:
## path: String with the path to the log files corresponding to the jobs that will be checked (i.e. single job: '/some_path/job1.log', job array: '/some_path/job1.log|/some_path/job2.log')
## secs: Number of seconds between subsequent checks
## Returns:
## 1: Once all the jobs have finished
## If an error is detected -> exit()
if "|" in path:
jobarray = path.split("|")
else:
jobarray = [path]
for path in jobarray:
while 1:
rt = manager.job_status(path)
if rt == 1:
break
elif rt == -1:
exit("Error on: " + path)
else:
time.sleep(secs)
return 1
def project_process(path_base, folder):
samples = get_samples(path_base, folder, path_base + "/" + folder + "/samples.list")
# Check main process
print "## MAIN PROCESS ###########################"
try:
f = open(path_base + "/" + folder + "/pid.txt", 'r')
i = f.readline().strip("\n").split("\t")[1]
f.close()
k = manager.job_status(i)
if k == 1:
st = "DONE"
elif k == 0:
st = "RUN"
else:
st = "ERROR"
print "- Main process (" + i + ") status: " + st
except:
print "- Main process not found or already finished"
# Check subprocesses
print "## SUBPROCESSES ###########################"
pids = dict()
try:
f = open(path_base + "/" + folder + "/temp/pids.txt", 'r')
for i in f:
i = i.strip("\n").split("\t")
pids[i[0]] = [i[1].split("|"),i[2].split("|")]
f.close()
except:
print "- No subprocesses file found"
f = open(path_base + "/" + folder + "/config.txt", 'r')
config = dict()
for i in f:
if not i.startswith("%"):
i = i.strip("\n").split("\t")
if i[0] in ["trimgalore", "fastqc", "star", "htseq-gene", "htseq-exon", 'sam2sortbam']:
i[1] = i[1].split("/")[0]
if i[1] != "0":
config[i[0]] = i[1]
if len(config) > 0:
for pg in ["trimgalore", "fastqc", "star", "htseq-gene", "htseq-exon", "sam2sortbam"]:
if config.has_key(pg):
if not pids.has_key(pg):
print "- Already done or waiting for previous module output"
else:
pid = pids[pg]
print "- ID: " + "|".join(pid[0])
n = list()
for i in pid[1]:
k = manager.job_status(i)
if k == 1:
n.append("DONE")
elif k == 0:
n.append("RUN")
else:
n.append("ERROR")
print "- Status: " + "|".join(n)
samples_v, stats = check_samples(samples, path_base, folder, pg, "update")
sok = str(round(100 * float(stats[1])/float(stats[0]),2))
sko = str(round(100 * float(stats[2])/float(stats[0]),2))
pending = str(round(100 * float(stats[0]-stats[1]-stats[2])/float(stats[0]),2))
print "- Progress: " + sok + "% succeeded / " + sko + "% exited / " + pending + "% pending"
def project_process(path_base, folder):
samples = get_samples(path_base, folder, path_base + "/" + folder + "/samples.list")
# Check main process
print "## MAIN PROCESS ###########################"
try:
f = open(path_base + "/" + folder + "/pid.txt", 'r')
i = f.readline().strip("\n").split("\t")[1]
f.close()
k = manager.job_status(i)
if k == 1:
st = "DONE"
elif k == 0:
st = "RUN"
else:
st = "ERROR"
print "- Main process (" + i + ") status: " + st
except:
print "- Main process not found or already finished"
# Check subprocesses
print "## SUBPROCESSES ###########################"
pids = dict()
try:
f = open(path_base + "/" + folder + "/temp/pids.txt", 'r')
for i in f:
i = i.strip("\n").split("\t")
pids[i[0]] = [i[1].split("|"),i[2].split("|")]
f.close()
except:
print "- No subprocesses file found"
f = open(path_base + "/" + folder + "/config.txt", 'r')
config = dict()
for i in f:
if not i.startswith("%"):
i = i.strip("\n").split("\t")
if i[0] in ["trimgalore", "fastqc", "kallisto", "star", "star-fusion", "picard", "htseq-gene", "htseq-exon", "varscan", "gatk"]:
i[1] = i[1].split("/")[0]
if i[1] != "0":
config[i[0]] = i[1]
if config.has_key("varscan") or config.has_key("gatk"):
config["sam2sortbam"] = 1
if len(config) > 0:
for pg in ["trimgalore", "fastqc", "kallisto", "star", "star-fusion", "picard", "htseq-gene", "htseq-exon", "sam2sortbam", "varscan", "gatk"]:
if config.has_key(pg):
print "Process: " + pg
if not pids.has_key(pg):
print "- Already done or waiting for previous module output"
else:
pid = pids[pg]
print "- ID: " + "|".join(pid[0])
n = list()
for i in pid[1]:
k = manager.job_status(i)
if k == 1:
n.append("DONE")
elif k == 0:
n.append("RUN")
else:
n.append("ERROR")
print "- Status: " + "|".join(n)
samples_v, stats = check_samples(samples, path_base, folder, pg, "update")
sok = str(round(100 * float(stats[1])/float(stats[0]),2))
sko = str(round(100 * float(stats[2])/float(stats[0]),2))
pending = str(round(100 * float(stats[0]-stats[1]-stats[2])/float(stats[0]),2))
print "- Progress: " + sok + "% succeeded / " + sko + "% exited / " + pending + "% pending"