def create_done_job(path):
status = 'd'
idx = job_done_id(path)
info = read_job_info(idx, path)
btime = info[1]
etime = info[2]
dtime = etime - btime
dtime_str = my.nice_sec2str(dtime.total_seconds())
ss1 = "Done"
ss2 = "Simulation time " + dtime_str
message = ss1 + "\n" + ss2
return Fjob(path, status, message)
def create_wait_job(qjob, path):
status = "qw"
idx = qjob.idx
ss1 = "Waiting " + idx
ctime = datetime.datetime.now() # current time
stime = qjob.btime # job submission time
dtime = ctime - stime
ctime_str = datetime.datetime.strftime(ctime, "%m/%d/%Y %H:%M:%S")
dtime_str = my.nice_sec2str(dtime.total_seconds())
ss2 = "Checked at " + ctime_str
ss3 = "Waiting time: " + dtime_str
message = ss1 + "\n" + ss2 + "\n" + ss3
return Fjob(path, status, message)
def create_run_job(qjob, path):
status = 'r'
idx = qjob.idx
info = read_job_info(idx, path)
stime = qjob.btime # job submission time
btime = info[1] # job beginning time
ctime = datetime.datetime.now() # current time
dtime = ctime - btime # job running time
ctime_str = datetime.datetime.strftime(ctime, "%m/%d/%Y %H:%M:%S")
dtime_str = my.nice_sec2str(dtime.total_seconds())
ss1 = "Running " + idx
ss2 = "Checked at " + ctime_str
ss3 = "Running time: " + dtime_str
message = ss1 + "\n" + ss2 + "\n" + ss3
return Fjob(path, status, message)
def main(path):
print("######################################")
running_jobs = Rjob_list()
running_jobs.myq()
print(running_jobs)
nUI = running_jobs.UI_usage()
print("There are {} UI jobs".format(nUI))
print("{} jobs running; {} jobs submitted".format(running_jobs.n_rjobs(),
running_jobs.n_jobs()))
###################################
# check folders for static jobs
###################################
os.chdir(path)
cwd = os.getcwd()
folders = [x[0] for x in os.walk(cwd)]
folders.sort()
# print(folders)
# generate the job list I want to monitor
# the folder should has job.begin
folder_jobs = Sjob_list()
for folder in folders:
os.chdir(folder)
# print(folder)
# working folder when it has job.begin file
binfo = read_job_begin(folder)
if binfo is not None:
idx = binfo[0]
# print(folder)
# now get idx status
status = running_jobs.checkstatus(idx)
info = read_job_info(folder, idx)
if status == "n":
if len(info) == 3:
status = "d"
else:
status = "nd"
# print(status)
if status == 'r':
rjob = running_jobs.find(idx)
stime = rjob.btime # job submission time
btime = info[1] # job beginning time
ctime = datetime.datetime.now() # current time
dtime = ctime - btime # job running time
ctime_str = datetime.datetime.strftime(ctime,
"%m/%d/%Y %H:%M:%S")
dtime_str = my.nice_sec2str(dtime.total_seconds())
ss1 = "Running " + idx
ss2 = "Checked at " + ctime_str
ss3 = "Running time: " + dtime_str
message = ss1 + "\n" + ss2 + "\n" + ss3
elif status == "qw":
ss1 = "Waiting " + idx
ctime = datetime.datetime.now() # current time
rjob = running_jobs.find(idx)
stime = rjob.btime # job submission time
dtime = ctime - stime
ctime_str = datetime.datetime.strftime(ctime,
"%m/%d/%Y %H:%M:%S")
dtime_str = my.nice_sec2str(dtime.total_seconds())
ss2 = "Checked at " + ctime_str
ss3 = "Waiting time: " + dtime_str
message = ss1 + "\n" + ss2 + "\n" + ss3
elif status == "eqw":
print(folder)
# kill it and resubmit
ctime = datetime.datetime.now()
ctime_str = datetime.datetime.strftime(ctime,
"%m/%d/%Y %H:%M:%S")
ss1 = "Error " + idx + " Kill and resubmit"
ss2 = "Checked at " + ctime_str
message = ss1 + "\n" + ss2
kill_job(idx)
nsmp = random.randint(2, 4)
if nUI < 2:
submit_job(server="UI", smp=nsmp)
nUI = nUI + 1
else:
submit_job(server="all.q", smp=nsmp)
elif status == "d":
btime = info[1]
etime = info[2]
dtime = etime - btime
dtime_str = my.nice_sec2str(dtime.total_seconds())
ss1 = "Done"
ss2 = "Simulation time " + dtime_str
message = ss1 + "\n" + ss2
else:
print(folder)
ctime = datetime.datetime.now()
ctime_str = datetime.datetime.strftime(ctime,
"%m/%d/%Y %H:%M:%S")
ss1 = "Not running, resubmit"
ss2 = "Checked at " + ctime_str
message = ss1 + "\n" + ss2
if nUI < 2:
submit_job(server="UI")
nUI = nUI + 1
else:
submit_job(server="all.q")
fjob = Sjob(folder, status, message)
folder_jobs.append(fjob)
# for job in folder_jobs:
# print(job)
print(folder_jobs.n_jobs_str())
print(folder_jobs)
print("End of the scan")
sys.stdout.flush()
os.chdir(cwd)