def submit(jobs, moni):
total_num = len(jobs)
count = 0
submitted_jobs = []
finished_jobs = []
max_calculations_dict = {'12': 5, '28': 3}
def test_finished(jobs):
nonlocal count
nonlocal count_dict
for job in jobs[:]:
if if_cal_finish(job):
finished_jobs.append(job)
num = str(len(finished_jobs)) + '/' + str(total_num)
rec = str(job) + '\n'
rec += num + 'calculation finished.\n'
rec += '---' * 25
print(rec)
record(job.root_path, rec)
jobs.remove(job)
count -= 1
count_dict[job.parameter['node']] -= 1
# test if there is some job which is already finished
for job in jobs[:]:
if if_cal_finish(job):
finished_jobs.append(job)
jobs.remove(job)
# test if there is some jobs which are already submitted but not finished
running_jobs = moni.get_running_jobs()
for job in jobs[:]:
if job in running_jobs:
submitted_jobs.append(job)
jobs.remove(job)
# categorize jobs according to the nodes number
jobs_dict = {}
count_dict = {}
nodes_list = []
for job in jobs:
node = job.parameter['node']
if node not in nodes_list:
nodes_list.append(node)
jobs_dict[node] = [job]
count_dict[node] = 0
else:
jobs_dict[node].append(job)
# submit and detect all jobs
j = 0
while True:
test_finished(submitted_jobs)
moni.update_status()
if len(finished_jobs) == total_num and len(submitted_jobs) == 0:
break
else:
for node in nodes_list:
if count_dict[node] < max_calculations_dict[node] and len(
jobs_dict[node]) > 0:
new_job = jobs_dict[node].pop()
os.chdir(new_job.path)
rename_file(new_job.path, '{}.out'.format(new_job.method))
out = submit_job(new_job, new_job.method)
count += 1
count_dict[node] += 1
submitted_jobs.append(new_job)
moni.insert_new_job(new_job, out)
rec = new_job.path + '\n'
rec += new_job.method + '\n'
rec += 'job submitted.'
rec += '\n' + out + '\n'
rec += '---' * 25
record(new_job.root_path, rec)
print(rec)
else:
# time.sleep(0.001)
time.sleep(500)
# test_calculation(j, jobs, finished_jobs)
j += 1
if j > 8:
rec = 'noting changes.\n'
rec += '---' * 25
record(submitted_jobs[0].root_path, rec)
j = 0
continue
return finished_jobs
def submit(jobs, moni):
job_num = len(jobs)
max_paralell = 5
count = 0
submitted_jobs = []
finished_jobs = []
def test_finished(jobs):
nonlocal count
for job in jobs[:]:
if if_cal_finish(job):
finished_jobs.append(job)
num = str(len(finished_jobs)) + '/' + str(job_num)
rec = str(job)
rec += '\n'
rec += num + ' calculation finished.\n'
rec += '---' * 25
print(rec)
record(job.root_path, rec)
jobs.remove(job)
count -= 1
# test if there is some job which is already finished
for job in jobs[:]:
if if_cal_finish(job):
finished_jobs.append(job)
jobs.remove(job)
# test if there is some jobs which are already submitted but not finished
running_jobs = moni.get_running_jobs()
for job in jobs[:]:
if job in running_jobs:
submitted_jobs.append(job)
jobs.remove(job)
# submit and detect all jobs
j = 0
while True:
test_finished(submitted_jobs)
moni.update_status()
if len(finished_jobs) == job_num and len(submitted_jobs) == 0:
break
else:
if count < max_paralell and len(jobs) > 0:
new_job = jobs.pop()
os.chdir(new_job.path)
rename_file(new_job.path, 'hf2.out')
out = submit_job(new_job, 'hf2')
count += 1
submitted_jobs.append(new_job)
moni.insert_new_job(new_job, out)
rec = str(new_job) + '\n'
rec += 'job submitted.'
rec += '\n' + out + '\n'
rec += '---' * 25
record(new_job.root_path, rec)
print(rec)
else:
time.sleep(500)
# time.sleep(200)
j += 1
# test_calculation(j, jobs, submitted_jobs, finished_jobs) # test function
if j > 15:
rec = 'noting changes.\n'
rec += '---' * 25
record(submitted_jobs[0].root_path, rec)
j = 0
continue
return finished_jobs
def submit(jobs, moni):
job_num = len(jobs)
max_paralell = 8
# max_paralell = 75
count = 0
submitted_jobs = []
finished_jobs = []
def test_finished(jobs):
"""
test jobs which have benn submittdt is finished or not
if a job finished, add it to list finished_jobs, and delete it from list submitted_jobs
:param jobs:
:return:
"""
nonlocal count
for job in jobs[:]:
if if_cal_finish(job):
finished_jobs.append(job)
num = str(len(finished_jobs)) + '/' + str(job_num)
rec = str(job)
rec += '\n'
rec += num + ' calculation finished.\n'
rec += '---' * 25
print(rec)
record(job.root_path, rec)
jobs.remove(job)
count -= 1
# test if there is some job which is already finished
for job in jobs[:]:
if if_cal_finish(job):
finished_jobs.append(job)
jobs.remove(job)
# test if there is some jobs which are already submitted but not finished
running_jobs = moni.get_running_jobs()
for job in jobs[:]:
if job in running_jobs:
submitted_jobs.append(job)
jobs.remove(job)
# submit and detect all jobs
j = 0
while True:
test_finished(submitted_jobs
) # update list finished_jobs and list submitted_jobs
moni.update_status()
if len(finished_jobs) == job_num and len(submitted_jobs) == 0:
break
else:
if count <= max_paralell and len(
jobs) > 0: # check the number of jobs which is running now
new_job = jobs.pop()
os.chdir(new_job.path)
out = submit_job(new_job, 'lmp2')
count += 1
submitted_jobs.append(new_job)
moni.insert_new_job(new_job, out)
rec = str(new_job) + '\n'
rec += 'job submitted.'
rec += '\n' + out + '\n'
rec += '---' * 25
record(new_job.root_path, rec)
print(rec)
else:
time.sleep(500)
# time.sleep(1)
j += 1
# test_calculation(j, jobs, submitted_jobs, finished_jobs) # test function
if j > 15:
rec = 'noting changes.\n'
rec += '---' * 25
record(submitted_jobs[0].root_path, rec)
j = 0
continue
return finished_jobs
def submit(jobs, nodes, crystal_path, moni):
job_numbers = len(jobs)
max_paralell = 5
count = 0
submitted_jobs = []
finished_jobs = []
# find and submit the initial job
loc = 0
for job in jobs:
if job.x == '0' and job.z == '0':
break
loc += 1
if loc < len(jobs):
job_init = jobs.pop(loc)
os.chdir(job_init.path)
if not if_cal_finish(job_init):
copy_submit_scr(job_init, nodes, crystal_path)
rename_file(job_init.path, 'geo_opt.out')
out = submit_job(job_init, 'geo_opt')
submitted_jobs.append(job_init)
moni.insert_new_job(job_init, out)
rec = job_init.path
print(rec)
rec += '\n'
rec += 'job submitted...'
rec += '\n' + out + '\n'
rec += '---' * 25
record(job_init.root_path, rec)
r = 0
while True:
moni.update_status()
if if_cal_finish(job_init):
rec = job_init.path
rec += '\n'
rec += 'calculation finished...'
record(job_init.root_path, rec)
submitted_jobs.remove(job_init)
finished_jobs.append(job_init)
break
else:
time.sleep(500)
r += 1
# test function
# test_init_job(job_init, r)
if r > 15:
rec = job_init.path
rec += '\n'
rec += 'initial calculation still not finished...'
record(job_init.root_path, rec)
r = 0
continue
else:
finished_jobs.append(job_init)
# test if there is some job which is already finished
for job in jobs[:]:
if if_cal_finish(job):
# print('Job already finished: ', job)
finished_jobs.append(job)
jobs.remove(job)
# test if there is some jobs which are already submitted but not finished
running_jobs = moni.get_running_jobs()
for job in jobs[:]:
if job in running_jobs:
submitted_jobs.append(job)
jobs.remove(job)
def test_finished(paths):
nonlocal count # debug: UnboundLocalError: local variable 'count' referenced before assignment
for path in paths[:]:
if if_cal_finish(path):
finished_jobs.append(path)
num = str(len(finished_jobs)) + '/' + str(job_numbers)
rec = path.path
rec += '\n'
rec += num + ' calculation finished.\n'
rec += '---' * 25
print(rec)
record(path.root_path, rec)
paths.remove(path)
count -= 1
if len(jobs) == 0:
return finished_jobs
else:
i = 0
j = 0
while True:
test_finished(submitted_jobs)
moni.update_status()
if len(finished_jobs) == job_numbers and len(submitted_jobs) == 0:
break
else:
if count <= max_paralell and i < len(jobs):
print(jobs[i].path)
nearest_job = obtain_nearest_job(jobs[i])
os.chdir(jobs[i].path)
copy_submit_scr(jobs[i], nodes, crystal_path, nearest_job)
# copy_fort9(jobs[i])
rename_file(jobs[i].path, 'geo_opt.out')
rename_file(jobs[i].path, 'fort.9')
out = submit_job(jobs[i], 'geo_opt')
count += 1
submitted_jobs.append(jobs[i])
moni.insert_new_job(jobs[i], out)
rec = jobs[i].path + '\n'
rec += 'job submitted.'
rec += '\n' + out + '\n'
rec += '---' * 25
record(jobs[i].root_path, rec)
i += 1
else:
time.sleep(500)
j += 1
# j = test_calculation(j, submitted_jobs) # test function
if j > 20:
rec = 'noting changes.\n'
rec += '---' * 25
# print(rec)
record(submitted_jobs[0].root_path, rec)
j = 0
continue
return finished_jobs
def submit(jobs, nodes, crystal_path, moni):
job_num = len(jobs)
max_paralell = 5
count = 0
submitted_jobs = []
finished_jobs = []
def test_finished(jobs):
nonlocal count
for job in jobs[:]:
if if_cal_finish(job):
finished_jobs.append(job)
num = str(len(finished_jobs)) + '/' + str(job_num)
rec = str(job)
rec += '\n'
rec += num + ' calculation finished.\n'
rec += '---' * 25
print(rec)
record(job.root_path, rec)
count -= 1
jobs.remove(job)
# test if there is some job which is already finished
for job in jobs[:]:
if if_cal_finish(job):
finished_jobs.append(job)
jobs.remove(job)
# test if there is some jobs which are already submitted but not finished
running_jobs = moni.get_running_jobs()
for job in jobs[:]:
if job in running_jobs:
submitted_jobs.append(job)
jobs.remove(job)
# find and submit the initial job
# print('number of jobs: ', len(jobs))
init_jobs = []
for job in jobs[:]:
if job.x == '0' and job.z == '0':
init_jobs.append(job)
jobs.remove(job)
for job in init_jobs[:]:
if not if_cal_finish(job):
os.chdir(job.path)
rename_file(job.path, 'hf.out')
out = submit_job(job, 'hf')
count += 1
submitted_jobs.append(job)
moni.insert_new_job(job, out)
rec = str(job)
print(rec)
rec += '\n'
rec += 'job submitted.'
rec += '\n' + out + '\n'
rec += '---' * 25
record(job.root_path, rec)
else:
finished_jobs.append(job)
# detect if init jobs finished
r = 0
while True:
# test_finished(submitted_jobs) # test function
moni.update_status()
if len(submitted_jobs) == 0:
break
else:
time.sleep(500)
r += 1
if r > 15:
rec = 'initial calculation still not finished.\n'
rec += '---' * 25
record(submitted_jobs[0].root_path, rec)
r = 0
# submit and detect the other jobs
j = 0
while True:
test_finished(submitted_jobs)
moni.update_status()
if len(finished_jobs) == job_num and len(submitted_jobs) == 0:
break
else:
if count < max_paralell and len(jobs) != 0:
new_job = jobs.pop()
os.chdir(new_job.path)
nearest_job = obtain_nearest_job(new_job)
rename_file(new_job.path, 'hf.out')
rename_file(new_job.path, 'fort.9')
# copy_fort9(new_job)
copy_submit_scr(new_job, nodes, crystal_path, nearest_job)
out = submit_job(new_job, 'hf')
count += 1
submitted_jobs.append(new_job)
moni.insert_new_job(new_job, out)
rec = str(new_job) + '\n'
rec += 'job submitted.'
rec += '\n' + out + '\n'
rec += '---' * 25
record(new_job.root_path, rec)
print(rec)
else:
# time.sleep(10)
time.sleep(500)
j += 1
# test_calculation(j, jobs, submitted_jobs, finished_jobs) # test function
if j > 15:
rec = 'noting changes.\n'
rec += '---' * 25
record(submitted_jobs[0].root_path, rec)
j = 0
continue
return finished_jobs