def submit(jobs, moni): total_num = len(jobs) count = 0 submitted_jobs = [] finished_jobs = [] max_calculations_dict = {'12': 5, '28': 3} def test_finished(jobs): nonlocal count nonlocal count_dict for job in jobs[:]: if if_cal_finish(job): finished_jobs.append(job) num = str(len(finished_jobs)) + '/' + str(total_num) rec = str(job) + '\n' rec += num + 'calculation finished.\n' rec += '---' * 25 print(rec) record(job.root_path, rec) jobs.remove(job) count -= 1 count_dict[job.parameter['node']] -= 1 # test if there is some job which is already finished for job in jobs[:]: if if_cal_finish(job): finished_jobs.append(job) jobs.remove(job) # test if there is some jobs which are already submitted but not finished running_jobs = moni.get_running_jobs() for job in jobs[:]: if job in running_jobs: submitted_jobs.append(job) jobs.remove(job) # categorize jobs according to the nodes number jobs_dict = {} count_dict = {} nodes_list = [] for job in jobs: node = job.parameter['node'] if node not in nodes_list: nodes_list.append(node) jobs_dict[node] = [job] count_dict[node] = 0 else: jobs_dict[node].append(job) # submit and detect all jobs j = 0 while True: test_finished(submitted_jobs) moni.update_status() if len(finished_jobs) == total_num and len(submitted_jobs) == 0: break else: for node in nodes_list: if count_dict[node] < max_calculations_dict[node] and len( jobs_dict[node]) > 0: new_job = jobs_dict[node].pop() os.chdir(new_job.path) rename_file(new_job.path, '{}.out'.format(new_job.method)) out = submit_job(new_job, new_job.method) count += 1 count_dict[node] += 1 submitted_jobs.append(new_job) moni.insert_new_job(new_job, out) rec = new_job.path + '\n' rec += new_job.method + '\n' rec += 'job submitted.' rec += '\n' + out + '\n' rec += '---' * 25 record(new_job.root_path, rec) print(rec) else: # time.sleep(0.001) time.sleep(500) # test_calculation(j, jobs, finished_jobs) j += 1 if j > 8: rec = 'noting changes.\n' rec += '---' * 25 record(submitted_jobs[0].root_path, rec) j = 0 continue return finished_jobs
def submit(jobs, moni): job_num = len(jobs) max_paralell = 5 count = 0 submitted_jobs = [] finished_jobs = [] def test_finished(jobs): nonlocal count for job in jobs[:]: if if_cal_finish(job): finished_jobs.append(job) num = str(len(finished_jobs)) + '/' + str(job_num) rec = str(job) rec += '\n' rec += num + ' calculation finished.\n' rec += '---' * 25 print(rec) record(job.root_path, rec) jobs.remove(job) count -= 1 # test if there is some job which is already finished for job in jobs[:]: if if_cal_finish(job): finished_jobs.append(job) jobs.remove(job) # test if there is some jobs which are already submitted but not finished running_jobs = moni.get_running_jobs() for job in jobs[:]: if job in running_jobs: submitted_jobs.append(job) jobs.remove(job) # submit and detect all jobs j = 0 while True: test_finished(submitted_jobs) moni.update_status() if len(finished_jobs) == job_num and len(submitted_jobs) == 0: break else: if count < max_paralell and len(jobs) > 0: new_job = jobs.pop() os.chdir(new_job.path) rename_file(new_job.path, 'hf2.out') out = submit_job(new_job, 'hf2') count += 1 submitted_jobs.append(new_job) moni.insert_new_job(new_job, out) rec = str(new_job) + '\n' rec += 'job submitted.' rec += '\n' + out + '\n' rec += '---' * 25 record(new_job.root_path, rec) print(rec) else: time.sleep(500) # time.sleep(200) j += 1 # test_calculation(j, jobs, submitted_jobs, finished_jobs) # test function if j > 15: rec = 'noting changes.\n' rec += '---' * 25 record(submitted_jobs[0].root_path, rec) j = 0 continue return finished_jobs
def submit(jobs, moni): job_num = len(jobs) max_paralell = 8 # max_paralell = 75 count = 0 submitted_jobs = [] finished_jobs = [] def test_finished(jobs): """ test jobs which have benn submittdt is finished or not if a job finished, add it to list finished_jobs, and delete it from list submitted_jobs :param jobs: :return: """ nonlocal count for job in jobs[:]: if if_cal_finish(job): finished_jobs.append(job) num = str(len(finished_jobs)) + '/' + str(job_num) rec = str(job) rec += '\n' rec += num + ' calculation finished.\n' rec += '---' * 25 print(rec) record(job.root_path, rec) jobs.remove(job) count -= 1 # test if there is some job which is already finished for job in jobs[:]: if if_cal_finish(job): finished_jobs.append(job) jobs.remove(job) # test if there is some jobs which are already submitted but not finished running_jobs = moni.get_running_jobs() for job in jobs[:]: if job in running_jobs: submitted_jobs.append(job) jobs.remove(job) # submit and detect all jobs j = 0 while True: test_finished(submitted_jobs ) # update list finished_jobs and list submitted_jobs moni.update_status() if len(finished_jobs) == job_num and len(submitted_jobs) == 0: break else: if count <= max_paralell and len( jobs) > 0: # check the number of jobs which is running now new_job = jobs.pop() os.chdir(new_job.path) out = submit_job(new_job, 'lmp2') count += 1 submitted_jobs.append(new_job) moni.insert_new_job(new_job, out) rec = str(new_job) + '\n' rec += 'job submitted.' rec += '\n' + out + '\n' rec += '---' * 25 record(new_job.root_path, rec) print(rec) else: time.sleep(500) # time.sleep(1) j += 1 # test_calculation(j, jobs, submitted_jobs, finished_jobs) # test function if j > 15: rec = 'noting changes.\n' rec += '---' * 25 record(submitted_jobs[0].root_path, rec) j = 0 continue return finished_jobs
def submit(jobs, nodes, crystal_path, moni): job_numbers = len(jobs) max_paralell = 5 count = 0 submitted_jobs = [] finished_jobs = [] # find and submit the initial job loc = 0 for job in jobs: if job.x == '0' and job.z == '0': break loc += 1 if loc < len(jobs): job_init = jobs.pop(loc) os.chdir(job_init.path) if not if_cal_finish(job_init): copy_submit_scr(job_init, nodes, crystal_path) rename_file(job_init.path, 'geo_opt.out') out = submit_job(job_init, 'geo_opt') submitted_jobs.append(job_init) moni.insert_new_job(job_init, out) rec = job_init.path print(rec) rec += '\n' rec += 'job submitted...' rec += '\n' + out + '\n' rec += '---' * 25 record(job_init.root_path, rec) r = 0 while True: moni.update_status() if if_cal_finish(job_init): rec = job_init.path rec += '\n' rec += 'calculation finished...' record(job_init.root_path, rec) submitted_jobs.remove(job_init) finished_jobs.append(job_init) break else: time.sleep(500) r += 1 # test function # test_init_job(job_init, r) if r > 15: rec = job_init.path rec += '\n' rec += 'initial calculation still not finished...' record(job_init.root_path, rec) r = 0 continue else: finished_jobs.append(job_init) # test if there is some job which is already finished for job in jobs[:]: if if_cal_finish(job): # print('Job already finished: ', job) finished_jobs.append(job) jobs.remove(job) # test if there is some jobs which are already submitted but not finished running_jobs = moni.get_running_jobs() for job in jobs[:]: if job in running_jobs: submitted_jobs.append(job) jobs.remove(job) def test_finished(paths): nonlocal count # debug: UnboundLocalError: local variable 'count' referenced before assignment for path in paths[:]: if if_cal_finish(path): finished_jobs.append(path) num = str(len(finished_jobs)) + '/' + str(job_numbers) rec = path.path rec += '\n' rec += num + ' calculation finished.\n' rec += '---' * 25 print(rec) record(path.root_path, rec) paths.remove(path) count -= 1 if len(jobs) == 0: return finished_jobs else: i = 0 j = 0 while True: test_finished(submitted_jobs) moni.update_status() if len(finished_jobs) == job_numbers and len(submitted_jobs) == 0: break else: if count <= max_paralell and i < len(jobs): print(jobs[i].path) nearest_job = obtain_nearest_job(jobs[i]) os.chdir(jobs[i].path) copy_submit_scr(jobs[i], nodes, crystal_path, nearest_job) # copy_fort9(jobs[i]) rename_file(jobs[i].path, 'geo_opt.out') rename_file(jobs[i].path, 'fort.9') out = submit_job(jobs[i], 'geo_opt') count += 1 submitted_jobs.append(jobs[i]) moni.insert_new_job(jobs[i], out) rec = jobs[i].path + '\n' rec += 'job submitted.' rec += '\n' + out + '\n' rec += '---' * 25 record(jobs[i].root_path, rec) i += 1 else: time.sleep(500) j += 1 # j = test_calculation(j, submitted_jobs) # test function if j > 20: rec = 'noting changes.\n' rec += '---' * 25 # print(rec) record(submitted_jobs[0].root_path, rec) j = 0 continue return finished_jobs
def submit(jobs, nodes, crystal_path, moni): job_num = len(jobs) max_paralell = 5 count = 0 submitted_jobs = [] finished_jobs = [] def test_finished(jobs): nonlocal count for job in jobs[:]: if if_cal_finish(job): finished_jobs.append(job) num = str(len(finished_jobs)) + '/' + str(job_num) rec = str(job) rec += '\n' rec += num + ' calculation finished.\n' rec += '---' * 25 print(rec) record(job.root_path, rec) count -= 1 jobs.remove(job) # test if there is some job which is already finished for job in jobs[:]: if if_cal_finish(job): finished_jobs.append(job) jobs.remove(job) # test if there is some jobs which are already submitted but not finished running_jobs = moni.get_running_jobs() for job in jobs[:]: if job in running_jobs: submitted_jobs.append(job) jobs.remove(job) # find and submit the initial job # print('number of jobs: ', len(jobs)) init_jobs = [] for job in jobs[:]: if job.x == '0' and job.z == '0': init_jobs.append(job) jobs.remove(job) for job in init_jobs[:]: if not if_cal_finish(job): os.chdir(job.path) rename_file(job.path, 'hf.out') out = submit_job(job, 'hf') count += 1 submitted_jobs.append(job) moni.insert_new_job(job, out) rec = str(job) print(rec) rec += '\n' rec += 'job submitted.' rec += '\n' + out + '\n' rec += '---' * 25 record(job.root_path, rec) else: finished_jobs.append(job) # detect if init jobs finished r = 0 while True: # test_finished(submitted_jobs) # test function moni.update_status() if len(submitted_jobs) == 0: break else: time.sleep(500) r += 1 if r > 15: rec = 'initial calculation still not finished.\n' rec += '---' * 25 record(submitted_jobs[0].root_path, rec) r = 0 # submit and detect the other jobs j = 0 while True: test_finished(submitted_jobs) moni.update_status() if len(finished_jobs) == job_num and len(submitted_jobs) == 0: break else: if count < max_paralell and len(jobs) != 0: new_job = jobs.pop() os.chdir(new_job.path) nearest_job = obtain_nearest_job(new_job) rename_file(new_job.path, 'hf.out') rename_file(new_job.path, 'fort.9') # copy_fort9(new_job) copy_submit_scr(new_job, nodes, crystal_path, nearest_job) out = submit_job(new_job, 'hf') count += 1 submitted_jobs.append(new_job) moni.insert_new_job(new_job, out) rec = str(new_job) + '\n' rec += 'job submitted.' rec += '\n' + out + '\n' rec += '---' * 25 record(new_job.root_path, rec) print(rec) else: # time.sleep(10) time.sleep(500) j += 1 # test_calculation(j, jobs, submitted_jobs, finished_jobs) # test function if j > 15: rec = 'noting changes.\n' rec += '---' * 25 record(submitted_jobs[0].root_path, rec) j = 0 continue return finished_jobs