def compute(self, output_file, print_file, path_to_work):
"""
Postprocess the finished tasks to compute the property.
Output the result to a json database
Parameters
----------
output_file:
The file to output the property in json format
print_file:
The file to output the property in txt format
path_to_work:
The working directory where the computational tasks locate.
"""
path_to_work = os.path.abspath(path_to_work)
task_dirs = glob.glob(os.path.join(path_to_work, 'task.[0-9]*[0-9]'))
task_dirs.sort()
all_res = []
for ii in task_dirs:
with open(os.path.join(ii, 'inter.json')) as fp:
idata = json.load(fp)
poscar = os.path.join(ii, 'POSCAR')
task = make_calculator(idata, poscar)
res = task.compute(ii)
dumpfn(res, os.path.join(ii, 'result_task.json'), indent=4)
# all_res.append(res)
all_res.append(os.path.join(ii, 'result_task.json'))
# cwd = os.getcwd()
# os.chdir(path_to_work)
res, ptr = self._compute_lower(output_file, task_dirs, all_res)
# with open(output_file, 'w') as fp:
# json.dump(fp, res, indent=4)
with open(print_file, 'w') as fp:
fp.write(ptr)
def make_property(confs, inter_param, property_list):
# find all POSCARs and their name like mp-xxx
# ...
# conf_dirs = glob.glob(confs)
# conf_dirs.sort()
conf_dirs = []
for conf in confs:
conf_dirs.extend(glob.glob(conf))
conf_dirs.sort()
for ii in conf_dirs:
sepline(ch=ii, screen=True)
for jj in property_list:
if jj.get("skip", False):
continue
if 'init_from_suffix' and 'output_suffix' in jj:
do_refine = True
suffix = jj['output_suffix']
elif 'reproduce' in jj and jj['reproduce']:
do_refine = False
suffix = 'reprod'
else:
do_refine = False
suffix = '00'
# generate working directory like mp-xxx/eos_00 if jj['type'] == 'eos'
# handel the exception that the working directory exists
# ...
# determine the suffix: from scratch or refine
# ...
property_type = jj['type']
path_to_equi = os.path.join(ii, 'relaxation', 'relax_task')
path_to_work = os.path.join(ii, property_type + '_' + suffix)
if os.path.exists(path_to_work):
dlog.warning('%s already exists' % path_to_work)
else:
os.makedirs(path_to_work)
prop = make_property_instance(jj)
task_list = prop.make_confs(path_to_work, path_to_equi, do_refine)
inter_param_prop = inter_param
if 'cal_setting' in jj and 'overwrite_interaction' in jj[
'cal_setting']:
inter_param_prop = jj['cal_setting']['overwrite_interaction']
for kk in task_list:
poscar = os.path.join(kk, 'POSCAR')
inter = make_calculator(inter_param_prop, poscar)
inter.make_potential_files(kk)
dlog.debug(prop.task_type()) ### debug
inter.make_input_file(kk, prop.task_type(), prop.task_param())
prop.post_process(
task_list
) # generate same KPOINTS file for elastic when doing VASP
def run_equi(confs,
inter_param,
mdata):
# find all POSCARs and their name like mp-xxx
# ...
conf_dirs = []
for conf in confs:
conf_dirs.extend(glob.glob(conf))
conf_dirs.sort()
# generate a list of task names like mp-xxx/relaxation/relax_task
# ...
work_path_list = []
for ii in conf_dirs:
work_path_list.append(os.path.abspath(os.path.join(ii, 'relaxation')))
all_task = []
for ii in work_path_list:
all_task.append(os.path.join(ii, 'relax_task'))
inter_type = inter_param['type']
# vasp
if inter_type == "vasp":
mdata = decide_fp_machine(mdata)
elif inter_type in lammps_task_type:
mdata = decide_model_devi_machine(mdata)
else:
raise RuntimeError("unknown task %s, something wrong" % inter_type)
# dispatch the tasks
# POSCAR here is useless
virtual_calculator = make_calculator(inter_param, "POSCAR")
forward_files = virtual_calculator.forward_files()
forward_common_files = virtual_calculator.forward_common_files()
backward_files = virtual_calculator.backward_files()
# backward_files += logs
# ...
run_tasks = util.collect_task(all_task, inter_type)
if len(run_tasks) == 0:
return
else:
run_tasks = [os.path.basename(ii) for ii in all_task]
machine, resources, command, group_size = util.get_machine_info(mdata, inter_type)
print('%d tasks will be submited '%len(run_tasks))
for ii in range(len(work_path_list)):
work_path = work_path_list[ii]
disp = make_dispatcher(machine, resources, work_path, [run_tasks[ii]], group_size)
print("%s --> Runing... "%(work_path))
disp.run_jobs(resources,
command,
work_path,
[run_tasks[ii]],
group_size,
forward_common_files,
forward_files,
backward_files,
outlog='outlog',
errlog='errlog')
def post_equi(confs, inter_param):
# find all POSCARs and their name like mp-xxx
# ...
conf_dirs = []
for conf in confs:
conf_dirs.extend(glob.glob(conf))
conf_dirs.sort()
task_dirs = []
for ii in conf_dirs:
task_dirs.append(os.path.abspath(os.path.join(ii, 'relaxation', 'relax_task')))
task_dirs.sort()
# generate a list of task names like mp-xxx/relaxation
# ...
# dump the relaxation result.
for ii in task_dirs:
poscar = os.path.join(ii, 'POSCAR')
inter = make_calculator(inter_param, poscar)
res = inter.compute(ii)
dumpfn(res, os.path.join(ii, 'result.json'), indent=4)
def run_property(confs, inter_param, property_list, mdata):
# find all POSCARs and their name like mp-xxx
# ...
# conf_dirs = glob.glob(confs)
# conf_dirs.sort()
processes = len(property_list)
pool = Pool(processes=processes)
print("Submit job via %d processes" % processes)
conf_dirs = []
for conf in confs:
conf_dirs.extend(glob.glob(conf))
conf_dirs.sort()
task_list = []
work_path_list = []
for ii in conf_dirs:
sepline(ch=ii, screen=True)
for jj in property_list:
# determine the suffix: from scratch or refine
# ...
if jj.get("skip", False):
continue
if 'init_from_suffix' and 'output_suffix' in jj:
suffix = jj['output_suffix']
elif 'reproduce' in jj and jj['reproduce']:
suffix = 'reprod'
else:
suffix = '00'
property_type = jj['type']
path_to_work = os.path.abspath(
os.path.join(ii, property_type + '_' + suffix))
work_path_list.append(path_to_work)
tmp_task_list = glob.glob(
os.path.join(path_to_work, 'task.[0-9]*[0-9]'))
tmp_task_list.sort()
task_list.append(tmp_task_list)
inter_param_prop = inter_param
if 'cal_setting' in jj and 'overwrite_interaction' in jj[
'cal_setting']:
inter_param_prop = jj['cal_setting']['overwrite_interaction']
# dispatch the tasks
# POSCAR here is useless
virtual_calculator = make_calculator(inter_param_prop, "POSCAR")
forward_files = virtual_calculator.forward_files(property_type)
forward_common_files = virtual_calculator.forward_common_files(
property_type)
backward_files = virtual_calculator.backward_files(property_type)
# backward_files += logs
# ...
inter_type = inter_param_prop['type']
# vasp
if inter_type == "vasp":
mdata = decide_fp_machine(mdata)
elif inter_type in lammps_task_type:
mdata = decide_model_devi_machine(mdata)
else:
raise RuntimeError("unknown task %s, something wrong" %
inter_type)
work_path = path_to_work
all_task = tmp_task_list
run_tasks = util.collect_task(all_task, inter_type)
if len(run_tasks) == 0:
return
else:
ret = pool.apply_async(worker, (
work_path,
all_task,
forward_common_files,
forward_files,
backward_files,
mdata,
inter_type,
))
# run_tasks = [os.path.basename(ii) for ii in all_task]
# machine, resources, command, group_size = util.get_machine_info(mdata, inter_type)
# disp = make_dispatcher(machine, resources, work_path, run_tasks, group_size)
# disp.run_jobs(resources,
# command,
# work_path,
# run_tasks,
# group_size,
# forward_common_files,
# forward_files,
# backward_files,
# outlog='outlog',
# errlog='errlog')
pool.close()
pool.join()
if ret.successful():
print('finished')
def make_equi(confs, inter_param, relax_param):
# find all POSCARs and their name like mp-xxx
# ...
dlog.debug('debug info make equi')
if 'type_map' in inter_param:
ele_list = [key for key in inter_param['type_map'].keys()]
else:
ele_list = [key for key in inter_param['potcars'].keys()]
# ele_list = inter_param['type_map']
dlog.debug("ele_list %s" % ':'.join(ele_list))
conf_dirs = []
for conf in confs:
conf_dirs.extend(glob.glob(conf))
conf_dirs.sort()
# generate a list of task names like mp-xxx/relaxation/relax_task
# ...
cwd = os.getcwd()
# generate poscar for single element crystal
if len(ele_list) == 1:
for ii in conf_dirs:
os.chdir(ii)
crys_type = ii.split('/')[-1]
dlog.debug('crys_type: %s' % crys_type)
dlog.debug('pwd: %s' % os.getcwd())
if crys_type == 'std-fcc':
if not os.path.exists('POSCAR'):
crys.fcc1(ele_list[0]).to('POSCAR', 'POSCAR')
elif crys_type == 'std-hcp':
if not os.path.exists('POSCAR'):
crys.hcp(ele_list[0]).to('POSCAR', 'POSCAR')
elif crys_type == 'std-dhcp':
if not os.path.exists('POSCAR'):
crys.dhcp(ele_list[0]).to('POSCAR', 'POSCAR')
elif crys_type == 'std-bcc':
if not os.path.exists('POSCAR'):
crys.bcc(ele_list[0]).to('POSCAR', 'POSCAR')
elif crys_type == 'std-diamond':
if not os.path.exists('POSCAR'):
crys.diamond(ele_list[0]).to('POSCAR', 'POSCAR')
elif crys_type == 'std-sc':
if not os.path.exists('POSCAR'):
crys.sc(ele_list[0]).to('POSCAR', 'POSCAR')
os.chdir(cwd)
task_dirs = []
# make task directories like mp-xxx/relaxation/relax_task
# if mp-xxx/exists then print a warning and exit.
# ...
for ii in conf_dirs:
crys_type = ii.split('/')[-1]
dlog.debug('crys_type: %s' % crys_type)
if 'mp-' in crys_type and not os.path.exists(os.path.join(
ii, 'POSCAR')):
get_structure(crys_type).to('POSCAR', os.path.join(ii, 'POSCAR'))
poscar = os.path.abspath(os.path.join(ii, 'POSCAR'))
if not os.path.exists(poscar):
raise FileNotFoundError('no configuration for autotest')
relax_dirs = os.path.abspath(
os.path.join(ii, 'relaxation', 'relax_task')
) # to be consistent with property in make dispatcher
if os.path.exists(relax_dirs):
dlog.warning('%s already exists' % relax_dirs)
else:
os.makedirs(relax_dirs)
task_dirs.append(relax_dirs)
os.chdir(relax_dirs)
# copy POSCARs to mp-xxx/relaxation/relax_task
# ...
if os.path.isfile('POSCAR'):
os.remove('POSCAR')
os.symlink(os.path.relpath(poscar), 'POSCAR')
os.chdir(cwd)
task_dirs.sort()
# generate task files
relax_param['cal_type'] = 'relaxation'
if 'cal_setting' not in relax_param:
relax_param['cal_setting'] = {
"relax_pos": True,
"relax_shape": True,
"relax_vol": True
}
elif "relax_pos" not in relax_param['cal_setting']:
relax_param['cal_setting']['relax_pos'] = True
elif "relax_shape" not in relax_param['cal_setting']:
relax_param['cal_setting']['relax_shape'] = True
elif "relax_vol" not in relax_param['cal_setting']:
relax_param['cal_setting']['relax_vol'] = True
for ii in task_dirs:
poscar = os.path.join(ii, 'POSCAR')
dlog.debug('task_dir %s' % ii)
inter = make_calculator(inter_param, poscar)
inter.make_potential_files(ii)
inter.make_input_file(ii, 'relaxation', relax_param)
def run_equi(confs, inter_param, mdata):
# find all POSCARs and their name like mp-xxx
# ...
conf_dirs = []
for conf in confs:
conf_dirs.extend(glob.glob(conf))
conf_dirs.sort()
# generate a list of task names like mp-xxx/relaxation/relax_task
# ...
work_path_list = []
for ii in conf_dirs:
work_path_list.append(os.path.abspath(os.path.join(ii, 'relaxation')))
all_task = []
for ii in work_path_list:
all_task.append(os.path.join(ii, 'relax_task'))
inter_type = inter_param['type']
# vasp
if inter_type == "vasp":
mdata = convert_mdata(mdata, ["fp"])
elif inter_type in lammps_task_type:
mdata = convert_mdata(mdata, ["model_devi"])
else:
raise RuntimeError("unknown task %s, something wrong" % inter_type)
# dispatch the tasks
# POSCAR here is useless
virtual_calculator = make_calculator(inter_param, "POSCAR")
forward_files = virtual_calculator.forward_files()
forward_common_files = virtual_calculator.forward_common_files()
backward_files = virtual_calculator.backward_files()
# backward_files += logs
# ...
run_tasks = util.collect_task(all_task, inter_type)
if len(run_tasks) == 0:
return
else:
# if LooseVersion()
run_tasks = [os.path.basename(ii) for ii in all_task]
machine, resources, command, group_size = util.get_machine_info(
mdata, inter_type)
print('%d tasks will be submited ' % len(run_tasks))
for ii in range(len(work_path_list)):
work_path = work_path_list[ii]
disp = make_dispatcher(machine, resources, work_path,
[run_tasks[ii]], group_size)
print("%s --> Runing... " % (work_path))
api_version = mdata.get('api_version', '0.9')
if LooseVersion(api_version) < LooseVersion('1.0'):
warnings.warn(
f"the dpdispatcher will be updated to new version."
f"And the interface may be changed. Please check the documents for more details"
)
disp.run_jobs(resources,
command,
work_path, [run_tasks[ii]],
group_size,
forward_common_files,
forward_files,
backward_files,
outlog='outlog',
errlog='errlog')
elif LooseVersion(api_version) >= LooseVersion('1.0'):
submission = make_submission(
mdata_machine=machine,
mdata_resource=resources,
commands=[command],
work_path=work_path,
run_tasks=run_tasks,
group_size=group_size,
forward_common_files=forward_common_files,
forward_files=forward_files,
backward_files=backward_files,
outlog='outlog',
errlog='errlog')
submission.run_submission()
def run_property(confs, inter_param, property_list, mdata):
# find all POSCARs and their name like mp-xxx
# ...
# conf_dirs = glob.glob(confs)
# conf_dirs.sort()
processes = len(property_list)
pool = Pool(processes=processes)
print("Submit job via %d processes" % processes)
conf_dirs = []
for conf in confs:
conf_dirs.extend(glob.glob(conf))
conf_dirs.sort()
task_list = []
work_path_list = []
multiple_ret = []
for ii in conf_dirs:
sepline(ch=ii, screen=True)
for jj in property_list:
# determine the suffix: from scratch or refine
# ...
if jj.get("skip", False):
continue
if 'init_from_suffix' and 'output_suffix' in jj:
suffix = jj['output_suffix']
elif 'reproduce' in jj and jj['reproduce']:
suffix = 'reprod'
else:
suffix = '00'
property_type = jj['type']
path_to_work = os.path.abspath(
os.path.join(ii, property_type + '_' + suffix))
work_path_list.append(path_to_work)
tmp_task_list = glob.glob(
os.path.join(path_to_work, 'task.[0-9]*[0-9]'))
tmp_task_list.sort()
task_list.append(tmp_task_list)
inter_param_prop = inter_param
if 'cal_setting' in jj and 'overwrite_interaction' in jj[
'cal_setting']:
inter_param_prop = jj['cal_setting']['overwrite_interaction']
# dispatch the tasks
# POSCAR here is useless
virtual_calculator = make_calculator(inter_param_prop, "POSCAR")
forward_files = virtual_calculator.forward_files(property_type)
forward_common_files = virtual_calculator.forward_common_files(
property_type)
backward_files = virtual_calculator.backward_files(property_type)
# backward_files += logs
# ...
inter_type = inter_param_prop['type']
# vasp
if inter_type == "vasp":
mdata = convert_mdata(mdata, ["fp"])
elif inter_type in lammps_task_type:
mdata = convert_mdata(mdata, ["model_devi"])
else:
raise RuntimeError("unknown task %s, something wrong" %
inter_type)
work_path = path_to_work
all_task = tmp_task_list
run_tasks = util.collect_task(all_task, inter_type)
if len(run_tasks) == 0:
continue
else:
ret = pool.apply_async(worker, (
work_path,
all_task,
forward_common_files,
forward_files,
backward_files,
mdata,
inter_type,
))
multiple_ret.append(ret)
pool.close()
pool.join()
for ii in range(len(multiple_ret)):
if not multiple_ret[ii].successful():
raise RuntimeError("Job %d is not successful!" % ii)
print('%d jobs are finished' % len(multiple_ret))
def run_equi(confs, inter_param, mdata):
# find all POSCARs and their name like mp-xxx
# ...
conf_dirs = []
for conf in confs:
conf_dirs.extend(glob.glob(conf))
conf_dirs.sort()
processes = len(conf_dirs)
pool = Pool(processes=processes)
print("Submit job via %d processes" % processes)
# generate a list of task names like mp-xxx/relaxation/relax_task
# ...
work_path_list = []
for ii in conf_dirs:
work_path_list.append(os.path.abspath(os.path.join(ii, 'relaxation')))
all_task = []
for ii in work_path_list:
all_task.append(os.path.join(ii, 'relax_task'))
inter_type = inter_param['type']
# vasp
if inter_type == "vasp":
mdata = convert_mdata(mdata, ["fp"])
elif inter_type in lammps_task_type:
mdata = convert_mdata(mdata, ["model_devi"])
else:
raise RuntimeError("unknown task %s, something wrong" % inter_type)
# dispatch the tasks
# POSCAR here is useless
virtual_calculator = make_calculator(inter_param, "POSCAR")
forward_files = virtual_calculator.forward_files()
forward_common_files = virtual_calculator.forward_common_files()
backward_files = virtual_calculator.backward_files()
# backward_files += logs
# ...
run_tasks = util.collect_task(all_task, inter_type)
if len(run_tasks) == 0:
return
else:
run_tasks = [os.path.basename(ii) for ii in all_task]
machine, resources, command, group_size = util.get_machine_info(
mdata, inter_type)
print('%d tasks will be submited ' % len(run_tasks))
multiple_ret = []
for ii in range(len(work_path_list)):
work_path = work_path_list[ii]
ret = pool.apply_async(worker, (
work_path,
run_tasks[ii],
forward_common_files,
forward_files,
backward_files,
mdata,
inter_type,
))
multiple_ret.append(ret)
pool.close()
pool.join()
for ii in range(len(multiple_ret)):
if not multiple_ret[ii].successful():
raise RuntimeError(
"Task %d is not successful! work_path: %s " %
(ii, work_path_list[ii]))
print('finished')
