def test_cal_cv(self):
dim_list = cal_cv_dim(self.mol_path + "/conf0.gro", self.cv_file)
self.assertTrue(dim_list[0] == 2)
self.assertTrue(dim_list[1] == 0)
dim_list = cal_cv_dim(self.mol_path + "/conf0.gro", self.cv_file_2)
self.assertTrue(dim_list[0] == 2)
self.assertTrue(dim_list[1] == 1)
def post_res(iter_index, json_file, machine_json, cv_file, base_dir="./"):
json_file = os.path.abspath(json_file)
machine_json = os.path.abspath(machine_json)
cv_file = os.path.abspath(cv_file)
base_dir = os.path.abspath(base_dir) + "/"
iter_name = make_iter_name(iter_index)
res_path = base_dir + iter_name + "/" + res_name + "/"
cwd = os.getcwd()
fp = open(json_file, 'r')
jdata = json.load(fp)
fp.close()
os.chdir(res_path)
all_task = glob.glob("/[0-9]*[0-9]")
all_task = list(
filter(lambda x: os.path.isdir(x), glob.glob("[0-9]*[0-9]")))
if len(all_task) == 0:
np.savetxt(res_path + 'data.raw', [], fmt="%.6e")
os.chdir(cwd)
return
all_task.sort()
all_task_reldir = [os.path.relpath(ii, res_path) for ii in all_task]
centers = []
force = []
ndim = 0
_conf_file = os.path.abspath(all_task[0] + "/conf.gro")
cv_dim_list = cal_cv_dim(_conf_file, cv_file)
cv_dih_dim = cv_dim_list[0]
cmpf_cmd = "python3 {}/cmpf.py".format(LIB_PATH)
cmpf_cmd += " -c %d" % cv_dih_dim
cmpf_log = "cmpf.log"
print("rid.post_res.post_res:cmpf_cmd:", cmpf_cmd)
cmpf_resources = set_resource(machine_json, target="cmpf")
machine = set_machine(machine_json, target="cmpf")
cmpf_task = [
Task(command=cmpf_cmd,
task_work_path="{}".format(ii),
outlog=cmpf_log,
errlog=cmpf_log) for ii in all_task_reldir
]
cmpf_submission = Submission(work_base=res_path,
machine=machine,
resources=cmpf_resources,
task_list=cmpf_task)
cmpf_submission.run_submission()
print('cmpf done')
abs_res_path = os.getcwd()
for work_path in all_task:
os.chdir(work_path)
this_centers = np.loadtxt('centers.out')
centers = np.append(centers, this_centers)
this_force = np.loadtxt('force.out')
force = np.append(force, this_force)
ndim = this_force.size
assert (ndim == this_centers.size
), "center size is diff to force size in " + work_path
os.chdir(abs_res_path)
os.chdir(cwd)
centers = np.reshape(centers, [-1, ndim])
force = np.reshape(force, [-1, ndim])
data = np.concatenate((centers, force), axis=1)
np.savetxt(res_path + 'data.raw', data, fmt="%.6e")
norm_force = np.linalg.norm(force, axis=1)
log_task("min|f| = %e max|f| = %e avg|f| = %e" %
(np.min(norm_force), np.max(norm_force), np.average(norm_force)))
print("min|f| = %e max|f| = %e avg|f| = %e" %
(np.min(norm_force), np.max(norm_force), np.average(norm_force)))
print('Saving cmpf finished!')
print("Post process of restrained MD finished.")
print(os.getcwd())
def make_res(iter_index,
json_file,
cv_file,
mol_path,
base_dir="./",
custom_mdp=None):
json_file = os.path.abspath(json_file)
fp = open(json_file, 'r')
jdata = json.load(fp)
fp.close()
cv_file = os.path.abspath(cv_file)
numb_walkers = jdata["numb_walkers"]
bias_nsteps = jdata["bias_nsteps"]
bias_frame_freq = jdata["bias_frame_freq"]
nsteps = jdata["res_nsteps"]
frame_freq = jdata["res_frame_freq"]
sel_threshold = jdata["sel_threshold"]
max_sel = jdata["max_sel"]
cluster_threshold = jdata["cluster_threshold"]
init_numb_cluster_upper = int(jdata["init_numb_cluster_upper"])
init_numb_cluster_lower = int(jdata["init_numb_cluster_lower"])
init_numb_cluster = [init_numb_cluster_lower, init_numb_cluster_upper]
base_dir = os.path.abspath(base_dir) + "/"
iter_name = make_iter_name(iter_index)
enhc_path = base_dir + iter_name + "/" + enhc_name + "/"
res_path = base_dir + iter_name + "/" + res_name + "/"
create_path(res_path)
cwd = os.getcwd()
_conf_file = enhc_path + make_walker_name(0) + "/" + "conf.gro"
cv_dim_list = cal_cv_dim(_conf_file, cv_file)
cv_dim = sum(cv_dim_list)
cv_dih_dim = cv_dim_list[0]
ret_list = [True for ii in range(numb_walkers)]
weight = jdata["cv_weight_for_cluster"]
if type(weight) == list:
assert len(
weight
) == cv_dim, "Number of values in the weight list is not equal to the number of CVs."
elif type(weight) == float or type(weight) == int:
assert weight != 0
else:
raise TypeError(
"Invalid type of weight of CVs for clustering. Please use int or list instead."
)
# check if we have graph in enhc
for walker_idx in range(numb_walkers):
cls_sel = None
walker_path = enhc_path + walker_format % walker_idx + "/"
graph_files = glob.glob(walker_path + "*.pb")
if len(graph_files) != 0:
cluster_threshold = np.loadtxt(base_dir + "cluster_threshold.dat")
os.chdir(walker_path)
models = glob.glob("*.pb")
std_message = make_std(cv_dim,
dataset=enhc_out_angle,
models=models,
threshold=sel_threshold,
output="sel.out",
output_angle="sel.angle.out")
os.system('echo "{}" > sel.log'.format(std_message))
log_task("select with threshold %f" % sel_threshold)
os.chdir(cwd)
sel_idx = []
sel_angles = np.array([])
with open(walker_path + "sel.out") as fp:
for line in fp:
sel_idx += [int(x) for x in line.split()]
if len(sel_idx) != 0:
sel_angles = np.reshape(
np.loadtxt(walker_path + 'sel.angle.out'), [-1, cv_dim])
elif len(sel_idx) == 0:
np.savetxt(walker_path + 'num_of_cluster.dat', [0], fmt='%d')
np.savetxt(walker_path + 'cls.sel.out', [], fmt='%d')
continue
else:
cluster_threshold = jdata["cluster_threshold"]
sel_idx = range(
len(glob.glob(walker_path + enhc_out_conf + "conf*gro")))
sel_angles = np.loadtxt(walker_path + enhc_out_angle)
sel_angles = np.reshape(sel_angles, [-1, cv_dim])
np.savetxt(walker_path + 'sel.out', sel_idx, fmt='%d')
np.savetxt(walker_path + 'sel.angle.out', sel_angles, fmt='%.6f')
cls_sel, cluster_threshold = make_threshold(
walker_idx, walker_path, base_dir, sel_angles,
cluster_threshold, init_numb_cluster, cv_dih_dim, weight)
if cls_sel is None:
print(sel_angles, cluster_threshold, cv_dih_dim)
cls_sel = sel_from_cluster(sel_angles, cluster_threshold,
cv_dih_dim, weight)
conf_start = 0
conf_every = 1
sel_idx = np.array(sel_idx, dtype=np.int)
assert (
len(sel_idx) == sel_angles.shape[0]
), "{} selected indexes don't match {} selected angles.".format(
len(sel_idx), sel_angles.shape[0])
sel_idx = config_cls(sel_idx, cls_sel, max_sel, walker_path,
cluster_threshold, sel_angles)
res_angles = np.loadtxt(walker_path + enhc_out_angle)
res_angles = np.reshape(res_angles, [-1, cv_dim])
res_angles = res_angles[sel_idx]
np.savetxt(walker_path + 'cls.sel.out', sel_idx, fmt='%d')
np.savetxt(walker_path + 'cls.sel.angle.out', res_angles, fmt='%.6f')
res_confs = []
for ii in sel_idx:
res_confs.append(walker_path + enhc_out_conf + ("conf%d.gro" % ii))
assert (len(res_confs) == res_angles.shape[0]
), "number of enhc out conf does not match out angle"
assert (len(sel_idx) == res_angles.shape[0]
), "number of enhc out conf does not match number sel"
nconf = len(res_confs)
if nconf == 0:
ret_list[walker_idx] = False
continue
sel_list = make_sel_list(nconf, sel_idx)
log_task("selected %d confs, indexes: %s" % (nconf, sel_list))
make_conf(nconf,
res_path,
walker_idx,
walker_path,
sel_idx,
jdata,
mol_path,
conf_start=0,
conf_every=1,
custom_mdp=custom_mdp)
make_res_plumed(nconf,
jdata,
res_path,
walker_idx,
sel_idx,
res_angles,
_conf_file,
cv_file,
conf_start=0,
conf_every=1)
print("Restrained MD has been prepared.")
def run_train(iter_index, json_file, machine_json, cv_file, base_dir="./"):
json_file = os.path.abspath(json_file)
cv_file = os.path.abspath(cv_file)
fp = open(json_file, 'r')
jdata = json.load(fp)
fp.close()
cv_file = os.path.abspath(cv_file)
numb_model = jdata["numb_model"]
train_thread = jdata["train_thread"]
res_iter = jdata["res_iter"]
base_dir = os.path.abspath(base_dir) + "/"
iter_name = make_iter_name(iter_index)
train_path = base_dir + iter_name + "/" + train_name + "/"
if check_new_data(iter_index, train_path, base_dir):
return
enhc_path = base_dir + iter_name + "/" + enhc_name + "/"
_conf_file = enhc_path + "000/conf.gro"
cv_dim_list = cal_cv_dim(_conf_file, cv_file)
cwd = os.getcwd()
neurons = jdata["neurons"]
batch_size = jdata["batch_size"]
if iter_index < res_iter:
numb_epoches = jdata["numb_epoches"]
starter_lr = jdata["starter_lr"]
decay_steps = jdata["decay_steps"]
decay_rate = jdata["decay_rate"]
cmdl_args = ""
else:
numb_epoches = jdata["res_numb_epoches"]
starter_lr = jdata["res_starter_lr"]
decay_steps = jdata["res_decay_steps"]
decay_rate = jdata["res_decay_rate"]
old_ratio = jdata["res_olddata_ratio"]
cmdl_args = " --restart --use-mix --old-ratio %f " % old_ratio
if jdata["resnet"]:
cmdl_args += " --resnet "
cmdl_args += " -n "
for nn in neurons:
cmdl_args += "%d " % nn
cmdl_args += " -c "
for cv_dim in cv_dim_list:
cmdl_args += "%d " % cv_dim
cmdl_args += " -b " + str(batch_size)
cmdl_args += " -e " + str(numb_epoches)
cmdl_args += " -l " + str(starter_lr)
cmdl_args += " --decay-steps " + str(decay_steps)
cmdl_args += " --decay-rate " + str(decay_rate)
train_cmd = "python3 {}/train.py -t {:d}".format(NN_PATH, train_thread)
train_cmd += cmdl_args
train_cmd = cmd_append_log(train_cmd, "train.log")
freez_cmd = "python3 {}/freeze.py -o graph.pb".format(NN_PATH)
freez_cmd = cmd_append_log(freez_cmd, "freeze.log")
task_dirs = [("%03d" % ii) for ii in range(numb_model)]
print('lib.modeling.run_train:train_cmd:', train_cmd)
print('lib.modeling.run_train:freez_cmd:', freez_cmd)
print('lib.modeling.run_train:train_path:', train_path)
print('lib.modeling.run_train:task_dirs:', task_dirs)
resources = set_resource(machine_json, target="train")
machine = set_machine(machine_json, target="train")
train_task = [
Task(command=train_cmd,
task_work_path=ii,
outlog='train.log',
errlog='train.log') for ii in task_dirs
]
train_submission = Submission(work_base=train_path,
machine=machine,
resources=resources,
task_list=train_task)
train_submission.run_submission()
freez_task = [
Task(command=freez_cmd,
task_work_path=ii,
outlog='freeze.log',
errlog='freeze.log') for ii in task_dirs
]
freez_submission = Submission(work_base=train_path,
machine=machine,
resources=resources,
task_list=freez_task)
freez_submission.run_submission()
os.chdir(train_path)
for ii in range(numb_model):
os.symlink("%03d/graph.pb" % ii, "graph.%03d.pb" % ii)
os.chdir(cwd)
print("Training finished!")