def run_optimizer(self,
check_result=True,
check_iter=True,
use_pvals=False):
optimizer = self.get_optimizer()
## Actually run the optimizer.
self.logger.debug("Done setting up! Running optimizer...\n")
result = optimizer.Run()
self.logger.debug("\nOptimizer finished. Final results:\n")
self.logger.debug(str(result) + '\n')
## Convert result to physical values if desired.
if use_pvals:
result = optimizer.FF.create_pvals(result)
if check_result:
msg = "\nCalculation results have changed from previously calculated values.\n " \
"If this seems reasonable, update %s in test_system.py with these values" % self.expected_results_name
np.testing.assert_allclose(self.expected_results,
result,
atol=self.absolute_tolerance,
err_msg=msg)
if check_iter:
# Fail if calculation takes longer than previously to converge
assert ITERATIONS_TO_CONVERGE >= Counter(), "Calculation took longer than expected to converge (%d iterations vs previous of %d)" %\
(ITERATIONS_TO_CONVERGE, Counter())
return result
def submit_jobs(self, mvals, AGrad=True, AHess=True):
# This routine is called by Objective.stage() will run before "get".
# It submits the jobs to the Work Queue and the stage() function will wait for jobs to complete.
#
# First dump the force field to a pickle file
lp_dump((self.FF, mvals, self.OptionDict, AGrad), 'forcebalance.p')
# Give the user an opportunity to copy over data from a previous (perhaps failed) run.
if Counter() == First() and self.manual:
warn_press_key(
"Now's our chance to fill the temp directory up with data!\n(Considering using 'read' or 'continue' for better checkpointing)",
timeout=7200)
# If self.save_traj == 1, delete the trajectory files from a previous good optimization step.
if Counter() > First() and GoodStep() and self.save_traj < 2:
for fn in self.last_traj:
if os.path.exists(fn):
os.remove(fn)
self.last_traj = []
# Set up and run the NPT simulations.
snum = 0
for label, pt in zip(self.Labels, self.PhasePoints):
T = pt[0]
P = pt[1]
Punit = pt[2]
if Punit == 'bar':
P *= 1.0 / 1.01325
if not os.path.exists(label):
os.makedirs(label)
os.chdir(label)
self.npt_simulation(T, P, snum)
os.chdir('..')
snum += 1
def meta_get(self, mvals, AGrad=False, AHess=False, customdir=None):
"""
Wrapper around the get function.
Create the directory for the target, and then calls 'get'.
If we are reading existing data, go into the appropriate read directory and call read() instead.
The 'get' method should not worry about the directory that it's running in.
"""
## Directory of the current iteration; if not None, then the simulation runs under
## temp/target_name/iteration_number
## The 'customdir' is customizable and can go below anything
cwd = os.getcwd()
absgetdir = os.path.join(self.root, self.tempdir)
if Counter() is not None:
# Not expecting more than ten thousand iterations
if Counter() > 10000:
logger.error(
'Cannot handle more than 10000 iterations due to current directory structure. Consider revising code.\n'
)
raise RuntimeError
iterdir = "iter_%04i" % Counter()
absgetdir = os.path.join(absgetdir, iterdir)
if customdir is not None:
absgetdir = os.path.join(absgetdir, customdir)
if not os.path.exists(absgetdir):
os.makedirs(absgetdir)
os.chdir(absgetdir)
self.link_from_tempdir(absgetdir)
self.rundir = absgetdir.replace(self.root + '/', '')
## Read existing information from disk (i.e. when recovering an aborted run)
# Note that reading information is not supported for custom folders (e.g. microiterations during search)
if self.rd is not None and (
not self.evaluated
) and self.read_objective and customdir is None:
os.chdir(self.absrd())
logger.info("Reading objective function information from %s\n" %
os.getcwd())
Answer = self.read(mvals, AGrad, AHess)
os.chdir(absgetdir)
else:
## Evaluate the objective function.
Answer = self.get(mvals, AGrad, AHess)
if self.write_objective:
forcebalance.nifty.lp_dump(Answer, 'objective.p')
## Save the force field files to this directory, so that it
## reflects the objective function and properties that were
## printed out.
if not in_fd():
self.FF.make(mvals)
os.chdir(cwd)
return Answer
def stage(self,
mvals,
AGrad=False,
AHess=False,
use_iterdir=True,
customdir=None,
firstIteration=False):
"""
Stages the directory for the target, and then launches Work Queue processes if any.
The 'get' method should not worry about the directory that it's running in.
"""
if self.sleepy > 0:
logger.info("Sleeping for %i seconds as directed...\n" %
self.sleepy)
time.sleep(self.sleepy)
## Directory of the current iteration; if not None, then the simulation runs under
## temp/target_name/iteration_number
## The 'customdir' is customizable and can go below anything
cwd = os.getcwd()
absgetdir = os.path.join(self.root, self.tempdir)
if use_iterdir and Counter() is not None:
## Not expecting more than ten thousand iterations
iterdir = "iter_%04i" % Counter()
absgetdir = os.path.join(absgetdir, iterdir)
if customdir is not None:
absgetdir = os.path.join(absgetdir, customdir)
## Go into the directory where get() will be executed.
if not os.path.exists(absgetdir):
os.makedirs(absgetdir)
os.chdir(absgetdir)
self.link_from_tempdir(absgetdir)
## Write mathematical parameters to file; will be used to checkpoint calculation.
if not in_fd():
np.savetxt('mvals.txt', mvals)
## Read in file that specifies which derivatives may be skipped.
if Counter() >= self.zerograd and self.zerograd >= 0:
self.read_0grads()
self.rundir = absgetdir.replace(self.root + '/', '')
## Submit jobs to the Work Queue.
if self.rd is None or (not firstIteration):
self.submit_jobs(mvals, AGrad, AHess)
elif customdir is not None:
# Allows us to submit micro-iteration jobs for remote targets
self.submit_jobs(mvals, AGrad, AHess)
os.chdir(cwd)
return
def get_G(self, mvals=None, customdir=None):
"""Computes the objective function contribution and its gradient.
First the low-level 'get' method is called with the analytic gradient
switch turned on. Then we loop through the fd1_pids and compute
the corresponding elements of the gradient by finite difference,
if the 'fdgrad' switch is turned on. Alternately we can compute
the gradient elements and diagonal Hessian elements at the same time
using central difference if 'fdhessdiag' is turned on.
In this function we also record which parameters cause a
nonzero change in the objective function contribution.
Parameters which do not change the objective function will
not be differentiated in subsequent calculations. This is
recorded in a text file in the targets directory.
"""
Ans = self.meta_get(mvals, 1, 0, customdir=customdir)
for i in self.pgrad:
if any([j in self.FF.plist[i]
for j in self.fd1_pids]) or 'ALL' in self.fd1_pids:
if self.fdhessdiag:
Ans['G'][i], Ans['H'][i,
i] = f12d3p(fdwrap_G(self, mvals, i),
self.h,
f0=Ans['X'])
elif self.fdgrad:
Ans['G'][i] = f1d2p(fdwrap_G(self, mvals, i),
self.h,
f0=Ans['X'])
self.gct += 1
if Counter() == self.zerograd and self.zerograd >= 0:
self.write_0grads(Ans)
return Ans
def submit_jobs(self, mvals, AGrad=False, AHess=False):
id_string = "%s_iter%04i" % (self.name, Counter())
self.serialize_ff(mvals, outside="forcefield-remote")
forcebalance.nifty.lp_dump((AGrad, AHess, id_string, self.r_options, self.r_tgt_opts, self.pgrad),'options.p')
# Link in the rpfx script.
if len(self.rpfx) > 0:
forcebalance.nifty.LinkFile(os.path.join(os.path.split(__file__)[0],"data",self.rpfx),self.rpfx)
forcebalance.nifty.LinkFile(os.path.join(os.path.split(__file__)[0],"data","rtarget.py"),"rtarget.py")
forcebalance.nifty.LinkFile(os.path.join(self.root, self.tempdir, "target.tar.bz2"),"target.tar.bz2")
wq = getWorkQueue()
# logger.info("Sending target '%s' to work queue for remote evaluation\n" % self.name)
# input:
# forcefield.p: pickled force field
# options.p: pickled mvals, options
# rtarget.py: remote target evaluation script
# target.tar.bz2: tarred target
# output:
# objective.p: pickled objective function dictionary
# indicate.log: results of target.indicate() written to file
# if len(self.rpfx) > 0 and self.rpfx not in ['rungmx.sh', 'runcuda.sh']:
# logger.error('Unsupported prefix script for launching remote target')
# raise RuntimeError
forcebalance.nifty.queue_up(wq, "%spython rtarget.py > rtarget.out 2>&1" % (("sh %s%s " % (self.rpfx, " -b" if self.rbak else ""))
if len(self.rpfx) > 0 else ""),
["forcefield.p", "options.p", "rtarget.py", "target.tar.bz2"] + ([self.rpfx] if len(self.rpfx) > 0 else []),
['objective.p', 'indicate.log', 'rtarget.out'],
tgt=self, tag=self.name, verbose=False)
def get_H(self,mvals=None,customdir=None):
"""Computes the objective function contribution and its gradient / Hessian.
First the low-level 'get' method is called with the analytic gradient
and Hessian both turned on. Then we loop through the fd1_pids and compute
the corresponding elements of the gradient by finite difference,
if the 'fdgrad' switch is turned on.
This is followed by looping through the fd2_pids and computing the corresponding
Hessian elements by finite difference. Forward finite difference is used
throughout for the sake of speed.
"""
Ans = self.meta_get(mvals,1,1,customdir=customdir)
if self.fdhess:
for i in self.pgrad:
if any([j in self.FF.plist[i] for j in self.fd1_pids]) or 'ALL' in self.fd1_pids:
Ans['G'][i] = f1d2p(fdwrap_G(self,mvals,i),self.h,f0 = Ans['X'])
for i in self.pgrad:
if any([j in self.FF.plist[i] for j in self.fd2_pids]) or 'ALL' in self.fd2_pids:
FDSlice = f1d2p(fdwrap_H(self,mvals,i),self.h,f0 = Ans['G'])
Ans['H'][i,:] = FDSlice
Ans['H'][:,i] = FDSlice
elif self.fdhessdiag:
for i in self.pgrad:
if any([j in self.FF.plist[i] for j in self.fd2_pids]) or 'ALL' in self.fd2_pids:
Ans['G'][i], Ans['H'][i,i] = f12d3p(fdwrap_G(self,mvals,i),self.h, f0 = Ans['X'])
if Counter() == self.zerograd and self.zerograd >= 0:
self.write_0grads(Ans)
self.hct += 1
return Ans
def compute(mvals_, indicate=False):
self.FF.make(mvals_)
M_opts = None
compute.emm = []
compute.rmsd = []
for i in range(self.ns):
energy, rmsd, M_opt = self.engine.optimize(shot=i, align=False)
# Create a molecule object to hold the MM-optimized structures
compute.emm.append(energy)
compute.rmsd.append(rmsd)
if M_opts is None:
M_opts = deepcopy(M_opt)
else:
M_opts += M_opt
compute.emm = np.array(compute.emm)
compute.emm -= compute.emm[self.smin]
compute.rmsd = np.array(compute.rmsd)
if indicate:
if self.writelevel > 0:
M_opts.write('mm_minimized.xyz')
if self.ndim == 1:
import matplotlib.pyplot as plt
plt.switch_backend('agg')
fig, ax = plt.subplots()
dihedrals = np.array(
[i[0] for i in self.metadata['torsion_grid_ids']])
dsort = np.argsort(dihedrals)
ax.plot(dihedrals[dsort], self.eqm[dsort], label='QM')
if hasattr(self, 'emm_orig'):
ax.plot(dihedrals[dsort],
compute.emm[dsort],
label='MM Current')
ax.plot(dihedrals[dsort],
self.emm_orig[dsort],
label='MM Initial')
else:
ax.plot(dihedrals[dsort],
compute.emm[dsort],
label='MM Initial')
self.emm_orig = compute.emm.copy()
ax.legend()
ax.set_xlabel('Dihedral (degree)')
ax.set_ylabel('Energy (kcal/mol)')
fig.suptitle(
'Torsion profile: iteration %i\nSystem: %s' %
(Counter(), self.name))
fig.savefig('plot_torsion.pdf')
return (np.sqrt(self.wts) / self.energy_denom) * (compute.emm -
self.eqm)
def draw_vibfreq_scatter_plot_n_overlap_matrix(name, engine, ref_eigvals, ref_eigvecs, freqs_rearr, normal_modes_rearr):
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import make_axes_locatable, axes_size
plt.switch_backend('agg')
fig, axs = plt.subplots(1,2, figsize=(10,6))
overlap_matrix = np.array([[(vib_overlap(engine, v1, v2)) for v2 in normal_modes_rearr] for v1 in ref_eigvecs])
qm_overlap_matrix = np.array([[(vib_overlap(engine,v1, v2)) for v2 in ref_eigvecs] for v1 in ref_eigvecs])
axs[0].scatter(ref_eigvals, freqs_rearr, label='MM vibrational frequencies(rearr.)')
axs[0].plot(ref_eigvals,ref_eigvals, 'k-')
axs[0].legend()
axs[0].set_xlabel(r'QM vibrational frequency ($cm^{-1}$)')
axs[0].set_ylabel(r'MM vibrational frequency ($cm^{-1}$)')
mae = np.sum(np.abs(ref_eigvals - freqs_rearr))/ len(ref_eigvals)
axs[0].set_title(f'QM vs. MM vibrational frequencies\n MAE= {mae:.2f}')
x0,x1 = axs[0].get_xlim()
y0,y1 = axs[0].get_ylim()
axs[0].set_aspect((x1-x0)/(y1-y0))
# move ax x axis to top
axs[1].xaxis.tick_top()
# move ax x ticks inside
axs[1].tick_params(axis="y", direction='in')
axs[1].tick_params(axis="x", direction='in')
# draw matrix
im = axs[1].imshow(overlap_matrix, cmap= 'OrRd', vmin=0,vmax=1)
# colorbar
aspect = 20
pad_fraction = 0.5
divider = make_axes_locatable(axs[1])
width = axes_size.AxesY(axs[1], aspect=1./aspect)
pad = axes_size.Fraction(pad_fraction, width)
cax = divider.append_axes("right", size=width, pad=pad)
cax.yaxis.tick_right()
cax.xaxis.set_visible(False)
plt.colorbar(im, cax=cax)
corr_coef = cal_corr_coef(overlap_matrix)
err = np.linalg.norm(qm_overlap_matrix - overlap_matrix)/np.linalg.norm(qm_overlap_matrix) # measure of error in matrix (Relative error)
axs[1].set_title(f'QM vs. MM normal modes\n Correlation coef. ={corr_coef:.4f}, Error={err:.4f}')
# # move ax x axis to top
# axs[2].xaxis.tick_top()
# # move ax x ticks inside
# axs[2].tick_params(axis="y", direction='in')
# axs[2].tick_params(axis="x", direction='in')
# # draw matrix
# im = axs[2].imshow(qm_overlap_matrix, cmap= 'OrRd', vmin=0,vmax=1)
# # colorbar
# aspect = 20
# pad_fraction = 0.5
# divider = make_axes_locatable(axs[2])
# width = axes_size.AxesY(axs[2], aspect=1./aspect)
# pad = axes_size.Fraction(pad_fraction, width)
# cax = divider.append_axes("right", size=width, pad=pad)
# cax.yaxis.tick_right()
# cax.xaxis.set_visible(False)
# plt.colorbar(im, cax=cax)
# axs[2].set_title(f'(QM normal modes for reference)')
plt.tight_layout()
plt.subplots_adjust(top=0.85)
fig.suptitle('Hessian: iteration %i\nSystem: %s' % (Counter(), name))
fig.savefig('vibfreq_scatter_plot_n_overlap_matrix.pdf')
def runTest(self):
"""Check water tutorial study runs without errors"""
self.logger.debug("\nSetting input file to 'very_simple.in'\n")
input_file = 'very_simple.in'
## The general options and target options that come from parsing the input file
self.logger.debug("Parsing inputs...\n")
options, tgt_opts = parse_inputs(input_file)
self.logger.debug("options:\n%s\n\ntgt_opts:\n%s\n\n" %
(str(options), str(tgt_opts)))
self.assertEqual(dict,
type(options),
msg="\nParser gave incorrect type for options")
self.assertEqual(list,
type(tgt_opts),
msg="\nParser gave incorrect type for tgt_opts")
for target in tgt_opts:
self.assertEqual(
dict,
type(target),
msg="\nParser gave incorrect type for target dict")
## The force field component of the project
forcefield = FF(options)
self.assertEqual(FF,
type(forcefield),
msg="\nExpected forcebalance forcefield object")
## The objective function
objective = Objective(options, tgt_opts, forcefield)
self.assertEqual(Objective,
type(objective),
msg="\nExpected forcebalance objective object")
## The optimizer component of the project
self.logger.debug("Creating optimizer: ")
optimizer = Optimizer(options, objective, forcefield)
self.assertEqual(Optimizer,
type(optimizer),
msg="\nExpected forcebalance optimizer object")
self.logger.debug(str(optimizer) + "\n")
## Actually run the optimizer.
self.logger.debug("Done setting up! Running optimizer...\n")
result = optimizer.Run()
self.logger.debug("\nOptimizer finished. Final results:\n")
self.logger.debug(str(result) + '\n')
self.assertNdArrayEqual(
EXPECTED_WATER_RESULTS,
result,
delta=0.001,
msg=
"\nCalculation results have changed from previously calculated values.\n"
"If this seems reasonable, update EXPECTED_WATER_RESULTS in test_system.py with these values"
)
# Fail if calculation takes longer than previously to converge
self.assertGreaterEqual(ITERATIONS_TO_CONVERGE, Counter(), msg="\nCalculation took longer than expected to converge (%d iterations vs previous of %d)" %\
(ITERATIONS_TO_CONVERGE, Counter()))
def get(self, mvals, AGrad=False, AHess=False):
"""
LPW 05-30-2012
This subroutine builds the objective function (and optionally
its derivatives) from a general software.
This subroutine interfaces with simulation software 'drivers'.
The driver is expected to give exact values, fitting values, and weights.
@param[in] mvals Mathematical parameter values
@param[in] AGrad Switch to turn on analytic gradient
@param[in] AHess Switch to turn on analytic Hessian
@return Answer Contribution to the objective function
"""
global LAST_MVALS, CHECK_BASIS
# print mvals
# print LAST_MVALS
# print mvals == LAST_MVALS
if LAST_MVALS == None or not (mvals == LAST_MVALS).all():
CHECK_BASIS = False
else:
CHECK_BASIS = False
Answer = {}
Fac = 1000000
## Dictionary for derivative terms
dM = {}
# Create the new force field!!
NP = len(mvals)
G = np.zeros(NP)
H = np.zeros((NP,NP))
pvals = self.FF.make(mvals)
if float('Inf') in pvals:
return {'X' : 1e10, 'G' : G, 'H' : H}
Ans = self.driver()
W = Ans[:,2]
M = Ans[:,1]
Q = Ans[:,0]
D = M - Q
self.MAQ = np.mean(np.abs(Q))
ns = len(M)
# Wrapper to the driver, which returns just the part that changes.
def callM(mvals_):
self.FF.make(mvals_)
Ans2 = self.driver()
M_ = Ans2[:,1]
D_ = M_ - Q
return Ans2[:,1]
if AGrad:
# Leaving comment here if we want to reintroduce second deriv someday.
# dM[p,:], ddM[p,:] = f12d3p(fdwrap(callM, mvals, p), h = self.h, f0 = M)
xgrad = []
for p in self.pgrad:
dM_arr = f1d2p(fdwrap(callM, mvals, p), h = self.h, f0 = M)
if np.max(np.abs(dM_arr)) == 0.0 and Counter() == First():
logger.info("\r Simulation %s will skip over parameter %i in subsequent steps\n" % (self.name, p))
xgrad.append(p)
else:
dM[p] = dM_arr.copy()
for p in xgrad:
self.pgrad.remove(p)
Objective = np.dot(W, D**2) * Fac
if AGrad:
for p in self.pgrad:
G[p] = 2 * np.dot(W, D*dM[p])
if not AHess: continue
H[p, p] = 2 * np.dot(W, dM[p]**2)
for q in range(p):
if q not in self.pgrad: continue
GNP = 2 * np.dot(W, dM[p] * dM[q])
H[q,p] = GNP
H[p,q] = GNP
G *= Fac
H *= Fac
Answer = {'X':Objective, 'G':G, 'H':H}
if not in_fd():
self.D = D
self.objective = Answer['X']
LAST_MVALS = mvals.copy()
return Answer
def get(self, mvals, AGrad=True, AHess=True):
"""
Fitting of liquid bulk properties. This is the current major
direction of development for ForceBalance. Basically, fitting
the QM energies / forces alone does not always give us the
best simulation behavior. In many cases it makes more sense
to try and reproduce some experimentally known data as well.
In order to reproduce experimentally known data, we need to
run a simulation and compare the simulation result to
experiment. The main challenge here is that the simulations
are computationally intensive (i.e. they require energy and
force evaluations), and furthermore the results are noisy. We
need to run the simulations automatically and remotely
(i.e. on clusters) and a good way to calculate the derivatives
of the simulation results with respect to the parameter values.
This function contains some experimentally known values of the
density and enthalpy of vaporization (Hvap) of liquid water.
It launches the density and Hvap calculations on the cluster,
and gathers the results / derivatives. The actual calculation
of results / derivatives is done in a separate file.
After the results come back, they are gathered together to form
an objective function.
@param[in] mvals Mathematical parameter values
@param[in] AGrad Switch to turn on analytic gradient
@param[in] AHess Switch to turn on analytic Hessian
@return Answer Contribution to the objective function
"""
unpack = forcebalance.nifty.lp_load('forcebalance.p')
mvals1 = unpack[1]
if (np.max(np.abs(mvals1 - mvals)) > 1e-3):
warn_press_key("mvals from forcebalance.p does not match up with internal values! (Are you reading data from a previous run?)\nmvals(call)=%s mvals(disk)=%s" % (mvals, mvals1))
mbar_verbose = False
Answer = {}
Results = {}
Points = [] # These are the phase points for which data exists.
BPoints = [] # These are the phase points for which we are doing MBAR for the condensed phase.
mBPoints = [] # These are the phase points for which we are doing MBAR for the monomers.
mPoints = [] # These are the phase points to use for enthalpy of vaporization; if we're scanning pressure then set hvap_wt for higher pressures to zero.
tt = 0
for label, PT in zip(self.Labels, self.PhasePoints):
if os.path.exists('./%s/npt_result.p' % label):
logger.info('Reading information from ./%s/npt_result.p\n' % label)
Points.append(PT)
Results[tt] = lp_load('./%s/npt_result.p' % label)
if 'hvap' in self.RefData and PT[0] not in [i[0] for i in mPoints]:
mPoints.append(PT)
if 'mbar' in self.RefData and PT in self.RefData['mbar'] and self.RefData['mbar'][PT]:
BPoints.append(PT)
if 'hvap' in self.RefData and PT[0] not in [i[0] for i in mBPoints]:
mBPoints.append(PT)
tt += 1
else:
logger.warning('In %s :\n' % os.getcwd())
logger.warning('The file ./%s/npt_result.p does not exist so we cannot read it\n' % label)
pass
if len(Points) == 0:
logger.error('The liquid simulations have terminated with \x1b[1;91mno readable data\x1b[0m - this is a problem!\n')
raise RuntimeError
# Assign variable names to all the stuff in npt_result.p
Rhos, Vols, Potentials, Energies, Dips, Grads, GDips, mPotentials, mEnergies, mGrads, \
Rho_errs, Hvap_errs, Alpha_errs, Kappa_errs, Cp_errs, Eps0_errs, NMols = ([Results[t][i] for t in range(len(Points))] for i in range(17))
# Determine the number of molecules
if len(set(NMols)) != 1:
logger.error(str(NMols))
logger.error('The above list should only contain one number - the number of molecules\n')
raise RuntimeError
else:
NMol = list(set(NMols))[0]
if not self.adapt_errors:
self.AllResults = defaultdict(lambda:defaultdict(list))
astrm = astr(mvals)
if len(Points) != len(self.Labels):
logger.info("Data sets is not full, will not use for concatenation.")
astrm += "_"*(Counter()+1)
self.AllResults[astrm]['Pts'].append(Points)
self.AllResults[astrm]['mPts'].append(Points)
self.AllResults[astrm]['E'].append(np.array(Energies))
self.AllResults[astrm]['V'].append(np.array(Vols))
self.AllResults[astrm]['R'].append(np.array(Rhos))
self.AllResults[astrm]['Dx'].append(np.array([d[:,0] for d in Dips]))
self.AllResults[astrm]['Dy'].append(np.array([d[:,1] for d in Dips]))
self.AllResults[astrm]['Dz'].append(np.array([d[:,2] for d in Dips]))
self.AllResults[astrm]['G'].append(np.array(Grads))
self.AllResults[astrm]['GDx'].append(np.array([gd[0] for gd in GDips]))
self.AllResults[astrm]['GDy'].append(np.array([gd[1] for gd in GDips]))
self.AllResults[astrm]['GDz'].append(np.array([gd[2] for gd in GDips]))
self.AllResults[astrm]['L'].append(len(Energies[0]))
self.AllResults[astrm]['Steps'].append(self.liquid_md_steps)
if len(mPoints) > 0:
self.AllResults[astrm]['mE'].append(np.array([i for pt, i in zip(Points,mEnergies) if pt in mPoints]))
self.AllResults[astrm]['mG'].append(np.array([i for pt, i in zip(Points,mGrads) if pt in mPoints]))
# Number of data sets belonging to this value of the parameters.
Nrpt = len(self.AllResults[astrm]['R'])
sumsteps = sum(self.AllResults[astrm]['Steps'])
if self.liquid_md_steps != sumsteps:
printcool("This objective function evaluation combines %i datasets\n" \
"Increasing simulation length: %i -> %i steps" % \
(Nrpt, self.liquid_md_steps, sumsteps), color=6)
if self.liquid_md_steps * 2 != sumsteps:
logger.error("Spoo!\n")
raise RuntimeError
self.liquid_eq_steps *= 2
self.liquid_md_steps *= 2
self.gas_eq_steps *= 2
self.gas_md_steps *= 2
# Concatenate along the data-set axis (more than 1 element if we've returned to these parameters.)
E, V, R, Dx, Dy, Dz = \
(np.hstack(tuple(self.AllResults[astrm][i])) for i in \
['E', 'V', 'R', 'Dx', 'Dy', 'Dz'])
G, GDx, GDy, GDz = \
(np.hstack((np.concatenate(tuple(self.AllResults[astrm][i]), axis=2))) for i in ['G', 'GDx', 'GDy', 'GDz'])
if len(mPoints) > 0:
mE = np.hstack(tuple(self.AllResults[astrm]['mE']))
mG = np.hstack((np.concatenate(tuple(self.AllResults[astrm]['mG']), axis=2)))
Rho_calc = OrderedDict([])
Rho_grad = OrderedDict([])
Rho_std = OrderedDict([])
Hvap_calc = OrderedDict([])
Hvap_grad = OrderedDict([])
Hvap_std = OrderedDict([])
Alpha_calc = OrderedDict([])
Alpha_grad = OrderedDict([])
Alpha_std = OrderedDict([])
Kappa_calc = OrderedDict([])
Kappa_grad = OrderedDict([])
Kappa_std = OrderedDict([])
Cp_calc = OrderedDict([])
Cp_grad = OrderedDict([])
Cp_std = OrderedDict([])
Eps0_calc = OrderedDict([])
Eps0_grad = OrderedDict([])
Eps0_std = OrderedDict([])
# The unit that converts atmospheres * nm**3 into kj/mol :)
pvkj=0.061019351687175
# Run MBAR using the total energies. Required for estimates that use the kinetic energy.
BSims = len(BPoints)
Shots = len(E[0])
N_k = np.ones(BSims)*Shots
# Use the value of the energy for snapshot t from simulation k at potential m
U_kln = np.zeros([BSims,BSims,Shots])
for m, PT in enumerate(BPoints):
T = PT[0]
P = PT[1] / 1.01325 if PT[2] == 'bar' else PT[1]
beta = 1. / (kb * T)
for k in range(BSims):
# The correct Boltzmann factors include PV.
# Note that because the Boltzmann factors are computed from the conditions at simulation "m",
# the pV terms must be rescaled to the pressure at simulation "m".
kk = Points.index(BPoints[k])
U_kln[k, m, :] = E[kk] + P*V[kk]*pvkj
U_kln[k, m, :] *= beta
W1 = None
if len(BPoints) > 1:
logger.info("Running MBAR analysis on %i states...\n" % len(BPoints))
mbar = pymbar.MBAR(U_kln, N_k, verbose=mbar_verbose, relative_tolerance=5.0e-8)
W1 = mbar.getWeights()
logger.info("Done\n")
elif len(BPoints) == 1:
W1 = np.ones((BPoints*Shots,BPoints))
W1 /= BPoints*Shots
def fill_weights(weights, phase_points, mbar_points, snapshots):
""" Fill in the weight matrix with MBAR weights where MBAR was run,
and equal weights otherwise. """
new_weights = np.zeros([len(phase_points)*snapshots,len(phase_points)])
for m, PT in enumerate(phase_points):
if PT in mbar_points:
mm = mbar_points.index(PT)
for kk, PT1 in enumerate(mbar_points):
k = phase_points.index(PT1)
logger.debug("Will fill W2[%i:%i,%i] with W1[%i:%i,%i]\n" % (k*snapshots,k*snapshots+snapshots,m,kk*snapshots,kk*snapshots+snapshots,mm))
new_weights[k*snapshots:(k+1)*snapshots,m] = weights[kk*snapshots:(kk+1)*snapshots,mm]
else:
logger.debug("Will fill W2[%i:%i,%i] with equal weights\n" % (m*snapshots,(m+1)*snapshots,m))
new_weights[m*snapshots:(m+1)*snapshots,m] = 1.0/snapshots
return new_weights
W2 = fill_weights(W1, Points, BPoints, Shots)
if len(mPoints) > 0:
# Run MBAR on the monomers. This is barely necessary.
mW1 = None
mShots = len(mE[0])
if len(mBPoints) > 0:
mBSims = len(mBPoints)
mN_k = np.ones(mBSims)*mShots
mU_kln = np.zeros([mBSims,mBSims,mShots])
for m, PT in enumerate(mBPoints):
T = PT[0]
beta = 1. / (kb * T)
for k in range(mBSims):
kk = Points.index(mBPoints[k])
mU_kln[k, m, :] = mE[kk]
mU_kln[k, m, :] *= beta
if np.abs(np.std(mE)) > 1e-6 and mBSims > 1:
mmbar = pymbar.MBAR(mU_kln, mN_k, verbose=False, relative_tolerance=5.0e-8, method='self-consistent-iteration')
mW1 = mmbar.getWeights()
elif len(mBPoints) == 1:
mW1 = np.ones((mBSims*mShots,mSims))
mW1 /= mBSims*mShots
mW2 = fill_weights(mW1, mPoints, mBPoints, mShots)
if self.do_self_pol:
EPol = self.polarization_correction(mvals)
GEPol = np.array([(f12d3p(fdwrap(self.polarization_correction, mvals, p), h = self.h, f0 = EPol)[0] if p in self.pgrad else 0.0) for p in range(self.FF.np)])
bar = printcool("Self-polarization correction to \nenthalpy of vaporization is % .3f kJ/mol%s" % (EPol, ", Derivative:" if AGrad else ""))
if AGrad:
self.FF.print_map(vals=GEPol)
logger.info(bar)
# Arrays must be flattened now for calculation of properties.
E = E.flatten()
V = V.flatten()
R = R.flatten()
Dx = Dx.flatten()
Dy = Dy.flatten()
Dz = Dz.flatten()
if len(mPoints) > 0: mE = mE.flatten()
for i, PT in enumerate(Points):
T = PT[0]
P = PT[1] / 1.01325 if PT[2] == 'bar' else PT[1]
PV = P*V*pvkj
H = E + PV
# The weights that we want are the last ones.
W = flat(W2[:,i])
C = weight_info(W, PT, np.ones(len(Points))*Shots, verbose=mbar_verbose)
Gbar = flat(np.matrix(G)*col(W))
mBeta = -1/kb/T
Beta = 1/kb/T
kT = kb*T
# Define some things to make the analytic derivatives easier.
def avg(vec):
return np.dot(W,vec)
def covde(vec):
return flat(np.matrix(G)*col(W*vec)) - avg(vec)*Gbar
def deprod(vec):
return flat(np.matrix(G)*col(W*vec))
## Density.
Rho_calc[PT] = np.dot(W,R)
Rho_grad[PT] = mBeta*(flat(np.matrix(G)*col(W*R)) - np.dot(W,R)*Gbar)
## Enthalpy of vaporization.
if PT in mPoints:
ii = mPoints.index(PT)
mW = flat(mW2[:,ii])
mGbar = flat(np.matrix(mG)*col(mW))
Hvap_calc[PT] = np.dot(mW,mE) - np.dot(W,E)/NMol + kb*T - np.dot(W, PV)/NMol
Hvap_grad[PT] = mGbar + mBeta*(flat(np.matrix(mG)*col(mW*mE)) - np.dot(mW,mE)*mGbar)
Hvap_grad[PT] -= (Gbar + mBeta*(flat(np.matrix(G)*col(W*E)) - np.dot(W,E)*Gbar)) / NMol
Hvap_grad[PT] -= (mBeta*(flat(np.matrix(G)*col(W*PV)) - np.dot(W,PV)*Gbar)) / NMol
if self.do_self_pol:
Hvap_calc[PT] -= EPol
Hvap_grad[PT] -= GEPol
if hasattr(self,'use_cni') and self.use_cni:
if not ('cni' in self.RefData and self.RefData['cni'][PT]):
logger.error('Asked for a nonideality correction but not provided in reference data (data.csv). Either disable the option in data.csv or add data.\n')
raise RuntimeError
logger.debug("Adding % .3f to enthalpy of vaporization at " % self.RefData['cni'][PT] + str(PT) + '\n')
Hvap_calc[PT] += self.RefData['cni'][PT]
if hasattr(self,'use_cvib_intra') and self.use_cvib_intra:
if not ('cvib_intra' in self.RefData and self.RefData['cvib_intra'][PT]):
logger.error('Asked for a quantum intramolecular vibrational correction but not provided in reference data (data.csv). Either disable the option in data.csv or add data.\n')
raise RuntimeError
logger.debug("Adding % .3f to enthalpy of vaporization at " % self.RefData['cvib_intra'][PT] + str(PT) + '\n')
Hvap_calc[PT] += self.RefData['cvib_intra'][PT]
if hasattr(self,'use_cvib_inter') and self.use_cvib_inter:
if not ('cvib_inter' in self.RefData and self.RefData['cvib_inter'][PT]):
logger.error('Asked for a quantum intermolecular vibrational correction but not provided in reference data (data.csv). Either disable the option in data.csv or add data.\n')
raise RuntimeError
logger.debug("Adding % .3f to enthalpy of vaporization at " % self.RefData['cvib_inter'][PT] + str(PT) + '\n')
Hvap_calc[PT] += self.RefData['cvib_inter'][PT]
else:
Hvap_calc[PT] = 0.0
Hvap_grad[PT] = np.zeros(self.FF.np)
## Thermal expansion coefficient.
Alpha_calc[PT] = 1e4 * (avg(H*V)-avg(H)*avg(V))/avg(V)/(kT*T)
GAlpha1 = -1 * Beta * deprod(H*V) * avg(V) / avg(V)**2
GAlpha2 = +1 * Beta * avg(H*V) * deprod(V) / avg(V)**2
GAlpha3 = deprod(V)/avg(V) - Gbar
GAlpha4 = Beta * covde(H)
Alpha_grad[PT] = 1e4 * (GAlpha1 + GAlpha2 + GAlpha3 + GAlpha4)/(kT*T)
## Isothermal compressibility.
bar_unit = 0.06022141793 * 1e6
Kappa_calc[PT] = bar_unit / kT * (avg(V**2)-avg(V)**2)/avg(V)
GKappa1 = +1 * Beta**2 * avg(V**2) * deprod(V) / avg(V)**2
GKappa2 = -1 * Beta**2 * avg(V) * deprod(V**2) / avg(V)**2
GKappa3 = +1 * Beta**2 * covde(V)
Kappa_grad[PT] = bar_unit*(GKappa1 + GKappa2 + GKappa3)
## Isobaric heat capacity.
Cp_calc[PT] = 1000/(4.184*NMol*kT*T) * (avg(H**2) - avg(H)**2)
if hasattr(self,'use_cvib_intra') and self.use_cvib_intra:
logger.debug("Adding " + str(self.RefData['devib_intra'][PT]) + " to the heat capacity\n")
Cp_calc[PT] += self.RefData['devib_intra'][PT]
if hasattr(self,'use_cvib_inter') and self.use_cvib_inter:
logger.debug("Adding " + str(self.RefData['devib_inter'][PT]) + " to the heat capacity\n")
Cp_calc[PT] += self.RefData['devib_inter'][PT]
GCp1 = 2*covde(H) * 1000 / 4.184 / (NMol*kT*T)
GCp2 = mBeta*covde(H**2) * 1000 / 4.184 / (NMol*kT*T)
GCp3 = 2*Beta*avg(H)*covde(H) * 1000 / 4.184 / (NMol*kT*T)
Cp_grad[PT] = GCp1 + GCp2 + GCp3
## Static dielectric constant.
prefactor = 30.348705333964077
D2 = avg(Dx**2)+avg(Dy**2)+avg(Dz**2)-avg(Dx)**2-avg(Dy)**2-avg(Dz)**2
Eps0_calc[PT] = 1.0 + prefactor*(D2/avg(V))/T
GD2 = 2*(flat(np.matrix(GDx)*col(W*Dx)) - avg(Dx)*flat(np.matrix(GDx)*col(W))) - Beta*(covde(Dx**2) - 2*avg(Dx)*covde(Dx))
GD2 += 2*(flat(np.matrix(GDy)*col(W*Dy)) - avg(Dy)*flat(np.matrix(GDy)*col(W))) - Beta*(covde(Dy**2) - 2*avg(Dy)*covde(Dy))
GD2 += 2*(flat(np.matrix(GDz)*col(W*Dz)) - avg(Dz)*flat(np.matrix(GDz)*col(W))) - Beta*(covde(Dz**2) - 2*avg(Dz)*covde(Dz))
Eps0_grad[PT] = prefactor*(GD2/avg(V) - mBeta*covde(V)*D2/avg(V)**2)/T
## Estimation of errors.
Rho_std[PT] = np.sqrt(sum(C**2 * np.array(Rho_errs)**2))
if PT in mPoints:
Hvap_std[PT] = np.sqrt(sum(C**2 * np.array(Hvap_errs)**2))
else:
Hvap_std[PT] = 0.0
Alpha_std[PT] = np.sqrt(sum(C**2 * np.array(Alpha_errs)**2)) * 1e4
Kappa_std[PT] = np.sqrt(sum(C**2 * np.array(Kappa_errs)**2)) * 1e6
Cp_std[PT] = np.sqrt(sum(C**2 * np.array(Cp_errs)**2))
Eps0_std[PT] = np.sqrt(sum(C**2 * np.array(Eps0_errs)**2))
# Get contributions to the objective function
X_Rho, G_Rho, H_Rho, RhoPrint = self.objective_term(Points, 'rho', Rho_calc, Rho_std, Rho_grad, name="Density")
X_Hvap, G_Hvap, H_Hvap, HvapPrint = self.objective_term(Points, 'hvap', Hvap_calc, Hvap_std, Hvap_grad, name="H_vap", SubAverage=self.hvap_subaverage)
X_Alpha, G_Alpha, H_Alpha, AlphaPrint = self.objective_term(Points, 'alpha', Alpha_calc, Alpha_std, Alpha_grad, name="Thermal Expansion")
X_Kappa, G_Kappa, H_Kappa, KappaPrint = self.objective_term(Points, 'kappa', Kappa_calc, Kappa_std, Kappa_grad, name="Compressibility")
X_Cp, G_Cp, H_Cp, CpPrint = self.objective_term(Points, 'cp', Cp_calc, Cp_std, Cp_grad, name="Heat Capacity")
X_Eps0, G_Eps0, H_Eps0, Eps0Print = self.objective_term(Points, 'eps0', Eps0_calc, Eps0_std, Eps0_grad, name="Dielectric Constant")
Gradient = np.zeros(self.FF.np)
Hessian = np.zeros((self.FF.np,self.FF.np))
if X_Rho == 0: self.w_rho = 0.0
if X_Hvap == 0: self.w_hvap = 0.0
if X_Alpha == 0: self.w_alpha = 0.0
if X_Kappa == 0: self.w_kappa = 0.0
if X_Cp == 0: self.w_cp = 0.0
if X_Eps0 == 0: self.w_eps0 = 0.0
if self.w_normalize:
w_tot = self.w_rho + self.w_hvap + self.w_alpha + self.w_kappa + self.w_cp + self.w_eps0
else:
w_tot = 1.0
w_1 = self.w_rho / w_tot
w_2 = self.w_hvap / w_tot
w_3 = self.w_alpha / w_tot
w_4 = self.w_kappa / w_tot
w_5 = self.w_cp / w_tot
w_6 = self.w_eps0 / w_tot
Objective = w_1 * X_Rho + w_2 * X_Hvap + w_3 * X_Alpha + w_4 * X_Kappa + w_5 * X_Cp + w_6 * X_Eps0
if AGrad:
Gradient = w_1 * G_Rho + w_2 * G_Hvap + w_3 * G_Alpha + w_4 * G_Kappa + w_5 * G_Cp + w_6 * G_Eps0
if AHess:
Hessian = w_1 * H_Rho + w_2 * H_Hvap + w_3 * H_Alpha + w_4 * H_Kappa + w_5 * H_Cp + w_6 * H_Eps0
if not in_fd():
self.Xp = {"Rho" : X_Rho, "Hvap" : X_Hvap, "Alpha" : X_Alpha,
"Kappa" : X_Kappa, "Cp" : X_Cp, "Eps0" : X_Eps0}
self.Wp = {"Rho" : w_1, "Hvap" : w_2, "Alpha" : w_3,
"Kappa" : w_4, "Cp" : w_5, "Eps0" : w_6}
self.Pp = {"Rho" : RhoPrint, "Hvap" : HvapPrint, "Alpha" : AlphaPrint,
"Kappa" : KappaPrint, "Cp" : CpPrint, "Eps0" : Eps0Print}
if AGrad:
self.Gp = {"Rho" : G_Rho, "Hvap" : G_Hvap, "Alpha" : G_Alpha,
"Kappa" : G_Kappa, "Cp" : G_Cp, "Eps0" : G_Eps0}
self.Objective = Objective
Answer = {'X':Objective, 'G':Gradient, 'H':Hessian}
return Answer
def read(self, mvals, AGrad=True, AHess=True):
"""
Read in time series for all previous iterations.
"""
unpack = forcebalance.nifty.lp_load('forcebalance.p')
mvals1 = unpack[1]
if (np.max(np.abs(mvals1 - mvals)) > 1e-3):
warn_press_key("mvals from forcebalance.p does not match up with internal values! (Are you reading data from a previous run?)\nmvals(call)=%s mvals(disk)=%s" % (mvals, mvals1))
for dn in range(Counter()-1, -1, -1):
cwd = os.getcwd()
os.chdir(self.absrd(inum=dn))
mprev = np.loadtxt('mvals.txt')
Results = {}
Points = [] # These are the phase points for which data exists.
mPoints = [] # These are the phase points to use for enthalpy of vaporization; if we're scanning pressure then set hvap_wt for higher pressures to zero.
tt = 0
logger.info('Reading liquid data from %s\n' % os.getcwd())
for label, PT in zip(self.Labels, self.PhasePoints):
if os.path.exists('./%s/npt_result.p' % label):
Points.append(PT)
Results[tt] = lp_load('./%s/npt_result.p' % label)
if 'hvap' in self.RefData and PT[0] not in [i[0] for i in mPoints]:
mPoints.append(PT)
tt += 1
else:
logger.warning('In %s :\n' % os.getcwd())
logger.warning('The file ./%s/npt_result.p does not exist so we cannot read it\n' % label)
pass
if len(Points) == 0:
logger.error('The liquid simulations have terminated with \x1b[1;91mno readable data\x1b[0m - this is a problem!\n')
raise RuntimeError
# Assign variable names to all the stuff in npt_result.p
Rhos, Vols, Potentials, Energies, Dips, Grads, GDips, mPotentials, mEnergies, mGrads, \
Rho_errs, Hvap_errs, Alpha_errs, Kappa_errs, Cp_errs, Eps0_errs, NMols = ([Results[t][i] for t in range(len(Points))] for i in range(17))
# Determine the number of molecules
if len(set(NMols)) != 1:
logger.error(str(NMols))
logger.error('The above list should only contain one number - the number of molecules\n')
raise RuntimeError
else:
NMol = list(set(NMols))[0]
if not self.adapt_errors:
self.AllResults = defaultdict(lambda:defaultdict(list))
astrm = astr(mprev)
if len(Points) != len(self.Labels):
logger.info("Data sets is not full, will not use for concatenation.\n")
astrm += "_"*(dn+1)
self.AllResults[astrm]['Pts'].append(Points)
self.AllResults[astrm]['mPts'].append(mPoints)
self.AllResults[astrm]['E'].append(np.array(Energies))
self.AllResults[astrm]['V'].append(np.array(Vols))
self.AllResults[astrm]['R'].append(np.array(Rhos))
self.AllResults[astrm]['Dx'].append(np.array([d[:,0] for d in Dips]))
self.AllResults[astrm]['Dy'].append(np.array([d[:,1] for d in Dips]))
self.AllResults[astrm]['Dz'].append(np.array([d[:,2] for d in Dips]))
self.AllResults[astrm]['G'].append(np.array(Grads))
self.AllResults[astrm]['GDx'].append(np.array([gd[0] for gd in GDips]))
self.AllResults[astrm]['GDy'].append(np.array([gd[1] for gd in GDips]))
self.AllResults[astrm]['GDz'].append(np.array([gd[2] for gd in GDips]))
self.AllResults[astrm]['L'].append(len(Energies[0]))
self.AllResults[astrm]['Steps'].append(self.liquid_md_steps)
if len(mPoints) > 0:
self.AllResults[astrm]['mE'].append(np.array([i for pt, i in zip(Points,mEnergies) if pt in mPoints]))
self.AllResults[astrm]['mG'].append(np.array([i for pt, i in zip(Points,mGrads) if pt in mPoints]))
os.chdir(cwd)
return self.get(mvals, AGrad, AHess)
def compute(mvals_, indicate=False):
self.FF.make(mvals_)
M_opts = None
compute.emm = []
compute.rmsd = []
for i in range(self.ns):
energy, rmsd, M_opt = self.engine.optimize(shot=i, align=False)
# Create a molecule object to hold the MM-optimized structures
compute.emm.append(energy)
compute.rmsd.append(rmsd)
if M_opts is None:
M_opts = deepcopy(M_opt)
else:
M_opts += M_opt
compute.emm = np.array(compute.emm)
compute.emm -= compute.emm[self.smin]
compute.rmsd = np.array(compute.rmsd)
if indicate:
if self.writelevel > 0:
energy_comparison = np.array([
self.eqm, compute.emm, compute.emm - self.eqm,
np.sqrt(self.wts) / self.energy_denom
]).T
np.savetxt(
"EnergyCompare.txt",
energy_comparison,
header="%11s %12s %12s %12s" %
("QMEnergy", "MMEnergy", "Delta(MM-QM)", "Weight"),
fmt="% 12.6e")
M_opts.write('mm_minimized.xyz')
if self.ndim == 1:
try:
import matplotlib.pyplot as plt
plt.switch_backend('agg')
fig, ax = plt.subplots()
dihedrals = np.array([
i[0] for i in self.metadata['torsion_grid_ids']
])
dsort = np.argsort(dihedrals)
ax.plot(dihedrals[dsort],
self.eqm[dsort],
label='QM')
if hasattr(self, 'emm_orig'):
ax.plot(dihedrals[dsort],
compute.emm[dsort],
label='MM Current')
ax.plot(dihedrals[dsort],
self.emm_orig[dsort],
label='MM Initial')
else:
ax.plot(dihedrals[dsort],
compute.emm[dsort],
label='MM Initial')
self.emm_orig = compute.emm.copy()
ax.legend()
ax.set_xlabel('Dihedral (degree)')
ax.set_ylabel('Energy (kcal/mol)')
fig.suptitle(
'Torsion profile: iteration %i\nSystem: %s' %
(Counter(), self.name))
fig.savefig('plot_torsion.pdf')
except ImportError:
logger.warning(
"matplotlib package is needed to make torsion profile plots\n"
)
return (np.sqrt(self.wts) / self.energy_denom) * (compute.emm -
self.eqm)