def to_task(self):
logger.info1('Creating task for', self.digest)
pxul.os.ensure_dir(self.outputdir)
logger.debug('Ensured', self.outputdir, 'exists')
cmd = 'bash %(script)s > %(log)s' % dict(script = SCRIPT_NAME, log = LOGFILE)
task = wq.Task(cmd)
# input files
task.specify_buffer(SCRIPT_CONTENTS, SCRIPT_NAME , cache=True)
task.specify_buffer(str(self._cpus), SCRIPT_INPUT_NAMES['cpus'],cache=True)
task.specify_input_file(self._x , SCRIPT_INPUT_NAMES['x'] , cache=False, name='x_i')
task.specify_input_file(self._v , SCRIPT_INPUT_NAMES['v'] , cache=False, name='v_i')
task.specify_input_file(self._t , SCRIPT_INPUT_NAMES['t'] , cache=False, name='t_i')
task.specify_input_file(self._tpr , SCRIPT_INPUT_NAMES['tpr'], cache=True , name='tpr')
# output files
task.specify_output_file(self.output_files['log'], LOGFILE , cache=False, name='log')
task.specify_output_file(self.output_files['x'], SCRIPT_OUTPUT_NAMES['x'] , cache=False, name='x_o')
task.specify_output_file(self.output_files['v'], SCRIPT_OUTPUT_NAMES['v'] , cache=False, name='v_o')
task.specify_output_file(self.output_files['t'], SCRIPT_OUTPUT_NAMES['t'] , cache=False, name='t_o')
self.check_binaries()
for path in self._binaries:
task.specify_input_file(path, cache=True)
for name in self._trajfiles:
task.specify_output_file(self.output_path(name), name, cache=False)
logger.debug('Created task:\n', str(task))
return task
def extend(self):
"""Set the file names to run the next generation"""
logger.debug('Extending generation:', self._generation, '->', self._generation + 1)
self._x = os.path.join(self.outputdir, SCRIPT_OUTPUT_NAMES['x'])
self._v = os.path.join(self.outputdir, SCRIPT_OUTPUT_NAMES['v'])
self._t = os.path.join(self.outputdir, SCRIPT_OUTPUT_NAMES['t'])
self._generation += 1
def to_task(self):
logger.info1('Creating task for', self.digest)
pxul.os.ensure_dir(self.outputdir)
logger.debug('Ensured', self.outputdir, 'exists')
cmd = 'bash %(script)s > %(log)s' % dict(script=SCRIPT_NAME,
log=LOGFILE)
task = wq.Task(cmd)
# input files
task.specify_buffer(SCRIPT_CONTENTS, SCRIPT_NAME, cache=True)
task.specify_buffer(str(self._cpus),
SCRIPT_INPUT_NAMES['cpus'],
cache=True)
task.specify_input_file(self._x,
SCRIPT_INPUT_NAMES['x'],
cache=False,
name='x_i')
task.specify_input_file(self._v,
SCRIPT_INPUT_NAMES['v'],
cache=False,
name='v_i')
task.specify_input_file(self._t,
SCRIPT_INPUT_NAMES['t'],
cache=False,
name='t_i')
task.specify_input_file(self._tpr,
SCRIPT_INPUT_NAMES['tpr'],
cache=True,
name='tpr')
# output files
task.specify_output_file(self.output_files['log'],
LOGFILE,
cache=False,
name='log')
task.specify_output_file(self.output_files['x'],
SCRIPT_OUTPUT_NAMES['x'],
cache=False,
name='x_o')
task.specify_output_file(self.output_files['v'],
SCRIPT_OUTPUT_NAMES['v'],
cache=False,
name='v_o')
task.specify_output_file(self.output_files['t'],
SCRIPT_OUTPUT_NAMES['t'],
cache=False,
name='t_o')
self.check_binaries()
for path in self._binaries:
task.specify_input_file(path, cache=True)
for name in self._trajfiles:
task.specify_output_file(self.output_path(name), name, cache=False)
logger.debug('Created task:\n', str(task))
return task
def relax(self, mdp, gammas=None, steps=None):
logger.info1('Iterative equilibration')
gammas = [1000, 100, 10, 1] if gammas is None else gammas
name = self.name + '_itr_posres_eq'
mdp_itr = suffix.mdp(name)
mdp.SETUP.define = '-DPOSRES'
if steps is not None:
mdp.nsteps = steps
mdp.set_velocity_generation()
for g in gammas:
mdp.dt = 0.001
mdp.set_gamma(g)
logger.debug('using gamma =', g)
logger.debug('using dt =', mdp.dt)
mdp.save(mdp_itr)
self._cn = name + '_gamma-%d' % g
gmx.grompp(f=mdp_itr,
c=suffix.gro(self.pn),
t=suffix.trr(self.pn),
p=self.top,
o=suffix.tpr(self.cn))
gmx.mdrun(s=suffix.tpr(self.cn), deffnm=self.cn, v=True)
self._pn = self.cn
mdp.unset_velocity_generation()
def relax(self, mdp, gammas=None, steps=None):
logger.info1('Iterative equilibration')
gammas = [1000, 100, 10, 1] if gammas is None else gammas
name = self.name + '_itr_posres_eq'
mdp_itr = suffix.mdp(name)
mdp.SETUP.define = '-DPOSRES'
if steps is not None:
mdp.nsteps = steps
mdp.set_velocity_generation()
for g in gammas:
mdp.dt = 0.001
mdp.set_gamma(g)
logger.debug('using gamma =', g)
logger.debug('using dt =', mdp.dt)
mdp.save(mdp_itr)
self._cn = name + '_gamma-%d' % g
gmx.grompp(
f = mdp_itr,
c = suffix.gro(self.pn),
t = suffix.trr(self.pn),
p = self.top,
o = suffix.tpr(self.cn)
)
gmx.mdrun(
s = suffix.tpr(self.cn),
deffnm = self.cn,
v = True
)
self._pn = self.cn
mdp.unset_velocity_generation()
def extend(self):
"""Set the file names to run the next generation"""
logger.debug('Extending generation:', self._generation, '->',
self._generation + 1)
self._x = os.path.join(self.outputdir, SCRIPT_OUTPUT_NAMES['x'])
self._v = os.path.join(self.outputdir, SCRIPT_OUTPUT_NAMES['v'])
self._t = os.path.join(self.outputdir, SCRIPT_OUTPUT_NAMES['t'])
self._generation += 1
def write(self, path=None):
path = path or CONFIG
if not os.path.exists(os.path.dirname(CONFIG)):
os.makedirs(os.path.dirname(CONFIG))
p = Persistent(path)
p['config'] = self
p.close()
logger.debug('Wrote:', path)
def write(self, path=None):
path = path or CONFIG
if not os.path.exists(os.path.dirname(CONFIG)):
os.makedirs(os.path.dirname(CONFIG))
p = Persistent(path)
p['config'] = self
p.close()
logger.debug('Wrote:', path)
def check_binaries(self):
"""Checks that the required executables (EXECUTABLES) have been added"""
logger.debug('Checking that all executables were added')
found = 0
notfound = list()
for path in self._binaries:
base = os.path.basename(path)
for name in EXECUTABLES:
if name == base:
logger.debug('Found', name, 'as', path)
found += 1
continue
notfound.append(name)
if not found == len(EXECUTABLES):
raise ValueError, 'Binaries for %s were not added' % ', '.join(notfound)
def get_mdp(self):
typ = type(self.mdp)
logger.debug('Loading MDP from', typ)
if typ is str:
return mdp.MDP.loads(self.mdp)
elif typ is types.FunctionType:
return self.mdp()
elif typ is mdp.MDP:
return self.mdp
else:
raise TypeError, 'Unknown mdp type %s' % typ
def get_mdp(self):
typ = type(self.mdp)
logger.debug('Loading MDP from', typ)
if typ is str:
return mdp.MDP.loads(self.mdp)
elif typ is types.FunctionType:
return self.mdp()
elif typ is mdp.MDP:
return self.mdp
else:
raise TypeError, 'Unknown mdp type %s' % typ
def check_binaries(self):
"""Checks that the required executables (EXECUTABLES) have been added"""
logger.debug('Checking that all executables were added')
found = 0
notfound = list()
for path in self._binaries:
base = os.path.basename(path)
for name in EXECUTABLES:
if name == base:
logger.debug('Found', name, 'as', path)
found += 1
continue
notfound.append(name)
if not found == len(EXECUTABLES):
raise ValueError, 'Binaries for %s were not added' % ', '.join(
notfound)
def set_velocity_generation(self):
logger.debug('Setting velocity generation')
g = velocity_generation_group(self.tau_t[0], self.ld_seed)
self.add(g)
def set_gamma(self, g):
logger.debug('setting gamma =', g)
self.set_tau_t(1 / float(g))
def set_tau_t(self, t):
logger.debug('setting tau_t =', t)
self.tau_t = self.tcgroups * [t]
def add_binary(self, path):
"""Add a binary file to cache"""
logger.debug('Adding binary', path)
self._binaries.append(path)
def seed(self, value):
for a in 'ld_seed gen_seed'.split():
if hasattr(self, a):
logger.debug('Setting', a, 'to', value)
setattr(self, a, value)
def unset_velocity_generation(self):
try:
del self._groups['VELOCITY_GENERATION']
logger.debug('Unsetting velocity generation')
except KeyError: pass
def set_velocity_generation(self):
logger.debug('Setting velocity generation')
g = velocity_generation_group(self.tau_t[0], self.ld_seed)
self.add(g)
def set_gamma(self, g):
logger.debug('setting gamma =', g)
self.set_tau_t(1/float(g))
def set_tau_t(self, t):
logger.debug('setting tau_t =', t)
self.tau_t = self.tcgroups * [t]
def _keep_XXX(self, suffix):
"""Mark a simulation output file to be transferred back from the worker"""
logger.debug('Task keeping', suffix, 'as', TRAJ_FILES[suffix])
self._trajfiles.append(TRAJ_FILES[suffix])
def unset_velocity_generation(self):
try:
del self._groups['VELOCITY_GENERATION']
logger.debug('Unsetting velocity generation')
except KeyError:
pass
def task(self, tpr, x=None, v=None, t=None, outputdir=None, seed=None, digest=None):
outdir = outputdir or tpr + '.mdq'
pxul.os.ensure_dir(outdir)
logger.debug('Ensured', outdir, 'exists')
tpr2 = os.path.join(outdir, 'topol.tpr')
shutil.copy(tpr, tpr2)
logger.debug(tpr, '->', tpr2)
gendir = os.path.join(outdir, '0')
pxul.os.ensure_dir(gendir)
logger.debug('Ensured', gendir, 'exists')
gps = dict(x = os.path.join(gendir, SCRIPT_INPUT_NAMES['x']),
v = os.path.join(gendir, SCRIPT_INPUT_NAMES['v']),
t = os.path.join(gendir, SCRIPT_INPUT_NAMES['t']))
if x is not None:
shutil.copy(x, gps['x'])
logger.debug(x, '->', gps['x'])
if v is not None:
shutil.copy(v, gps['v'])
logger.debug(v, '->', gps['v'])
if t is not None:
if type(t) is float:
with open(gps['t'], 'w') as fd: fd.write(str(t))
logger.debug('Wrote', t, 'to', gps['t'])
elif type(t) is str:
shutil.copy(t, gps['t'])
logger.debug(t, '->', gps['t'])
else: raise ValueError, 'Illegal state: invalid time spec %s' % t
for sel, key in SELECTIONS.iteritems():
logger.info1('Getting', sel, 'from', tpr2)
guamps_get(f=tpr2, s=sel, o=gps[key])
if seed:
logger.info1('Setting seed', seed)
tpr_set_scalar(tpr2, 'ld_seed', seed)
dt = tpr_get_scalar(tpr2, 'deltat', float)
if self._picoseconds:
nsteps = int(self._picoseconds / dt)
logger.info1('Running for', self._picoseconds, 'ps as', nsteps, 'nsteps')
tpr_set_scalar(tpr2, 'nsteps', nsteps)
if self._outputfreq:
freq = int(self._outputfreq / dt)
# FIXME nstenergy, see badi/guamps#27
for attr in 'nstxout nstxtcout nstfout nstvout nstlog'.split():
logger.info1('Setting output frequency', self._outputfreq,
'for', attr, 'as', freq, 'steps', 'in', tpr2)
tpr_set_scalar(tpr2, attr, freq)
if not digest:
logger.info1('Computing digest for', tpr2)
sha256 = hashlib.sha256()
sha256.update(open(tpr2, 'rb').read())
digest = sha256.hexdigest()
task = Task(x=gps['x'], v=gps['v'], t=gps['t'], tpr=tpr2,
outputdir=outdir, cpus=self._cpus, digest=digest)
task.add_binary(self._mdrun)
task.add_binary(self._guamps_get)
task.add_binary(self._guamps_set)
if self._keep_trajfiles:
task.keep_trajfiles()
logger.info('Prepared', digest, 'from', tpr)
for k in self.__dict__:
logger.info(10*' ', k.lstrip('_'), '=', getattr(self, k))
return task
def _keep_XXX(self, suffix):
"""Mark a simulation output file to be transferred back from the worker"""
logger.debug('Task keeping', suffix, 'as', TRAJ_FILES[suffix])
self._trajfiles.append(TRAJ_FILES[suffix])
def add_binary(self, path):
"""Add a binary file to cache"""
logger.debug('Adding binary', path)
self._binaries.append(path)
def seed(self, value):
for a in 'ld_seed gen_seed'.split():
if hasattr(self, a):
logger.debug('Setting', a, 'to', value)
setattr(self, a, value)
def _persist(self, taskable):
if self._persist_to is not None:
logger.debug('%-15s' % 'Persisting', taskable.uuid)
run_stream((lambda: (yield taskable))(),
PersistTaskStream,
extra_args=[self._persist_to])
def submit(self, taskable):
logger.debug('%-15s' % 'Submitting', taskable.uuid)
task = taskable.to_task()
self._table[task.uuid] = taskable
return self.wq.submit(task)
def task(self,
tpr,
x=None,
v=None,
t=None,
outputdir=None,
seed=None,
digest=None):
outdir = outputdir or tpr + '.mdq'
pxul.os.ensure_dir(outdir)
logger.debug('Ensured', outdir, 'exists')
tpr2 = os.path.join(outdir, 'topol.tpr')
shutil.copy(tpr, tpr2)
logger.debug(tpr, '->', tpr2)
gendir = os.path.join(outdir, '0')
pxul.os.ensure_dir(gendir)
logger.debug('Ensured', gendir, 'exists')
gps = dict(x=os.path.join(gendir, SCRIPT_INPUT_NAMES['x']),
v=os.path.join(gendir, SCRIPT_INPUT_NAMES['v']),
t=os.path.join(gendir, SCRIPT_INPUT_NAMES['t']))
if x is not None:
shutil.copy(x, gps['x'])
logger.debug(x, '->', gps['x'])
if v is not None:
shutil.copy(v, gps['v'])
logger.debug(v, '->', gps['v'])
if t is not None:
if type(t) is float:
with open(gps['t'], 'w') as fd:
fd.write(str(t))
logger.debug('Wrote', t, 'to', gps['t'])
elif type(t) is str:
shutil.copy(t, gps['t'])
logger.debug(t, '->', gps['t'])
else:
raise ValueError, 'Illegal state: invalid time spec %s' % t
for sel, key in SELECTIONS.iteritems():
logger.info1('Getting', sel, 'from', tpr2)
guamps_get(f=tpr2, s=sel, o=gps[key])
if seed:
logger.info1('Setting seed', seed)
tpr_set_scalar(tpr2, 'ld_seed', seed)
dt = tpr_get_scalar(tpr2, 'deltat', float)
if self._picoseconds:
nsteps = int(self._picoseconds / dt)
logger.info1('Running for', self._picoseconds, 'ps as', nsteps,
'nsteps')
tpr_set_scalar(tpr2, 'nsteps', nsteps)
if self._outputfreq:
freq = int(self._outputfreq / dt)
# FIXME nstenergy, see badi/guamps#27
for attr in 'nstxout nstxtcout nstfout nstvout nstlog'.split():
logger.info1('Setting output frequency', self._outputfreq,
'for', attr, 'as', freq, 'steps', 'in', tpr2)
tpr_set_scalar(tpr2, attr, freq)
if not digest:
logger.info1('Computing digest for', tpr2)
sha256 = hashlib.sha256()
sha256.update(open(tpr2, 'rb').read())
digest = sha256.hexdigest()
task = Task(x=gps['x'],
v=gps['v'],
t=gps['t'],
tpr=tpr2,
outputdir=outdir,
cpus=self._cpus,
digest=digest)
task.add_binary(self._mdrun)
task.add_binary(self._guamps_get)
task.add_binary(self._guamps_set)
if self._keep_trajfiles:
task.keep_trajfiles()
logger.info('Prepared', digest, 'from', tpr)
for k in self.__dict__:
logger.info(10 * ' ', k.lstrip('_'), '=', getattr(self, k))
return task