def add_md_stage(self):
rid = self._rid
cycle = self._cycle
sbox = self._sbox
cores = self._cores
exe = self._exe
self._log.debug('=== %s add md (cycle %s)', rid, cycle)
task = re.Task()
task.name = 'mdtsk-%s-%s' % (rid, cycle)
## task.link_input_data = ['%s/inpcrd > inpcrd-%s-%s' % (sbox, rid, cycle),
## '%s/prmtop' % (sbox),
## '%s/mdin-%s-%s > mdin' % (sbox, rid, cycle)]
task.arguments = [
'-O', '-i', 'mdin', '-p', 'prmtop', '-c',
'inpcrd-%s-%s' % (rid, cycle), '-o', 'out', '-x', 'mdcrd', '-r',
'%s/inpcrd-%s-%s' % (sbox, rid, cycle), '-inf',
'%s/mdinfo-%s-%s' % (sbox, rid, cycle)
]
task.executable = exe
task.cpu_reqs = {'processes': cores}
task.pre_exec = ['echo $SHARED']
stage = re.Stage()
stage.add_tasks(task)
stage.post_exec = self.after_md
self.add_stages(stage)
def add_ex_stg(rid, cycle):
# ex stg here
ex_tsk = re.Task()
ex_stg = re.Stage()
ex_tsk.name = 'extsk-{replica}-{cycle}'.format(replica=rid,
cycle=cycle)
for rid in range(len(waiting_replicas)):
ex_tsk.link_input_data += [
'%s/mdinfo-{replica}-{cycle}'.format(
replica=rid, cycle=self.cycle) % replica_sandbox
]
ex_tsk.arguments = ['t_ex_gibbs.py',
len(waiting_replicas)
] # This needs to be fixed
ex_tsk.executable = ['python']
ex_tsk.cpu_reqs = {
'processes': 1,
'process_type': '',
'threads_per_process': 1,
'thread_type': None
}
ex_stg.add_tasks(ex_tsk)
ex_stg.post_exec = {
'condition': post_ex,
'on_true': terminate_replicas,
'on_false': continue_md
}
return ex_stg
def add_ex_stage(self, exchange_list):
self._log.debug('=== %s add ex: %s', self.rid,
[r.rid for r in exchange_list])
self._ex_list = exchange_list
self._res_list = exchange_list
task = re.Task()
task.name = 'extsk'
task.executable = ['/home/scm177/VirtualEnvs/Env_RepEx/bin/python']
task.upload_input_data = ['t_ex_gibbs.py']
task.arguments = ['t_ex_gibbs.py', self._sbox]
for replica in exchange_list:
rid = replica.rid
cycle = replica.cycle
task.link_input_data.append(
'%s/mdinfo-%s-%s' %
(self._sbox, rid, cycle - 1)) # % (self._sbox, rid, cycle))
stage = re.Stage()
stage.add_tasks(task)
stage.post_exec = self.check_resume #_after_ex
self.add_stages(stage)
def generate_pipeline(master=False):
global pipes
if master:
def func_post_1():
for p in pipes[1:]:
p.suspend()
def func_post_2():
for p in pipes[1:]:
p.resume()
# return list of resumed pipeline IDs
return [p.uid for p in pipes[1:]]
else:
def func_post_1(): pass
def func_post_2(): pass
# --------------------------------------------------------------------------
# create a pipeline, stage and tasks
t1 = re.Task()
t1.executable = '/bin/sleep'
if master: t1.arguments = [' 1']
else : t1.arguments = ['10']
s1 = re.Stage()
s1.add_tasks(t1)
s1.post_exec = func_post_1
t2 = re.Task()
t2.executable = '/bin/sleep'
t2.arguments = ['1']
s2 = re.Stage()
s2.add_tasks(t2)
s2.post_exec = func_post_2
p = re. Pipeline()
p.add_stages(s1)
p.add_stages(s2)
return p
def add_md_stage(self):
self._log.debug('=== %s add md', self.rid)
task = re.Task(from_dict=self._workload['md'])
# task.name = 'mdtsk-%s-%s' % (self.rid, self.cycle)
stage = re.Stage()
stage.add_tasks(task)
stage.post_exec = self.check_exchange
self.add_stages(stage)
def generate_discover_pipe(self, filetype='csv', img_ftype='tif'):
'''
This function takes as an input paths on Bridges and returns a pipeline
that will provide a file for all the images that exist in that path.
'''
pipeline = re.Pipeline()
pipeline.name = 'Disc'
stage = re.Stage()
stage.name = 'Disc.S0'
if self._paths is None:
raise RuntimeError('Images paths are not set.')
# Create the module load list
modules_load = list()
if self._modules:
for module in self._modules:
tmp_load = 'module load %s' % module
modules_load.append(tmp_load)
tmp_pre_execs = ['unset PYTHONPATH']
if self._pre_execs:
tmp_pre_execs = tmp_pre_execs + modules_load + self._pre_execs
else:
tmp_pre_execs = tmp_pre_execs + modules_load
for i in range(len(self._paths)):
task = re.Task()
task.name = 'Disc.T%d' % i
task.pre_exec = tmp_pre_execs
task.executable = 'python' # Assign executable to the task
task.arguments = [
'image_disc.py',
'%s' % self._paths[i],
'--image_ftype=%s' % img_ftype,
'--filename=images%d' % i,
'--filetype=%s' % filetype, '--filesize'
]
task.download_output_data = ['images%d.csv' % i]
task.upload_input_data = [
os.path.dirname(os.path.abspath(__file__)) + '/image_disc.py'
]
task.cpu_reqs = {
'cpu_processes': 1,
'cpu_process_type': '',
'cpu_threads': 1,
'cpu_thread_type': 'OpenMP'
}
stage.add_tasks(task)
# Add Stage to the Pipeline
pipeline.add_stages(stage)
return pipeline
def add_md_stage(self):
self._log.debug('=== %s add md', self.rid)
task = re.Task()
task.name = 'mdtsk-%s-%s' % (self.rid, self.cycle)
task.executable = 'sleep'
task.arguments = [str(random.randint(0, 20) / 10.0)]
stage = re.Stage()
stage.add_tasks(task)
stage.post_exec = self.check_exchange
self.add_stages(stage)
def add_ex_stage(self, exchange_list, ex_alg):
self._log.debug('=== %s add ex: %s', self.rid,
[r.rid for r in exchange_list])
self._ex_list = exchange_list
task = re.Task(from_dict=self._workload['ex'])
task.arguments = [ex_alg, len(exchange_list), self._cycle]
task.upload_input_data = [ex_alg]
stage = re.Stage()
stage.add_tasks(task)
stage.post_exec = self.check_resume
self.add_stages(stage)
def add_ex_stage(self, exchange_list):
self._log.debug('=== %s add ex: %s', self.rid,
[r.rid for r in exchange_list])
self._ex_list = exchange_list
task = re.Task()
task.name = 'extsk'
task.executable = 'date'
stage = re.Stage()
stage.add_tasks(task)
stage.post_exec = self.check_resume
self.add_stages(stage)
def setup(self):
# prepare input for all replicas
writeInputs.writeInputs(max_temp=self._max_temp,
min_temp=self._min_temp,
replicas=self._en_size,
timesteps=self._timesteps,
basename=self._basename)
# and tar it up
tar = tarfile.open("input_files.tar", "w")
for name in [self._basename + ".prmtop",
self._basename + ".inpcrd",
self._basename + ".mdin"]:
tar.add(name)
for replica in self._replicas:
tar.add ('mdin-%s-0' % replica.rid) #how does this work
os.remove('mdin-%s-0' % replica.rid)
tar.close()
# create a single pipeline with one stage to transfer the tarball
task = re.Task()
task.name = 'untarTsk'
task.executable = ['python']
task.upload_input_data = ['untar_input_files.py', 'input_files.tar']
task.arguments = ['untar_input_files.py', 'input_files.tar']
task.cpu_reqs = {'processes' : 1,
'thread_type' : None,
'threads_per_process': 1,
'process_type': None}
task.post_exec = []
stage = re.Stage()
stage.name = 'untarStg'
stage.add_tasks(task)
#setup = re.Pipeline()
#setup.name = 'untarPipe'
#setup.add_stages(stage)
setup_pipeline = re.Pipeline()
setup_pipeline.name = 'untarPipe'
setup_pipeline.add_stages(stage)
return [setup_pipeline]
def _generate_pipeline(self, name, pre_execs, image, gpu_id):
'''
This function creates a pipeline for an image that will be analyzed.
:Arguments:
:name: Pipeline name, str
:image: image path, str
:model_path: Path to the model file, str
:model_arch: Prediction Model Architecture, str
:model_name: Prediction Model Name, str
:hyperparam_set: Which hyperparameter set to use, str
'''
# Create a Pipeline object
entk_pipeline = re.Pipeline()
entk_pipeline.name = name
# Create a Stage object
stage0 = re.Stage()
stage0.name = '%s-S0' % (name)
# Create Task 1, training
task1 = re.Task()
task1.name = '%s-T0' % stage0.name
task1.pre_exec = pre_execs
task1.executable = 'iceberg_penguins.detect' # Assign task executable
# Assign arguments for the task executable
task1.arguments = ['--gpu_ids', gpu_id,
'--name', self._model_name,
'--epoch', self._epoch,
'--checkpoints_dir', self._model_path,
'--output', self._output_path,
'--testset', 'GE',
'--input_im', image.split('/')[-1]]
task1.link_input_data = ['%s' % image]
task1.cpu_reqs = {'cpu_processes': 1, 'cpu_threads': 1,
'cpu_process_type': None, 'cpu_thread_type': 'OpenMP'}
task1.gpu_reqs = {'gpu_processes': 1, 'gpu_threads': 1,
'gpu_process_type': None, 'gpu_thread_type': 'OpenMP'}
# Download resuting images
# task1.download_output_data = ['%s/ > %s' % (image.split('/')[-1].
# split('.')[0],
# image.split('/')[-1])]
# task1.tag = task0.name
stage0.add_tasks(task1)
# Add Stage to the Pipeline
entk_pipeline.add_stages(stage0)
return entk_pipeline
def add_ex_stage(self, exchange_list, ex_alg, sid):
self._prof.prof('add_ex_start', uid=self.rid)
self._log.debug('%5s add ex: %s', self.rid,
[r.rid for r in exchange_list])
self._ex_list = exchange_list
task = re.Task()
task.executable = 'python3'
task.arguments = [ex_alg, '-r', self.rid, '-c', self.cycle] \
+ ['-e'] + [r.rid for r in exchange_list] \
+ ['-d'] + [d for d in self._workload.exchange.ex_data]
if self._workload.pre_exec:
task.pre_exec = self._workload.pre_exec
# link alg
link_inputs = ['pilot:///%s/%s' % (self._workload.data.inputs, ex_alg)]
# link exchange data
for r in exchange_list:
t = last_task(r)
self._log.debug('Exchage: %s, Task Name: %s Sandbox %s', r.name,
t.name, t.sandbox)
link_inputs += expand_ln(self._workload.exchange.md_2_ex,
# FIXME: how to get absolute task sbox?
# rep.0000.0000:/// ...
# i.e., use task ID as schema
'pilot:///%s' % t.sandbox,
'task://', r.rid, r.cycle)
task.link_input_data = link_inputs
task.name = '%s.%04d.ex' % (self.rid, self.cycle)
task.sandbox = '%s.%04d.ex' % (self.rid, self.cycle)
self._log.debug('%5s added ex: %s, input data: %s',
self.rid, task.name, task.link_input_data)
stage = re.Stage()
stage.add_tasks(task)
stage.post_exec = self.check_resume
self.add_stages(stage)
self._prof.prof('add_ex_stop', uid=self.rid)
def setup_replicas(replicas, min_temp, max_temp, timesteps, basename):
writeInputs.writeInputs(max_temp=max_temp,
min_temp=min_temp,
replicas=replicas,
timesteps=timesteps,
basename=basename)
tar = tarfile.open("input_files.tar", "w")
for name in [
basename + ".prmtop", basename + ".inpcrd", basename + ".mdin"
]:
tar.add(name)
for r in range(replicas):
tar.add('mdin-{replica}-{cycle}'.format(replica=r, cycle=0))
os.remove('mdin-{replica}-{cycle}'.format(replica=r, cycle=0))
tar.close()
setup_p = re.Pipeline()
setup_p.name = 'untarPipe'
# # unused
# repo = git.Repo('.', search_parent_directories=True)
# aux_function_path = repo.working_tree_dir
untar_stg = re.Stage()
untar_stg.name = 'untarStg'
# Untar Task
untar_tsk = re.Task()
untar_tsk.name = 'untarTsk'
untar_tsk.executable = ['python']
untar_tsk.upload_input_data = ['untar_input_files.py', 'input_files.tar']
untar_tsk.arguments = ['untar_input_files.py', 'input_files.tar']
untar_tsk.cpu_reqs = 1
untar_tsk.post_exec = []
untar_stg.add_tasks(untar_tsk)
setup_p.add_stages(untar_stg)
replica_sandbox = '$Pipeline_%s_Stage_%s_Task_%s' \
% (setup_p.name, untar_stg.name, untar_tsk.name)
return setup_p, replica_sandbox
def add_md_stage(self):
"""
Problem: add_md_stage only adds md tasks that initiate from "inpcrd" i.e. cycle 0.
Needs to accept input from previous MD stage.
"""
rid = self._rid
cycle = self._cycle
sbox = self._sbox
cores = self._cores
exe = self._exe
self._log.debug('=== %s add md (cycle %s)', rid, cycle)
task = re.Task()
task.name = 'mdtsk-%s-%s' % (rid, cycle)
if cycle == 0:
task.link_input_data = [
'%s/inpcrd > inpcrd-%s-%s' % (sbox, rid, cycle),
'%s/prmtop' % (sbox),
'%s/mdin-%s-%s > mdin' % (sbox, rid, cycle)
]
else:
cycle_0 = '0'
task.link_input_data = [
'%s/inpcrd-%s-%s' % (sbox, rid, cycle),
'%s/prmtop' % (sbox),
'%s/mdin-%s-%s > mdin' % (sbox, rid, cycle_0)
]
#['%s/mdcrd-out-%s-%s > inpcrd-%s-%s' % (sbox, rid, cycle, rid, cycle),
task.arguments = [
'-O', '-i', 'mdin', '-p', 'prmtop', '-c',
'inpcrd-%s-%s' % (rid, cycle), '-o', 'out', '-x',
'%s/mdcrd-%s-%s' % (sbox, rid, cycle + 1), '-r',
'%s/inpcrd-%s-%s' % (sbox, rid, cycle + 1), '-inf',
'%s/mdinfo-%s-%s' % (sbox, rid, cycle)
]
task.executable = SANDER #[exe]
task.cpu_reqs = {'processes': cores}
task.pre_exec = ['echo $SHARED'
] #This will be different for different MD engines.
stage = re.Stage()
stage.add_tasks(task)
stage.post_exec = self._after_md
self.add_stages(stage)
def add_md_stage(self):
self._log.debug('=== %s add md', self.rid)
rid = self._rid
cycle = self._cycle
sbox = self._sbox
cores = self._cores
exe = self._exe
task = re.Task()
task.name = 'mdtsk-%s-%s' % ( rid, cycle)
if cycle == 0:
task.link_input_data = ['%s/inpcrd > inpcrd-%s-%s' % (sbox, rid, cycle),
'%s/prmtop' % (sbox),
'%s/mdin-%s-%s > mdin' % (sbox, rid, cycle)]
else:
cycle_0 = '0'
task.link_input_data = ['%s/inpcrd-%s-%s' % (sbox, rid, cycle),
'%s/prmtop' % (sbox),
'%s/mdin-%s-%s > mdin' % (sbox, rid, cycle_0)]
#['%s/mdcrd-out-%s-%s > inpcrd-%s-%s' % (sbox, rid, cycle, rid, cycle),
task.arguments = ['-O',
'-i', 'mdin',
'-p', 'prmtop',
'-c', 'inpcrd-%s-%s' % ( rid, cycle),
'-o', 'out',
'-x', '%s/mdcrd-%s-%s' % (sbox, rid, cycle+1),
'-r', '%s/inpcrd-%s-%s' % (sbox, rid, cycle+1),
'-inf', '%s/mdinfo-%s-%s' % (sbox, rid, cycle)]
task.executable = SANDER #[exe]
task.cpu_reqs = {'processes' : cores,
'thread_type' : None,
'threads_per_process': 1,
'process_type': None}
task.pre_exec = ['module load intel/19.3', 'module load python3/intel_3.6.3'] #This will be different for different MD engines.
stage = re.Stage()
stage.add_tasks(task)
stage.post_exec = self.check_exchange #_after_md
self.add_stages(stage)
def setup(self):
self._log.debug('=== data staging')
# prepare input for all replicas
writeInputs.writeInputs(max_temp=self._max_temp,
min_temp=self._min_temp,
replicas=self._size,
timesteps=self._timesteps,
basename=self._basename)
# and tar it up
tar = tarfile.open("input_files.tar", "w")
for name in [
self._basename + ".prmtop", self._basename + ".inpcrd",
self._basename + ".mdin"
]:
tar.add(name)
for replica in self._replicas:
tar.add('mdin-%s-0' % replica.rid)
os.remove('mdin-%s-0' % replica.rid)
tar.close()
# create a single pipeline with one stage to transfer the tarball
task = re.Task()
task.name = 'untarTsk'
task.executable = 'python'
task.upload_input_data = ['untar_input_files.py', 'input_files.tar']
task.arguments = ['untar_input_files.py', 'input_files.tar']
task.cpu_reqs = 1
task.post_exec = []
stage = re.Stage()
stage.name = 'untarStg'
stage.add_tasks(task)
setup = re.Pipeline()
setup.name = 'untarPipe'
setup.add_stages(stage)
# run the setup pipeline
self.workflow = set([setup])
self.run()
def get_wf3_input(appman, cfg):
# Assuming shared filesystem on login node this can be executed by the
# script instead of EnTK.
p = entk.Pipeline()
p.name = 'get_wf3_input'
s = entk.Stage()
t = entk.Task()
t.executable = ['python3']
t.arguments = [
'gather.py', '-f', cfg['outlier_path'], '-p', cfg['top_path']
]
s.add_tasks(t)
p.add_stages(s)
appman.workflow = [p]
appman.run()
def add_md_stg(rid, cycle):
# md stg here
print 'cycle: ', self.cycle
md_tsk = re.Task()
md_stg = re.Stage()
md_tsk.name = 'mdtsk-{replica}-{cycle}'.format(replica=rid,
cycle=self.cycle)
md_tsk.link_input_data = [
'%s/inpcrd > inpcrd-{replica}-{cycle}'.format(
replica=rid, cycle=self.cycle) % replica_sandbox,
'%s/prmtop' % replica_sandbox,
'%s/mdin-{replica}-{cycle} > mdin'.format(
replica=rid, cycle=self.cycle) % replica_sandbox
]
md_tsk.arguments = [
'-O', '-i', 'mdin', '-p', 'prmtop', '-c',
'inpcrd-{replica}-{cycle}'.format(replica=rid,
cycle=self.cycle), '-o',
'out', '-x', 'mdcrd', '-r',
'%s/inpcrd-{replica}-{cycle}'.format(
replica=rid, cycle=self.cycle + 1) % replica_sandbox,
'-inf', '%s/mdinfo-{replica}-{cycle}'.format(
replica=rid, cycle=self.cycle) % replica_sandbox
]
md_tsk.executable = [md_executable]
md_tsk.cpu_reqs = {
'processes': replica_cores,
'process_type': '',
'threads_per_process': 1,
'thread_type': None
}
md_tsk.pre_exec = ['echo $SHARED']
md_stg.add_tasks(md_tsk)
md_stg.post_exec = {
'condition': post_md,
'on_true': start_ex,
'on_false': suspend_replica
}
return md_stg
def add_md_stage(self):
rid = self._rid
cycle = self._cycle
sbox = self._sbox
cores = self._cores
exe = self._exe
self._log.debug('=== %s add md (cycle %s)', rid, cycle)
task = re.Task()
task.name = 'mdtsk-%s-%s' % (rid, cycle)
task.link_input_data = [
'%s/inpcrd > inpcrd-%s-%s' % (sbox, rid, cycle),
'%s/prmtop' % (sbox),
'%s/mdin-%s-%s > mdin' % (sbox, rid, cycle)
]
task.arguments = [
'-O', '-i', 'mdin', '-p', 'prmtop', '-c',
'inpcrd-%s-%s' % (rid, cycle), '-o', 'out', '-x', 'mdcrd', '-r',
'%s/inpcrd-%s-%s' % (sbox, rid, cycle), '-inf',
'%s/mdinfo-%s-%s' % (sbox, rid, cycle)
]
task.executable = SANDER #[exe]
task.cpu_reqs = {'processes': cores}
task.pre_exec = ['echo $SHARED'
] #This will be different for different MD engines.
stage = re.Stage()
stage.add_tasks(task)
try:
stage.post_exec = self.after_md
except:
stage.post_exec = {
'condition': self.after_md,
'on_true': void,
'on_false': void
}
self.add_stages(stage)
def _check_exchange(self, replica):
'''
add this replica to the wait list, and check if we have sufficient
replicas for an exchange. If no, just wait for the next one.
If yes, we request an exchange over the collected suspended replicas
in the waitlist. We abuse the current replica to run that exchange
stage for us.
'''
# mark this replica for the next exchange
self._waitlist.append(replica)
self._log.debug('=== EX %s check exchange (%d >= %d?)', replica.rid,
len(self._waitlist), self._max_wait)
if len(self._waitlist) < self._max_wait:
# just suspend this replica and wait for the next
self._log.debug('=== EX %s suspend', replica.rid)
replica.suspend()
else:
# we are in for a wile ride!
self._log.debug('=== EX %s exchange', replica.rid)
task = re.Task()
task.name = 'extsk'
## task.executable = 'python'
task.executable = 'echo'
task.arguments = ['t_ex_gibbs.py', len(self._waitlist)]
## for replica in self._waitlist:
## rid = replica.rid
## cycle = replica.cycle
## task.link_input_data.append('%s/mdinfo-%s-%s'
## % (self._sbox, rid, cycle))
stage = re.Stage()
stage.add_tasks(task)
stage.post_exec = self._after_exchange
replica.add_stages(stage)
def sim_esmacs_py(self,
rep_count=24,
structures=None,
trajectory=None,
component="com",
nanoseconds=6):
for i in range(1, int(rep_count) + 1):
t = entk.Task()
pre_exec = """export OMP_NUM_THREADS=1
export WDIR="$MEMBERWORK/chm155/Model-generation"
. /gpfs/alpine/scratch/apbhati/chm155/miniconda/etc/profile.d/conda.sh
conda activate conda-entk
module load cuda gcc spectrum-mpi"""
t.pre_exec = [x.strip() for x in pre_exec.split("\n")]
t.executable = '/gpfs/alpine/scratch/apbhati/chm155/miniconda/envs/conda-entk/bin/python'
t.arguments = [
'$WDIR/sim_esmacs.py', '-i{}'.format(structures),
'-o{}'.format(trajectory), '-n{}'.format(nanoseconds),
'-c{}'.format(component), '-r{}'.format(i)
]
t.post_exec = []
t.cpu_reqs = {
'processes': 1,
'process_type': None,
'threads_per_process': 4,
'thread_type': 'OpenMP'
}
t.gpu_reqs = {
'processes': 1,
'process_type': None,
'threads_per_process': 1,
'thread_type': 'CUDA'
}
self.s.add_tasks(t)
self.p.add_stages(self.s)
def ties(self,
calc,
ncores,
rct_stage="s4",
stage="eq0",
outdir="equilibration",
name=None):
for l in [
0.00, 0.05, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80,
0.90, 0.95, 1.00
]:
# for l in [0.50]: # For quick testing
for i in range(1, 6):
# for i in range(1, 2): # For quick testing
t = entk.Task()
t.pre_exec = [
"module load spectrum-mpi/10.3.1.2-20200121 fftw/3.3.8",
"cd $MEMBERWORK/med110/test_hybridwf/{}/{}/replica-confs".
format(name, calc),
"mkdir -p ../LAMBDA_{:.2f}/rep{}/{}".format(l, i, outdir),
"export OMP_NUM_THREADS=1"
]
t.executable = '/gpfs/alpine/world-shared/bip115/NAMD_binaries/summit/NAMD_LATEST_Linux-POWER-MPI-smp-Summit/namd2'
t.arguments = [
'+ppn', '41', '--tclmain', '{}.conf'.format(stage),
'{:.2f}'.format(l), '{}'.format(i)
] #'{}'.format(ncores)
t.post_exec = []
t.cpu_reqs = {
'processes': 1,
'process_type': None, #'MPI'
'threads_per_process': 4 * int(
ncores
), #amounts to 35 cores per node (out of 41 usable cores)
'thread_type': 'OpenMP'
}
#self.rct_stage.add_tasks(t)
getattr(self, rct_stage).add_tasks(t)
def esmacs(self,
rct_stage="s1",
stage="eq1",
outdir="equilibration",
name=None):
for i in range(1, 13):
# for i in range(1, 7): # currently set to 6 replicas for coarse grained ESMACS
t = entk.Task()
t.pre_exec = [
"export WDIR=\"$MEMBERWORK/med110/test_hybridwf/{}\"".format(
name),
". /ccs/home/litan/miniconda3/etc/profile.d/conda.sh",
"conda activate wf3",
"module load cuda/10.1.243 gcc/7.4.0 spectrum-mpi/10.3.1.2-20200121",
"mkdir -p $WDIR/replicas/rep{}/{}".format(i, outdir),
"cd $WDIR/replicas/rep{}/{}".format(i, outdir),
"rm -f {}.log {}.xml {}.dcd {}.chk".format(
stage, stage, stage, stage), "export OMP_NUM_THREADS=1"
]
t.executable = '/ccs/home/litan/miniconda3/envs/wf3/bin/python3.7'
t.arguments = ['$WDIR/{}.py'.format(stage)]
t.post_exec = []
t.cpu_reqs = {
'processes': 1,
'process_type': None,
'threads_per_process': 4,
'thread_type': 'OpenMP'
}
t.gpu_reqs = {
'processes': 1,
'process_type': None,
'threads_per_process': 1,
'thread_type': 'CUDA'
}
#self.rct_stage.add_tasks(t)
getattr(self, rct_stage).add_tasks(t)
def _check_exchange(self, replica):
'''
add this replica to the wait list, and check if we have sufficient
replicas for an exchange. If no, just wait for the next one.
If yes, we request an exchange over the collected suspended replicas
in the waitlist. We abuse the current replica to run that exchange
stage for us.
Sliding Window model: We are to only perform exchange on a sublist
from the waitlist of replicas. We select this sublist based on parameter
(T in this case) proximity. Width of the sliding window is user defined.
This is just exchange_size.
'''
# mark this replica for the next exchange
self._waitlist.append(replica)
self._log.debug('=== %s check exchange (%d >= %d?)', replica.rid,
len(self._waitlist), self._exchange_size)
print "waitlist is: "
for i in (self._waitlist):
print i.rid
# Sort the waitlist as soon as a new replica is added.
self._sorted_waitlist = list()
for replica in self._waitlist:
self._sorted_waitlist.append(
[replica, replica.rid]
) # We're sorting by RID here since RID's are assigned in sorted order with
# Temperature (or whatever paramater is of interest to us)
self._sorted_waitlist = sorted(
self._sorted_waitlist,
key=lambda x: x[1]) #Taken from Andre's example
print "sorted waitlist is: "
for i in (self._sorted_waitlist):
print i[0].rid
# Now we generate a sublist called exchange_list, within which an exchange is performed. This is done with
# the sliding_window function
self._exchange_list = self._sliding_window(self._sorted_waitlist,
self._exchange_size,
self._window_size)
# Now check if the proposed exchange list is big enough (it should be, this seems slightly redundant)
print "exchange size is ", self._exchange_size, " and exchange list length is ", len(
self._exchange_list)
if len(self._exchange_list) < self._exchange_size:
# just suspend this replica and wait for the next
self._log.debug('=== %s suspend', replica.rid)
print "replica ", replica.rid, " should suspend now"
replica.suspend()
print "BUT IT ISN'T AAAAAARGH"
else:
# we are in for a wild ride!
self._log.debug('=== %s exchange')
task = re.Task()
task.name = 'extsk'
task.executable = 'python'
task.arguments = ['t_ex_gibbs.py', len(self._waitlist)]
for replica in self._waitlist:
rid = replica.rid
cycle = replica.cycle
task.link_input_data.append('%s/mdinfo-%s-%s' %
(self._sbox, rid, cycle))
stage = re.Stage()
stage.add_tasks(task)
try:
stage.post_exec = self._after_ex
except:
stage.post_exec = {
'condition': self.after_ex,
'on_true': void,
'on_false': void
}
replica.add_stages(stage)
# Here we remove the replicas participating in the triggered exchange from the waitlist.
for replica in self._exchange_list:
self._sorted_waitlist.remove(
[replica,
replica.rid]) #Sorted_Waitlist is a list of tuples
def esmacs_py(self,
rep_count=24,
component="com",
structures=None,
trajectory=None,
nanoseconds=6):
for i in range(1, int(rep_count) + 1):
t = entk.Task()
t.pre_exec = [
"export OMP_NUM_THREADS=1",
". /gpfs/alpine/scratch/apbhati/chm155/miniconda/etc/profile.d/conda.sh",
"conda activate conda-entk",
"module load cuda/10.1.243 gcc/6.4.0 spectrum-mpi/10.3.1.2-20200121",
"export CUDA_HOME=/sw/summit/cuda/10.1.243",
"export WDIR=\"$MEMBERWORK/chm155/Model-generation\"",
"source /gpfs/alpine/scratch/apbhati/chm155/AmberTools19/amber18/amber.sh",
"export LD_LIBRARY_PATH=\"/sw/summit/cuda/10.1.243/lib:${LD_LIBRARY_PATH}\"",
"export INPATH={}".format(structures),
"export COM={}".format(component), "cd {}/{}/rep{}".format(
trajectory, component, i)
]
# Amber
t.executable = "MMPBSA.py.MPI"
if component == "com":
t.arguments = ("-O -i $WDIR/impress_md/esmacs.in -sp " + \
"$INPATH/${COM}_sol.prmtop -cp $INPATH/com.prmtop -rp " + \
"$INPATH/apo.prmtop -lp $INPATH/lig.prmtop -y traj.dcd " + \
"> mmpbsa.log").split()
post_exec = """ls _MMPBSA_complex_gb.mdout.* | sort -V | xargs cat > _MMPBSA_complex_gb.mdout.all
ls _MMPBSA_complex_gb_surf.dat.* | sort -V | xargs cat > _MMPBSA_complex_gb_surf.dat.all
ls _MMPBSA_complex_pb.mdout.* | sort -V | xargs cat > _MMPBSA_complex_pb.mdout.all
ls _MMPBSA_ligand_gb.mdout.* | sort -V | xargs cat > _MMPBSA_ligand_gb.mdout.all
ls _MMPBSA_ligand_gb_surf.dat.* | sort -V | xargs cat > _MMPBSA_ligand_gb_surf.dat.all
ls _MMPBSA_ligand_pb.mdout.* | sort -V | xargs cat > _MMPBSA_ligand_pb.mdout.all
ls _MMPBSA_receptor_gb.mdout.* | sort -V | xargs cat > _MMPBSA_receptor_gb.mdout.all
ls _MMPBSA_receptor_gb_surf.dat.* | sort -V | xargs cat > _MMPBSA_receptor_gb_surf.dat.all
ls _MMPBSA_receptor_pb.mdout.* | sort -V | xargs cat > _MMPBSA_receptor_pb.mdout.all
rm _MMPBSA_*.[0-9]* reference.frc *.inpcrd *.mdin* *.out"""
else:
t.arguments = ("-O -i $WDIR/impress_md/esmacs.in -sp " + \
"$INPATH/${COM}_sol.prmtop -cp $INPATH/$COM.prmtop " + \
"-y traj.dcd > mmpbsa.log").split()
post_exec = """ls _MMPBSA_complex_gb.mdout.* | sort -V | xargs cat > _MMPBSA_complex_gb.mdout.all
ls _MMPBSA_complex_gb_surf.dat.* | sort -V | xargs cat > _MMPBSA_complex_gb_surf.dat.all
ls _MMPBSA_complex_pb.mdout.* | sort -V | xargs cat > _MMPBSA_complex_pb.mdout.all
rm _MMPBSA_*.[0-9]* reference.frc *.inpcrd *.mdin* *.out"""
t.post_exec = [x.strip() for x in post_exec.split("\n")]
t.post_exec = ["cd {}/{}/rep{}".format(trajectory, component, i)
] + t.post_exec
t.cpu_reqs = {
'processes': (int(nanoseconds) - 2) * 10,
'process_type': 'MPI',
'threads_per_process': 4,
'thread_type': 'OpenMP'
}
t.gpu_reqs = {
'processes': 0,
'process_type': None,
'threads_per_process': 1,
'thread_type': 'CUDA'
}
self.s.add_tasks(t)
self.p.add_stages(self.s)
def _generate_pipeline(self, name, pre_execs, image, image_size):
'''
This function creates a pipeline for an image that will be analyzed.
:Arguments:
:name: Pipeline name, str
:image: image path, str
:image_size: image size in MBs, int
:tile_size: The size of each tile, int
:model_path: Path to the model file, str
:model_arch: Prediction Model Architecture, str
:model_name: Prediction Model Name, str
:hyperparam_set: Which hyperparameter set to use, str
'''
# Create a Pipeline object
entk_pipeline = re.Pipeline()
entk_pipeline.name = name
# Create a Stage object
stage0 = re.Stage()
stage0.name = '%s.S0' % (name)
# Create Task 1, training
task0 = re.Task()
task0.name = '%s.T0' % stage0.name
task0.pre_exec = pre_execs
task0.executable = 'iceberg_seals.tiling' # Assign tak executable
# Assign arguments for the task executable
task0.arguments = [
'--input_image=%s' % image.split('/')[-1],
'--output_folder=$NODE_LFS_PATH/%s' % task0.name,
'--bands=%s' % self._bands,
'--stride=%s' % self._stride,
'--patch_size=%s' % self._patch_size,
'--geotiff=%s' % self._geotiff
]
task0.link_input_data = [image]
task0.cpu_reqs = {
'cpu_processes': 1,
'cpu_threads': 4,
'cpu_process_type': None,
'cpu_thread_type': 'OpenMP'
}
task0.lfs_per_process = image_size
stage0.add_tasks(task0)
# Add Stage to the Pipeline
entk_pipeline.add_stages(stage0)
# Create a Stage object
stage1 = re.Stage()
stage1.name = '%s.S1' % (name)
# Create Task 1, training
task1 = re.Task()
task1.name = '%s.T1' % stage1.name
task1.pre_exec = pre_execs
task1.executable = 'iceberg_seals.predicting' # Assign task executable
# Assign arguments for the task executable
task1.arguments = [
'--input_dir=$NODE_LFS_PATH/%s' % task0.name,
'--model_architecture=%s' % self._model_arch,
'--hyperparameter_set=%s' % self._hyperparam,
'--model_name=%s' % self._model_name,
'--models_folder=./',
'--output_dir=./%s' % image.split('/')[-1].split('.')[0],
]
task1.link_input_data = ['$SHARED/%s' % self._model_name]
task1.cpu_reqs = {
'cpu_processes': 1,
'cpu_threads': 1,
'cpu_process_type': None,
'cpu_thread_type': 'OpenMP'
}
task1.gpu_reqs = {
'gpu_processes': 1,
'gpu_threads': 1,
'gpu_process_type': None,
'gpu_thread_type': 'OpenMP'
}
# Download resulting images
# task1.download_output_data = ['%s/ > %s' % (image.split('/')[-1].
# split('.')[0],
# image.split('/')[-1])]
task1.tags = {'colocate': task0.name}
stage1.add_tasks(task1)
# Add Stage to the Pipeline
entk_pipeline.add_stages(stage1)
return entk_pipeline
def _generate_pipeline(self, name, pre_execs, image, image_size):
'''
This function creates a pipeline for an image that will be analyzed.
:Arguments:
:name: Pipeline name, str
:image: image path, str
:image_size: image size in MBs, int
:tile_size: The size of each tile, int
:model_path: Path to the model file, str
:model_arch: Prediction Model Architecture, str
:model_name: Prediction Model Name, str
:hyperparam_set: Which hyperparameter set to use, str
'''
# Create a Pipeline object
entk_pipeline = re.Pipeline()
entk_pipeline.name = name
# Create a Stage object
stage0 = re.Stage()
stage0.name = '%s-S0' % (name)
# Create Task 1, training
task0 = re.Task()
task0.name = '%s-T0' % stage0.name
task0.pre_exec = pre_execs
task0.executable = 'iceberg_seals.tiling' # Assign tak executable
# Assign arguments for the task executable
task0.arguments = ['--scale_bands=%s' % self._scale_bands,
'--input_image=%s' % image.split('/')[-1],
# This line points to the local filesystem of the
# node that the tiling of the image happened.
'--output_folder=$NODE_LFS_PATH/%s' % task0.name]
task0.link_input_data = [image]
task0.cpu_reqs = {'processes': 1, 'threads_per_process': 4,
'process_type': None, 'thread_type': 'OpenMP'}
task0.lfs_per_process = image_size
stage0.add_tasks(task0)
# Add Stage to the Pipeline
entk_pipeline.add_stages(stage0)
# Create a Stage object
stage1 = re.Stage()
stage1.name = '%s-S1' % (name)
# Create Task 1, training
task1 = re.Task()
task1.name = '%s-T1' % stage1.name
task1.pre_exec = pre_execs
task1.executable = 'iceberg_seals.predicting' # Assign task executable
# Assign arguments for the task executable
task1.arguments = ['--input_image', image.split('/')[-1],
'--model_architecture', self._model_arch,
'--hyperparameter_set', self._hyperparam,
'--training_set', 'test_vanilla',
'--test_folder', '$NODE_LFS_PATH/%s' % task0.name,
'--model_path', './',
'--output_folder', './%s' % image.split('/')[-1].
split('.')[0]]
task1.link_input_data = ['$SHARED/%s' % self._model_name]
task1.cpu_reqs = {'processes': 1, 'threads_per_process': 1,
'process_type': None, 'thread_type': 'OpenMP'}
task1.gpu_reqs = {'processes': 1, 'threads_per_process': 1,
'process_type': None, 'thread_type': 'OpenMP'}
# Download resuting images
task1.download_output_data = ['%s/ > %s' % (image.split('/')[-1].
split('.')[0],
image.split('/')[-1])]
task1.tag = task0.name
stage1.add_tasks(task1)
# Add Stage to the Pipeline
entk_pipeline.add_stages(stage1)
return entk_pipeline
def add_md_stage(self, exchanged_from=None, sid=None, last=False):
self._prof.prof('add_md_start', uid=self.rid)
self._cycle += 1
self._log.debug('%5s %s add md', self.rid, self._uid)
# task = re.Task(from_dict=self._workload['md'])
# task.name = 'mdtsk-%s-%s' % (self.rid, self.cycle)
sandbox = '%s.%04d.md' % (self.rid, self.cycle)
link_inputs = list()
# link initial data
link_inputs += expand_ln(self._workload.md.inputs,
'pilot:///%s' % self._workload.data.inputs,
'task://', self.rid, self.cycle)
if self._cycle == 0:
# link initial data
link_inputs += expand_ln(self._workload.md.inputs_0,
'pilot:///%s' % self._workload.data.inputs,
'task://',
self.rid, self.cycle)
else:
# get data from previous task
t = last_task(self)
if exchanged_from:
self._log.debug('Exchange from %s', exchanged_from.name)
link_inputs += expand_ln(self._workload.md.ex_2_md,
'pilot:///%s' % (exchanged_from.sandbox),
'task://',
self.rid, self.cycle)
else:
# FIXME: this apparently can't happen
link_inputs += expand_ln(self._workload.md.md_2_md,
'resource:///%s' % (t.sandbox),
'task://',
self.rid, self.cycle)
copy_outputs = expand_ln(self._workload.md.outputs,
'task://',
'client:///%s' % self._workload.data.outputs,
self.rid, self.cycle)
if last:
copy_outputs += expand_ln(self._workload.md.outputs_n,
'task://',
'client:///%s' % self._workload.data.outputs,
self.rid, self.cycle)
# TODO: filter out custom keys from that dict before deepcopy
env = {'REPEX_RID' : str(self.rid),
'REPEX_CYCLE' : str(self.cycle)}
tds = copy.deepcopy(self._workload['md']['descriptions'])
first = 0
last = len(tds) - 1
for idx, td in enumerate(tds):
stage = re.Stage()
task = re.Task()
td = ru.expand_env(td, env=env)
for k,v in td.items():
setattr(task, k, v)
if self._workload.pre_exec:
if task.pre_exec:
task.pre_exec.extend(self._workload.pre_exec)
else:
task.pre_exec.extend = self._workload.pre_exec
task.name = '%s.%04d.%02d.md' % (self.rid, self.cycle, idx)
task.sandbox = sandbox
if idx == first:
task.link_input_data = link_inputs
if idx == last:
task.download_output_data = copy_outputs
stage.post_exec = self.check_exchange
stage.add_tasks(task)
self.add_stages(stage)
self._log.debug('%5s add md: %s', self.rid, task.name)
self._prof.prof('add_md_stop', uid=self.rid)
def _generate_pipeline(self, name, pre_execs, image, image_size):
'''
This function creates a pipeline for an image that will be analyzed.
:Arguments:
:name: Pipeline name, str
:pre_execs: things need to happen before execution
:image: image path, str
:image_size: image size in MBs, int
'''
# Create a Pipeline object
entk_pipeline = re.Pipeline()
entk_pipeline.name = name
# Create a Stage object
stage0 = re.Stage()
stage0.name = '%s.S0' % (name)
# Create Task 1, training
task0 = re.Task()
task0.name = '%s.T0' % stage0.name
task0.pre_exec = pre_execs
task0.executable = 'iceberg_rivers.tiling' # Assign tak executable
task0.arguments = [
'--input=%s' % image.split('/')[-1],
'--output=$NODE_LFS_PATH/%s/' % task0.name,
'--tile_size=%s' % self._tile_size,
'--step=%s' % self._step
]
task0.link_input_data = [image]
task0.cpu_reqs = {
'cpu_processes': 1,
'cpu_threads': 4,
'cpu_process_type': None,
'cpu_thread_type': None
}
task0.lfs_per_process = image_size
stage0.add_tasks(task0)
# Add Stage to the Pipeline
entk_pipeline.add_stages(stage0)
# Create a Stage object
stage1 = re.Stage()
stage1.name = '%s.S1' % (name)
# Create Task 1, training
task1 = re.Task()
task1.name = '%s.T1' % stage1.name
task1.pre_exec = pre_execs
task1.executable = 'iceberg_rivers.predicting' # Assign task executable
# # Assign arguments for the task executable
task1.arguments = [
'--input=$NODE_LFS_PATH/%s/' % task0.name,
'--weights_path=%s' % self._weights_path,
'--output_folder=$NODE_LFS_PATH/%s/' % task1.name
]
# # task1.link_input_data = ['$SHARED/%s' % self._model_name]
task1.cpu_reqs = {
'processes': 1,
'threads_per_process': 1,
'process_type': None,
'thread_type': None
}
task1.gpu_reqs = {
'processes': 1,
'threads_per_process': 1,
'process_type': None,
'thread_type': None
}
task0.lfs_per_process = image_size
# Download resulting images
# task1.download_output_data = ['%s/ > %s' % (image.split('/')[-1].
# split('.')[0],
# image.split('/')[-1])]
task1.tags = {'colocate': task0.name}
stage1.add_tasks(task1)
# Add Stage to the Pipeline
entk_pipeline.add_stages(stage1)
# Create a Stage object
stage2 = re.Stage()
stage2.name = '%s.S2' % (name)
# Create Task 1, training
task2 = re.Task()
task2.name = '%s.T2' % stage2.name
task2.pre_exec = pre_execs
task2.executable = 'iceberg_rivers.mosaic' # Assign task executable
# # Assign arguments for the task executable
task2.arguments = [
'--input=$NODE_LFS_PATH/%s/' % task1.name,
'--input_WV=%s' % image.split('/')[-1],
'--tile_size=%s' % self._tile_size,
'--step=%s' % self._step, '--output_folder=./'
]
task2.cpu_reqs = {
'processes': 1,
'threads_per_process': 1,
'process_type': None,
'thread_type': None
}
task2.link_input_data = [image]
task2.tags = {'colocate': task0.name}
stage2.add_tasks(task2)
# Add Stage to the Pipeline
entk_pipeline.add_stages(stage2)
return entk_pipeline
