def generate_ML_stage(num_ML=1):
"""
Function to generate the learning stage
"""
s3 = Stage()
s3.name = 'learning'
# learn task
for i in range(num_ML):
t3 = Task()
# https://github.com/radical-collaboration/hyperspace/blob/MD/microscope/experiments/CVAE_exps/train_cvae.py
t3.pre_exec = []
t3.pre_exec = ['module reset']
t3.pre_exec += [
'. /sw/summit/python/2.7/anaconda2/5.3.0/etc/profile.d/conda.sh'
]
t3.pre_exec += ['module load cuda/9.1.85']
t3.pre_exec += ['conda activate rp.copy']
t3.pre_exec += ['export CUDA_VISIBLE_DEVICES=0']
t3.pre_exec += [
'export PYTHONPATH=/gpfs/alpine/scratch/hrlee/bip179/hyperspace/microscope/experiments/CVAE_exps:$PYTHONPATH'
]
t3.pre_exec += [
'cd /gpfs/alpine/scratch/hrlee/bip179/hyperspace/microscope/experiments/CVAE_exps'
]
time_stamp = int(time.time())
dim = i + 3
cvae_dir = 'cvae_runs_%.2d_%d' % (dim, time_stamp)
t3.pre_exec += ['mkdir -p {0} && cd {0}'.format(cvae_dir)]
t3.executable = ['/ccs/home/hrlee/.conda/envs/rp.copy/bin/python'
] # train_cvae.py
t3.arguments = [
'/gpfs/alpine/bip179/scratch/hrlee/hyperspace/microscope/experiments/CVAE_exps/train_cvae.py',
'-f',
'/gpfs/alpine/bip179/scratch/hrlee/hyperspace/microscope/experiments/MD_to_CVAE/cvae_input.h5',
'-d', dim
]
t3.cpu_reqs = {
'processes': 1,
'process_type': None,
'threads_per_process': 4,
'thread_type': 'OpenMP'
}
t3.gpu_reqs = {
'processes': 1,
'process_type': None,
'threads_per_process': 1,
'thread_type': 'CUDA'
}
# Add the learn task to the learning stage
s3.add_tasks(t3)
time.sleep(1)
return s3
def get_pipeline(tasks):
# Create a Pipeline object
p = Pipeline()
# Create a Stage 1
s1 = Stage()
# Create a Task object according to the app_name
t1 = Task()
t1.pre_exec = ['module load gromacs/5.0/INTEL-140-MVAPICH2-2.0']
t1.executable = app_coll['grompp']['executable']
t1.arguments = app_coll['grompp']['arguments']
t1.cores = app_coll['grompp']['cores']
t1.link_input_data = [
'$SHARED/grompp.mdp > grompp.mdp', '$SHARED/input.gro > input.gro',
'$SHARED/topol.top > topol.top'
]
# Add the Task to the Stage
s1.add_tasks(t1)
# Add Stage to the Pipeline
p.add_stages(s1)
# Create a Stage 2
s2 = Stage()
for cnt in range(tasks):
# Create a Task object according to the app_name
t2 = Task()
t2.pre_exec = [
'module load gromacs/5.0/INTEL-140-MVAPICH2-2.0',
'export OMP_NUM_THREADS=%s' % num_cores
]
t2.executable = app_coll['mdrun']['executable']
t2.arguments = app_coll['mdrun']['arguments']
#t2.cores = app_coll['mdrun']['cores']
t2.cores = num_cores
t2.copy_input_data = [
'$Pipeline_%s_Stage_%s_Task_%s/topol.tpr' % (p.uid, s1.uid, t1.uid)
]
# Add the Task to the Stage
s2.add_tasks(t2)
# Add Stage to the Pipeline
p.add_stages(s2)
return p
def generate_interfacing_stage():
s4 = Stage()
s4.name = 'scanning'
# Scaning for outliers and prepare the next stage of MDs
t4 = Task()
t4.pre_exec = []
t4.pre_exec = ['module reset']
t4.pre_exec += [
'. /sw/summit/python/2.7/anaconda2/5.3.0/etc/profile.d/conda.sh'
]
t4.pre_exec += ['module load cuda/9.1.85']
t4.pre_exec += ['conda activate rp.copy']
t4.pre_exec += ['export CUDA_VISIBLE_DEVICES=0']
t4.pre_exec += [
'export PYTHONPATH=/gpfs/alpine/bip179/scratch/hrlee/hyperspace/microscope/experiments/CVAE_exps:$PYTHONPATH'
]
t4.pre_exec += [
'cd /gpfs/alpine/bip179/scratch/hrlee/hyperspace/microscope/experiments/Outlier_search'
]
# python outlier_locator.py -m ../MD_exps/fs-pep -c ../CVAE_exps -p ../MD_exps/fs-pep/pdb/100-fs-peptide-400K.pdb
t4.executable = ['/ccs/home/hrlee/.conda/envs/rp.copy/bin/python']
t4.arguments = [
'outlier_locator.py', '--md', '../MD_exps/fs-pep', '--cvae',
'../CVAE_exps --pdb',
'../MD_exps/fs-pep/pdb/100-fs-peptide-400K.pdb'
]
# t4.arguments = ['/gpfs/alpine/bip179/scratch/hrlee/hyperspace/microscope/experiments/Outlier_search/outlier_locator.py',
# '-m', '/gpfs/alpine/bip179/scratch/hrlee/hyperspace/microscope/experiments/MD_exps/fs-pep',
# '-c', '/gpfs/alpine/bip179/scratch/hrlee/hyperspace/microscope/experiments/CVAE_exps',
# '-p', '/gpfs/alpine/bip179/scratch/hrlee/hyperspace/microscope/experiments/MD_exps/fs-pep/pdb/100-fs-peptide-400K.pdb'
# ]
t4.cpu_reqs = {
'processes': 1,
'process_type': None,
'threads_per_process': 12,
'thread_type': 'OpenMP'
}
t4.gpu_reqs = {
'processes': 1,
'process_type': None,
'threads_per_process': 1,
'thread_type': 'CUDA'
}
s4.add_tasks(t4)
s4.post_exec = func_condition
return s4
def generate_pipeline():
global CUR_TASKS, CUR_CORES, duration, MAX_NEW_STAGE
def func_condition():
global CUR_NEW_STAGE, MAX_NEW_STAGE
if CUR_NEW_STAGE < MAX_NEW_STAGE:
return True
return False
def func_on_true():
global CUR_NEW_STAGE
CUR_NEW_STAGE += 1
for t in p.stages[CUR_NEW_STAGE].tasks:
cores = randint(1,16)
t.arguments = ['-c', str(cores), '-t', str(duration)]
def func_on_false():
print 'Done'
# Create a Pipeline object
p = Pipeline()
for s in range(MAX_NEW_STAGE+1):
# Create a Stage object
s1 = Stage()
for i in range(CUR_TASKS):
t1 = Task()
t1.pre_exec = ['export PATH=/u/sciteam/balasubr/modules/stress-ng-0.09.34:$PATH']
t1.executable = ['stress-ng']
t1.arguments = [ '-c', str(CUR_CORES), '-t', str(duration)]
t1.cpu_reqs = {
'processes': 1,
'process_type': '',
'threads_per_process': CUR_CORES,
'thread_type': ''
}
# Add the Task to the Stage
s1.add_tasks(t1)
# Add post-exec to the Stage
s1.post_exec = {
'condition': func_condition,
'on_true': func_on_true,
'on_false': func_on_false
}
# Add Stage to the Pipeline
p.add_stages(s1)
return p
def func_on_true():
global CUR_NEW_STAGE, CUR_TASKS, CUR_CORES, duration
CUR_NEW_STAGE += 1
s = Stage()
for i in range(CUR_TASKS):
t = Task()
t.pre_exec = [
'export PATH=/u/sciteam/balasubr/modules/stress-ng-0.09.34:$PATH'
]
t.executable = ['stress-ng']
t.arguments = ['-c', str(CUR_CORES), '-t', str(duration)]
t.cpu_reqs = {
'processes': 1,
'process_type': '',
'threads_per_process': CUR_CORES,
'thread_type': ''
}
# Add the Task to the Stage
s.add_tasks(t)
# Add post-exec to the Stage
s.post_exec = {
'condition': func_condition,
'on_true': func_on_true,
'on_false': func_on_false
}
p.add_stages(s)
def generate_aggregating_stage():
"""
Function to concatenate the MD trajectory (h5 contact map)
"""
s2 = Stage()
s2.name = 'aggregating'
# Aggregation task
t2 = Task()
# https://github.com/radical-collaboration/hyperspace/blob/MD/microscope/experiments/MD_to_CVAE/MD_to_CVAE.py
t2.pre_exec = []
#t2.pre_exec += ['. /sw/summit/python/2.7/anaconda2/5.3.0/etc/profile.d/conda.sh']
#t2.pre_exec += ['conda activate %s' % conda_path]
t2.pre_exec += ['module unload python']
t2.pre_exec += ['module load ibm-wml-ce']
t2.pre_exec += ['cd %s' % agg_path]
#t2.executable = ['%s/bin/python' % conda_path] # MD_to_CVAE.py
t2.executable = [
'/sw/summit/ibm-wml-ce/anaconda-base/envs/ibm-wml-ce-1.7.0-2/bin/python'
]
t2.arguments = ['%s/MD_to_CVAE.py' % agg_path, '--sim_path', md_path]
# Add the aggregation task to the aggreagating stage
s2.add_tasks(t2)
return s2
def add_ex_stg(rid, cycle):
#ex stg here
ex_tsk = Task()
ex_stg = Stage()
ex_tsk.name = 'extsk-{replica}-{cycle}'.format(replica=rid, cycle=cycle)
for rid in range(len(waiting_replicas)):
ex_tsk.link_input_data += ['%s/mdinfo-{replica}-{cycle}'.format(replica=rid, cycle=self.cycle)%replica_sandbox]
ex_tsk.arguments = ['t_ex_gibbs.py', len(waiting_replicas)] #This needs to be fixed
ex_tsk.executable = ['python']
ex_tsk.cpu_reqs = {
'processes': 1,
'process_type': '',
'threads_per_process': 1,
'thread_type': None
}
ex_tsk.pre_exec = ['export dummy_variable=19']
ex_stg.add_tasks(ex_tsk)
ex_stg.post_exec = {
'condition': post_ex,
'on_true': terminate_replicas,
'on_false': continue_md
}
return ex_stg
def add_md_stg(rid,cycle):
#md stg h
md_tsk = Task()
md_stg = Stage()
md_tsk.name = 'mdtsk-{replica}-{cycle}'.format(replica=rid, cycle=cycle)
md_tsk.link_input_data += ['%s/inpcrd' %replica_sandbox,
'%s/prmtop' %replica_sandbox,
'%s/mdin-{replica}-{cycle}'.format(replica=rid, cycle=0) %replica_sandbox]
md_tsk.arguments = ['-O',
'-i', 'mdin-{replica}-{cycle}'.format(replica=rid, cycle=0),
'-p', 'prmtop',
'-c', 'inpcrd',
'-o', 'out',
'-r', '%s/restrt-{replica}-{cycle}'.format(replica=rid, cycle=cycle) %replica_sandbox,
'-x', 'mdcrd',
'-inf', '%s/mdinfo-{replica}-{cycle}'.format(replica=rid, cycle=cycle) %replica_sandbox]
md_tsk.executable = ['/home/scm177/mantel/AMBER/amber14/bin/sander']
md_tsk.cpu_reqs = {
'processes': replica_cores,
'process_type': '',
'threads_per_process': 1,
'thread_type': None
}
md_tsk.pre_exec = ['export dummy_variable=19', 'echo $SHARED']
md_stg.add_tasks(md_tsk)
md_stg.post_exec = {
'condition': md_post,
'on_true': suspend,
'on_false': exchange_stg
}
return md_stg
def generate_aggregating_stage(self) -> Stage:
stage = Stage()
stage.name = "aggregating"
cfg = self.cfg.aggregation_stage
# Aggregation task
task = Task()
task.cpu_reqs = cfg.cpu_reqs.dict()
task.pre_exec = cfg.pre_exec
task.executable = cfg.executable
task.arguments = cfg.arguments
# Update base parameters
cfg.run_config.experiment_directory = self.cfg.experiment_directory
cfg.run_config.output_path = self.aggregated_data_path(
self.cur_iteration)
cfg_path = self.experiment_dirs["aggregation_runs"].joinpath(
f"aggregation_{self.cur_iteration:03d}.yaml")
cfg.run_config.dump_yaml(cfg_path)
task.arguments += ["-c", cfg_path]
stage.add_tasks(task)
return stage
def generate_ml_stage(self) -> Stage:
stage = Stage()
stage.name = "learning"
cfg = self.cfg.ml_stage
task = Task()
task.cpu_reqs = cfg.cpu_reqs.dict()
task.gpu_reqs = cfg.gpu_reqs.dict()
task.pre_exec = cfg.pre_exec
task.executable = cfg.executable
task.arguments = cfg.arguments
# Update base parameters
cfg.run_config.input_path = self.aggregated_data_path(
self.cur_iteration)
cfg.run_config.output_path = self.model_path(self.cur_iteration)
if self.cur_iteration > 0:
cfg.run_config.init_weights_path = self.latest_ml_checkpoint_path(
self.cur_iteration - 1)
cfg_path = self.experiment_dirs["ml_runs"].joinpath(
f"ml_{self.cur_iteration:03d}.yaml")
cfg.run_config.dump_yaml(cfg_path)
task.arguments += ["-c", cfg_path]
stage.add_tasks(task)
return stage
def generate_aggregating_stage():
"""
Function to concatenate the MD trajectory (h5 contact map)
"""
s2 = Stage()
s2.name = 'aggregating'
# Aggregation task
t2 = Task()
# https://github.com/radical-collaboration/hyperspace/blob/MD/microscope/experiments/MD_to_CVAE/MD_to_CVAE.py
t2.pre_exec = []
t2.pre_exec += [
'. /sw/summit/python/2.7/anaconda2/5.3.0/etc/profile.d/conda.sh'
]
t2.pre_exec += ['conda activate rp.copy']
t2.pre_exec += [
'cd /gpfs/alpine/bip179/scratch/hrlee/hyperspace/microscope/experiments/MD_to_CVAE'
]
t2.executable = ['/ccs/home/hrlee/.conda/envs/rp.copy/bin/python'
] # MD_to_CVAE.py
t2.arguments = [
'/gpfs/alpine/bip179/scratch/hrlee/hyperspace/microscope/experiments/MD_to_CVAE/MD_to_CVAE.py',
'-f',
'/gpfs/alpine/bip179/scratch/hrlee/hyperspace/microscope/experiments/MD_exps/fs-pep'
]
# Add the aggregation task to the aggreagating stage
s2.add_tasks(t2)
return s2
def generate_outlier_detection_stage(self) -> Stage:
stage = Stage()
stage.name = "outlier_detection"
cfg = self.cfg.od_stage
task = Task()
task.cpu_reqs = cfg.cpu_reqs.dict()
task.gpu_reqs = cfg.gpu_reqs.dict()
task.pre_exec = cfg.pre_exec
task.executable = cfg.executable
task.arguments = cfg.arguments
self.outlier_pdbs_path(self.cur_iteration).mkdir()
# Update base parameters
cfg.run_config.experiment_directory = self.cfg.experiment_directory
cfg.run_config.input_path = self.aggregated_data_path(
self.cur_iteration)
cfg.run_config.output_path = self.outlier_pdbs_path(self.cur_iteration)
cfg.run_config.weights_path = self.latest_ml_checkpoint_path(
self.cur_iteration)
cfg.run_config.restart_points_path = self.restart_points_path(
self.cur_iteration)
cfg_path = self.experiment_dirs["od_runs"].joinpath(
f"od_{self.cur_iteration:03d}.yaml")
cfg.run_config.dump_yaml(cfg_path)
task.arguments += ["-c", cfg_path]
stage.add_tasks(task)
return stage
def generate_aggregating_task(self):
"""
Function to concatenate the MD trajectory (h5 contact map)
"""
p = Pipeline()
p.name = 'aggragating'
s2 = Stage()
s2.name = 'aggregating'
# Aggregation task
t2 = Task()
# https://github.com/radical-collaboration/hyperspace/blob/MD/microscope/experiments/MD_to_CVAE/MD_to_CVAE.py
t2.pre_exec = []
t2.pre_exec += ['. /sw/summit/python/2.7/anaconda2/5.3.0/etc/profile.d/conda.sh']
t2.pre_exec += ['conda activate %s' % conda_path]
t2.pre_exec += ['cd %s' % agg_path]
t2.executable = ['%s/bin/python' % conda_path] # MD_to_CVAE.py
t2.arguments = [
'%s/MD_to_CVAE.py' % agg_path,
'--sim_path', md_path,
'--train_frames', 100000]
# assign hardware the task
t2.cpu_reqs = {
'processes': 1,
'process_type': None,
'threads_per_process': 4,
'thread_type': 'OpenMP'
}
# Add the aggregation task to the aggreagating stage
s2.add_tasks(t2)
p.add_stages(s2)
return p
def esmacs(self, rct_stage, stage, outdir="equilibration", name=None):
for i in range(1, 13):
t = Task()
t.pre_exec = [
"export WDIR=\"{}/{}\"".format(self.run_dir, name),
". {}".format(self.conda_init),
"conda activate {}".format(self.esmacs_tenv),
"module load {}".format(self.esmacs_tmodules),
"mkdir -p $WDIR/replicas/rep{}/{}".format(i, outdir),
"cd $WDIR/replicas/rep{}/{}".format(i, outdir),
"rm -f {}.log {}.xml {}.dcd {}.chk".format(
stage, stage, stage, stage), "export OMP_NUM_THREADS=1"
]
# t.executable = '/ccs/home/litan/miniconda3/envs/wf3/bin/python3.7'
t.executable = 'python3'
t.arguments = ['$WDIR/{}.py'.format(stage)]
t.post_exec = []
t.cpu_reqs = {
'processes': 1,
'process_type': None,
'threads_per_process': 4,
'thread_type': 'OpenMP'
}
t.gpu_reqs = {
'processes': 1,
'process_type': None,
'threads_per_process': 1,
'thread_type': 'CUDA'
}
getattr(self, rct_stage).add_tasks(t)
print(getattr(self, rct_stage).to_dict())
def create_inversion_dict_stage(cmt_file_db, param_path, task_counter):
"""Creates stage for the creation of the inversion files. This stage is
tiny, but required before the actual inversion.
:param cmt_file_db:
:param param_path:
:param task_counter:
:return:
"""
# Get database parameter path
databaseparam_path = os.path.join(param_path,
"Database/DatabaseParameters.yml")
# Load Parameters
DB_params = read_yaml_file(databaseparam_path)
# Earthquake specific database parameters: Dir and Cid
Cdir, Cid = get_Centry_path(DB_params["databasedir"], cmt_file_db)
# Function
inv_dict_func = os.path.join(bin_path, "write_inversion_dicts.py")
# Create Process Paths Stage (CPP)
# Create a Stage object
inv_dict_stage = Stage()
inv_dict_stage.name = "Creating"
# Create Task
inv_dict_task = Task()
# This way the task gets the name of the path file
inv_dict_task.name = "Inversion-Dictionaries"
inv_dict_task.pre_exec = [ # Conda activate
DB_params["conda-activate"]
]
inv_dict_task.executable = [DB_params["bin-python"]] # Assign exec
# to the task
inv_dict_task.arguments = [
inv_dict_func, "-f", cmt_file_db, "-p", param_path
]
# In the future maybe to database dir as a total log?
inv_dict_task.stdout = os.path.join(
"%s" % Cdir, "logs", "stdout.pipeline_%s.task_%s.%s" %
(Cid, str(task_counter).zfill(4), inv_dict_task.name))
inv_dict_task.stderr = os.path.join(
"%s" % Cdir, "logs", "stderr.pipeline_%s.task_%s.%s" %
(Cid, str(task_counter).zfill(4), inv_dict_task.name))
inv_dict_stage.add_tasks(inv_dict_task)
task_counter += 1
return inv_dict_stage, task_counter
def generate_ML_pipeline():
p = Pipeline()
p.name = 'ML'
s1 = Stage()
s1.name = 'Generator-ML'
# the generator/ML Pipeline will consist of 1 Stage, 2 Tasks Task 1 :
# Generator; Task 2: ConvNet/Active Learning Model
# NOTE: Generator and ML/AL are alive across the whole workflow execution.
# For local testing, sleep time is longer than the total execution time of
# the MD pipelines.
t1 = Task()
t1.name = "generator"
t1.pre_exec = [
# 'module load python/2.7.15-anaconda2-5.3.0',
# 'module load cuda/9.1.85',
# 'module load gcc/6.4.0',
# 'source activate snakes'
]
# t1.executable = ['python']
# t1.arguments = ['/ccs/home/jdakka/tf.py']
t1.executable = ['sleep']
t1.arguments = ['5']
s1.add_tasks(t1)
t2 = Task()
t2.name = "ml-al"
t2.pre_exec = [
# 'module load python/2.7.15-anaconda2-5.3.0',
# 'module load cuda/9.1.85',
# 'module load gcc/6.4.0',
# 'source activate snakes'
]
# t2.executable = ['python']
# t2.arguments = ['/ccs/home/jdakka/tf.py']
t2.executable = ['sleep']
t2.arguments = ['10']
s1.add_tasks(t2)
# Add Stage to the Pipeline
p.add_stages(s1)
return p
def generate_task(cfg: BaseStageConfig) -> Task:
task = Task()
task.cpu_reqs = cfg.cpu_reqs.dict().copy()
task.gpu_reqs = cfg.gpu_reqs.dict().copy()
task.pre_exec = cfg.pre_exec.copy()
task.executable = cfg.executable
task.arguments = cfg.arguments.copy()
return task
def main():
cmd = "{0} 'ls {1}'".format(ssh, dir_)
p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
out, _ = p.communicate()
out = out.decode('utf-8').strip().split(linesep)
fullpaths = [op.join(dir_, p) for p in out]
print(fullpaths)
# Start radical entk pipeline
p = Pipeline()
for i in range(iterations):
s = Stage()
for fp in fullpaths:
t = Task()
t.name = 'Incrementation {}'.format(i)
t.pre_exec = [
'source /home/vhayot/miniconda3/etc/profile.d/conda.sh',
'conda activate radenv'
]
t.executable = 'python /home/vhayot/inc.py'
if i == 0:
t.arguments = [fp, out_dir, i]
else:
# Note: assuming all data is accessible through shared dir
# radical entk functions without sharedfs, however
t.arguments = [
op.join(out_dir,
"it-{0}-{1}".format(i - 1, op.basename(fp))),
out_dir, i
]
s.add_tasks(t)
# Create a new stage everytime there's a dependency
p.add_stages(s)
appman = AppManager(hostname=hostname, port=port)
appman.resource_desc = {
'resource': 'xsede.bridges',
'walltime': 20,
'cpus': 5,
'project': 'mc3bggp',
'schema': 'gsissh'
}
appman.workflow = set([p])
appman.run()
def create_inversion_stage(cmt_file_db, param_path, task_counter):
"""Creates inversion stage.
:param cmt_file_db:
:param param_path:
:return:
"""
# Get database parameter path
databaseparam_path = os.path.join(param_path,
"Database/DatabaseParameters.yml")
# Load Parameters
DB_params = read_yaml_file(databaseparam_path)
# Earthquake specific database parameters: Dir and Cid
Cdir, Cid = get_Centry_path(DB_params["databasedir"], cmt_file_db)
# Function
inversion_func = os.path.join(bin_path, "inversion.py")
# Create a Stage object
inversion_stage = Stage()
inversion_stage.name = "CMT3D"
# Create Task
inversion_task = Task()
# This way the task gets the name of the path file
inversion_task.name = "Inversion"
inversion_task.pre_exec = [ # Conda activate
DB_params["conda-activate"]
]
inversion_task.executable = DB_params["bin-python"] # Assign exec
# to the task
inversion_task.arguments = [
inversion_func, "-f", cmt_file_db, "-p", param_path
]
# In the future maybe to database dir as a total log?
inversion_task.stdout = os.path.join(
"%s" % Cdir, "logs", "stdout.pipeline_%s.task_%s.%s" %
(Cid, str(task_counter).zfill(4), inversion_task.name))
inversion_task.stderr = os.path.join(
"%s" % Cdir, "logs", "stderr.pipeline_%s.task_%s.%s" %
(Cid, str(task_counter).zfill(4), inversion_task.name))
inversion_stage.add_tasks(inversion_task)
return inversion_stage
def write_sources(cmt_file_db, param_path, task_counter):
""" This function creates a stage that modifies the CMTSOLUTION files
before the simulations are run.
:param cmt_file_db: cmtfile in the database
:param param_path: path to parameter file directory
:param task_counter: total task count up until now in pipeline
:return: EnTK Stage
"""
# Get Database parameters
databaseparam_path = os.path.join(param_path,
"Database/DatabaseParameters.yml")
DB_params = read_yaml_file(databaseparam_path)
# Earthquake specific database parameters: Dir and Cid
Cdir, Cid = get_Centry_path(DB_params["databasedir"], cmt_file_db)
# Path to function
write_source_func = os.path.join(bin_path, "write_sources.py")
# Create a Stage object
w_sources = Stage()
w_sources.name = "Write-Sources"
# Create Task for stage
w_sources_t = Task()
w_sources_t.name = "Task-Sources"
w_sources_t.pre_exec = [ # Conda activate
DB_params["conda-activate"]
]
w_sources_t.executable = DB_params["bin-python"] #
# Assign executable
# to the task
w_sources_t.arguments = [write_source_func, cmt_file_db]
# In the future maybe to database dir as a total log?
w_sources_t.stdout = os.path.join(
"%s" % Cdir, "logs", "stdout.pipeline_%s.task_%s.%s" %
(Cid, str(task_counter).zfill(4), w_sources_t.name))
w_sources_t.stderr = os.path.join(
"%s" % Cdir, "logs", "stderr.pipeline_%s.task_%s.%s" %
(Cid, str(task_counter).zfill(4), w_sources_t.name))
# Add Task to the Stage
w_sources.add_tasks(w_sources_t)
task_counter += 1
return w_sources, task_counter
def create_process_path_files(cmt_file_db, param_path, task_counter):
"""This function creates the path files used for processing both
synthetic and observed data in ASDF format, as well as the following
windowing procedure.
:param cmt_file_db: cmtfile in the database
:param param_path: path to parameter file directory
:param pipelinedir: path to pipeline directory
:return: EnTK Stage
"""
# Get database parameter path
databaseparam_path = os.path.join(param_path,
"Database/DatabaseParameters.yml")
# Load Parameters
DB_params = read_yaml_file(databaseparam_path)
# Earthquake specific database parameters: Dir and Cid
Cdir, Cid = get_Centry_path(DB_params["databasedir"], cmt_file_db)
# Process path function
create_process_path_bin = os.path.join(bin_path, "create_path_files.py")
# Create Process Paths Stage (CPP)
# Create a Stage object
cpp = Stage()
cpp.name = "CreateProcessPaths"
# Create Task
cpp_t = Task()
cpp_t.name = "CPP-Task"
cpp_t.pre_exec = [ # Conda activate
DB_params["conda-activate"]
]
cpp_t.executable = DB_params["bin-python"] # Assign executable
# to the task
cpp_t.arguments = [create_process_path_bin, cmt_file_db]
# In the future maybe to database dir as a total log?
cpp_t.stdout = os.path.join(
"%s" % Cdir, "logs", "stdout.pipeline_%s.task_%s.%s" %
(Cid, str(task_counter).zfill(4), cpp_t.name))
cpp_t.stderr = os.path.join(
"%s" % Cdir, "logs", "stderr.pipeline_%s.task_%s.%s" %
(Cid, str(task_counter).zfill(4), cpp_t.name))
task_counter += 1
cpp.add_tasks(cpp_t)
return cpp, task_counter
def generate_interfacing_stage():
s4 = Stage()
s4.name = 'scanning'
# Scaning for outliers and prepare the next stage of MDs
t4 = Task()
t4.pre_exec = []
#t4.pre_exec += ['. /sw/summit/python/2.7/anaconda2/5.3.0/etc/profile.d/conda.sh']
#t4.pre_exec += ['module load cuda/9.1.85']
#t4.pre_exec += ['conda activate %s' % conda_path]
t4.pre_exec += [
'module unload prrte', 'module unload python', 'module load xl',
'module load xalt', 'module load spectrum-mpi', 'module load cuda',
'module list'
]
t4.pre_exec += [
'. /sw/summit/ibm-wml-ce/anaconda-base/etc/profile.d/conda.sh',
'source /sw/summit/ibm-wml-ce/anaconda-base/etc/profile.d/conda.sh',
'conda deactivate', 'conda deactivate',
'conda activate /gpfs/alpine/proj-shared/med110/wf-2/conda/envs/ibm-wml-ce-cloned'
]
#'conda activate /sw/summit/ibm-wml-ce/anaconda-base/envs/ibm-wml-ce-1.7.0-2']
t4.pre_exec += [
'export PYTHONPATH=%s/CVAE_exps:%s/CVAE_exps/cvae:$PYTHONPATH' %
(base_path, base_path)
]
t4.pre_exec += ['cd %s/Outlier_search' % base_path]
#t4.executable = ['%s/bin/python' % conda_path]
t4.executable = ['python']
t4.arguments = [
'outlier_locator.py', '--md', md_path, '--cvae', cvae_path,
'--pdb', pdb_file
]
#'--ref', ref_pdb_file]
t4.cpu_reqs = {
'processes': 1,
'process_type': None,
'threads_per_process': 16,
'thread_type': 'OpenMP'
}
t4.gpu_reqs = {
'processes': 1,
'process_type': None,
'threads_per_process': 1,
'thread_type': 'CUDA'
}
s4.add_tasks(t4)
s4.post_exec = func_condition
return s4
def selection(self, ps_file, select_file):
tasks = []
t = Task()
t.pre_exec = ['/bin/cp {0} {1}'.format(self.param_space, ps_file)]
t.executable = self.analysis
t.arguments = [ps_file, self.n_samples, select_file]
t.download_output_data = [select_file]
tasks.append(t)
return tasks
def specfem_clean_up(cmt_file_db, param_path, task_counter):
""" Cleaning up the simulation directories since we don"t need all the
files for the future.
:param cmt_file_db: cmtfile in the database
:param param_path: path to parameter file directory
:param pipelinedir: path to pipeline directory
:return: EnTK Stage
"""
# Get Database parameters
databaseparam_path = os.path.join(param_path,
"Database/DatabaseParameters.yml")
# Database parameters.
DB_params = read_yaml_file(databaseparam_path)
# Earthquake specific database parameters: Dir and Cid
Cdir, Cid = get_Centry_path(DB_params["databasedir"], cmt_file_db)
# Path to function
clean_up_func = os.path.join(bin_path, "clean_up_simdirs.py")
# Create a Stage object
clean_up = Stage()
clean_up.name = "Clean-Up"
# Create Task for stage
clean_up_t = Task()
clean_up_t.name = "Task-Clean-Up"
clean_up_t.pre_exec = [ # Conda activate
DB_params["conda-activate"]
]
clean_up_t.executable = DB_params["bin-python"] # Assign executable
# to the task
clean_up_t.arguments = [clean_up_func, cmt_file_db]
# In the future maybe to database dir as a total log?
clean_up_t.stdout = os.path.join(
"%s" % Cdir, "logs", "stdout.pipeline_%s.task_%s.%s" %
(Cid, str(task_counter).zfill(4), clean_up_t.name))
clean_up_t.stderr = os.path.join(
"%s" % Cdir, "logs", "stderr.pipeline_%s.task_%s.%s" %
(Cid, str(task_counter).zfill(4), clean_up_t.name))
# Add Task to the Stage
clean_up.add_tasks(clean_up_t)
return clean_up, task_counter
def generate_MD_stage(num_MD=1):
"""
Function to generate MD stage.
"""
s1 = Stage()
s1.name = 'MD'
# MD tasks
time_stamp = int(time.time())
for i in range(num_MD):
t1 = Task()
t1.pre_exec = [
'. /sw/summit/python/2.7/anaconda2/5.3.0/etc/profile.d/conda.sh'
]
t1.pre_exec += ['module load cuda/9.1.85']
t1.pre_exec += ['conda activate %s' % conda_path]
t1.pre_exec += [
'export PYTHONPATH=%s/MD_exps:$PYTHONPATH' % base_path
]
t1.pre_exec += ['cd %s/MD_exps/fs-pep' % base_path]
t1.pre_exec += [
'mkdir -p omm_runs_%d && cd omm_runs_%d' %
(time_stamp + i, time_stamp + i)
]
t1.executable = ['%s/bin/python' % conda_path] # run_openmm.py
t1.arguments = ['%s/MD_exps/fs-pep/run_openmm.py' % base_path]
# t1.arguments += ['--topol', '%s/MD_exps/fs-pep/pdb/topol.top' % base_path]
t1.arguments += [
'--pdb_file',
'%s/MD_exps/fs-pep/pdb/100-fs-peptide-400K.pdb' % base_path,
'--length', LEN_sim
]
# assign hardware the task
t1.cpu_reqs = {
'processes': 1,
'process_type': None,
'threads_per_process': 4,
'thread_type': 'OpenMP'
}
t1.gpu_reqs = {
'processes': 1,
'process_type': None,
'threads_per_process': 1,
'thread_type': 'CUDA'
}
# Add the MD task to the simulating stage
s1.add_tasks(t1)
return s1
def data_request(cmt_file_db, param_path, task_counter):
""" This function creates the request for the observed data and returns
it as an EnTK Stage
:param cmt_file_db: cmt_file in the database
:param param_path: path to parameter file directory
:param task_counter: total task count up until now in pipeline
:return: EnTK Stage
"""
# Get Database parameters
databaseparam_path = os.path.join(param_path,
"Database/DatabaseParameters.yml")
DB_params = read_yaml_file(databaseparam_path)
# Earthquake specific database parameters: Dir and Cid
Cdir, Cid = get_Centry_path(DB_params["databasedir"], cmt_file_db)
# # Path to function
request_data_func = os.path.join(bin_path, "request_data.py")
# Create a Stage object
datarequest = Stage()
datarequest_t = Task()
datarequest_t.name = "data-request"
datarequest_t.pre_exec = [ # Conda activate
DB_params["conda-activate"]
]
datarequest_t.executable = DB_params["bin-python"] # Assign executable
# to the task
datarequest_t.arguments = [request_data_func, cmt_file_db]
# In the future maybe to database dir as a total log?
datarequest_t.stdout = os.path.join(
"%s" % Cdir, "logs", "stdout.pipeline_%s.task_%s.%s" %
(Cid, str(task_counter).zfill(4), datarequest_t.name))
datarequest_t.stderr = os.path.join(
"%s" % Cdir, "logs", "stderr.pipeline_%s.task_%s.%s" %
(Cid, str(task_counter).zfill(4), datarequest_t.name))
# Add Task to the Stage
datarequest.add_tasks(datarequest_t)
# Increase task-counter
task_counter += 1
return datarequest, task_counter
def generate_ML_stage(num_ML=1):
"""
Function to generate the learning stage
"""
s3 = Stage()
s3.name = 'learning'
# learn task
time_stamp = int(time.time())
for i in range(num_ML):
t3 = Task()
# https://github.com/radical-collaboration/hyperspace/blob/MD/microscope/experiments/CVAE_exps/train_cvae.py
t3.pre_exec = []
t3.pre_exec += [
'. /sw/summit/python/2.7/anaconda2/5.3.0/etc/profile.d/conda.sh'
]
t3.pre_exec += ['module load cuda/9.1.85']
t3.pre_exec += ['conda activate %s' % conda_path]
t3.pre_exec += [
'export PYTHONPATH=%s/CVAE_exps:$PYTHONPATH' % base_path
]
t3.pre_exec += ['cd %s' % cvae_path]
dim = i + 3
cvae_dir = 'cvae_runs_%.2d_%d' % (dim, time_stamp + i)
t3.pre_exec += ['mkdir -p {0} && cd {0}'.format(cvae_dir)]
t3.executable = ['%s/bin/python' % conda_path] # train_cvae.py
t3.arguments = [
'%s/train_cvae.py' % cvae_path, '--h5_file',
'%s/cvae_input.h5' % agg_path, '--dim', dim
]
t3.cpu_reqs = {
'processes': 1,
'process_type': None,
'threads_per_process': 4,
'thread_type': 'OpenMP'
}
t3.gpu_reqs = {
'processes': 1,
'process_type': None,
'threads_per_process': 1,
'thread_type': 'CUDA'
}
# Add the learn task to the learning stage
s3.add_tasks(t3)
return s3
def generate_interfacing_stage():
s4 = Stage()
s4.name = 'scanning'
# Scaning for outliers and prepare the next stage of MDs
t4 = Task()
t4.pre_exec = ['. /sw/summit/python/3.6/anaconda3/5.3.0/etc/profile.d/conda.sh']
t4.pre_exec += ['conda activate %s' % cfg['conda_pytorch']]
t4.pre_exec += ['mkdir -p %s/Outlier_search/outlier_pdbs' % cfg['base_path']]
t4.pre_exec += ['export models=""; for i in `ls -d %s/CVAE_exps/model-cvae_runs*/`; do if [ "$models" != "" ]; then models=$models","$i; else models=$i; fi; done;cat /dev/null' % cfg['base_path']]
t4.pre_exec += ['export LANG=en_US.utf-8', 'export LC_ALL=en_US.utf-8']
t4.pre_exec += ['unset CUDA_VISIBLE_DEVICES', 'export OMP_NUM_THREADS=4']
cmd_cat = 'cat /dev/null'
cmd_jsrun = 'jsrun -n %s -a 6 -g 6 -r 1 -c 7' % cfg['node_counts']
#molecules_path = '/gpfs/alpine/world-shared/ven201/tkurth/molecules/'
t4.executable = [' %s; %s %s/examples/outlier_detection/run_optics_dist_summit_entk.sh' % (cmd_cat, cmd_jsrun, cfg['molecules_path'])]
t4.arguments = ['%s/bin/python' % cfg['conda_pytorch']]
t4.arguments += ['%s/examples/outlier_detection/optics.py' % cfg['molecules_path'],
'--sim_path', '%s/MD_exps/%s' % (cfg['base_path'], cfg['system_name']),
'--pdb_out_path', '%s/Outlier_search/outlier_pdbs' % cfg['base_path'],
'--restart_points_path',
'%s/Outlier_search/restart_points.json' % cfg['base_path'],
'--data_path', '%s/MD_to_CVAE/cvae_input.h5' % cfg['base_path'],
'--model_paths', '$models',
'--model_type', cfg['model_type'],
'--min_samples', 10,
'--n_outliers', 500,
'--dim1', cfg['residues'],
'--dim2', cfg['residues'],
'--cm_format', 'sparse-concat',
'--batch_size', cfg['batch_size'],
'--distributed',
'-iw', cfg['init_weights']]
t4.cpu_reqs = {'processes' : 1,
'process_type' : None,
'threads_per_process': 12,
'thread_type' : 'OpenMP'}
t4.gpu_reqs = {'processes' : 1,
'process_type' : None,
'threads_per_process': 1,
'thread_type' : 'CUDA'}
s4.add_tasks(t4)
s4.post_exec = func_condition
return s4
def esmacs(cfg, names, stage, outdir):
s = Stage()
s.name = 'S3.%s' % stage
#print("DEBUG:instantiation: %s" % len(s._tasks))
for comp in names:
#print("DEBUG:first loop: %s" % len(s._tasks))
for i in range(1, cfg['n_replicas']):
#print("DEBUG:second loop:start: %s" % len(s._tasks))
t = Task()
# RCT native
t.pre_exec = [
#". /sw/summit/lmod/lmod/init/profile",
"export WDIR=\"{}\"".format(comp),
". {}".format(cfg['conda_init']),
"conda activate {}".format(cfg['conda_esmacs_task_env']),
"module load {}".format(cfg['esmacs_task_modules']),
"mkdir -p $WDIR/replicas/rep{}/{}".format(i, outdir),
"cd $WDIR/replicas/rep{}/{}".format(i, outdir),
#"rm -f {}.log {}.xml {}.dcd {}.chk".format(stage, stage, stage, stage),
"export OMP_NUM_THREADS=1"]
t.executable = 'python3'
t.arguments = ['$WDIR/{}.py'.format(stage)]
# Bash wrapper
#t.executable = '%s/wf3.sh' % comp
#t.arguments = [comp, i, outdir, stage,
# cfg['conda_init'],
# cfg['conda_esmacs_task_env'],
# cfg['esmacs_task_modules']]
t.cpu_reqs = {
'processes': 1,
'process_type': None,
'threads_per_process': 4,
'thread_type': 'OpenMP'}
t.gpu_reqs = {
'processes': 1,
'process_type': None,
'threads_per_process': 1,
'thread_type': 'CUDA'}
s.add_tasks(t)
#print("DEBUG:second loop:end: %s" % len(s._tasks))
return s
def describe_MD_pipeline():
p = Pipeline()
p.name = 'MD'
# MD stage
s1 = Stage()
s1.name = 'OpenMM'
# Each Task() is an OpenMM executable that will run on a single GPU.
# Set sleep time for local testing
# for i in range(18):
task = Task()
task.name = 'md'
task.pre_exec = []
# task.pre_exec += ['export MINICONDA=/gpfs/alpine/scratch/jdakka/bip178/miniconda']
# task.pre_exec += ['export PATH=$MINICONDA/bin:$PATH']
# task.pre_exec += ['export LD_LIBRARY_PATH=$MINICONDA/lib:$LD_LIBRARY_PATH']
task.pre_exec += ['module load python/2.7.15-anaconda2-5.3.0']
task.pre_exec += ['module load cuda/9.1.85']
task.pre_exec += ['module load gcc/6.4.0']
task.pre_exec += ['source activate openmm']
task.pre_exec += ['cd /gpfs/alpine/scratch/jdakka/bip178/benchmarks/MD_exps/fs-pep/results_2']
task.executable = '/ccs/home/jdakka/.conda/envs/openmm/bin/python'
task.arguments = ['run_openmm.py', '-f',
'/gpfs/alpine/scratch/jdakka/bip178/benchmarks/MD_exps/fs-pep/pdb/100-fs-peptide-400K.pdb']
task.cpu_reqs = {'processes': 1,
'process_type': None,
'threads_per_process': 1,
'thread_type': None
}
task.gpu_reqs = {'processes': 1,
'process_type': None,
'threads_per_process': 1,
'thread_type': 'CUDA'
}
# Add the MD task to the Docking Stage
s1.add_tasks(task)
# Add MD stage to the MD Pipeline
p.add_stages(s1)
return p
# Create a Pipeline object
p = Pipeline()
# Bookkeeping
stage_uids = list()
task_uids = dict()
Stages = 3
Replicas = 4
for N_Stg in range(Stages):
stg = Stage() ## initialization
task_uids['Stage_%s'%N_Stg] = list()
if N_Stg == 0:
for n0 in range(Replicas):
t = Task()
t.executable = ['/usr/local/packages/gromacs/5.1.4/INTEL-140-MVAPICH2-2.0/bin/gmx_mpi_d'] #MD Engine
t.upload_input_data = ['in.gro', 'in.top', 'FNF.itp', 'martini_v2.2.itp', 'in.mdp']
t.pre_exec = ['module load gromacs', '/usr/local/packages/gromacs/5.1.4/INTEL-140-MVAPICH2-2.0/bin/gmx_mpi_d grompp -f in.mdp -c in.gro -o in.tpr -p in.top']
t.arguments = ['mdrun', '-s', 'in.tpr', '-deffnm', 'out']
t.cores = 32
stg.add_tasks(t)
task_uids['Stage_%s'%N_Stg].append(t.uid)
p.add_stages(stg)
stage_uids.append(stg.uid)
else:
for n0 in range(Replicas):
t = Task()
t.executable = ['/usr/local/packages/gromacs/5.1.4/INTEL-140-MVAPICH2-2.0/bin/gmx_mpi_d'] #MD Engine
t.copy_input_data = ['$Pipeline_%s_Stage_%s_Task_%s/out.gro > in.gro'%(p.uid, stage_uids[N_Stg-1], task_uids['Stage_%s'%(N_Stg-1)][n0]), '$Pipeline_%s_Stage_%s_Task_%s/in.top'%(p.uid, stage_uids[N_Stg-1], task_uids['Stage_%s'%(N_Stg-1)][n0]), '$Pipeline_%s_Stage_%s_Task_%s/FNF.itp'%(p.uid, stage_uids[N_Stg-1], task_uids['Stage_%s'%(N_Stg-1)][n0]), '$Pipeline_%s_Stage_%s_Task_%s/martini_v2.2.itp'%(p.uid, stage_uids[N_Stg-1], task_uids['Stage_%s'%(N_Stg-1)][n0]), '$Pipeline_%s_Stage_%s_Task_%s/in.mdp'%(p.uid, stage_uids[N_Stg-1], task_uids['Stage_%s'%(N_Stg-1)][n0])]
def test_create_cud_from_task():
"""
**Purpose**: Test if the 'create_cud_from_task' function generates a RP ComputeUnitDescription with the complete
Task description
"""
pipeline = 'p1'
stage = 's1'
task = 't1'
placeholder_dict = {
pipeline: {
stage: {
task: '/home/vivek/some_file.txt'
}
}
}
t1 = Task()
t1.name = 't1'
t1.pre_exec = ['module load gromacs']
t1.executable = ['grompp']
t1.arguments = ['hello']
t1.cpu_reqs = {'processes': 4,
'process_type': 'MPI',
'threads_per_process': 1,
'thread_type': 'OpenMP'
}
t1.gpu_reqs = {'processes': 4,
'process_type': 'MPI',
'threads_per_process': 2,
'thread_type': 'OpenMP'
}
t1.post_exec = ['echo test']
t1.upload_input_data = ['upload_input.dat']
t1.copy_input_data = ['copy_input.dat']
t1.link_input_data = ['link_input.dat']
t1.copy_output_data = ['copy_output.dat']
t1.download_output_data = ['download_output.dat']
p = Pipeline()
p.name = 'p1'
s = Stage()
s.name = 's1'
s.tasks = t1
p.stages = s
p._assign_uid('test')
cud = create_cud_from_task(t1, placeholder_dict)
assert cud.name == '%s,%s,%s,%s,%s,%s' % (t1.uid, t1.name,
t1.parent_stage['uid'], t1.parent_stage['name'],
t1.parent_pipeline['uid'], t1.parent_pipeline['name'])
assert cud.pre_exec == t1.pre_exec
# rp returns executable as a string regardless of whether assignment was using string or list
assert cud.executable == t1.executable
assert cud.arguments == t1.arguments
assert cud.cpu_processes == t1.cpu_reqs['processes']
assert cud.cpu_threads == t1.cpu_reqs['threads_per_process']
assert cud.cpu_process_type == t1.cpu_reqs['process_type']
assert cud.cpu_thread_type == t1.cpu_reqs['thread_type']
assert cud.gpu_processes == t1.gpu_reqs['processes']
assert cud.gpu_threads == t1.gpu_reqs['threads_per_process']
assert cud.gpu_process_type == t1.gpu_reqs['process_type']
assert cud.gpu_thread_type == t1.gpu_reqs['thread_type']
assert cud.post_exec == t1.post_exec
assert {'source': 'upload_input.dat', 'target': 'upload_input.dat'} in cud.input_staging
assert {'source': 'copy_input.dat', 'action': rp.COPY, 'target': 'copy_input.dat'} in cud.input_staging
assert {'source': 'link_input.dat', 'action': rp.LINK, 'target': 'link_input.dat'} in cud.input_staging
assert {'source': 'copy_output.dat', 'action': rp.COPY, 'target': 'copy_output.dat'} in cud.output_staging
assert {'source': 'download_output.dat', 'target': 'download_output.dat'} in cud.output_staging
def test_task_to_dict():
"""
**Purpose**: Test if the 'to_dict' function of Task class converts all expected attributes of the Task into a
dictionary
"""
t = Task()
d = t.to_dict()
assert d == { 'uid': None,
'name': None,
'state': states.INITIAL,
'state_history': [states.INITIAL],
'pre_exec': [],
'executable': str(),
'arguments': [],
'post_exec': [],
'cpu_reqs': { 'processes': 1,
'process_type': None,
'threads_per_process': 1,
'thread_type': None
},
'gpu_reqs': { 'processes': 0,
'process_type': None,
'threads_per_process': 0,
'thread_type': None
},
'lfs_per_process': 0,
'upload_input_data': [],
'copy_input_data': [],
'link_input_data': [],
'move_input_data': [],
'copy_output_data': [],
'move_output_data': [],
'download_output_data': [],
'stdout': None,
'stderr': None,
'exit_code': None,
'path': None,
'tag': None,
'parent_stage': {'uid':None, 'name': None},
'parent_pipeline': {'uid':None, 'name': None}}
t = Task()
t.uid = 'test.0000'
t.name = 'new'
t.pre_exec = ['module load abc']
t.executable = ['sleep']
t.arguments = ['10']
t.cpu_reqs['processes'] = 10
t.cpu_reqs['threads_per_process'] = 2
t.gpu_reqs['processes'] = 5
t.gpu_reqs['threads_per_process'] = 3
t.lfs_per_process = 1024
t.upload_input_data = ['test1']
t.copy_input_data = ['test2']
t.link_input_data = ['test3']
t.move_input_data = ['test4']
t.copy_output_data = ['test5']
t.move_output_data = ['test6']
t.download_output_data = ['test7']
t.stdout = 'out'
t.stderr = 'err'
t.exit_code = 1
t.path = 'a/b/c'
t.tag = 'task.0010'
t.parent_stage = {'uid': 's1', 'name': 'stage1'}
t.parent_pipeline = {'uid': 'p1', 'name': 'pipeline1'}
d = t.to_dict()
assert d == { 'uid': 'test.0000',
'name': 'new',
'state': states.INITIAL,
'state_history': [states.INITIAL],
'pre_exec': ['module load abc'],
'executable': 'sleep',
'arguments': ['10'],
'post_exec': [],
'cpu_reqs': { 'processes': 10,
'process_type': None,
'threads_per_process': 2,
'thread_type': None
},
'gpu_reqs': { 'processes': 5,
'process_type': None,
'threads_per_process': 3,
'thread_type': None
},
'lfs_per_process': 1024,
'upload_input_data': ['test1'],
'copy_input_data': ['test2'],
'link_input_data': ['test3'],
'move_input_data': ['test4'],
'copy_output_data': ['test5'],
'move_output_data': ['test6'],
'download_output_data': ['test7'],
'stdout': 'out',
'stderr': 'err',
'exit_code': 1,
'path': 'a/b/c',
'tag': 'task.0010',
'parent_stage': {'uid': 's1', 'name': 'stage1'},
'parent_pipeline': {'uid': 'p1', 'name': 'pipeline1'}}
t.executable = 'sleep'
d = t.to_dict()
assert d == { 'uid': 'test.0000',
'name': 'new',
'state': states.INITIAL,
'state_history': [states.INITIAL],
'pre_exec': ['module load abc'],
'executable': 'sleep',
'arguments': ['10'],
'post_exec': [],
'cpu_reqs': { 'processes': 10,
'process_type': None,
'threads_per_process': 2,
'thread_type': None
},
'gpu_reqs': { 'processes': 5,
'process_type': None,
'threads_per_process': 3,
'thread_type': None
},
'lfs_per_process': 1024,
'upload_input_data': ['test1'],
'copy_input_data': ['test2'],
'link_input_data': ['test3'],
'move_input_data': ['test4'],
'copy_output_data': ['test5'],
'move_output_data': ['test6'],
'download_output_data': ['test7'],
'stdout': 'out',
'stderr': 'err',
'exit_code': 1,
'path': 'a/b/c',
'tag': 'task.0010',
'parent_stage': {'uid': 's1', 'name': 'stage1'},
'parent_pipeline': {'uid': 'p1', 'name': 'pipeline1'}}
def test_task_exceptions(s,l,i,b):
"""
**Purpose**: Test if all attribute assignments raise exceptions for invalid values
"""
t = Task()
data_type = [s,l,i,b]
for data in data_type:
if not isinstance(data,str):
with pytest.raises(TypeError):
t.name = data
with pytest.raises(TypeError):
t.path = data
with pytest.raises(TypeError):
t.parent_stage = data
with pytest.raises(TypeError):
t.parent_pipeline = data
with pytest.raises(TypeError):
t.stdout = data
with pytest.raises(TypeError):
t.stderr = data
if not isinstance(data,list):
with pytest.raises(TypeError):
t.pre_exec = data
with pytest.raises(TypeError):
t.arguments = data
with pytest.raises(TypeError):
t.post_exec = data
with pytest.raises(TypeError):
t.upload_input_data = data
with pytest.raises(TypeError):
t.copy_input_data = data
with pytest.raises(TypeError):
t.link_input_data = data
with pytest.raises(TypeError):
t.move_input_data = data
with pytest.raises(TypeError):
t.copy_output_data = data
with pytest.raises(TypeError):
t.download_output_data = data
with pytest.raises(TypeError):
t.move_output_data = data
if not isinstance(data, str) and not isinstance(data, list):
with pytest.raises(TypeError):
t.executable = data
if not isinstance(data, str) and not isinstance(data, unicode):
with pytest.raises(ValueError):
t.cpu_reqs = {
'processes': 1,
'process_type': data,
'threads_per_process': 1,
'thread_type': None
}
t.cpu_reqs = {
'processes': 1,
'process_type': None,
'threads_per_process': 1,
'thread_type': data
}
t.gpu_reqs = {
'processes': 1,
'process_type': data,
'threads_per_process': 1,
'thread_type': None
}
t.gpu_reqs = {
'processes': 1,
'process_type': None,
'threads_per_process': 1,
'thread_type': data
}
if not isinstance(data, int):
with pytest.raises(TypeError):
t.cpu_reqs = {
'processes': data,
'process_type': None,
'threads_per_process': 1,
'thread_type': None
}
t.cpu_reqs = {
'processes': 1,
'process_type': None,
'threads_per_process': data,
'thread_type': None
}
t.gpu_reqs = {
'processes': data,
'process_type': None,
'threads_per_process': 1,
'thread_type': None
}
t.gpu_reqs = {
'processes': 1,
'process_type': None,
'threads_per_process': data,
'thread_type': None
}
# Bookkeeping
stage_uids = list()
task_uids = dict()
Stages = 1
Replicas = 2
for N_Stg in range(Stages):
stg = Stage() ## initialization
task_uids['Stage_%s'%N_Stg] = list()
if N_Stg == 0:
for n0 in range(Replicas):
t = Task()
t.executable = ['/u/sciteam/mushnoor/amber/amber14/bin/sander.MPI'] #MD Engine
t.upload_input_data = ['inpcrd', 'prmtop', 'mdin']
t.pre_exec = ['export AMBERHOME=$HOME/amber/amber14/']
t.arguments = ['-O', '-i', 'mdin', '-p', 'prmtop', '-c', 'inpcrd', '-o', 'out']
t.cores = 32
t.mpi = True
stg.add_tasks(t)
task_uids['Stage_%s'%N_Stg].append(t.uid)
p.add_stages(stg)
stage_uids.append(stg.uid)
else:
for n0 in range(Replicas):
t = Task()
t.executable = ['/u/sciteam/mushnoor/amber/amber14/bin/sander.MPI'] #MD Engine
t.copy_input_data = ['$Pipeline_%s_Stage_%s_Task_%s/out.gro > in.gro'%(p.uid, stage_uids[N_Stg-1], task_uids['Stage_%s'%(N_Stg-1)][n0]), '$Pipeline_%s_Stage_%s_Task_%s/in.top'%(p.uid, stage_uids[N_Stg-1], task_uids['Stage_%s'%(N_Stg-1)][n0]), '$Pipeline_%s_Stage_%s_Task_%s/FNF.itp'%(p.uid, stage_uids[N_Stg-1], task_uids['Stage_%s'%(N_Stg-1)][n0]), '$Pipeline_%s_Stage_%s_Task_%s/martini_v2.2.itp'%(p.uid, stage_uids[N_Stg-1], task_uids['Stage_%s'%(N_Stg-1)][n0]), '$Pipeline_%s_Stage_%s_Task_%s/in.mdp'%(p.uid, stage_uids[N_Stg-1], task_uids['Stage_%s'%(N_Stg-1)][n0])]
def Cycle(Replicas, Replica_Cores, Cycles, MD_Executable, ExchangeMethod):
"""
All cycles after the initial cycle
"""
with open("exchangePairs.dat","r") as f: # Read exchangePairs.dat
ExchangeArray = []
for line in f:
ExchangeArray.append(int(line.split()[1]))
#ExchangeArray.append(line)
#print ExchangeArray
q = Pipeline()
#Bookkeeping
stage_uids = list()
task_uids = list() ## = dict()
md_dict = dict()
#Create initial MD stage
md_stg = Stage()
for r in range (Replicas):
md_tsk = Task()
md_tsk.executable = [MD_Executable] #MD Engine, Blue Waters
md_tsk.link_input_data = ['%s/restrt > inpcrd'%(Book[Cycle-1][ExchangeArray[r]]),
'%s/prmtop'%(Book[Cycle-1][r]),
#'%s/mdin_{0}'.format(r)%(Book[k-1][r])]
'%s/mdin'%(Book[Cycle-1][r])]
md_tsk.pre_exec = ['export AMBERHOME=$HOME/amber/amber14/'] # Should be abstracted from user?
#md_tsk.pre_exec = ['module load amber']
#md_tsk.arguments = ['-O', '-i', 'mdin_{0}'.format(n0), '-p', 'prmtop', '-c', 'inpcrd', '-o', 'out_{0}'.format(n0),'-inf', 'mdinfo_{0}'.format(n0)]
md_tsk.arguments = ['-O', '-i', 'mdin', '-p', 'prmtop', '-c', 'inpcrd', '-o', 'out_{0}'.format(r),'-inf', 'mdinfo_{0}'.format(r)]
md_tsk.cores = Replica_Cores
md_tsk.mpi = True
md_dict[r] = '$Pipeline_%s_Stage_%s_Task_%s'%(p.uid, md_stg.uid, md_tsk.uid)
md_stg.add_tasks(md_tsk)
#task_uids.append(md_tsk.uid)
q.add_stages(md_stg)
ex_stg= Stage()
#Create Exchange Task
ex_tsk = Task()
ex_tsk.executable = ['python']
ex_tsk.upload_input_data = ['exchangeMethods/TempEx.py']
for n1 in range (Replicas):
#print d[n1]
ex_tsk.link_input_data += ['%s/mdinfo_%s'%(d[n1],n1)]
ex_tsk.arguments = ['TempEx.py','{0}'.format(Replicas)]
ex_tsk.cores = 1
ex_tsk.mpi = False
ex_tsk.download_output_data = ['exchangePairs.dat']
ex_stg.add_tasks(ex_tsk)
#task_uids.append(ex_tsk.uid)
q.add_stages(ex_stg)
#stage_uids.append(ex_stg.uid)
Book.append(md_dict)
#print d
#print Book
return q
def cycle(k):
#read exchangePairs.dat
#
with open("exchangePairs.dat","r") as f:
ExchangeArray = []
for line in f:
ExchangeArray.append(int(line.split()[1]))
#ExchangeArray.append(line)
#print ExchangeArray
p = Pipeline()
#Bookkeeping
stage_uids = list()
task_uids = list() ## = dict()
d = dict()
#Create initial MD stage
md_stg = Stage()
#Create MD task
for n0 in range (Replicas):
md_tsk = Task()
md_tsk.executable = ['/u/sciteam/mushnoor/amber/amber14/bin/sander.MPI'] #MD Engine, Blue Waters
#md_tsk.executable = ['/usr/local/packages/amber/16/INTEL-140-MVAPICH2-2.0/bin/pmemd.MPI'] #MD Engine, SuperMIC
#md_tsk.executable = ['/opt/amber/bin/pmemd.MPI']
md_tsk.link_input_data = ['%s/restrt > inpcrd'%(Book[k-1][ExchangeArray[n0]]),
'%s/prmtop'%(Book[k-1][n0]),
#'%s/mdin_{0}'.format(n0)%(Book[k-1][n0])]
'%s/mdin'%(Book[k-1][n0])]
##Above: Copy from previous PIPELINE, make sure bookkeeping is correct
md_tsk.pre_exec = ['export AMBERHOME=$HOME/amber/amber14/'] #Preexec, BLue Waters
#md_tsk.pre_exec = ['module load amber']
#md_tsk.arguments = ['-O', '-i', 'mdin_{0}'.format(n0), '-p', 'prmtop', '-c', 'inpcrd', '-o', 'out_{0}'.format(n0),'-inf', 'mdinfo_{0}'.format(n0)]
md_tsk.arguments = ['-O', '-i', 'mdin', '-p', 'prmtop', '-c', 'inpcrd', '-o', 'out_{0}'.format(n0),'-inf', 'mdinfo_{0}'.format(n0)]
md_tsk.cores = Replica_Cores
md_tsk.mpi = True
d[n0] = '$Pipeline_%s_Stage_%s_Task_%s'%(p.uid, md_stg.uid, md_tsk.uid)
#print d
md_stg.add_tasks(md_tsk)
task_uids.append(md_tsk.uid)
p.add_stages(md_stg)
stage_uids.append(md_stg.uid)
#Create exchange stage
ex_stg= Stage()
#Create Exchange Task
ex_tsk = Task()
ex_tsk.executable = ['python']
ex_tsk.upload_input_data = ['exchangeMethods/TempEx.py']
for n1 in range (Replicas):
#print d[n1]
ex_tsk.link_input_data += ['%s/mdinfo_%s'%(d[n1],n1)]
ex_tsk.arguments = ['TempEx.py','{0}'.format(Replicas)]
ex_tsk.cores = 1
ex_tsk.mpi = False
ex_tsk.download_output_data = ['exchangePairs.dat']
ex_stg.add_tasks(ex_tsk)
task_uids.append(ex_tsk.uid)
p.add_stages(ex_stg)
stage_uids.append(ex_stg.uid)
Book.append(d)
#print d
#print Book
return p
Pilot_Cores = Replicas * Replica_Cores
for N_Stg in range(Stages):
stg = Stage() ## initialization
task_uids['Stage_%s'%N_Stg] = list()
#####Initial MD stage
if N_Stg == 0:
for n0 in range(Replicas):
t = Task()
t.executable = ['/u/sciteam/mushnoor/amber/amber14/bin/sander.MPI'] #MD Engine
t.upload_input_data = ['inpcrd', 'prmtop', 'mdin_{0}'.format(n0)]
t.pre_exec = ['export AMBERHOME=$HOME/amber/amber14/']
t.arguments = ['-O', '-i', 'mdin_{0}'.format(n0), '-p', 'prmtop', '-c', 'inpcrd', '-o', 'out']
t.cores = Replica_Cores
stg.add_tasks(t)
task_uids['Stage_%s'%N_Stg].append(t.uid)
p.add_stages(stg)
stage_uids.append(stg.uid)
#####Exchange Stages
elif N_Stg != 0 and N_Stg%2 = 1:
t = Task()
t.executable = ['python']
t.upload_input_data = ['exchangeMethods/RandEx.py']
#t.link_input_data = ['']
def init_cycle(self, replicas, replica_cores, python_path, md_executable, exchange_method, min_temp, max_temp, timesteps, basename, pre_exec): # "cycle" = 1 MD stage plus the subsequent exchange computation
"""
Initial cycle consists of:
1) Create tarball of MD input data
2) Transfer the tarball to pilot sandbox
3) Untar the tarball
4) Run first cycle
"""
#Initialize Pipeline
self._prof.prof('InitTar', uid=self._uid)
p = Pipeline()
p.name = 'initpipeline'
md_dict = dict() #bookkeeping
tar_dict = dict() #bookkeeping
#Write the input files
self._prof.prof('InitWriteInputs', uid=self._uid)
writeInputs.writeInputs(
max_temp=max_temp,
min_temp=min_temp,
replicas=replicas,
timesteps=timesteps,
basename=basename)
self._prof.prof('EndWriteInputs', uid=self._uid)
self._prof.prof('InitTar', uid=self._uid)
#Create Tarball of input data
tar = tarfile.open("input_files.tar", "w")
for name in [
basename + ".prmtop", basename + ".inpcrd", basename + ".mdin"
]:
tar.add(name)
for r in range(replicas):
tar.add('mdin_{0}'.format(r))
tar.close()
#delete all input files outside the tarball
for r in range(replicas):
os.remove('mdin_{0}'.format(r))
self._prof.prof('EndTar', uid=self._uid)
#Create Untar Stage
repo = git.Repo('.', search_parent_directories=True)
aux_function_path = repo.working_tree_dir
untar_stg = Stage()
untar_stg.name = 'untarStg'
#Untar Task
untar_tsk = Task()
untar_tsk.name = 'untartsk'
untar_tsk.executable = ['python']
untar_tsk.upload_input_data = [
str(aux_function_path)+'/repex/untar_input_files.py', 'input_files.tar'
]
untar_tsk.arguments = ['untar_input_files.py', 'input_files.tar']
untar_tsk.cpu_reqs = 1
#untar_tsk.post_exec = ['']
untar_stg.add_tasks(untar_tsk)
p.add_stages(untar_stg)
tar_dict[0] = '$Pipeline_%s_Stage_%s_Task_%s' % (
p.name, untar_stg.name, untar_tsk.name)
# First MD stage: needs to be defined separately since workflow is not built from a predetermined order, also equilibration needs to happen first.
md_stg = Stage()
md_stg.name = 'mdstg0'
self._prof.prof('InitMD_0', uid=self._uid)
# MD tasks
for r in range(replicas):
md_tsk = AMBERTask(cores=replica_cores, md_executable=md_executable, pre_exec=pre_exec)
md_tsk.name = 'mdtsk-{replica}-{cycle}'.format(replica=r, cycle=0)
md_tsk.link_input_data += [
'%s/inpcrd' % tar_dict[0],
'%s/prmtop' % tar_dict[0],
'%s/mdin_{0}'.format(r) %
tar_dict[0] #Use for full temperature exchange
]
md_tsk.arguments = [
'-O',
'-p',
'prmtop',
'-i',
'mdin_{0}'.format(r),
'-c',
'inpcrd',
'-o',
'out-{replica}-{cycle}'.format(replica=r, cycle=0),
'-r',
'restrt'.format(replica=r, cycle=0),
#'-r', 'rstrt-{replica}-{cycle}'.format(replica=r,cycle=0),
'-x',
'mdcrd-{replica}-{cycle}'.format(replica=r, cycle=0),
#'-o', '$NODE_LFS_PATH/out-{replica}-{cycle}'.format(replica=r,cycle=0),
#'-r', '$NODE_LFS_PATH/rstrt-{replica}-{cycle}'.format(replica=r,cycle=0),
#'-x', '$NODE_LFS_PATH/mdcrd-{replica}-{cycle}'.format(replica=r,cycle=0),
'-inf',
'mdinfo_{0}'.format(r)
]
md_dict[r] = '$Pipeline_%s_Stage_%s_Task_%s' % (
p.name, md_stg.name, md_tsk.name)
md_stg.add_tasks(md_tsk)
self.md_task_list.append(md_tsk)
#print md_tsk.uid
p.add_stages(md_stg)
#stage_uids.append(md_stg.uid)
# First Exchange Stage
ex_stg = Stage()
ex_stg.name = 'exstg0'
self._prof.prof('InitEx_0', uid=self._uid)
# Create Exchange Task
ex_tsk = Task()
ex_tsk.name = 'extsk0'
#ex_tsk.pre_exec = ['module load python/2.7.10']
ex_tsk.executable = [python_path]
ex_tsk.upload_input_data = [exchange_method]
for r in range(replicas):
ex_tsk.link_input_data += ['%s/mdinfo_%s' % (md_dict[r], r)]
ex_tsk.pre_exec = ['mv *.py exchange_method.py']
ex_tsk.arguments = ['exchange_method.py', '{0}'.format(replicas), '0']
ex_tsk.cores = 1
ex_tsk.mpi = False
ex_tsk.download_output_data = ['exchangePairs_0.dat']
ex_stg.add_tasks(ex_tsk)
#task_uids.append(ex_tsk.uid)
p.add_stages(ex_stg)
self.ex_task_list.append(ex_tsk)
#self.ex_task_uids.append(ex_tsk.uid)
self.book.append(md_dict)
return p
def general_cycle(self, replicas, replica_cores, cycle, python_path, md_executable, exchange_method, pre_exec):
"""
All cycles after the initial cycle
Pulls up exchange pairs file and generates the new workflow
"""
self._prof.prof('InitcreateMDwokflow_{0}'.format(cycle), uid=self._uid)
with open('exchangePairs_{0}.dat'.format(cycle),
'r') as f: # Read exchangePairs.dat
exchange_array = []
for line in f:
exchange_array.append(int(line.split()[1]))
#exchange_array.append(line)
#print exchange_array
q = Pipeline()
q.name = 'genpipeline{0}'.format(cycle)
#bookkeeping
stage_uids = list()
task_uids = list() ## = dict()
md_dict = dict()
#Create MD stage
md_stg = Stage()
md_stg.name = 'mdstage{0}'.format(cycle)
self._prof.prof('InitMD_{0}'.format(cycle), uid=self._uid)
for r in range(replicas):
md_tsk = AMBERTask(cores=replica_cores, md_executable=md_executable, pre_exec=pre_exec)
md_tsk.name = 'mdtsk-{replica}-{cycle}'.format(
replica=r, cycle=cycle)
md_tsk.link_input_data = [
'%s/restrt > inpcrd' %
(self.book[cycle - 1][exchange_array[r]]),
'%s/prmtop' % (self.book[0][r]),
'%s/mdin_{0}'.format(r) % (self.book[0][r])
]
### The Following softlinking scheme is to be used ONLY if node local file system is to be used: not fully supported yet.
#md_tsk.link_input_data = ['$NODE_LFS_PATH/rstrt-{replica}-{cycle}'.format(replica=exchange_array[r],cycle=cycle-1) > '$NODE_LFS_PATH/inpcrd',
# #'%s/restrt > inpcrd'%(self.book[cycle-1][exchange_array[r]]),
# '%s/prmtop'%(self.book[0][r]),
# '%s/mdin_{0}'.format(r)%(self.Book[0][r])]
md_tsk.arguments = [
'-O',
'-i',
'mdin_{0}'.format(r),
'-p',
'prmtop',
'-c',
'inpcrd',
#'-c', 'rstrt-{replica}-{cycle}'.format(replica=r,cycle=cycle-1),
'-o',
'out-{replica}-{cycle}'.format(replica=r, cycle=cycle),
'-r',
'restrt',
#'-r', 'rstrt-{replica}-{cycle}'.format(replica=r,cycle=cycle),
'-x',
'mdcrd-{replica}-{cycle}'.format(replica=r, cycle=cycle),
'-inf',
'mdinfo_{0}'.format(r)
]
#md_tsk.tag = 'mdtsk-{replica}-{cycle}'.format(replica=r,cycle=0)
md_dict[r] = '$Pipeline_%s_Stage_%s_Task_%s' % (
q.name, md_stg.name, md_tsk.name)
self.md_task_list.append(md_tsk)
md_stg.add_tasks(md_tsk)
q.add_stages(md_stg)
ex_stg = Stage()
ex_stg.name = 'exstg{0}'.format(cycle + 1)
#Create Exchange Task
ex_tsk = Task()
ex_tsk.name = 'extsk{0}'.format(cycle + 1)
ex_tsk.executable = [python_path]#['/usr/bin/python'] #['/opt/python/bin/python']
ex_tsk.upload_input_data = [exchange_method]
for r in range(replicas):
ex_tsk.link_input_data += ['%s/mdinfo_%s' % (md_dict[r], r)]
ex_tsk.pre_exec = ['mv *.py exchange_method.py']
ex_tsk.arguments = [
'exchange_method.py', '{0}'.format(replicas), '{0}'.format(cycle + 1)
]
ex_tsk.cores = 1
ex_tsk.mpi = False
ex_tsk.download_output_data = [
'exchangePairs_{0}.dat'.format(cycle + 1)
] # Finds exchange partners, also Generates exchange history trace
ex_stg.add_tasks(ex_tsk)
#task_uids.append(ex_tsk.uid)
self.ex_task_list.append(ex_tsk)
q.add_stages(ex_stg)
#stage_uids.append(ex_stg.uid)
self.book.append(md_dict)
#self._prof.prof('EndEx_{0}'.format(cycle), uid=self._uid)
#print d
#print self.book
return q
def init_cycle():
# Create Pipeline Obj
p = Pipeline()
#Bookkeeping
stage_uids = list()
task_uids = list() ## = dict()
d = dict()
dict_tarball = dict()
#Create Tarball stage
tar_stg = Stage()
#Create Tar/untar task
tar_tsk = Task()
tar_tsk.executable = ['python']
tar_tsk.upload_input_data = ['Input_Files.tar', 'untar_input_files.py']
tar_tsk.arguments = ['untar_input_files.py','Input_Files.tar']
tar_tsk.cores = 1
tar_stg.add_tasks(tar_tsk)
#task_uids.append(tar_tsk.uid)
p.add_stages(tar_stg)
#stage_uids.append(tar_stg.uid)
dict_tarball[0] = '$Pipeline_%s_Stage_%s_Task_%s'%(p.uid,tar_stg.uid,tar_tsk.uid)
#Create initial MD stage
md_stg = Stage()
#Create MD task
for n0 in range (Replicas):
md_tsk = Task()
md_tsk.executable = ['/u/sciteam/mushnoor/amber/amber14/bin/sander.MPI'] #MD Engine, BW
#md_tsk.executable = ['/usr/local/packages/amber/16/INTEL-140-MVAPICH2-2.0/bin/pmemd.MPI'] #MD Engine, SuperMIC
#md_tsk.executable = ['/opt/amber/bin/pmemd.MPI']
#md_tsk.upload_input_data = ['inpcrd', 'prmtop', 'mdin_{0}'.format(n0)]
#md_tsk.upload_input_data = ['inpcrd','prmtop','mdin']
md_tsk.link_input_data += ['%s/inpcrd'%dict_tarball[0],
'%s/prmtop'%dict_tarball[0],
'%s/mdin'%dict_tarball[0]]
md_tsk.pre_exec = ['export AMBERHOME=$HOME/amber/amber14/']
#md_tsk.pre_exec = ['module load amber']
#md_tsk.arguments = ['-O', '-i', 'mdin_{0}'.format(n0), '-p', 'prmtop', '-c', 'inpcrd', '-o', 'out_{0}'.format(n0), '-inf', 'mdinfo_{0}'.format(n0)]
md_tsk.arguments = ['-O', '-i', 'mdin', '-p', 'prmtop', '-c', 'inpcrd', '-o', 'out_{0}'.format(n0), '-inf', 'mdinfo_{0}'.format(n0)]
md_tsk.cores = Replica_Cores
md_tsk.mpi = True
d[n0] = '$Pipeline_%s_Stage_%s_Task_%s'%(p.uid, md_stg.uid, md_tsk.uid)
md_stg.add_tasks(md_tsk)
task_uids.append(md_tsk.uid)
p.add_stages(md_stg)
stage_uids.append(md_stg.uid)
#print d
#Create Exchange Stage
ex_stg = Stage()
#Create Exchange Task
ex_tsk = Task()
ex_tsk.executable = ['python']
ex_tsk.upload_input_data = ['exchangeMethods/TempEx.py']
for n1 in range (Replicas):
ex_tsk.link_input_data += ['%s/mdinfo_%s'%(d[n1],n1)]
ex_tsk.arguments = ['TempEx.py','{0}'.format(Replicas)]
ex_tsk.cores = 1
ex_tsk.mpi = False
ex_tsk.download_output_data = ['exchangePairs.dat']
ex_stg.add_tasks(ex_tsk)
task_uids.append(ex_tsk.uid)
p.add_stages(ex_stg)
stage_uids.append(ex_stg.uid)
Book.append(d)
#print Book
return p
def InitCycle(Replicas, Replica_Cores, MD_Executable, ExchangeMethod): # "Cycle" = 1 MD stage plus the subsequent exchange computation
#Initialize Pipeline
p = Pipeline()
md_dict = dict() #Bookkeeping
tar_dict = dict() #Bookkeeping
#Create Tarball of input data
#Create Untar Stage
untar_stg = Stage()
#Untar Task
untar_tsk = Task()
untar_tsk.executable = ['python']
untar_tsk.upload_input_data = ['untar_input_files.py','../../Input_Files.tar']
untar_tsk.arguments = ['untar_input_files.py','Input_Files.tar']
untar_tsk.cores = 1
untar_stg.add_tasks(untar_tsk)
p.add_stages(untar_stg)
tar_dict[0] = '$Pipeline_%s_Stage_%s_Task_%s'%(p.uid,
untar_stg.uid,
untar_tsk.uid)
print tar_dict[0]
# First MD stage: needs to be defined separately since workflow is not built from a predetermined order
md_stg = Stage()
# MD tasks
for r in range (Replicas):
md_tsk = Task()
md_tsk.executable = [MD_Executable]
md_tsk.link_input_data += ['%s/inpcrd'%tar_dict[0],
'%s/prmtop'%tar_dict[0],
#'%s/mdin_{0}'.format(r)%tar_dict[0]
'%s/mdin'%tar_dict[0]
]
md_tsk.pre_exec = ['export AMBERHOME=$HOME/amber/amber14/'] #Should be abstracted from the user?
md_tsk.arguments = ['-O','-p','prmtop', '-i', 'mdin', #'mdin_{0}'.format(r), # Use this for full Temperature Exchange
'-c','inpcrd','-o','out_{0}'.format(r),
'-inf','mdinfo_{0}'.format(r)]
md_tsk.cores = Replica_Cores
md_tsk.mpi = True
md_dict[r] = '$Pipeline_%s_Stage_%s_Task_%s'%(p.uid, md_stg.uid, md_tsk.uid)
md_stg.add_tasks(md_tsk)
#task_uids.append(md_tsk.uid)
p.add_stages(md_stg)
#stage_uids.append(md_stg.uid)
# First Exchange Stage
ex_stg = Stage()
# Create Exchange Task. Exchange task performs a Metropolis Hastings thermodynamic balance condition
# and spits out the exchangePairs.dat file that contains a sorted list of ordered pairs.
# Said pairs then exchange configurations by linking output configuration files appropriately.
ex_tsk = Task()
ex_tsk.executable = ['python']
#ex_tsk.upload_input_data = ['exchangeMethods/TempEx.py']
ex_tsk.upload_input_data = [ExchangeMethod]
for r in range (Replicas):
ex_tsk.link_input_data += ['%s/mdinfo_%s'%(md_dict[r],r)]
ex_tsk.arguments = ['TempEx.py','{0}'.format(Replicas)]
ex_tsk.cores = 1
ex_tsk.mpi = False
ex_tsk.download_output_data = ['exchangePairs.dat']
ex_stg.add_tasks(ex_tsk)
#task_uids.append(ex_tsk.uid)
p.add_stages(ex_stg)
#stage_uids.append(ex_stg.uid)
Book.append(md_dict)
#print Book
return p
