def test_issue_239(): t = Task() t.cpu_reqs = {'processes': 1} assert t.cpu_reqs == { 'processes': 1, 'thread_type': None, 'threads_per_process': 1, 'process_type': None} t.cpu_reqs = {'threads_per_process': 1} assert t.cpu_reqs == { 'processes': 1, 'thread_type': None, 'threads_per_process': 1, 'process_type': None} t.gpu_reqs = {'processes': 1} assert t.gpu_reqs == { 'processes': 1, 'thread_type': None, 'threads_per_process': 1, 'process_type': None} t.gpu_reqs = {'threads_per_process': 1} assert t.gpu_reqs == { 'processes': 1, 'thread_type': None, 'threads_per_process': 1, 'process_type': None}
def test_issue_239(): t = Task() t.cpu_reqs = {'processes': 1} assert t.cpu_reqs == { 'processes': 1, 'thread_type': None, 'threads_per_process': 1, 'process_type': None } t.cpu_reqs = {'threads_per_process': 1} assert t.cpu_reqs == { 'processes': 1, 'thread_type': None, 'threads_per_process': 1, 'process_type': None } t.gpu_reqs = {'processes': 1} assert t.gpu_reqs == { 'processes': 1, 'thread_type': None, 'threads_per_process': 1, 'process_type': None } t.gpu_reqs = {'threads_per_process': 1} assert t.gpu_reqs == { 'processes': 1, 'thread_type': None, 'threads_per_process': 1, 'process_type': None }
def generate_pipeline(): global CUR_TASKS, CUR_CORES, duration, MAX_NEW_STAGE def func_condition(): global CUR_NEW_STAGE, MAX_NEW_STAGE if CUR_NEW_STAGE < MAX_NEW_STAGE: return True return False def func_on_true(): global CUR_NEW_STAGE CUR_NEW_STAGE += 1 for t in p.stages[CUR_NEW_STAGE].tasks: cores = randint(1,16) t.arguments = ['-c', str(cores), '-t', str(duration)] def func_on_false(): print 'Done' # Create a Pipeline object p = Pipeline() for s in range(MAX_NEW_STAGE+1): # Create a Stage object s1 = Stage() for i in range(CUR_TASKS): t1 = Task() t1.pre_exec = ['export PATH=/u/sciteam/balasubr/modules/stress-ng-0.09.34:$PATH'] t1.executable = ['stress-ng'] t1.arguments = [ '-c', str(CUR_CORES), '-t', str(duration)] t1.cpu_reqs = { 'processes': 1, 'process_type': '', 'threads_per_process': CUR_CORES, 'thread_type': '' } # Add the Task to the Stage s1.add_tasks(t1) # Add post-exec to the Stage s1.post_exec = { 'condition': func_condition, 'on_true': func_on_true, 'on_false': func_on_false } # Add Stage to the Pipeline p.add_stages(s1) return p
def generate_aggregating_task(self): """ Function to concatenate the MD trajectory (h5 contact map) """ p = Pipeline() p.name = 'aggragating' s2 = Stage() s2.name = 'aggregating' # Aggregation task t2 = Task() # https://github.com/radical-collaboration/hyperspace/blob/MD/microscope/experiments/MD_to_CVAE/MD_to_CVAE.py t2.pre_exec = [] t2.pre_exec += ['. /sw/summit/python/2.7/anaconda2/5.3.0/etc/profile.d/conda.sh'] t2.pre_exec += ['conda activate %s' % conda_path] t2.pre_exec += ['cd %s' % agg_path] t2.executable = ['%s/bin/python' % conda_path] # MD_to_CVAE.py t2.arguments = [ '%s/MD_to_CVAE.py' % agg_path, '--sim_path', md_path, '--train_frames', 100000] # assign hardware the task t2.cpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 4, 'thread_type': 'OpenMP' } # Add the aggregation task to the aggreagating stage s2.add_tasks(t2) p.add_stages(s2) return p
def func_on_true(): global CUR_NEW_STAGE, CUR_TASKS, CUR_CORES, duration CUR_NEW_STAGE += 1 s = Stage() for i in range(CUR_TASKS): t = Task() t.pre_exec = [ 'export PATH=/u/sciteam/balasubr/modules/stress-ng-0.09.34:$PATH' ] t.executable = ['stress-ng'] t.arguments = ['-c', str(CUR_CORES), '-t', str(duration)] t.cpu_reqs = { 'processes': 1, 'process_type': '', 'threads_per_process': CUR_CORES, 'thread_type': '' } # Add the Task to the Stage s.add_tasks(t) # Add post-exec to the Stage s.post_exec = { 'condition': func_condition, 'on_true': func_on_true, 'on_false': func_on_false } p.add_stages(s)
def add_ex_stg(rid, cycle): #ex stg here ex_tsk = Task() ex_stg = Stage() ex_tsk.name = 'extsk-{replica}-{cycle}'.format(replica=rid, cycle=cycle) for rid in range(len(waiting_replicas)): ex_tsk.link_input_data += ['%s/mdinfo-{replica}-{cycle}'.format(replica=rid, cycle=self.cycle)%replica_sandbox] ex_tsk.arguments = ['t_ex_gibbs.py', len(waiting_replicas)] #This needs to be fixed ex_tsk.executable = ['python'] ex_tsk.cpu_reqs = { 'processes': 1, 'process_type': '', 'threads_per_process': 1, 'thread_type': None } ex_tsk.pre_exec = ['export dummy_variable=19'] ex_stg.add_tasks(ex_tsk) ex_stg.post_exec = { 'condition': post_ex, 'on_true': terminate_replicas, 'on_false': continue_md } return ex_stg
def add_md_stg(rid,cycle): #md stg h md_tsk = Task() md_stg = Stage() md_tsk.name = 'mdtsk-{replica}-{cycle}'.format(replica=rid, cycle=cycle) md_tsk.link_input_data += ['%s/inpcrd' %replica_sandbox, '%s/prmtop' %replica_sandbox, '%s/mdin-{replica}-{cycle}'.format(replica=rid, cycle=0) %replica_sandbox] md_tsk.arguments = ['-O', '-i', 'mdin-{replica}-{cycle}'.format(replica=rid, cycle=0), '-p', 'prmtop', '-c', 'inpcrd', '-o', 'out', '-r', '%s/restrt-{replica}-{cycle}'.format(replica=rid, cycle=cycle) %replica_sandbox, '-x', 'mdcrd', '-inf', '%s/mdinfo-{replica}-{cycle}'.format(replica=rid, cycle=cycle) %replica_sandbox] md_tsk.executable = ['/home/scm177/mantel/AMBER/amber14/bin/sander'] md_tsk.cpu_reqs = { 'processes': replica_cores, 'process_type': '', 'threads_per_process': 1, 'thread_type': None } md_tsk.pre_exec = ['export dummy_variable=19', 'echo $SHARED'] md_stg.add_tasks(md_tsk) md_stg.post_exec = { 'condition': md_post, 'on_true': suspend, 'on_false': exchange_stg } return md_stg
def generate_aggregating_stage(self) -> Stage: stage = Stage() stage.name = "aggregating" cfg = self.cfg.aggregation_stage # Aggregation task task = Task() task.cpu_reqs = cfg.cpu_reqs.dict() task.pre_exec = cfg.pre_exec task.executable = cfg.executable task.arguments = cfg.arguments # Update base parameters cfg.run_config.experiment_directory = self.cfg.experiment_directory cfg.run_config.output_path = self.aggregated_data_path( self.cur_iteration) cfg_path = self.experiment_dirs["aggregation_runs"].joinpath( f"aggregation_{self.cur_iteration:03d}.yaml") cfg.run_config.dump_yaml(cfg_path) task.arguments += ["-c", cfg_path] stage.add_tasks(task) return stage
def generate_ml_stage(self) -> Stage: stage = Stage() stage.name = "learning" cfg = self.cfg.ml_stage task = Task() task.cpu_reqs = cfg.cpu_reqs.dict() task.gpu_reqs = cfg.gpu_reqs.dict() task.pre_exec = cfg.pre_exec task.executable = cfg.executable task.arguments = cfg.arguments # Update base parameters cfg.run_config.input_path = self.aggregated_data_path( self.cur_iteration) cfg.run_config.output_path = self.model_path(self.cur_iteration) if self.cur_iteration > 0: cfg.run_config.init_weights_path = self.latest_ml_checkpoint_path( self.cur_iteration - 1) cfg_path = self.experiment_dirs["ml_runs"].joinpath( f"ml_{self.cur_iteration:03d}.yaml") cfg.run_config.dump_yaml(cfg_path) task.arguments += ["-c", cfg_path] stage.add_tasks(task) return stage
def generate_outlier_detection_stage(self) -> Stage: stage = Stage() stage.name = "outlier_detection" cfg = self.cfg.od_stage task = Task() task.cpu_reqs = cfg.cpu_reqs.dict() task.gpu_reqs = cfg.gpu_reqs.dict() task.pre_exec = cfg.pre_exec task.executable = cfg.executable task.arguments = cfg.arguments self.outlier_pdbs_path(self.cur_iteration).mkdir() # Update base parameters cfg.run_config.experiment_directory = self.cfg.experiment_directory cfg.run_config.input_path = self.aggregated_data_path( self.cur_iteration) cfg.run_config.output_path = self.outlier_pdbs_path(self.cur_iteration) cfg.run_config.weights_path = self.latest_ml_checkpoint_path( self.cur_iteration) cfg.run_config.restart_points_path = self.restart_points_path( self.cur_iteration) cfg_path = self.experiment_dirs["od_runs"].joinpath( f"od_{self.cur_iteration:03d}.yaml") cfg.run_config.dump_yaml(cfg_path) task.arguments += ["-c", cfg_path] stage.add_tasks(task) return stage
def post_stage(): if (not os.path.exists(f'{run_dir}/aggregator/stop.aggregator')): nstages = len(p.stages) s = Stage() s.name = f"{nstages}" t = Task() t.cpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 4, 'thread_type': 'OpenMP' } t.gpu_reqs = { 'processes': 0, 'process_type': None, 'threads_per_process': 0, 'thread_type': None } t.name = f" {i}_{nstages} " t.executable = PYTHON t.arguments = [ f'{current_dir}/simulation.py', f'{run_dir}/simulations/all/{i}_{nstages}', ADIOS_XML ] subprocess.getstatusoutput( f'ln -s {run_dir}/simulations/all/{i}_{nstages} {run_dir}/simulations/new/{i}_{nstages}' ) s.add_tasks(t) s.post_exec = post_stage p.add_stages(s)
def esmacs(self, rct_stage, stage, outdir="equilibration", name=None): for i in range(1, 13): t = Task() t.pre_exec = [ "export WDIR=\"{}/{}\"".format(self.run_dir, name), ". {}".format(self.conda_init), "conda activate {}".format(self.esmacs_tenv), "module load {}".format(self.esmacs_tmodules), "mkdir -p $WDIR/replicas/rep{}/{}".format(i, outdir), "cd $WDIR/replicas/rep{}/{}".format(i, outdir), "rm -f {}.log {}.xml {}.dcd {}.chk".format( stage, stage, stage, stage), "export OMP_NUM_THREADS=1" ] # t.executable = '/ccs/home/litan/miniconda3/envs/wf3/bin/python3.7' t.executable = 'python3' t.arguments = ['$WDIR/{}.py'.format(stage)] t.post_exec = [] t.cpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 4, 'thread_type': 'OpenMP' } t.gpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 1, 'thread_type': 'CUDA' } getattr(self, rct_stage).add_tasks(t) print(getattr(self, rct_stage).to_dict())
def generate_task(cfg: BaseStageConfig) -> Task: task = Task() task.cpu_reqs = cfg.cpu_reqs.dict().copy() task.gpu_reqs = cfg.gpu_reqs.dict().copy() task.pre_exec = cfg.pre_exec.copy() task.executable = cfg.executable task.arguments = cfg.arguments.copy() return task
def generate_ML_stage(num_ML=1): """ Function to generate the learning stage """ s3 = Stage() s3.name = 'learning' # learn task for i in range(num_ML): t3 = Task() # https://github.com/radical-collaboration/hyperspace/blob/MD/microscope/experiments/CVAE_exps/train_cvae.py t3.pre_exec = [] t3.pre_exec = ['module reset'] t3.pre_exec += [ '. /sw/summit/python/2.7/anaconda2/5.3.0/etc/profile.d/conda.sh' ] t3.pre_exec += ['module load cuda/9.1.85'] t3.pre_exec += ['conda activate rp.copy'] t3.pre_exec += ['export CUDA_VISIBLE_DEVICES=0'] t3.pre_exec += [ 'export PYTHONPATH=/gpfs/alpine/scratch/hrlee/bip179/hyperspace/microscope/experiments/CVAE_exps:$PYTHONPATH' ] t3.pre_exec += [ 'cd /gpfs/alpine/scratch/hrlee/bip179/hyperspace/microscope/experiments/CVAE_exps' ] time_stamp = int(time.time()) dim = i + 3 cvae_dir = 'cvae_runs_%.2d_%d' % (dim, time_stamp) t3.pre_exec += ['mkdir -p {0} && cd {0}'.format(cvae_dir)] t3.executable = ['/ccs/home/hrlee/.conda/envs/rp.copy/bin/python' ] # train_cvae.py t3.arguments = [ '/gpfs/alpine/bip179/scratch/hrlee/hyperspace/microscope/experiments/CVAE_exps/train_cvae.py', '-f', '/gpfs/alpine/bip179/scratch/hrlee/hyperspace/microscope/experiments/MD_to_CVAE/cvae_input.h5', '-d', dim ] t3.cpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 4, 'thread_type': 'OpenMP' } t3.gpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 1, 'thread_type': 'CUDA' } # Add the learn task to the learning stage s3.add_tasks(t3) time.sleep(1) return s3
def generate_interfacing_stage(): s4 = Stage() s4.name = 'scanning' # Scaning for outliers and prepare the next stage of MDs t4 = Task() t4.pre_exec = [] #t4.pre_exec += ['. /sw/summit/python/2.7/anaconda2/5.3.0/etc/profile.d/conda.sh'] #t4.pre_exec += ['module load cuda/9.1.85'] #t4.pre_exec += ['conda activate %s' % conda_path] t4.pre_exec += [ 'module unload prrte', 'module unload python', 'module load xl', 'module load xalt', 'module load spectrum-mpi', 'module load cuda', 'module list' ] t4.pre_exec += [ '. /sw/summit/ibm-wml-ce/anaconda-base/etc/profile.d/conda.sh', 'source /sw/summit/ibm-wml-ce/anaconda-base/etc/profile.d/conda.sh', 'conda deactivate', 'conda deactivate', 'conda activate /gpfs/alpine/proj-shared/med110/wf-2/conda/envs/ibm-wml-ce-cloned' ] #'conda activate /sw/summit/ibm-wml-ce/anaconda-base/envs/ibm-wml-ce-1.7.0-2'] t4.pre_exec += [ 'export PYTHONPATH=%s/CVAE_exps:%s/CVAE_exps/cvae:$PYTHONPATH' % (base_path, base_path) ] t4.pre_exec += ['cd %s/Outlier_search' % base_path] #t4.executable = ['%s/bin/python' % conda_path] t4.executable = ['python'] t4.arguments = [ 'outlier_locator.py', '--md', md_path, '--cvae', cvae_path, '--pdb', pdb_file ] #'--ref', ref_pdb_file] t4.cpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 16, 'thread_type': 'OpenMP' } t4.gpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 1, 'thread_type': 'CUDA' } s4.add_tasks(t4) s4.post_exec = func_condition return s4
def generate_task(self, **ensembles): """ Generate a `radical.entk` task. Parameters ---------- ensembles: dict, OrderedDict Dictionary of the *current* values of variables that are ensembles. All the variables that were declared with `add_ensemble` should be specified here so that a correct task object can be generated. """ [setattr(self, k, w) for k, w in ensembles.iteritems()] if not self.all_variables_defined(): raise ValueError('Some variables are not defined!') task = Task() task.name = ensembles['task_name'] task.pre_exec += self.engine.pre_exec task.executable += str(self.engine.executable) task.arguments += self.engine.arguments task.cpu_reqs = { 'processes': self._processes, 'process_type': 'MPI' if self.engine.uses_mpi else None, 'threads_per_process': self._threads_per_process, 'thread_type': None } task.gpu_reqs = { 'processes': self._gpu_processes, 'process_type': 'MPI' if self.engine.gpu_uses_mpi else None, 'threads_per_process': self._gpu_threads_per_process, 'thread_type': None } task.arguments.extend(self.arguments) task.copy_input_data.extend(self.copied_files) task.copy_input_data.extend(self.system.copied_files) task.post_exec.append('echo "{}" > sim_desc.txt'.format(task.name)) task.link_input_data.extend(self.input_data(**ensembles)) task.link_input_data.extend(self.system.linked_files) task.pre_exec.extend( self._sed.format(n, v, f) for f, vs in self.get_variables().items() for n, v in vs) return task
def generate_MD_stage(num_MD=1): """ Function to generate MD stage. """ s1 = Stage() s1.name = 'MD' # MD tasks time_stamp = int(time.time()) for i in range(num_MD): t1 = Task() t1.pre_exec = [ '. /sw/summit/python/2.7/anaconda2/5.3.0/etc/profile.d/conda.sh' ] t1.pre_exec += ['module load cuda/9.1.85'] t1.pre_exec += ['conda activate %s' % conda_path] t1.pre_exec += [ 'export PYTHONPATH=%s/MD_exps:$PYTHONPATH' % base_path ] t1.pre_exec += ['cd %s/MD_exps/fs-pep' % base_path] t1.pre_exec += [ 'mkdir -p omm_runs_%d && cd omm_runs_%d' % (time_stamp + i, time_stamp + i) ] t1.executable = ['%s/bin/python' % conda_path] # run_openmm.py t1.arguments = ['%s/MD_exps/fs-pep/run_openmm.py' % base_path] # t1.arguments += ['--topol', '%s/MD_exps/fs-pep/pdb/topol.top' % base_path] t1.arguments += [ '--pdb_file', '%s/MD_exps/fs-pep/pdb/100-fs-peptide-400K.pdb' % base_path, '--length', LEN_sim ] # assign hardware the task t1.cpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 4, 'thread_type': 'OpenMP' } t1.gpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 1, 'thread_type': 'CUDA' } # Add the MD task to the simulating stage s1.add_tasks(t1) return s1
def generate_interfacing_stage(): s4 = Stage() s4.name = 'scanning' # Scaning for outliers and prepare the next stage of MDs t4 = Task() t4.pre_exec = [] t4.pre_exec = ['module reset'] t4.pre_exec += [ '. /sw/summit/python/2.7/anaconda2/5.3.0/etc/profile.d/conda.sh' ] t4.pre_exec += ['module load cuda/9.1.85'] t4.pre_exec += ['conda activate rp.copy'] t4.pre_exec += ['export CUDA_VISIBLE_DEVICES=0'] t4.pre_exec += [ 'export PYTHONPATH=/gpfs/alpine/bip179/scratch/hrlee/hyperspace/microscope/experiments/CVAE_exps:$PYTHONPATH' ] t4.pre_exec += [ 'cd /gpfs/alpine/bip179/scratch/hrlee/hyperspace/microscope/experiments/Outlier_search' ] # python outlier_locator.py -m ../MD_exps/fs-pep -c ../CVAE_exps -p ../MD_exps/fs-pep/pdb/100-fs-peptide-400K.pdb t4.executable = ['/ccs/home/hrlee/.conda/envs/rp.copy/bin/python'] t4.arguments = [ 'outlier_locator.py', '--md', '../MD_exps/fs-pep', '--cvae', '../CVAE_exps --pdb', '../MD_exps/fs-pep/pdb/100-fs-peptide-400K.pdb' ] # t4.arguments = ['/gpfs/alpine/bip179/scratch/hrlee/hyperspace/microscope/experiments/Outlier_search/outlier_locator.py', # '-m', '/gpfs/alpine/bip179/scratch/hrlee/hyperspace/microscope/experiments/MD_exps/fs-pep', # '-c', '/gpfs/alpine/bip179/scratch/hrlee/hyperspace/microscope/experiments/CVAE_exps', # '-p', '/gpfs/alpine/bip179/scratch/hrlee/hyperspace/microscope/experiments/MD_exps/fs-pep/pdb/100-fs-peptide-400K.pdb' # ] t4.cpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 12, 'thread_type': 'OpenMP' } t4.gpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 1, 'thread_type': 'CUDA' } s4.add_tasks(t4) s4.post_exec = func_condition return s4
def esmacs(cfg, names, stage, outdir): s = Stage() s.name = 'S3.%s' % stage #print("DEBUG:instantiation: %s" % len(s._tasks)) for comp in names: #print("DEBUG:first loop: %s" % len(s._tasks)) for i in range(1, cfg['n_replicas']): #print("DEBUG:second loop:start: %s" % len(s._tasks)) t = Task() # RCT native t.pre_exec = [ #". /sw/summit/lmod/lmod/init/profile", "export WDIR=\"{}\"".format(comp), ". {}".format(cfg['conda_init']), "conda activate {}".format(cfg['conda_esmacs_task_env']), "module load {}".format(cfg['esmacs_task_modules']), "mkdir -p $WDIR/replicas/rep{}/{}".format(i, outdir), "cd $WDIR/replicas/rep{}/{}".format(i, outdir), #"rm -f {}.log {}.xml {}.dcd {}.chk".format(stage, stage, stage, stage), "export OMP_NUM_THREADS=1"] t.executable = 'python3' t.arguments = ['$WDIR/{}.py'.format(stage)] # Bash wrapper #t.executable = '%s/wf3.sh' % comp #t.arguments = [comp, i, outdir, stage, # cfg['conda_init'], # cfg['conda_esmacs_task_env'], # cfg['esmacs_task_modules']] t.cpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 4, 'thread_type': 'OpenMP'} t.gpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 1, 'thread_type': 'CUDA'} s.add_tasks(t) #print("DEBUG:second loop:end: %s" % len(s._tasks)) return s
def generate_ML_stage(num_ML=1): """ Function to generate the learning stage """ s3 = Stage() s3.name = 'learning' # learn task time_stamp = int(time.time()) for i in range(num_ML): t3 = Task() # https://github.com/radical-collaboration/hyperspace/blob/MD/microscope/experiments/CVAE_exps/train_cvae.py t3.pre_exec = [] t3.pre_exec += [ '. /sw/summit/python/2.7/anaconda2/5.3.0/etc/profile.d/conda.sh' ] t3.pre_exec += ['module load cuda/9.1.85'] t3.pre_exec += ['conda activate %s' % conda_path] t3.pre_exec += [ 'export PYTHONPATH=%s/CVAE_exps:$PYTHONPATH' % base_path ] t3.pre_exec += ['cd %s' % cvae_path] dim = i + 3 cvae_dir = 'cvae_runs_%.2d_%d' % (dim, time_stamp + i) t3.pre_exec += ['mkdir -p {0} && cd {0}'.format(cvae_dir)] t3.executable = ['%s/bin/python' % conda_path] # train_cvae.py t3.arguments = [ '%s/train_cvae.py' % cvae_path, '--h5_file', '%s/cvae_input.h5' % agg_path, '--dim', dim ] t3.cpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 4, 'thread_type': 'OpenMP' } t3.gpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 1, 'thread_type': 'CUDA' } # Add the learn task to the learning stage s3.add_tasks(t3) return s3
def generate_interfacing_stage(): s4 = Stage() s4.name = 'scanning' # Scaning for outliers and prepare the next stage of MDs t4 = Task() t4.pre_exec = ['. /sw/summit/python/3.6/anaconda3/5.3.0/etc/profile.d/conda.sh'] t4.pre_exec += ['conda activate %s' % cfg['conda_pytorch']] t4.pre_exec += ['mkdir -p %s/Outlier_search/outlier_pdbs' % cfg['base_path']] t4.pre_exec += ['export models=""; for i in `ls -d %s/CVAE_exps/model-cvae_runs*/`; do if [ "$models" != "" ]; then models=$models","$i; else models=$i; fi; done;cat /dev/null' % cfg['base_path']] t4.pre_exec += ['export LANG=en_US.utf-8', 'export LC_ALL=en_US.utf-8'] t4.pre_exec += ['unset CUDA_VISIBLE_DEVICES', 'export OMP_NUM_THREADS=4'] cmd_cat = 'cat /dev/null' cmd_jsrun = 'jsrun -n %s -a 6 -g 6 -r 1 -c 7' % cfg['node_counts'] #molecules_path = '/gpfs/alpine/world-shared/ven201/tkurth/molecules/' t4.executable = [' %s; %s %s/examples/outlier_detection/run_optics_dist_summit_entk.sh' % (cmd_cat, cmd_jsrun, cfg['molecules_path'])] t4.arguments = ['%s/bin/python' % cfg['conda_pytorch']] t4.arguments += ['%s/examples/outlier_detection/optics.py' % cfg['molecules_path'], '--sim_path', '%s/MD_exps/%s' % (cfg['base_path'], cfg['system_name']), '--pdb_out_path', '%s/Outlier_search/outlier_pdbs' % cfg['base_path'], '--restart_points_path', '%s/Outlier_search/restart_points.json' % cfg['base_path'], '--data_path', '%s/MD_to_CVAE/cvae_input.h5' % cfg['base_path'], '--model_paths', '$models', '--model_type', cfg['model_type'], '--min_samples', 10, '--n_outliers', 500, '--dim1', cfg['residues'], '--dim2', cfg['residues'], '--cm_format', 'sparse-concat', '--batch_size', cfg['batch_size'], '--distributed', '-iw', cfg['init_weights']] t4.cpu_reqs = {'processes' : 1, 'process_type' : None, 'threads_per_process': 12, 'thread_type' : 'OpenMP'} t4.gpu_reqs = {'processes' : 1, 'process_type' : None, 'threads_per_process': 1, 'thread_type' : 'CUDA'} s4.add_tasks(t4) s4.post_exec = func_condition return s4
def describe_MD_pipeline(): p = Pipeline() p.name = 'MD' # MD stage s1 = Stage() s1.name = 'OpenMM' # Each Task() is an OpenMM executable that will run on a single GPU. # Set sleep time for local testing # for i in range(18): task = Task() task.name = 'md' task.pre_exec = [] # task.pre_exec += ['export MINICONDA=/gpfs/alpine/scratch/jdakka/bip178/miniconda'] # task.pre_exec += ['export PATH=$MINICONDA/bin:$PATH'] # task.pre_exec += ['export LD_LIBRARY_PATH=$MINICONDA/lib:$LD_LIBRARY_PATH'] task.pre_exec += ['module load python/2.7.15-anaconda2-5.3.0'] task.pre_exec += ['module load cuda/9.1.85'] task.pre_exec += ['module load gcc/6.4.0'] task.pre_exec += ['source activate openmm'] task.pre_exec += ['cd /gpfs/alpine/scratch/jdakka/bip178/benchmarks/MD_exps/fs-pep/results_2'] task.executable = '/ccs/home/jdakka/.conda/envs/openmm/bin/python' task.arguments = ['run_openmm.py', '-f', '/gpfs/alpine/scratch/jdakka/bip178/benchmarks/MD_exps/fs-pep/pdb/100-fs-peptide-400K.pdb'] task.cpu_reqs = {'processes': 1, 'process_type': None, 'threads_per_process': 1, 'thread_type': None } task.gpu_reqs = {'processes': 1, 'process_type': None, 'threads_per_process': 1, 'thread_type': 'CUDA' } # Add the MD task to the Docking Stage s1.add_tasks(task) # Add MD stage to the MD Pipeline p.add_stages(s1) return p
def generate_ML_tasks(self): """ Function to generate the learning stage """ p = Pipeline() p.name = 'learning' s3 = Stage() s3.name = 'training' # learn task for i in range(self.num_ML): t3 = Task() # https://github.com/radical-collaboration/hyperspace/blob/MD/microscope/experiments/CVAE_exps/train_cvae.py t3.pre_exec = [] t3.pre_exec += ['. /sw/summit/python/2.7/anaconda2/5.3.0/etc/profile.d/conda.sh'] t3.pre_exec += ['module load cuda/10.1.168'] t3.pre_exec += ['conda activate %s' % conda_path] t3.pre_exec += ['export PYTHONPATH=%s/CVAE_exps:$PYTHONPATH' % base_path] t3.pre_exec += ['cd %s' % cvae_path] t3.pre_exec += [f"sleep {i}"] dim = i + 3 t3.executable = ['%s/bin/python' % conda_path] # train_cvae.py t3.arguments = [ '%s/train_cvae.py' % cvae_path, '--h5_file', '%s/cvae_input.h5' % agg_path, '--dim', dim] t3.cpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 4, 'thread_type': 'OpenMP' } t3.gpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 1, 'thread_type': 'CUDA' } # Add the learn task to the learning stage s3.add_tasks(t3) p.add_stages(s3) return p
def generate_MD_tasks(self): """ Function to generate MD tasks. """ p = Pipeline() p.name = "MD" s1 = Stage() s1.name = 'MD' # MD tasks for i in range(self.num_MD): t1 = Task() # https://github.com/radical-collaboration/hyperspace/blob/MD/microscope/experiments/MD_exps/fs-pep/run_openmm.py t1.pre_exec = ['. /sw/summit/python/2.7/anaconda2/5.3.0/etc/profile.d/conda.sh'] t1.pre_exec += ['module load cuda/10.1.168'] t1.pre_exec += ['conda activate %s' % conda_path] t1.pre_exec += ['export PYTHONPATH=%s/MD_exps:$PYTHONPATH' % base_path] t1.pre_exec += ['cd %s' % md_path] # t1.pre_exec += [f"sleep {i}"] t1.executable = ['%s/bin/python' % conda_path] # run_openmm.py t1.arguments = ['%s/run_openmm.py' % md_path] t1.arguments += ['--pdb_file', pdb_file] if top_file: t1.arguments += ['--topol', top_file] t1.arguments += ['--length', 1000] # assign hardware the task t1.cpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 4, 'thread_type': 'OpenMP' } t1.gpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 1, 'thread_type': 'CUDA' } # Add the MD task to the simulating stage s1.add_tasks(t1) p.add_stages(s1) return p
def generate_interfacing_task(self): p = Pipeline() p.name = 'interfacing' s4 = Stage() s4.name = 'scanning' # Scaning for outliers and prepare the next stage of MDs t4 = Task() t4.pre_exec = [] t4.pre_exec += ['. /sw/summit/python/2.7/anaconda2/5.3.0/etc/profile.d/conda.sh'] t4.pre_exec += ['module load cuda/10.1.168'] t4.pre_exec += ['conda activate %s' % conda_path] t4.pre_exec += ['export PYTHONPATH=%s/CVAE_exps:$PYTHONPATH' % base_path] t4.pre_exec += ['cd %s/Outlier_search' % base_path] t4.executable = ['%s/bin/python' % conda_path] t4.arguments = [ 'outlier_locator.py', '--md', md_path, '--cvae', cvae_path, '--pdb', pdb_file, '--ref', ref_pdb_file, '--n_out', self.num_outliers, '--timeout', self.t_timeout] t4.cpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 12, 'thread_type': 'OpenMP' } t4.gpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 1, 'thread_type': 'CUDA' } s4.add_tasks(t4) p.add_stages(s4) return p
def setup_replicas(replicas, min_temp, max_temp, timesteps, basename): writeInputs.writeInputs(max_temp=max_temp, min_temp=min_temp, replicas=replicas, timesteps=timesteps, basename=basename) tar = tarfile.open("input_files.tar", "w") for name in [basename + ".prmtop", basename + ".inpcrd", basename + ".mdin"]: tar.add(name) for r in range(replicas): tar.add('mdin-{replica}-{cycle}'.format(replica=r, cycle=0)) tar.close() for r in range(replicas): os.remove('mdin-{replica}-{cycle}'.format(replica=r, cycle=0)) setup_p = Pipeline() setup_p.name = 'untarPipe' repo = git.Repo('.', search_parent_directories=True) aux_function_path = repo.working_tree_dir untar_stg = Stage() untar_stg.name = 'untarStg' #Untar Task untar_tsk = Task() untar_tsk.name = 'untarTsk' untar_tsk.executable = ['python'] untar_tsk.upload_input_data = ['untar_input_files.py', 'input_files.tar'] untar_tsk.arguments = ['untar_input_files.py', 'input_files.tar'] untar_tsk.cpu_reqs = 1 untar_tsk.post_exec = [] untar_stg.add_tasks(untar_tsk) setup_p.add_stages(untar_stg) global replica_sandbox replica_sandbox='$Pipeline_%s_Stage_%s_Task_%s'%(setup_p.name, untar_stg.name, untar_tsk.name) print replica_sandbox return setup_p
def generate_task(self): task = Task() task.name = self.name task.executable = [NAMD_TI_ANALYSIS] task.arguments = ['-f', '>', self.output] task.cpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 1, 'thread_type': None } links = [ self.input_data([self.extension], **x) for x in self._input_sim._ensemble_product() ] links = [l for link in links for l in link] task.link_input_data.extend(links) task.download_output_data = [self.output_name] return task
def generate_task(self): task = Task() task.name = self.name task.executable = ["tar", "czvfh"] task.arguments = [self.output_name, "*{}".format(self.extension)] task.cpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 1, 'thread_type': None } links = [ self.input_data([self.extension], **x) for x in self._input_sim._ensemble_product() ] links = [l for link in links for l in link] task.link_input_data.extend(links) task.download_output_data = [self.output_name] return task
def generate_task(self): task = Task() task.name = self.name task.pre_exec = [ 'env > env.log', 'export PATH=/home/dakka/miniconda3/bin:$PATH', 'export LD_LIBRARY_PATH=/home/dakka/miniconda3/lib:$LD_LIBRARY_PATH', 'source activate ve_hyperspace' ] task.executable = ['python'] task.arguments = [ 'optimize.py', '--data_path', self.data_path, '--results_dir', self.results_dir ] task.cpu_reqs = { 'processes': self.hyperparameters**2, 'process_type': None, 'threads_per_process': 32, 'thread_type': 'MPI' } task.upload_input_data = [self.optimization_file] return task
def generate_md_stage(self) -> Stage: stage = Stage() stage.name = "MD" cfg = self.cfg.md_stage if self.cur_iteration > 0: outlier_filename = self.restart_points_path(self.cur_iteration - 1) pdb_filenames = get_outlier_pdbs(outlier_filename) else: pdb_filenames = get_initial_pdbs(cfg.run_config.initial_pdb_dir) for i, pdb_filename in zip(range(cfg.num_jobs), cycle(pdb_filenames)): task = Task() task.cpu_reqs = cfg.cpu_reqs.dict() task.gpu_reqs = cfg.gpu_reqs.dict() task.pre_exec = cfg.pre_exec task.executable = cfg.executable task.arguments = cfg.arguments # Set unique output directory name for task dir_prefix = f"md_{self.cur_iteration:03d}_{i:04d}" # Update base parameters cfg.run_config.result_dir = self.experiment_dirs["md_runs"] cfg.run_config.dir_prefix = dir_prefix cfg.run_config.pdb_file = pdb_filename # Write MD yaml to tmp directory to be picked up and moved by MD job cfg_path = self.experiment_dirs["tmp"].joinpath( f"{dir_prefix}.yaml") cfg.run_config.dump_yaml(cfg_path) task.arguments += ["-c", cfg_path] stage.add_tasks(task) return stage
def generate_MD_stage(num_MD=1): """ Function to generate MD stage. """ s1 = Stage() s1.name = 'MD' initial_MD = True outlier_filepath = '%s/Outlier_search/restart_points.json' % cfg['base_path'] if os.path.exists(outlier_filepath): initial_MD = False outlier_file = open(outlier_filepath, 'r') outlier_list = json.load(outlier_file) outlier_file.close() # MD tasks time_stamp = int(time.time()) for i in range(num_MD): t1 = Task() # https://github.com/radical-collaboration/hyperspace/blob/MD/microscope/experiments/MD_exps/fs-pep/run_openmm.py t1.pre_exec = ['. /sw/summit/python/3.6/anaconda3/5.3.0/etc/profile.d/conda.sh'] t1.pre_exec += ['module load cuda/9.1.85'] t1.pre_exec += ['conda activate %s' % cfg['conda_openmm']] t1.pre_exec += ['export PYTHONPATH=%s/MD_exps:%s/MD_exps/MD_utils:$PYTHONPATH' % (cfg['base_path'], cfg['base_path'])] t1.pre_exec += ['cd %s/MD_exps/%s' % (cfg['base_path'], cfg['system_name'])] t1.pre_exec += ['mkdir -p omm_runs_%d && cd omm_runs_%d' % (time_stamp+i, time_stamp+i)] t1.executable = ['%s/bin/python' % cfg['conda_openmm']] # run_openmm.py t1.arguments = ['%s/MD_exps/%s/run_openmm.py' % (cfg['base_path'], cfg['system_name'])] #t1.arguments += ['--topol', '%s/MD_exps/fs-pep/pdb/topol.top' % cfg['base_path']] if 'top_file' in cfg: t1.arguments += ['--topol', cfg['top_file']] # pick initial point of simulation if initial_MD or i >= len(outlier_list): t1.arguments += ['--pdb_file', cfg['pdb_file'] ] elif outlier_list[i].endswith('pdb'): t1.arguments += ['--pdb_file', outlier_list[i]] t1.pre_exec += ['cp %s ./' % outlier_list[i]] elif outlier_list[i].endswith('chk'): t1.arguments += ['--pdb_file', cfg['pdb_file'], '-c', outlier_list[i]] t1.pre_exec += ['cp %s ./' % outlier_list[i]] # how long to run the simulation if initial_MD: t1.arguments += ['--length', cfg['LEN_initial']] else: t1.arguments += ['--length', cfg['LEN_iter']] # assign hardware the task t1.cpu_reqs = {'processes' : 1, 'process_type' : None, 'threads_per_process': 4, 'thread_type' : 'OpenMP'} t1.gpu_reqs = {'processes' : 1, 'process_type' : None, 'threads_per_process': 1, 'thread_type' : 'CUDA'} # Add the MD task to the simulating stage s1.add_tasks(t1) return s1
def test_create_cud_from_task(): """ **Purpose**: Test if the 'create_cud_from_task' function generates a RP ComputeUnitDescription with the complete Task description """ pipeline = 'p1' stage = 's1' task = 't1' placeholder_dict = { pipeline: { stage: { task: '/home/vivek/some_file.txt' } } } t1 = Task() t1.name = 't1' t1.pre_exec = ['module load gromacs'] t1.executable = ['grompp'] t1.arguments = ['hello'] t1.cpu_reqs = {'processes': 4, 'process_type': 'MPI', 'threads_per_process': 1, 'thread_type': 'OpenMP' } t1.gpu_reqs = {'processes': 4, 'process_type': 'MPI', 'threads_per_process': 2, 'thread_type': 'OpenMP' } t1.post_exec = ['echo test'] t1.upload_input_data = ['upload_input.dat'] t1.copy_input_data = ['copy_input.dat'] t1.link_input_data = ['link_input.dat'] t1.copy_output_data = ['copy_output.dat'] t1.download_output_data = ['download_output.dat'] p = Pipeline() p.name = 'p1' s = Stage() s.name = 's1' s.tasks = t1 p.stages = s p._assign_uid('test') cud = create_cud_from_task(t1, placeholder_dict) assert cud.name == '%s,%s,%s,%s,%s,%s' % (t1.uid, t1.name, t1.parent_stage['uid'], t1.parent_stage['name'], t1.parent_pipeline['uid'], t1.parent_pipeline['name']) assert cud.pre_exec == t1.pre_exec # rp returns executable as a string regardless of whether assignment was using string or list assert cud.executable == t1.executable assert cud.arguments == t1.arguments assert cud.cpu_processes == t1.cpu_reqs['processes'] assert cud.cpu_threads == t1.cpu_reqs['threads_per_process'] assert cud.cpu_process_type == t1.cpu_reqs['process_type'] assert cud.cpu_thread_type == t1.cpu_reqs['thread_type'] assert cud.gpu_processes == t1.gpu_reqs['processes'] assert cud.gpu_threads == t1.gpu_reqs['threads_per_process'] assert cud.gpu_process_type == t1.gpu_reqs['process_type'] assert cud.gpu_thread_type == t1.gpu_reqs['thread_type'] assert cud.post_exec == t1.post_exec assert {'source': 'upload_input.dat', 'target': 'upload_input.dat'} in cud.input_staging assert {'source': 'copy_input.dat', 'action': rp.COPY, 'target': 'copy_input.dat'} in cud.input_staging assert {'source': 'link_input.dat', 'action': rp.LINK, 'target': 'link_input.dat'} in cud.input_staging assert {'source': 'copy_output.dat', 'action': rp.COPY, 'target': 'copy_output.dat'} in cud.output_staging assert {'source': 'download_output.dat', 'target': 'download_output.dat'} in cud.output_staging
def init_cycle(self, replicas, replica_cores, python_path, md_executable, exchange_method, min_temp, max_temp, timesteps, basename, pre_exec): # "cycle" = 1 MD stage plus the subsequent exchange computation """ Initial cycle consists of: 1) Create tarball of MD input data 2) Transfer the tarball to pilot sandbox 3) Untar the tarball 4) Run first cycle """ #Initialize Pipeline self._prof.prof('InitTar', uid=self._uid) p = Pipeline() p.name = 'initpipeline' md_dict = dict() #bookkeeping tar_dict = dict() #bookkeeping #Write the input files self._prof.prof('InitWriteInputs', uid=self._uid) writeInputs.writeInputs( max_temp=max_temp, min_temp=min_temp, replicas=replicas, timesteps=timesteps, basename=basename) self._prof.prof('EndWriteInputs', uid=self._uid) self._prof.prof('InitTar', uid=self._uid) #Create Tarball of input data tar = tarfile.open("input_files.tar", "w") for name in [ basename + ".prmtop", basename + ".inpcrd", basename + ".mdin" ]: tar.add(name) for r in range(replicas): tar.add('mdin_{0}'.format(r)) tar.close() #delete all input files outside the tarball for r in range(replicas): os.remove('mdin_{0}'.format(r)) self._prof.prof('EndTar', uid=self._uid) #Create Untar Stage repo = git.Repo('.', search_parent_directories=True) aux_function_path = repo.working_tree_dir untar_stg = Stage() untar_stg.name = 'untarStg' #Untar Task untar_tsk = Task() untar_tsk.name = 'untartsk' untar_tsk.executable = ['python'] untar_tsk.upload_input_data = [ str(aux_function_path)+'/repex/untar_input_files.py', 'input_files.tar' ] untar_tsk.arguments = ['untar_input_files.py', 'input_files.tar'] untar_tsk.cpu_reqs = 1 #untar_tsk.post_exec = [''] untar_stg.add_tasks(untar_tsk) p.add_stages(untar_stg) tar_dict[0] = '$Pipeline_%s_Stage_%s_Task_%s' % ( p.name, untar_stg.name, untar_tsk.name) # First MD stage: needs to be defined separately since workflow is not built from a predetermined order, also equilibration needs to happen first. md_stg = Stage() md_stg.name = 'mdstg0' self._prof.prof('InitMD_0', uid=self._uid) # MD tasks for r in range(replicas): md_tsk = AMBERTask(cores=replica_cores, md_executable=md_executable, pre_exec=pre_exec) md_tsk.name = 'mdtsk-{replica}-{cycle}'.format(replica=r, cycle=0) md_tsk.link_input_data += [ '%s/inpcrd' % tar_dict[0], '%s/prmtop' % tar_dict[0], '%s/mdin_{0}'.format(r) % tar_dict[0] #Use for full temperature exchange ] md_tsk.arguments = [ '-O', '-p', 'prmtop', '-i', 'mdin_{0}'.format(r), '-c', 'inpcrd', '-o', 'out-{replica}-{cycle}'.format(replica=r, cycle=0), '-r', 'restrt'.format(replica=r, cycle=0), #'-r', 'rstrt-{replica}-{cycle}'.format(replica=r,cycle=0), '-x', 'mdcrd-{replica}-{cycle}'.format(replica=r, cycle=0), #'-o', '$NODE_LFS_PATH/out-{replica}-{cycle}'.format(replica=r,cycle=0), #'-r', '$NODE_LFS_PATH/rstrt-{replica}-{cycle}'.format(replica=r,cycle=0), #'-x', '$NODE_LFS_PATH/mdcrd-{replica}-{cycle}'.format(replica=r,cycle=0), '-inf', 'mdinfo_{0}'.format(r) ] md_dict[r] = '$Pipeline_%s_Stage_%s_Task_%s' % ( p.name, md_stg.name, md_tsk.name) md_stg.add_tasks(md_tsk) self.md_task_list.append(md_tsk) #print md_tsk.uid p.add_stages(md_stg) #stage_uids.append(md_stg.uid) # First Exchange Stage ex_stg = Stage() ex_stg.name = 'exstg0' self._prof.prof('InitEx_0', uid=self._uid) # Create Exchange Task ex_tsk = Task() ex_tsk.name = 'extsk0' #ex_tsk.pre_exec = ['module load python/2.7.10'] ex_tsk.executable = [python_path] ex_tsk.upload_input_data = [exchange_method] for r in range(replicas): ex_tsk.link_input_data += ['%s/mdinfo_%s' % (md_dict[r], r)] ex_tsk.pre_exec = ['mv *.py exchange_method.py'] ex_tsk.arguments = ['exchange_method.py', '{0}'.format(replicas), '0'] ex_tsk.cores = 1 ex_tsk.mpi = False ex_tsk.download_output_data = ['exchangePairs_0.dat'] ex_stg.add_tasks(ex_tsk) #task_uids.append(ex_tsk.uid) p.add_stages(ex_stg) self.ex_task_list.append(ex_tsk) #self.ex_task_uids.append(ex_tsk.uid) self.book.append(md_dict) return p
def test_task_exceptions(s,l,i,b): """ **Purpose**: Test if all attribute assignments raise exceptions for invalid values """ t = Task() data_type = [s,l,i,b] for data in data_type: if not isinstance(data,str): with pytest.raises(TypeError): t.name = data with pytest.raises(TypeError): t.path = data with pytest.raises(TypeError): t.parent_stage = data with pytest.raises(TypeError): t.parent_pipeline = data with pytest.raises(TypeError): t.stdout = data with pytest.raises(TypeError): t.stderr = data if not isinstance(data,list): with pytest.raises(TypeError): t.pre_exec = data with pytest.raises(TypeError): t.arguments = data with pytest.raises(TypeError): t.post_exec = data with pytest.raises(TypeError): t.upload_input_data = data with pytest.raises(TypeError): t.copy_input_data = data with pytest.raises(TypeError): t.link_input_data = data with pytest.raises(TypeError): t.move_input_data = data with pytest.raises(TypeError): t.copy_output_data = data with pytest.raises(TypeError): t.download_output_data = data with pytest.raises(TypeError): t.move_output_data = data if not isinstance(data, str) and not isinstance(data, list): with pytest.raises(TypeError): t.executable = data if not isinstance(data, str) and not isinstance(data, unicode): with pytest.raises(ValueError): t.cpu_reqs = { 'processes': 1, 'process_type': data, 'threads_per_process': 1, 'thread_type': None } t.cpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 1, 'thread_type': data } t.gpu_reqs = { 'processes': 1, 'process_type': data, 'threads_per_process': 1, 'thread_type': None } t.gpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 1, 'thread_type': data } if not isinstance(data, int): with pytest.raises(TypeError): t.cpu_reqs = { 'processes': data, 'process_type': None, 'threads_per_process': 1, 'thread_type': None } t.cpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': data, 'thread_type': None } t.gpu_reqs = { 'processes': data, 'process_type': None, 'threads_per_process': 1, 'thread_type': None } t.gpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': data, 'thread_type': None }