def test_wfp_check_processor(): p = Pipeline() s = Stage() t = Task() t.executable = ['/bin/date'] s.add_tasks(t) p.add_stages(s) amgr = Amgr(hostname=hostname, port=port) amgr._setup_mqs() wfp = WFprocessor(sid=amgr._sid, workflow=[p], pending_queue=amgr._pending_queue, completed_queue=amgr._completed_queue, mq_hostname=amgr._mq_hostname, port=amgr._port, resubmit_failed=False) wfp.start_processor() assert wfp.check_processor() wfp.terminate_processor() assert not wfp.check_processor()
def add_md_stg(rid,cycle): #md stg h md_tsk = Task() md_stg = Stage() md_tsk.name = 'mdtsk-{replica}-{cycle}'.format(replica=rid, cycle=cycle) md_tsk.link_input_data += ['%s/inpcrd' %replica_sandbox, '%s/prmtop' %replica_sandbox, '%s/mdin-{replica}-{cycle}'.format(replica=rid, cycle=0) %replica_sandbox] md_tsk.arguments = ['-O', '-i', 'mdin-{replica}-{cycle}'.format(replica=rid, cycle=0), '-p', 'prmtop', '-c', 'inpcrd', '-o', 'out', '-r', '%s/restrt-{replica}-{cycle}'.format(replica=rid, cycle=cycle) %replica_sandbox, '-x', 'mdcrd', '-inf', '%s/mdinfo-{replica}-{cycle}'.format(replica=rid, cycle=cycle) %replica_sandbox] md_tsk.executable = ['/home/scm177/mantel/AMBER/amber14/bin/sander'] md_tsk.cpu_reqs = { 'processes': replica_cores, 'process_type': '', 'threads_per_process': 1, 'thread_type': None } md_tsk.pre_exec = ['export dummy_variable=19', 'echo $SHARED'] md_stg.add_tasks(md_tsk) md_stg.post_exec = { 'condition': md_post, 'on_true': suspend, 'on_false': exchange_stg } return md_stg
def test_wfp_initialization(s, i, b, l): p = Pipeline() st = Stage() t = Task() t.executable = ['/bin/date'] st.add_tasks(t) p.add_stages(st) wfp = WFprocessor(sid='rp.session.local.0000', workflow=set([p]), pending_queue=['pending'], completed_queue=['completed'], mq_hostname=hostname, port=port, resubmit_failed=True) assert len(wfp._uid.split('.')) == 2 assert 'wfprocessor' == wfp._uid.split('.')[0] assert wfp._pending_queue == ['pending'] assert wfp._completed_queue == ['completed'] assert wfp._mq_hostname == hostname assert wfp._port == port assert wfp._wfp_process == None assert wfp._workflow == set([p]) if not isinstance(s, unicode): wfp = WFprocessor(sid=s, workflow=set([p]), pending_queue=l, completed_queue=l, mq_hostname=s, port=i, resubmit_failed=b)
def test_stage_post_exec(): global p1 p1.name = 'p1' s = Stage() s.name = 's1' for t in range(NUM_TASKS): s.add_tasks(create_single_task()) s.post_exec = condition p1.add_stages(s) res_dict = { 'resource': 'local.localhost', 'walltime': 30, 'cpus': 1, } os.environ['RADICAL_PILOT_DBURL'] = MLAB appman = AppManager(rts='radical.pilot', hostname=hostname, port=port) appman.resource_desc = res_dict appman.workflow = [p1] appman.run()
def generate_aggregating_stage(): """ Function to concatenate the MD trajectory (h5 contact map) """ s2 = Stage() s2.name = 'aggregating' # Aggregation task t2 = Task() # https://github.com/radical-collaboration/hyperspace/blob/MD/microscope/experiments/MD_to_CVAE/MD_to_CVAE.py t2.pre_exec = [] t2.pre_exec += [ '. /sw/summit/python/2.7/anaconda2/5.3.0/etc/profile.d/conda.sh' ] t2.pre_exec += ['conda activate rp.copy'] t2.pre_exec += [ 'cd /gpfs/alpine/bip179/scratch/hrlee/hyperspace/microscope/experiments/MD_to_CVAE' ] t2.executable = ['/ccs/home/hrlee/.conda/envs/rp.copy/bin/python' ] # MD_to_CVAE.py t2.arguments = [ '/gpfs/alpine/bip179/scratch/hrlee/hyperspace/microscope/experiments/MD_to_CVAE/MD_to_CVAE.py', '-f', '/gpfs/alpine/bip179/scratch/hrlee/hyperspace/microscope/experiments/MD_exps/fs-pep' ] # Add the aggregation task to the aggreagating stage s2.add_tasks(t2) return s2
def add_ex_stg(rid, cycle): #ex stg here ex_tsk = Task() ex_stg = Stage() ex_tsk.name = 'extsk-{replica}-{cycle}'.format(replica=rid, cycle=cycle) for rid in range(len(waiting_replicas)): ex_tsk.link_input_data += ['%s/mdinfo-{replica}-{cycle}'.format(replica=rid, cycle=self.cycle)%replica_sandbox] ex_tsk.arguments = ['t_ex_gibbs.py', len(waiting_replicas)] #This needs to be fixed ex_tsk.executable = ['python'] ex_tsk.cpu_reqs = { 'processes': 1, 'process_type': '', 'threads_per_process': 1, 'thread_type': None } ex_tsk.pre_exec = ['export dummy_variable=19'] ex_stg.add_tasks(ex_tsk) ex_stg.post_exec = { 'condition': post_ex, 'on_true': terminate_replicas, 'on_false': continue_md } return ex_stg
def generate_agent_stage(self) -> Stage: stage = Stage() stage.name = self.AGENT_STAGE_NAME cfg = self.cfg.agent_stage stage_api = self.api.agent_stage task_idx = 0 output_path = stage_api.task_dir(self.stage_idx, task_idx, mkdir=True) assert output_path is not None # Update base parameters cfg.task_config.experiment_directory = self.cfg.experiment_directory cfg.task_config.stage_idx = self.stage_idx cfg.task_config.task_idx = task_idx cfg.task_config.node_local_path = self.cfg.node_local_path cfg.task_config.output_path = output_path # Write yaml configuration cfg_path = stage_api.config_path(self.stage_idx, task_idx) cfg.task_config.dump_yaml(cfg_path) task = generate_task(cfg) task.arguments += ["-c", cfg_path.as_posix()] stage.add_tasks(task) return stage
def generate_machine_learning_stage(self) -> Stage: stage = Stage() stage.name = self.MACHINE_LEARNING_STAGE_NAME cfg = self.cfg.machine_learning_stage stage_api = self.api.machine_learning_stage task_idx = 0 output_path = stage_api.task_dir(self.stage_idx, task_idx, mkdir=True) assert output_path is not None # Update base parameters cfg.task_config.experiment_directory = self.cfg.experiment_directory cfg.task_config.stage_idx = self.stage_idx cfg.task_config.task_idx = task_idx cfg.task_config.node_local_path = self.cfg.node_local_path cfg.task_config.output_path = output_path cfg.task_config.model_tag = stage_api.unique_name(output_path) if self.stage_idx > 0: # Machine learning should use model selection API cfg.task_config.init_weights_path = None # Write yaml configuration cfg_path = stage_api.config_path(self.stage_idx, task_idx) cfg.task_config.dump_yaml(cfg_path) task = generate_task(cfg) task.arguments += ["-c", cfg_path.as_posix()] stage.add_tasks(task) return stage
def test_pipeline_pass_uid(): p = Pipeline() p._uid = 'test' p.name = 'p1' s1 = Stage() s2 = Stage() p.add_stages([s1, s2]) p._pass_uid() assert s1.parent_pipeline['uid'] == p.uid assert s1.parent_pipeline['name'] == p.name assert s2.parent_pipeline['uid'] == p.uid assert s2.parent_pipeline['name'] == p.name
def test_pipeline_validate_entities(t, l, i, b, se): p = Pipeline() data_type = [t, l, i, b, se] for data in data_type: with pytest.raises(TypeError): p._validate_entities(data) s = Stage() assert isinstance(p._validate_entities(s), list) s1 = Stage() s2 = Stage() assert [s1, s2] == p._validate_entities([s1, s2])
def get_pipeline(tasks): # Create a Pipeline object p = Pipeline() # Create a Stage 1 s1 = Stage() # Create a Task object according to the app_name t1 = Task() t1.pre_exec = ['module load gromacs/5.0/INTEL-140-MVAPICH2-2.0'] t1.executable = app_coll['grompp']['executable'] t1.arguments = app_coll['grompp']['arguments'] t1.cores = app_coll['grompp']['cores'] t1.link_input_data = [ '$SHARED/grompp.mdp > grompp.mdp', '$SHARED/input.gro > input.gro', '$SHARED/topol.top > topol.top' ] # Add the Task to the Stage s1.add_tasks(t1) # Add Stage to the Pipeline p.add_stages(s1) # Create a Stage 2 s2 = Stage() for cnt in range(tasks): # Create a Task object according to the app_name t2 = Task() t2.pre_exec = ['module load gromacs/5.0/INTEL-140-MVAPICH2-2.0','export OMP_NUM_THREADS=1'] t2.executable = app_coll['mdrun']['executable'] t2.arguments = app_coll['mdrun']['arguments'] t2.cores = app_coll['mdrun']['cores'] t2.copy_input_data = ['$Pipeline_%s_Stage_%s_Task_%s/topol.tpr'%(p.uid, s1.uid,t1.uid)] # Add the Task to the Stage s2.add_tasks(t2) # Add Stage to the Pipeline p.add_stages(s2) return p
def test_sync_with_master(self, mocked_init, mocked_Logger, mocked_Profiler): # -------------------------------------------------------------------------- # def component_execution(packets, conn_params, queue): tmgr = BaseTmgr(None, None, None, None, None, None) tmgr._log = mocked_Logger tmgr._prof = mocked_Profiler mq_connection2 = pika.BlockingConnection(rmq_conn_params) mq_channel2 = mq_connection2.channel() for obj_type, obj, in packets: tmgr._sync_with_master(obj, obj_type, mq_channel2, conn_params, queue) if mq_channel2.is_open: mq_channel2.close() task = Task() task.parent_stage = {'uid':'stage.0000', 'name': 'stage.0000'} packets = [('Task', task)] stage = Stage() stage.parent_pipeline = {'uid':'pipe.0000', 'name': 'pipe.0000'} packets.append(('Stage', stage)) hostname = os.environ.get('RMQ_HOSTNAME', 'localhost') port = int(os.environ.get('RMQ_PORT', '5672')) username = os.environ.get('RMQ_USERNAME','guest') password = os.environ.get('RMQ_PASSWORD','guest') credentials = pika.PlainCredentials(username, password) rmq_conn_params = pika.ConnectionParameters(host=hostname, port=port, credentials=credentials) mq_connection = pika.BlockingConnection(rmq_conn_params) mq_channel = mq_connection.channel() mq_channel.queue_declare(queue='master') master_thread = mt.Thread(target=component_execution, name='tmgr_sync', args=(packets, rmq_conn_params, 'master')) master_thread.start() time.sleep(1) try: while packets: packet = packets.pop(0) _, _, body = mq_channel.basic_get(queue='master') msg = json.loads(body) self.assertEqual(msg['object'], packet[1].to_dict()) self.assertEqual(msg['type'], packet[0]) except Exception as ex: print(ex) print(json.loads(body)) master_thread.join() mq_channel.queue_delete(queue='master') mq_channel.close() mq_connection.close() raise ex else: master_thread.join() mq_channel.queue_delete(queue='master') mq_channel.close() mq_connection.close()
def test_amgr_synchronizer(): amgr = Amgr(hostname=hostname, port=port) amgr._setup_mqs() p = Pipeline() s = Stage() # Create and add 10 tasks to the stage for cnt in range(10): t = Task() t.executable = 'some-executable-%s' % cnt s.add_tasks(t) p.add_stages(s) p._assign_uid(amgr._sid) p._validate() amgr.workflow = [p] sid = 'test.0016' rmgr = BaseRmgr({}, sid, None, {}) tmgr = BaseTmgr(sid=sid, pending_queue=['pending-1'], completed_queue=['completed-1'], rmgr=rmgr, mq_hostname=hostname, port=port, rts=None) amgr._rmgr = rmgr rmgr._task_manager = tmgr for t in p.stages[0].tasks: assert t.state == states.INITIAL assert p.stages[0].state == states.INITIAL assert p.state == states.INITIAL # Start the synchronizer method in a thread amgr._terminate_sync = mt.Event() sync_thread = mt.Thread(target=amgr._synchronizer, name='synchronizer-thread') sync_thread.start() # Start the synchronizer method in a thread proc = mp.Process(target=func_for_synchronizer_test, name='temp-proc', args=(amgr._sid, p, tmgr)) proc.start() proc.join() amgr._terminate_sync.set() sync_thread.join() for t in p.stages[0].tasks: assert t.state == states.COMPLETED
def create_inversion_dict_stage(cmt_file_db, param_path, task_counter): """Creates stage for the creation of the inversion files. This stage is tiny, but required before the actual inversion. :param cmt_file_db: :param param_path: :param task_counter: :return: """ # Get database parameter path databaseparam_path = os.path.join(param_path, "Database/DatabaseParameters.yml") # Load Parameters DB_params = read_yaml_file(databaseparam_path) # Earthquake specific database parameters: Dir and Cid Cdir, Cid = get_Centry_path(DB_params["databasedir"], cmt_file_db) # Function inv_dict_func = os.path.join(bin_path, "write_inversion_dicts.py") # Create Process Paths Stage (CPP) # Create a Stage object inv_dict_stage = Stage() inv_dict_stage.name = "Creating" # Create Task inv_dict_task = Task() # This way the task gets the name of the path file inv_dict_task.name = "Inversion-Dictionaries" inv_dict_task.pre_exec = [ # Conda activate DB_params["conda-activate"] ] inv_dict_task.executable = [DB_params["bin-python"]] # Assign exec # to the task inv_dict_task.arguments = [ inv_dict_func, "-f", cmt_file_db, "-p", param_path ] # In the future maybe to database dir as a total log? inv_dict_task.stdout = os.path.join( "%s" % Cdir, "logs", "stdout.pipeline_%s.task_%s.%s" % (Cid, str(task_counter).zfill(4), inv_dict_task.name)) inv_dict_task.stderr = os.path.join( "%s" % Cdir, "logs", "stderr.pipeline_%s.task_%s.%s" % (Cid, str(task_counter).zfill(4), inv_dict_task.name)) inv_dict_stage.add_tasks(inv_dict_task) task_counter += 1 return inv_dict_stage, task_counter
def generate_stage(self): s = Stage() s.name = self.name s.add_tasks( {self.generate_task(**x) for x in self._ensemble_product()}) return s
def generate_pipeline(nid): # Create a Pipeline object p = Pipeline() p.name = 'p%s' % nid # Create a Stage object s1 = Stage() s1.name = 's1' # Create a Task object which creates a file named 'output.txt' of size 1 MB t1 = Task() t1.name = 't2' t1.executable = ['/bin/echo'] t1.arguments = ['hello'] # Add the Task to the Stage s1.add_tasks(t1) # Add Stage to the Pipeline p.add_stages(s1) # Create another Stage object to hold character count tasks s2 = Stage() s2.name = 's2' s2_task_uids = [] for cnt in range(10): # Create a Task object t2 = Task() t2.name = 't%s' % (cnt + 1) t2.executable = ['/bin/echo'] t2.arguments = ['world'] # Copy data from the task in the first stage to the current task's location t2.copy_input_data = [ '$Pipeline_%s_Stage_%s_Task_%s/output.txt' % (p.name, s1.name, t1.name)] # Add the Task to the Stage s2.add_tasks(t2) s2_task_uids.append(t2.name) # Add Stage to the Pipeline p.add_stages(s2) return p
def main(): cmd = "{0} 'ls {1}'".format(ssh, dir_) p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) out, _ = p.communicate() out = out.decode('utf-8').strip().split(linesep) fullpaths = [op.join(dir_, p) for p in out] print(fullpaths) # Start radical entk pipeline p = Pipeline() for i in range(iterations): s = Stage() for fp in fullpaths: t = Task() t.name = 'Incrementation {}'.format(i) t.pre_exec = [ 'source /home/vhayot/miniconda3/etc/profile.d/conda.sh', 'conda activate radenv' ] t.executable = 'python /home/vhayot/inc.py' if i == 0: t.arguments = [fp, out_dir, i] else: # Note: assuming all data is accessible through shared dir # radical entk functions without sharedfs, however t.arguments = [ op.join(out_dir, "it-{0}-{1}".format(i - 1, op.basename(fp))), out_dir, i ] s.add_tasks(t) # Create a new stage everytime there's a dependency p.add_stages(s) appman = AppManager(hostname=hostname, port=port) appman.resource_desc = { 'resource': 'xsede.bridges', 'walltime': 20, 'cpus': 5, 'project': 'mc3bggp', 'schema': 'gsissh' } appman.workflow = set([p]) appman.run()
def test_wfp_workflow_incomplete(): p = Pipeline() s = Stage() t = Task() t.executable = ['/bin/date'] s.add_tasks(t) p.add_stages(s) amgr = Amgr(hostname=hostname, port=port) amgr._setup_mqs() wfp = WFprocessor(sid=amgr._sid, workflow=[p], pending_queue=amgr._pending_queue, completed_queue=amgr._completed_queue, mq_hostname=amgr._mq_hostname, port=amgr._port, resubmit_failed=False) wfp._initialize_workflow() assert wfp.workflow_incomplete() amgr.workflow = [p] profiler = ru.Profiler(name='radical.entk.temp') p.stages[0].state == states.SCHEDULING p.state == states.SCHEDULED for t in p.stages[0].tasks: t.state = states.COMPLETED import json import pika task_as_dict = json.dumps(t.to_dict()) mq_connection = pika.BlockingConnection( pika.ConnectionParameters(host=amgr._mq_hostname, port=amgr._port)) mq_channel = mq_connection.channel() mq_channel.basic_publish(exchange='', routing_key='%s-completedq-1' % amgr._sid, body=task_as_dict) amgr._terminate_sync = Event() sync_thread = Thread(target=amgr._synchronizer, name='synchronizer-thread') sync_thread.start() proc = Process(target=func_for_dequeue_test, name='temp-proc', args=(wfp, )) proc.start() proc.join() amgr._terminate_sync.set() sync_thread.join() assert not wfp.workflow_incomplete()
def test_wfp_dequeue(): p = Pipeline() s = Stage() t = Task() t.executable = '/bin/date' s.add_tasks(t) p.add_stages(s) amgr = Amgr(hostname=hostname, port=port) amgr._setup_mqs() wfp = WFprocessor(sid=amgr._sid, workflow=[p], pending_queue=amgr._pending_queue, completed_queue=amgr._completed_queue, mq_hostname=amgr._hostname, port=amgr._port, resubmit_failed=False) wfp.initialize_workflow() assert p.state == states.INITIAL assert p.stages[0].state == states.INITIAL for t in p.stages[0].tasks: assert t.state == states.INITIAL p.state == states.SCHEDULED p.stages[0].state == states.SCHEDULING for t in p.stages[0].tasks: t.state = states.COMPLETED task_as_dict = json.dumps(t.to_dict()) mq_connection = pika.BlockingConnection( pika.ConnectionParameters(host=amgr._hostname, port=amgr._port)) mq_channel = mq_connection.channel() mq_channel.basic_publish(exchange='', routing_key='%s' % amgr._completed_queue[0], body=task_as_dict) wfp.start_processor() th = mt.Thread(target=func_for_dequeue_test, name='temp-proc', args=(p, )) th.start() th.join() wfp.terminate_processor() assert p.state == states.DONE assert p.stages[0].state == states.DONE for t in p.stages[0].tasks: assert t.state == states.DONE
def test_state_order(): """ **Purpose**: Test if the Pipeline, Stage and Task are assigned their states in the correct order """ def create_single_task(): t1 = Task() t1.name = 'simulation' t1.executable = ['/bin/date'] t1.copy_input_data = [] t1.copy_output_data = [] return t1 p1 = Pipeline() p1.name = 'p1' s = Stage() s.name = 's1' s.tasks = create_single_task() s.add_tasks(create_single_task()) p1.add_stages(s) res_dict = { 'resource': 'local.localhost', 'walltime': 5, 'cpus': 1, 'project': '' } os.environ['RADICAL_PILOT_DBURL'] = MLAB os.environ['RP_ENABLE_OLD_DEFINES'] = 'True' appman = Amgr(hostname=hostname, port=port) appman.resource_desc = res_dict appman.workflow = [p1] appman.run() p_state_hist = p1.state_history assert p_state_hist == ['DESCRIBED', 'SCHEDULING', 'DONE'] s_state_hist = p1.stages[0].state_history assert s_state_hist == ['DESCRIBED', 'SCHEDULING', 'SCHEDULED', 'DONE'] tasks = p1.stages[0].tasks for t in tasks: t_state_hist = t.state_history assert t_state_hist == ['DESCRIBED', 'SCHEDULING', 'SCHEDULED', 'SUBMITTING', 'SUBMITTED', 'EXECUTED', 'DEQUEUEING', 'DEQUEUED', 'DONE']
def test_pipeline_stage_addition(): p = Pipeline() s1 = Stage() t = Task() t.executable = '/bin/date' s1.tasks = t s2 = Stage() t = Task() t.executable = '/bin/date' s2.tasks = t p.add_stages([s1, s2]) assert type(p.stages) == list assert p._stage_count == 2 assert p._cur_stage == 1 assert p.stages[0] == s1 assert p.stages[1] == s2
def describe_MD_pipline(): p = Pipeline() p.name = 'MD' # Docking stage s1 = Stage() s1.name = 'Docking' # Docking task t1 = Task() t1.executable = ['sleep'] t1.arguments = ['30'] # Add the Docking task to the Docking Stage s1.add_tasks(t1) # Add Docking stage to the pipeline p.add_stages(s1) # MD stage s2 = Stage() s2.name = 'Simulation' # Each Task() is an OpenMM executable that will run on a single GPU. # Set sleep time for local testing for i in range(6): t2 = Task() t2.executable = ['sleep'] t2.arguments = ['60'] # Add the MD task to the Docking Stage s2.add_tasks(t2) # Add post-exec to the Stage s2.post_exec = { 'condition': func_condition, 'on_true': func_on_true, 'on_false': func_on_false } # Add MD stage to the MD Pipeline p.add_stages(s2) return p
def generate_pipeline(): # Create a Pipeline object p = Pipeline() p.name = 'p1' # Create a Stage object s1 = Stage() s1.name = 's1' # Create a Task object which creates a file named 'output.txt' of size 1 MB t1 = Task() t1.name = 't1' t1.executable = ['/bin/echo'] t1.arguments = ['"Hello World"'] t1.stdout = 'temp.txt' # Add the Task to the Stage s1.add_tasks(t1) # Add Stage to the Pipeline p.add_stages(s1) # Create a Stage object s2 = Stage() s2.name = 's2' # Create a Task object which creates a file named 'output.txt' of size 1 MB t2 = Task() t2.name = 't2' t2.executable = ['/bin/cat'] t2.arguments = [ '$Pipeline_%s_Stage_%s_Task_%s/temp.txt' % (p.name, s1.name, t1.name) ] t2.stdout = 'output.txt' t2.download_output_data = ['output.txt'] # Add the Task to the Stage s2.add_tasks(t2) # Add Stage to the Pipeline p.add_stages(s2) return p
def generate_ML_stage(num_ML=1): """ Function to generate the learning stage """ s3 = Stage() s3.name = 'learning' # learn task for i in range(num_ML): t3 = Task() # https://github.com/radical-collaboration/hyperspace/blob/MD/microscope/experiments/CVAE_exps/train_cvae.py t3.pre_exec = [] t3.pre_exec = ['module reset'] t3.pre_exec += [ '. /sw/summit/python/2.7/anaconda2/5.3.0/etc/profile.d/conda.sh' ] t3.pre_exec += ['module load cuda/9.1.85'] t3.pre_exec += ['conda activate rp.copy'] t3.pre_exec += ['export CUDA_VISIBLE_DEVICES=0'] t3.pre_exec += [ 'export PYTHONPATH=/gpfs/alpine/scratch/hrlee/bip179/hyperspace/microscope/experiments/CVAE_exps:$PYTHONPATH' ] t3.pre_exec += [ 'cd /gpfs/alpine/scratch/hrlee/bip179/hyperspace/microscope/experiments/CVAE_exps' ] time_stamp = int(time.time()) dim = i + 3 cvae_dir = 'cvae_runs_%.2d_%d' % (dim, time_stamp) t3.pre_exec += ['mkdir -p {0} && cd {0}'.format(cvae_dir)] t3.executable = ['/ccs/home/hrlee/.conda/envs/rp.copy/bin/python' ] # train_cvae.py t3.arguments = [ '/gpfs/alpine/bip179/scratch/hrlee/hyperspace/microscope/experiments/CVAE_exps/train_cvae.py', '-f', '/gpfs/alpine/bip179/scratch/hrlee/hyperspace/microscope/experiments/MD_to_CVAE/cvae_input.h5', '-d', dim ] t3.cpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 4, 'thread_type': 'OpenMP' } t3.gpu_reqs = { 'processes': 1, 'process_type': None, 'threads_per_process': 1, 'thread_type': 'CUDA' } # Add the learn task to the learning stage s3.add_tasks(t3) time.sleep(1) return s3
def test_stage_to_dict(): s = Stage() d = s.to_dict() assert d == {'uid': 'stage.0000', 'name': None, 'state': states.INITIAL, 'state_history': [states.INITIAL], 'parent_pipeline': {'uid': None, 'name': None}}
def generate_pipeline(): def func_condition(): global CUR_NEW_STAGE, MAX_NEW_STAGE if CUR_NEW_STAGE <= MAX_NEW_STAGE: func_on_true() func_on_false() def func_on_true(): global CUR_NEW_STAGE CUR_NEW_STAGE += 1 s = Stage() for i in range(10): t = Task() t.executable = '/bin/sleep' t.arguments = ['30'] s.add_tasks(t) # Add post-exec to the Stage s.post_exec = func_condition p.add_stages(s) def func_on_false(): print('Done') # Create a Pipeline object p = Pipeline() # Create a Stage object s1 = Stage() for i in range(10): t1 = Task() t1.executable = 'sleep' t1.arguments = ['30'] # Add the Task to the Stage s1.add_tasks(t1) # Add post-exec to the Stage s1.post_exec = func_condition # Add Stage to the Pipeline p.add_stages(s1) return p
def test_amgr_synchronizer(): logger = ru.get_logger('radical.entk.temp_logger') profiler = ru.Profiler(name='radical.entk.temp') amgr = Amgr(hostname=hostname, port=port) mq_connection = pika.BlockingConnection(pika.ConnectionParameters(host=hostname, port=port)) mq_channel = mq_connection.channel() amgr._setup_mqs() p = Pipeline() s = Stage() # Create and add 100 tasks to the stage for cnt in range(100): t = Task() t.executable = ['some-executable-%s' % cnt] s.add_tasks(t) p.add_stages(s) p._assign_uid(amgr._sid) p._validate() amgr.workflow = [p] for t in p.stages[0].tasks: assert t.state == states.INITIAL assert p.stages[0].state == states.INITIAL assert p.state == states.INITIAL # Start the synchronizer method in a thread amgr._terminate_sync = Event() sync_thread = Thread(target=amgr._synchronizer, name='synchronizer-thread') sync_thread.start() # Start the synchronizer method in a thread proc = Process(target=func_for_synchronizer_test, name='temp-proc', args=(amgr._sid, p, logger, profiler)) proc.start() proc.join() for t in p.stages[0].tasks: assert t.state == states.SCHEDULING assert p.stages[0].state == states.SCHEDULING assert p.state == states.SCHEDULING amgr._terminate_sync.set() sync_thread.join()
def write_sources(cmt_file_db, param_path, task_counter): """ This function creates a stage that modifies the CMTSOLUTION files before the simulations are run. :param cmt_file_db: cmtfile in the database :param param_path: path to parameter file directory :param task_counter: total task count up until now in pipeline :return: EnTK Stage """ # Get Database parameters databaseparam_path = os.path.join(param_path, "Database/DatabaseParameters.yml") DB_params = read_yaml_file(databaseparam_path) # Earthquake specific database parameters: Dir and Cid Cdir, Cid = get_Centry_path(DB_params["databasedir"], cmt_file_db) # Path to function write_source_func = os.path.join(bin_path, "write_sources.py") # Create a Stage object w_sources = Stage() w_sources.name = "Write-Sources" # Create Task for stage w_sources_t = Task() w_sources_t.name = "Task-Sources" w_sources_t.pre_exec = [ # Conda activate DB_params["conda-activate"] ] w_sources_t.executable = DB_params["bin-python"] # # Assign executable # to the task w_sources_t.arguments = [write_source_func, cmt_file_db] # In the future maybe to database dir as a total log? w_sources_t.stdout = os.path.join( "%s" % Cdir, "logs", "stdout.pipeline_%s.task_%s.%s" % (Cid, str(task_counter).zfill(4), w_sources_t.name)) w_sources_t.stderr = os.path.join( "%s" % Cdir, "logs", "stderr.pipeline_%s.task_%s.%s" % (Cid, str(task_counter).zfill(4), w_sources_t.name)) # Add Task to the Stage w_sources.add_tasks(w_sources_t) task_counter += 1 return w_sources, task_counter
def create_process_path_files(cmt_file_db, param_path, task_counter): """This function creates the path files used for processing both synthetic and observed data in ASDF format, as well as the following windowing procedure. :param cmt_file_db: cmtfile in the database :param param_path: path to parameter file directory :param pipelinedir: path to pipeline directory :return: EnTK Stage """ # Get database parameter path databaseparam_path = os.path.join(param_path, "Database/DatabaseParameters.yml") # Load Parameters DB_params = read_yaml_file(databaseparam_path) # Earthquake specific database parameters: Dir and Cid Cdir, Cid = get_Centry_path(DB_params["databasedir"], cmt_file_db) # Process path function create_process_path_bin = os.path.join(bin_path, "create_path_files.py") # Create Process Paths Stage (CPP) # Create a Stage object cpp = Stage() cpp.name = "CreateProcessPaths" # Create Task cpp_t = Task() cpp_t.name = "CPP-Task" cpp_t.pre_exec = [ # Conda activate DB_params["conda-activate"] ] cpp_t.executable = DB_params["bin-python"] # Assign executable # to the task cpp_t.arguments = [create_process_path_bin, cmt_file_db] # In the future maybe to database dir as a total log? cpp_t.stdout = os.path.join( "%s" % Cdir, "logs", "stdout.pipeline_%s.task_%s.%s" % (Cid, str(task_counter).zfill(4), cpp_t.name)) cpp_t.stderr = os.path.join( "%s" % Cdir, "logs", "stderr.pipeline_%s.task_%s.%s" % (Cid, str(task_counter).zfill(4), cpp_t.name)) task_counter += 1 cpp.add_tasks(cpp_t) return cpp, task_counter
def create_inversion_stage(cmt_file_db, param_path, task_counter): """Creates inversion stage. :param cmt_file_db: :param param_path: :return: """ # Get database parameter path databaseparam_path = os.path.join(param_path, "Database/DatabaseParameters.yml") # Load Parameters DB_params = read_yaml_file(databaseparam_path) # Earthquake specific database parameters: Dir and Cid Cdir, Cid = get_Centry_path(DB_params["databasedir"], cmt_file_db) # Function inversion_func = os.path.join(bin_path, "inversion.py") # Create a Stage object inversion_stage = Stage() inversion_stage.name = "CMT3D" # Create Task inversion_task = Task() # This way the task gets the name of the path file inversion_task.name = "Inversion" inversion_task.pre_exec = [ # Conda activate DB_params["conda-activate"] ] inversion_task.executable = DB_params["bin-python"] # Assign exec # to the task inversion_task.arguments = [ inversion_func, "-f", cmt_file_db, "-p", param_path ] # In the future maybe to database dir as a total log? inversion_task.stdout = os.path.join( "%s" % Cdir, "logs", "stdout.pipeline_%s.task_%s.%s" % (Cid, str(task_counter).zfill(4), inversion_task.name)) inversion_task.stderr = os.path.join( "%s" % Cdir, "logs", "stderr.pipeline_%s.task_%s.%s" % (Cid, str(task_counter).zfill(4), inversion_task.name)) inversion_stage.add_tasks(inversion_task) return inversion_stage