def add_ex_stg(rid, cycle): # ex stg here ex_tsk = re.Task() ex_stg = re.Stage() ex_tsk.name = 'extsk-{replica}-{cycle}'.format(replica=rid, cycle=cycle) for rid in range(len(waiting_replicas)): ex_tsk.link_input_data += [ '%s/mdinfo-{replica}-{cycle}'.format( replica=rid, cycle=self.cycle) % replica_sandbox ] ex_tsk.arguments = ['t_ex_gibbs.py', len(waiting_replicas) ] # This needs to be fixed ex_tsk.executable = ['python'] ex_tsk.cpu_reqs = { 'processes': 1, 'process_type': '', 'threads_per_process': 1, 'thread_type': None } ex_stg.add_tasks(ex_tsk) ex_stg.post_exec = { 'condition': post_ex, 'on_true': terminate_replicas, 'on_false': continue_md } return ex_stg
def add_ex_stage(self, exchange_list): self._log.debug('=== %s add ex: %s', self.rid, [r.rid for r in exchange_list]) self._ex_list = exchange_list self._res_list = exchange_list task = re.Task() task.name = 'extsk' task.executable = ['/home/scm177/VirtualEnvs/Env_RepEx/bin/python'] task.upload_input_data = ['t_ex_gibbs.py'] task.arguments = ['t_ex_gibbs.py', self._sbox] for replica in exchange_list: rid = replica.rid cycle = replica.cycle task.link_input_data.append( '%s/mdinfo-%s-%s' % (self._sbox, rid, cycle - 1)) # % (self._sbox, rid, cycle)) stage = re.Stage() stage.add_tasks(task) stage.post_exec = self.check_resume #_after_ex self.add_stages(stage)
def __init__(self): self.set_argparse() self._set_rmq() self.am = entk.AppManager(hostname=self.rmq_hostname, port=self.rmq_port) self.p = entk.Pipeline() self.s = entk.Stage()
def add_md_stage(self): rid = self._rid cycle = self._cycle sbox = self._sbox cores = self._cores exe = self._exe self._log.debug('=== %s add md (cycle %s)', rid, cycle) task = re.Task() task.name = 'mdtsk-%s-%s' % (rid, cycle) ## task.link_input_data = ['%s/inpcrd > inpcrd-%s-%s' % (sbox, rid, cycle), ## '%s/prmtop' % (sbox), ## '%s/mdin-%s-%s > mdin' % (sbox, rid, cycle)] task.arguments = [ '-O', '-i', 'mdin', '-p', 'prmtop', '-c', 'inpcrd-%s-%s' % (rid, cycle), '-o', 'out', '-x', 'mdcrd', '-r', '%s/inpcrd-%s-%s' % (sbox, rid, cycle), '-inf', '%s/mdinfo-%s-%s' % (sbox, rid, cycle) ] task.executable = exe task.cpu_reqs = {'processes': cores} task.pre_exec = ['echo $SHARED'] stage = re.Stage() stage.add_tasks(task) stage.post_exec = self.after_md self.add_stages(stage)
def generate_pipeline(master=False): global pipes if master: def func_post_1(): for p in pipes[1:]: p.suspend() def func_post_2(): for p in pipes[1:]: p.resume() # return list of resumed pipeline IDs return [p.uid for p in pipes[1:]] else: def func_post_1(): pass def func_post_2(): pass # -------------------------------------------------------------------------- # create a pipeline, stage and tasks t1 = re.Task() t1.executable = '/bin/sleep' if master: t1.arguments = [' 1'] else : t1.arguments = ['10'] s1 = re.Stage() s1.add_tasks(t1) s1.post_exec = func_post_1 t2 = re.Task() t2.executable = '/bin/sleep' t2.arguments = ['1'] s2 = re.Stage() s2.add_tasks(t2) s2.post_exec = func_post_2 p = re. Pipeline() p.add_stages(s1) p.add_stages(s2) return p
def add_md_stage(self): self._log.debug('=== %s add md', self.rid) task = re.Task(from_dict=self._workload['md']) # task.name = 'mdtsk-%s-%s' % (self.rid, self.cycle) stage = re.Stage() stage.add_tasks(task) stage.post_exec = self.check_exchange self.add_stages(stage)
def generate_discover_pipe(self, filetype='csv', img_ftype='tif'): ''' This function takes as an input paths on Bridges and returns a pipeline that will provide a file for all the images that exist in that path. ''' pipeline = re.Pipeline() pipeline.name = 'Disc' stage = re.Stage() stage.name = 'Disc.S0' if self._paths is None: raise RuntimeError('Images paths are not set.') # Create the module load list modules_load = list() if self._modules: for module in self._modules: tmp_load = 'module load %s' % module modules_load.append(tmp_load) tmp_pre_execs = ['unset PYTHONPATH'] if self._pre_execs: tmp_pre_execs = tmp_pre_execs + modules_load + self._pre_execs else: tmp_pre_execs = tmp_pre_execs + modules_load for i in range(len(self._paths)): task = re.Task() task.name = 'Disc.T%d' % i task.pre_exec = tmp_pre_execs task.executable = 'python' # Assign executable to the task task.arguments = [ 'image_disc.py', '%s' % self._paths[i], '--image_ftype=%s' % img_ftype, '--filename=images%d' % i, '--filetype=%s' % filetype, '--filesize' ] task.download_output_data = ['images%d.csv' % i] task.upload_input_data = [ os.path.dirname(os.path.abspath(__file__)) + '/image_disc.py' ] task.cpu_reqs = { 'cpu_processes': 1, 'cpu_process_type': '', 'cpu_threads': 1, 'cpu_thread_type': 'OpenMP' } stage.add_tasks(task) # Add Stage to the Pipeline pipeline.add_stages(stage) return pipeline
def add_md_stage(self): self._log.debug('=== %s add md', self.rid) task = re.Task() task.name = 'mdtsk-%s-%s' % (self.rid, self.cycle) task.executable = 'sleep' task.arguments = [str(random.randint(0, 20) / 10.0)] stage = re.Stage() stage.add_tasks(task) stage.post_exec = self.check_exchange self.add_stages(stage)
def add_ex_stage(self, exchange_list, ex_alg): self._log.debug('=== %s add ex: %s', self.rid, [r.rid for r in exchange_list]) self._ex_list = exchange_list task = re.Task(from_dict=self._workload['ex']) task.arguments = [ex_alg, len(exchange_list), self._cycle] task.upload_input_data = [ex_alg] stage = re.Stage() stage.add_tasks(task) stage.post_exec = self.check_resume self.add_stages(stage)
def add_ex_stage(self, exchange_list): self._log.debug('=== %s add ex: %s', self.rid, [r.rid for r in exchange_list]) self._ex_list = exchange_list task = re.Task() task.name = 'extsk' task.executable = 'date' stage = re.Stage() stage.add_tasks(task) stage.post_exec = self.check_resume self.add_stages(stage)
def _generate_pipeline(self, name, pre_execs, image, gpu_id): ''' This function creates a pipeline for an image that will be analyzed. :Arguments: :name: Pipeline name, str :image: image path, str :model_path: Path to the model file, str :model_arch: Prediction Model Architecture, str :model_name: Prediction Model Name, str :hyperparam_set: Which hyperparameter set to use, str ''' # Create a Pipeline object entk_pipeline = re.Pipeline() entk_pipeline.name = name # Create a Stage object stage0 = re.Stage() stage0.name = '%s-S0' % (name) # Create Task 1, training task1 = re.Task() task1.name = '%s-T0' % stage0.name task1.pre_exec = pre_execs task1.executable = 'iceberg_penguins.detect' # Assign task executable # Assign arguments for the task executable task1.arguments = ['--gpu_ids', gpu_id, '--name', self._model_name, '--epoch', self._epoch, '--checkpoints_dir', self._model_path, '--output', self._output_path, '--testset', 'GE', '--input_im', image.split('/')[-1]] task1.link_input_data = ['%s' % image] task1.cpu_reqs = {'cpu_processes': 1, 'cpu_threads': 1, 'cpu_process_type': None, 'cpu_thread_type': 'OpenMP'} task1.gpu_reqs = {'gpu_processes': 1, 'gpu_threads': 1, 'gpu_process_type': None, 'gpu_thread_type': 'OpenMP'} # Download resuting images # task1.download_output_data = ['%s/ > %s' % (image.split('/')[-1]. # split('.')[0], # image.split('/')[-1])] # task1.tag = task0.name stage0.add_tasks(task1) # Add Stage to the Pipeline entk_pipeline.add_stages(stage0) return entk_pipeline
def setup(self): # prepare input for all replicas writeInputs.writeInputs(max_temp=self._max_temp, min_temp=self._min_temp, replicas=self._en_size, timesteps=self._timesteps, basename=self._basename) # and tar it up tar = tarfile.open("input_files.tar", "w") for name in [self._basename + ".prmtop", self._basename + ".inpcrd", self._basename + ".mdin"]: tar.add(name) for replica in self._replicas: tar.add ('mdin-%s-0' % replica.rid) #how does this work os.remove('mdin-%s-0' % replica.rid) tar.close() # create a single pipeline with one stage to transfer the tarball task = re.Task() task.name = 'untarTsk' task.executable = ['python'] task.upload_input_data = ['untar_input_files.py', 'input_files.tar'] task.arguments = ['untar_input_files.py', 'input_files.tar'] task.cpu_reqs = {'processes' : 1, 'thread_type' : None, 'threads_per_process': 1, 'process_type': None} task.post_exec = [] stage = re.Stage() stage.name = 'untarStg' stage.add_tasks(task) #setup = re.Pipeline() #setup.name = 'untarPipe' #setup.add_stages(stage) setup_pipeline = re.Pipeline() setup_pipeline.name = 'untarPipe' setup_pipeline.add_stages(stage) return [setup_pipeline]
def add_md_stage(self): """ Problem: add_md_stage only adds md tasks that initiate from "inpcrd" i.e. cycle 0. Needs to accept input from previous MD stage. """ rid = self._rid cycle = self._cycle sbox = self._sbox cores = self._cores exe = self._exe self._log.debug('=== %s add md (cycle %s)', rid, cycle) task = re.Task() task.name = 'mdtsk-%s-%s' % (rid, cycle) if cycle == 0: task.link_input_data = [ '%s/inpcrd > inpcrd-%s-%s' % (sbox, rid, cycle), '%s/prmtop' % (sbox), '%s/mdin-%s-%s > mdin' % (sbox, rid, cycle) ] else: cycle_0 = '0' task.link_input_data = [ '%s/inpcrd-%s-%s' % (sbox, rid, cycle), '%s/prmtop' % (sbox), '%s/mdin-%s-%s > mdin' % (sbox, rid, cycle_0) ] #['%s/mdcrd-out-%s-%s > inpcrd-%s-%s' % (sbox, rid, cycle, rid, cycle), task.arguments = [ '-O', '-i', 'mdin', '-p', 'prmtop', '-c', 'inpcrd-%s-%s' % (rid, cycle), '-o', 'out', '-x', '%s/mdcrd-%s-%s' % (sbox, rid, cycle + 1), '-r', '%s/inpcrd-%s-%s' % (sbox, rid, cycle + 1), '-inf', '%s/mdinfo-%s-%s' % (sbox, rid, cycle) ] task.executable = SANDER #[exe] task.cpu_reqs = {'processes': cores} task.pre_exec = ['echo $SHARED' ] #This will be different for different MD engines. stage = re.Stage() stage.add_tasks(task) stage.post_exec = self._after_md self.add_stages(stage)
def add_ex_stage(self, exchange_list, ex_alg, sid): self._prof.prof('add_ex_start', uid=self.rid) self._log.debug('%5s add ex: %s', self.rid, [r.rid for r in exchange_list]) self._ex_list = exchange_list task = re.Task() task.executable = 'python3' task.arguments = [ex_alg, '-r', self.rid, '-c', self.cycle] \ + ['-e'] + [r.rid for r in exchange_list] \ + ['-d'] + [d for d in self._workload.exchange.ex_data] if self._workload.pre_exec: task.pre_exec = self._workload.pre_exec # link alg link_inputs = ['pilot:///%s/%s' % (self._workload.data.inputs, ex_alg)] # link exchange data for r in exchange_list: t = last_task(r) self._log.debug('Exchage: %s, Task Name: %s Sandbox %s', r.name, t.name, t.sandbox) link_inputs += expand_ln(self._workload.exchange.md_2_ex, # FIXME: how to get absolute task sbox? # rep.0000.0000:/// ... # i.e., use task ID as schema 'pilot:///%s' % t.sandbox, 'task://', r.rid, r.cycle) task.link_input_data = link_inputs task.name = '%s.%04d.ex' % (self.rid, self.cycle) task.sandbox = '%s.%04d.ex' % (self.rid, self.cycle) self._log.debug('%5s added ex: %s, input data: %s', self.rid, task.name, task.link_input_data) stage = re.Stage() stage.add_tasks(task) stage.post_exec = self.check_resume self.add_stages(stage) self._prof.prof('add_ex_stop', uid=self.rid)
def setup_replicas(replicas, min_temp, max_temp, timesteps, basename): writeInputs.writeInputs(max_temp=max_temp, min_temp=min_temp, replicas=replicas, timesteps=timesteps, basename=basename) tar = tarfile.open("input_files.tar", "w") for name in [ basename + ".prmtop", basename + ".inpcrd", basename + ".mdin" ]: tar.add(name) for r in range(replicas): tar.add('mdin-{replica}-{cycle}'.format(replica=r, cycle=0)) os.remove('mdin-{replica}-{cycle}'.format(replica=r, cycle=0)) tar.close() setup_p = re.Pipeline() setup_p.name = 'untarPipe' # # unused # repo = git.Repo('.', search_parent_directories=True) # aux_function_path = repo.working_tree_dir untar_stg = re.Stage() untar_stg.name = 'untarStg' # Untar Task untar_tsk = re.Task() untar_tsk.name = 'untarTsk' untar_tsk.executable = ['python'] untar_tsk.upload_input_data = ['untar_input_files.py', 'input_files.tar'] untar_tsk.arguments = ['untar_input_files.py', 'input_files.tar'] untar_tsk.cpu_reqs = 1 untar_tsk.post_exec = [] untar_stg.add_tasks(untar_tsk) setup_p.add_stages(untar_stg) replica_sandbox = '$Pipeline_%s_Stage_%s_Task_%s' \ % (setup_p.name, untar_stg.name, untar_tsk.name) return setup_p, replica_sandbox
def add_md_stage(self): self._log.debug('=== %s add md', self.rid) rid = self._rid cycle = self._cycle sbox = self._sbox cores = self._cores exe = self._exe task = re.Task() task.name = 'mdtsk-%s-%s' % ( rid, cycle) if cycle == 0: task.link_input_data = ['%s/inpcrd > inpcrd-%s-%s' % (sbox, rid, cycle), '%s/prmtop' % (sbox), '%s/mdin-%s-%s > mdin' % (sbox, rid, cycle)] else: cycle_0 = '0' task.link_input_data = ['%s/inpcrd-%s-%s' % (sbox, rid, cycle), '%s/prmtop' % (sbox), '%s/mdin-%s-%s > mdin' % (sbox, rid, cycle_0)] #['%s/mdcrd-out-%s-%s > inpcrd-%s-%s' % (sbox, rid, cycle, rid, cycle), task.arguments = ['-O', '-i', 'mdin', '-p', 'prmtop', '-c', 'inpcrd-%s-%s' % ( rid, cycle), '-o', 'out', '-x', '%s/mdcrd-%s-%s' % (sbox, rid, cycle+1), '-r', '%s/inpcrd-%s-%s' % (sbox, rid, cycle+1), '-inf', '%s/mdinfo-%s-%s' % (sbox, rid, cycle)] task.executable = SANDER #[exe] task.cpu_reqs = {'processes' : cores, 'thread_type' : None, 'threads_per_process': 1, 'process_type': None} task.pre_exec = ['module load intel/19.3', 'module load python3/intel_3.6.3'] #This will be different for different MD engines. stage = re.Stage() stage.add_tasks(task) stage.post_exec = self.check_exchange #_after_md self.add_stages(stage)
def setup(self): self._log.debug('=== data staging') # prepare input for all replicas writeInputs.writeInputs(max_temp=self._max_temp, min_temp=self._min_temp, replicas=self._size, timesteps=self._timesteps, basename=self._basename) # and tar it up tar = tarfile.open("input_files.tar", "w") for name in [ self._basename + ".prmtop", self._basename + ".inpcrd", self._basename + ".mdin" ]: tar.add(name) for replica in self._replicas: tar.add('mdin-%s-0' % replica.rid) os.remove('mdin-%s-0' % replica.rid) tar.close() # create a single pipeline with one stage to transfer the tarball task = re.Task() task.name = 'untarTsk' task.executable = 'python' task.upload_input_data = ['untar_input_files.py', 'input_files.tar'] task.arguments = ['untar_input_files.py', 'input_files.tar'] task.cpu_reqs = 1 task.post_exec = [] stage = re.Stage() stage.name = 'untarStg' stage.add_tasks(task) setup = re.Pipeline() setup.name = 'untarPipe' setup.add_stages(stage) # run the setup pipeline self.workflow = set([setup]) self.run()
def get_wf3_input(appman, cfg): # Assuming shared filesystem on login node this can be executed by the # script instead of EnTK. p = entk.Pipeline() p.name = 'get_wf3_input' s = entk.Stage() t = entk.Task() t.executable = ['python3'] t.arguments = [ 'gather.py', '-f', cfg['outlier_path'], '-p', cfg['top_path'] ] s.add_tasks(t) p.add_stages(s) appman.workflow = [p] appman.run()
def add_md_stg(rid, cycle): # md stg here print 'cycle: ', self.cycle md_tsk = re.Task() md_stg = re.Stage() md_tsk.name = 'mdtsk-{replica}-{cycle}'.format(replica=rid, cycle=self.cycle) md_tsk.link_input_data = [ '%s/inpcrd > inpcrd-{replica}-{cycle}'.format( replica=rid, cycle=self.cycle) % replica_sandbox, '%s/prmtop' % replica_sandbox, '%s/mdin-{replica}-{cycle} > mdin'.format( replica=rid, cycle=self.cycle) % replica_sandbox ] md_tsk.arguments = [ '-O', '-i', 'mdin', '-p', 'prmtop', '-c', 'inpcrd-{replica}-{cycle}'.format(replica=rid, cycle=self.cycle), '-o', 'out', '-x', 'mdcrd', '-r', '%s/inpcrd-{replica}-{cycle}'.format( replica=rid, cycle=self.cycle + 1) % replica_sandbox, '-inf', '%s/mdinfo-{replica}-{cycle}'.format( replica=rid, cycle=self.cycle) % replica_sandbox ] md_tsk.executable = [md_executable] md_tsk.cpu_reqs = { 'processes': replica_cores, 'process_type': '', 'threads_per_process': 1, 'thread_type': None } md_tsk.pre_exec = ['echo $SHARED'] md_stg.add_tasks(md_tsk) md_stg.post_exec = { 'condition': post_md, 'on_true': start_ex, 'on_false': suspend_replica } return md_stg
def add_md_stage(self): rid = self._rid cycle = self._cycle sbox = self._sbox cores = self._cores exe = self._exe self._log.debug('=== %s add md (cycle %s)', rid, cycle) task = re.Task() task.name = 'mdtsk-%s-%s' % (rid, cycle) task.link_input_data = [ '%s/inpcrd > inpcrd-%s-%s' % (sbox, rid, cycle), '%s/prmtop' % (sbox), '%s/mdin-%s-%s > mdin' % (sbox, rid, cycle) ] task.arguments = [ '-O', '-i', 'mdin', '-p', 'prmtop', '-c', 'inpcrd-%s-%s' % (rid, cycle), '-o', 'out', '-x', 'mdcrd', '-r', '%s/inpcrd-%s-%s' % (sbox, rid, cycle), '-inf', '%s/mdinfo-%s-%s' % (sbox, rid, cycle) ] task.executable = SANDER #[exe] task.cpu_reqs = {'processes': cores} task.pre_exec = ['echo $SHARED' ] #This will be different for different MD engines. stage = re.Stage() stage.add_tasks(task) try: stage.post_exec = self.after_md except: stage.post_exec = { 'condition': self.after_md, 'on_true': void, 'on_false': void } self.add_stages(stage)
def _check_exchange(self, replica): ''' add this replica to the wait list, and check if we have sufficient replicas for an exchange. If no, just wait for the next one. If yes, we request an exchange over the collected suspended replicas in the waitlist. We abuse the current replica to run that exchange stage for us. ''' # mark this replica for the next exchange self._waitlist.append(replica) self._log.debug('=== EX %s check exchange (%d >= %d?)', replica.rid, len(self._waitlist), self._max_wait) if len(self._waitlist) < self._max_wait: # just suspend this replica and wait for the next self._log.debug('=== EX %s suspend', replica.rid) replica.suspend() else: # we are in for a wile ride! self._log.debug('=== EX %s exchange', replica.rid) task = re.Task() task.name = 'extsk' ## task.executable = 'python' task.executable = 'echo' task.arguments = ['t_ex_gibbs.py', len(self._waitlist)] ## for replica in self._waitlist: ## rid = replica.rid ## cycle = replica.cycle ## task.link_input_data.append('%s/mdinfo-%s-%s' ## % (self._sbox, rid, cycle)) stage = re.Stage() stage.add_tasks(task) stage.post_exec = self._after_exchange replica.add_stages(stage)
def __init__(self): self.set_argparse() self._set_rmq() self.am = entk.AppManager(hostname=self.rmq_hostname, port=self.rmq_port, username=self.rmq_username, password=self.rmq_password) #pipelines = [] self.pipelines = [] self.p1 = entk.Pipeline() self.p2 = entk.Pipeline() #pipelines.append(self.p1) #pipelines.append(self.p2) self.s1 = entk.Stage() self.s2 = entk.Stage() self.s3 = entk.Stage() self.s4 = entk.Stage() self.s5 = entk.Stage() self.s6 = entk.Stage() self.s7 = entk.Stage()
def _check_exchange(self, replica): ''' add this replica to the wait list, and check if we have sufficient replicas for an exchange. If no, just wait for the next one. If yes, we request an exchange over the collected suspended replicas in the waitlist. We abuse the current replica to run that exchange stage for us. Sliding Window model: We are to only perform exchange on a sublist from the waitlist of replicas. We select this sublist based on parameter (T in this case) proximity. Width of the sliding window is user defined. This is just exchange_size. ''' # mark this replica for the next exchange self._waitlist.append(replica) self._log.debug('=== %s check exchange (%d >= %d?)', replica.rid, len(self._waitlist), self._exchange_size) print "waitlist is: " for i in (self._waitlist): print i.rid # Sort the waitlist as soon as a new replica is added. self._sorted_waitlist = list() for replica in self._waitlist: self._sorted_waitlist.append( [replica, replica.rid] ) # We're sorting by RID here since RID's are assigned in sorted order with # Temperature (or whatever paramater is of interest to us) self._sorted_waitlist = sorted( self._sorted_waitlist, key=lambda x: x[1]) #Taken from Andre's example print "sorted waitlist is: " for i in (self._sorted_waitlist): print i[0].rid # Now we generate a sublist called exchange_list, within which an exchange is performed. This is done with # the sliding_window function self._exchange_list = self._sliding_window(self._sorted_waitlist, self._exchange_size, self._window_size) # Now check if the proposed exchange list is big enough (it should be, this seems slightly redundant) print "exchange size is ", self._exchange_size, " and exchange list length is ", len( self._exchange_list) if len(self._exchange_list) < self._exchange_size: # just suspend this replica and wait for the next self._log.debug('=== %s suspend', replica.rid) print "replica ", replica.rid, " should suspend now" replica.suspend() print "BUT IT ISN'T AAAAAARGH" else: # we are in for a wild ride! self._log.debug('=== %s exchange') task = re.Task() task.name = 'extsk' task.executable = 'python' task.arguments = ['t_ex_gibbs.py', len(self._waitlist)] for replica in self._waitlist: rid = replica.rid cycle = replica.cycle task.link_input_data.append('%s/mdinfo-%s-%s' % (self._sbox, rid, cycle)) stage = re.Stage() stage.add_tasks(task) try: stage.post_exec = self._after_ex except: stage.post_exec = { 'condition': self.after_ex, 'on_true': void, 'on_false': void } replica.add_stages(stage) # Here we remove the replicas participating in the triggered exchange from the waitlist. for replica in self._exchange_list: self._sorted_waitlist.remove( [replica, replica.rid]) #Sorted_Waitlist is a list of tuples
def _generate_pipeline(self, name, pre_execs, image, image_size): ''' This function creates a pipeline for an image that will be analyzed. :Arguments: :name: Pipeline name, str :image: image path, str :image_size: image size in MBs, int :tile_size: The size of each tile, int :model_path: Path to the model file, str :model_arch: Prediction Model Architecture, str :model_name: Prediction Model Name, str :hyperparam_set: Which hyperparameter set to use, str ''' # Create a Pipeline object entk_pipeline = re.Pipeline() entk_pipeline.name = name # Create a Stage object stage0 = re.Stage() stage0.name = '%s-S0' % (name) # Create Task 1, training task0 = re.Task() task0.name = '%s-T0' % stage0.name task0.pre_exec = pre_execs task0.executable = 'iceberg_seals.tiling' # Assign tak executable # Assign arguments for the task executable task0.arguments = ['--scale_bands=%s' % self._scale_bands, '--input_image=%s' % image.split('/')[-1], # This line points to the local filesystem of the # node that the tiling of the image happened. '--output_folder=$NODE_LFS_PATH/%s' % task0.name] task0.link_input_data = [image] task0.cpu_reqs = {'processes': 1, 'threads_per_process': 4, 'process_type': None, 'thread_type': 'OpenMP'} task0.lfs_per_process = image_size stage0.add_tasks(task0) # Add Stage to the Pipeline entk_pipeline.add_stages(stage0) # Create a Stage object stage1 = re.Stage() stage1.name = '%s-S1' % (name) # Create Task 1, training task1 = re.Task() task1.name = '%s-T1' % stage1.name task1.pre_exec = pre_execs task1.executable = 'iceberg_seals.predicting' # Assign task executable # Assign arguments for the task executable task1.arguments = ['--input_image', image.split('/')[-1], '--model_architecture', self._model_arch, '--hyperparameter_set', self._hyperparam, '--training_set', 'test_vanilla', '--test_folder', '$NODE_LFS_PATH/%s' % task0.name, '--model_path', './', '--output_folder', './%s' % image.split('/')[-1]. split('.')[0]] task1.link_input_data = ['$SHARED/%s' % self._model_name] task1.cpu_reqs = {'processes': 1, 'threads_per_process': 1, 'process_type': None, 'thread_type': 'OpenMP'} task1.gpu_reqs = {'processes': 1, 'threads_per_process': 1, 'process_type': None, 'thread_type': 'OpenMP'} # Download resuting images task1.download_output_data = ['%s/ > %s' % (image.split('/')[-1]. split('.')[0], image.split('/')[-1])] task1.tag = task0.name stage1.add_tasks(task1) # Add Stage to the Pipeline entk_pipeline.add_stages(stage1) return entk_pipeline
def _generate_pipeline(self, name, pre_execs, image, image_size): ''' This function creates a pipeline for an image that will be analyzed. :Arguments: :name: Pipeline name, str :image: image path, str :image_size: image size in MBs, int :tile_size: The size of each tile, int :model_path: Path to the model file, str :model_arch: Prediction Model Architecture, str :model_name: Prediction Model Name, str :hyperparam_set: Which hyperparameter set to use, str ''' # Create a Pipeline object entk_pipeline = re.Pipeline() entk_pipeline.name = name # Create a Stage object stage0 = re.Stage() stage0.name = '%s.S0' % (name) # Create Task 1, training task0 = re.Task() task0.name = '%s.T0' % stage0.name task0.pre_exec = pre_execs task0.executable = 'iceberg_seals.tiling' # Assign tak executable # Assign arguments for the task executable task0.arguments = [ '--input_image=%s' % image.split('/')[-1], '--output_folder=$NODE_LFS_PATH/%s' % task0.name, '--bands=%s' % self._bands, '--stride=%s' % self._stride, '--patch_size=%s' % self._patch_size, '--geotiff=%s' % self._geotiff ] task0.link_input_data = [image] task0.cpu_reqs = { 'cpu_processes': 1, 'cpu_threads': 4, 'cpu_process_type': None, 'cpu_thread_type': 'OpenMP' } task0.lfs_per_process = image_size stage0.add_tasks(task0) # Add Stage to the Pipeline entk_pipeline.add_stages(stage0) # Create a Stage object stage1 = re.Stage() stage1.name = '%s.S1' % (name) # Create Task 1, training task1 = re.Task() task1.name = '%s.T1' % stage1.name task1.pre_exec = pre_execs task1.executable = 'iceberg_seals.predicting' # Assign task executable # Assign arguments for the task executable task1.arguments = [ '--input_dir=$NODE_LFS_PATH/%s' % task0.name, '--model_architecture=%s' % self._model_arch, '--hyperparameter_set=%s' % self._hyperparam, '--model_name=%s' % self._model_name, '--models_folder=./', '--output_dir=./%s' % image.split('/')[-1].split('.')[0], ] task1.link_input_data = ['$SHARED/%s' % self._model_name] task1.cpu_reqs = { 'cpu_processes': 1, 'cpu_threads': 1, 'cpu_process_type': None, 'cpu_thread_type': 'OpenMP' } task1.gpu_reqs = { 'gpu_processes': 1, 'gpu_threads': 1, 'gpu_process_type': None, 'gpu_thread_type': 'OpenMP' } # Download resulting images # task1.download_output_data = ['%s/ > %s' % (image.split('/')[-1]. # split('.')[0], # image.split('/')[-1])] task1.tags = {'colocate': task0.name} stage1.add_tasks(task1) # Add Stage to the Pipeline entk_pipeline.add_stages(stage1) return entk_pipeline
def add_md_stage(self, exchanged_from=None, sid=None, last=False): self._prof.prof('add_md_start', uid=self.rid) self._cycle += 1 self._log.debug('%5s %s add md', self.rid, self._uid) # task = re.Task(from_dict=self._workload['md']) # task.name = 'mdtsk-%s-%s' % (self.rid, self.cycle) sandbox = '%s.%04d.md' % (self.rid, self.cycle) link_inputs = list() # link initial data link_inputs += expand_ln(self._workload.md.inputs, 'pilot:///%s' % self._workload.data.inputs, 'task://', self.rid, self.cycle) if self._cycle == 0: # link initial data link_inputs += expand_ln(self._workload.md.inputs_0, 'pilot:///%s' % self._workload.data.inputs, 'task://', self.rid, self.cycle) else: # get data from previous task t = last_task(self) if exchanged_from: self._log.debug('Exchange from %s', exchanged_from.name) link_inputs += expand_ln(self._workload.md.ex_2_md, 'pilot:///%s' % (exchanged_from.sandbox), 'task://', self.rid, self.cycle) else: # FIXME: this apparently can't happen link_inputs += expand_ln(self._workload.md.md_2_md, 'resource:///%s' % (t.sandbox), 'task://', self.rid, self.cycle) copy_outputs = expand_ln(self._workload.md.outputs, 'task://', 'client:///%s' % self._workload.data.outputs, self.rid, self.cycle) if last: copy_outputs += expand_ln(self._workload.md.outputs_n, 'task://', 'client:///%s' % self._workload.data.outputs, self.rid, self.cycle) # TODO: filter out custom keys from that dict before deepcopy env = {'REPEX_RID' : str(self.rid), 'REPEX_CYCLE' : str(self.cycle)} tds = copy.deepcopy(self._workload['md']['descriptions']) first = 0 last = len(tds) - 1 for idx, td in enumerate(tds): stage = re.Stage() task = re.Task() td = ru.expand_env(td, env=env) for k,v in td.items(): setattr(task, k, v) if self._workload.pre_exec: if task.pre_exec: task.pre_exec.extend(self._workload.pre_exec) else: task.pre_exec.extend = self._workload.pre_exec task.name = '%s.%04d.%02d.md' % (self.rid, self.cycle, idx) task.sandbox = sandbox if idx == first: task.link_input_data = link_inputs if idx == last: task.download_output_data = copy_outputs stage.post_exec = self.check_exchange stage.add_tasks(task) self.add_stages(stage) self._log.debug('%5s add md: %s', self.rid, task.name) self._prof.prof('add_md_stop', uid=self.rid)
def _generate_pipeline(self, name, pre_execs, image, image_size): ''' This function creates a pipeline for an image that will be analyzed. :Arguments: :name: Pipeline name, str :pre_execs: things need to happen before execution :image: image path, str :image_size: image size in MBs, int ''' # Create a Pipeline object entk_pipeline = re.Pipeline() entk_pipeline.name = name # Create a Stage object stage0 = re.Stage() stage0.name = '%s.S0' % (name) # Create Task 1, training task0 = re.Task() task0.name = '%s.T0' % stage0.name task0.pre_exec = pre_execs task0.executable = 'iceberg_rivers.tiling' # Assign tak executable task0.arguments = [ '--input=%s' % image.split('/')[-1], '--output=$NODE_LFS_PATH/%s/' % task0.name, '--tile_size=%s' % self._tile_size, '--step=%s' % self._step ] task0.link_input_data = [image] task0.cpu_reqs = { 'cpu_processes': 1, 'cpu_threads': 4, 'cpu_process_type': None, 'cpu_thread_type': None } task0.lfs_per_process = image_size stage0.add_tasks(task0) # Add Stage to the Pipeline entk_pipeline.add_stages(stage0) # Create a Stage object stage1 = re.Stage() stage1.name = '%s.S1' % (name) # Create Task 1, training task1 = re.Task() task1.name = '%s.T1' % stage1.name task1.pre_exec = pre_execs task1.executable = 'iceberg_rivers.predicting' # Assign task executable # # Assign arguments for the task executable task1.arguments = [ '--input=$NODE_LFS_PATH/%s/' % task0.name, '--weights_path=%s' % self._weights_path, '--output_folder=$NODE_LFS_PATH/%s/' % task1.name ] # # task1.link_input_data = ['$SHARED/%s' % self._model_name] task1.cpu_reqs = { 'processes': 1, 'threads_per_process': 1, 'process_type': None, 'thread_type': None } task1.gpu_reqs = { 'processes': 1, 'threads_per_process': 1, 'process_type': None, 'thread_type': None } task0.lfs_per_process = image_size # Download resulting images # task1.download_output_data = ['%s/ > %s' % (image.split('/')[-1]. # split('.')[0], # image.split('/')[-1])] task1.tags = {'colocate': task0.name} stage1.add_tasks(task1) # Add Stage to the Pipeline entk_pipeline.add_stages(stage1) # Create a Stage object stage2 = re.Stage() stage2.name = '%s.S2' % (name) # Create Task 1, training task2 = re.Task() task2.name = '%s.T2' % stage2.name task2.pre_exec = pre_execs task2.executable = 'iceberg_rivers.mosaic' # Assign task executable # # Assign arguments for the task executable task2.arguments = [ '--input=$NODE_LFS_PATH/%s/' % task1.name, '--input_WV=%s' % image.split('/')[-1], '--tile_size=%s' % self._tile_size, '--step=%s' % self._step, '--output_folder=./' ] task2.cpu_reqs = { 'processes': 1, 'threads_per_process': 1, 'process_type': None, 'thread_type': None } task2.link_input_data = [image] task2.tags = {'colocate': task0.name} stage2.add_tasks(task2) # Add Stage to the Pipeline entk_pipeline.add_stages(stage2) return entk_pipeline