def SimpleJobExample(configuration_item_name, userid, userpw=None): """ Dummy workflow to test the install Parameters ---------- configuration_item_name: str the name of the configuration item (ex. "Gabriel") userid: str user name on the server side userpw: str (optional) user password to login the server using ssh. If you want to use "id_rsa.pub", just leave userpw to None To copy the public key on the server use ssh-copy-id -i name@server. """ job_1 = Job(command=["sleep", "5"], name="job 1") job_2 = Job(command=["sleep", "5"], name="job 2") job_3 = Job(command=["sleep", "5"], name="job 3") job_4 = Job(command=["sleep", "5"], name="job 4") jobs = [job_1, job_2, job_3, job_4] dependencies = [(job_1, job_2), (job_1, job_3), (job_2, job_4), (job_3, job_4)] workflow = Workflow(jobs=jobs, dependencies=dependencies) controller = WorkflowController(configuration_item_name, userid, userpw) controller.submit_workflow(workflow=workflow, name="TestConnectionExample")
def iteration_run(self): # this method should be replaced by a call to # pipeline_workflow.workflow_from_pipeline() # (but first, the iteration has to be an actual pipeline) from soma_workflow.client import Job, Workflow, WorkflowController print 'ITERATION RUN' jobs = {} i = 0 for process in self.list_process_iteration: jobs['job'+str(i)] = Job(command=process.command()) i = i+1 wf = Workflow(jobs=[value for value in \ jobs.itervalues()], name='test') # Helper.serialize('/tmp/test_wf',wf) controller = WorkflowController() controller.submit_workflow(workflow=wf, name='test run')
def setup_connection(cls, resource_id, login, password): cls.login = login cls.password = password cls.resource_id = resource_id cls.wf_ctrl = WorkflowController(resource_id, login, password) cls.transfer_timeout = -24 cls.jobs_timeout = 1 cls.job_examples = JobExamples(cls.wf_ctrl, 'python', cls.transfer_timeout, cls.jobs_timeout)
def _setup_soma_workflow_controller(self, create_new=False): resource_id, login, password, rsa_key_pass \ = self.get_soma_workflow_credentials() config_file_path = swconf.Configuration.search_config_path() try: sw_config = swconf.Configuration.load_from_file( resource_id, config_file_path) except swconf.ConfigurationError: sw_config = None resource_id = None if self._workflow_controller is None or create_new: self._workflow_controller = WorkflowController( resource_id, login, password=None, config=sw_config, rsa_key_pass=None) self._delete_old_workflows()
def remote_map_marshal(func, largs=None, lkwargs=None, mode='local'): if largs is None: if lkwargs is not None: largs = [[]] * len(lkwargs) else: largs = [] if lkwargs is None: lkwargs = [{}] * len(largs) lkwargs = [merge_default_kwargs(func, kw) for kw in lkwargs] assert len(lkwargs) == len(largs) all_args = zip(largs, lkwargs) if mode == 'local': return [func(*args, **kwargs) for args, kwargs in all_args] elif mode == 'local_with_dumps': func_fn = './func.marshal' dump_func(func, func_fn) results = [] for i, params in enumerate(all_args): print 'params:', params params_fn = 'params_%d.pck' % i fparams = open(params_fn, 'wb') cPickle.dump(params, fparams) fparams.close() output_fn = 'output_%d.pck' % i print 'call subprocess ...' subprocess.call(['python', '-c', cfunc_marshal, params_fn, func_fn, output_fn]) print 'Read outputs ...' fout = open(output_fn) results.append(cPickle.load(fout)) return results elif mode == 'remote_cluster': # FileTransfer creation for input files #data_dir = './rmap_data' data_dir = mkdtemp(prefix="sw_rmap") func_fn = op.join(data_dir, 'func.marshal') dump_func(func, func_fn) func_file = FileTransfer(is_input=True, client_path=func_fn, name="func_file") all_jobs = [] param_files = [] for i, params in enumerate(all_args): params_fn = op.join(data_dir, 'params_%d.pck' % i) fparams = open(params_fn, 'wb') cPickle.dump(params, fparams) fparams.close() param_file = FileTransfer(is_input=True, client_path=params_fn, name='params_file_%d' % i) param_files.append(param_file) output_fn = op.join(data_dir, 'output_%d.pck' % i) output_file = FileTransfer(is_input=False, client_path=output_fn, name='output_file_%d' % i) job = Job(command=['python', '-c', cfunc, param_file, func_file, output_file], name="rmap, item %d" % i, referenced_input_files=[func_file, param_file], referenced_output_files=[output_file]) all_jobs.append(job) workflow = Workflow(jobs=all_jobs, dependencies=[]) # submit the workflow cfg = pyhrf.cfg['parallel-cluster'] controller = WorkflowController(cfg['server_id'], cfg['user']) # controller.transfer_files(fids_to_transfer) wf_id = controller.submit_workflow( workflow=workflow, name="remote_map") Helper.transfer_input_files(wf_id, controller) Helper.wait_workflow(wf_id, controller) Helper.transfer_output_files(wf_id, controller) results = [] for i in xrange(len(all_args)): fout = open(op.join(data_dir, 'output_%d.pck' % i)) results.append(cPickle.load(fout)) fout.close() return results
group_elements = [] first_job = Job(command=["sleep", "10"], name="first job") last_job = Job(command=["sleep", "10"], name="last job") jobs.append(first_job) jobs.append(last_job) for i in range(0, 30): job = Job(command=["sleep", "60"], name="job " + repr(i)) jobs.append(job) dependencies.append((first_job, job)) dependencies.append((job, last_job)) group_elements.append(job) thirty_jobs_group = Group(elements=group_elements, name="my 30 jobs") workflow = Workflow(jobs=jobs, dependencies=dependencies, root_group=[first_job, thirty_jobs_group, last_job]) login = '******' password = '******' controller = WorkflowController("DSV_cluster", login, password) controller.submit_workflow(workflow=workflow, name="Simple workflow with group")
def morphologist_all(t1file, sid, outdir, study="morphologist", waittime=10, somaworkflow=False, spmexec="/i2bm/local/spm8-standalone/run_spm8.sh", spmdir="/i2bm/local/spm8-standalone"): """ Performs all the Morphologist steps. Steps: 1- Ensure image orientation and reorient it if needed (Prepare Subject for Anatomical Pipeline). 2- Computation of a brain mask (Brain Mask Segmentation). 3- Computation of a mask for each hemisphere (Split Brain Mask). 4- A grey/white classification of each hemisphere to perform "Voxel Based Morphometry" (Grey White Classification) and spherical triangulation of cortical hemispheres (Grey White Surface). 5- Spherical triangulation of the external interface of the cortex of one or two hemispheres (Get Spherical Hemi Surface). 6- Computation of a graph representing the cortical fold topography (Cortical Fold Graph). 7- Automatic identification of the cortical sulci (Automatic Sulci Recognition), located in the "sulci" toolbox. The execution is performed with soma_workflow that has to be installed in the bv_env environment. To check the worklow submission, use the 'soma_workflow_gui' command. If the input 't1file' has no the expected extension, an Exception will be raised. If the $outdir/$study/$sid has already been created, an Exception will be raised. Parameters ---------- t1file: str (mandatory) the path to a ".nii.gz" anatomical T1 weighted file. sid: str (mandatory) a subject identifier. outdir: str (mandatory) the morphologist output files will be written in $outdir/$study/$sid. study: str (mandatory) the name of the study. waittime: float (optional, default 10) a delay (in seconds) used to check the worflow status. somaworkflow: bool (optional, default False) if True use somaworkflow for the execution. spmexec: str (optional) the path to the standalone SPM execution file. spmdir: str (optional) the standalone SPM directory. Returns ------- wffile: str a file containing the submitted workflow. wfid: int the submitted workflow identifier. wfstatus: str the submited worflow status afer 'waittime' seconds. """ # Check roughly the input file extension if not t1file.endswith(".nii.gz"): raise Exception("'{0}' is not a COMPRESSED NIFTI file.".format(t1file)) # Create a configuration for the morphologist study study_config = StudyConfig( modules=StudyConfig.default_modules + ["FomConfig", "BrainVISAConfig"]) study_dict = { "name": "morphologist_fom", "input_directory": outdir, "output_directory": outdir, "input_fom": "morphologist-auto-nonoverlap-1.0", "output_fom": "morphologist-auto-nonoverlap-1.0", "shared_fom": "shared-brainvisa-1.0", "spm_directory": spmdir, "use_soma_workflow": True, "use_fom": True, "spm_standalone": True, "use_matlab": False, "volumes_format": "NIFTI gz", "meshes_format": "GIFTI", "use_spm": True, "spm_exec": spmexec, "study_config.somaworkflow_computing_resource": "localhost", "somaworkflow_computing_resources_config": { "localhost": { } } } study_config.set_study_configuration(study_dict) # Create the morphologist pipeline pipeline = get_process_instance( "morphologist.capsul.morphologist.Morphologist") morphologist_pipeline = process_with_fom.ProcessWithFom( pipeline, study_config) morphologist_pipeline.attributes = dict( (trait_name, getattr(morphologist_pipeline, trait_name)) for trait_name in morphologist_pipeline.user_traits()) morphologist_pipeline.attributes["center"] = "morphologist" morphologist_pipeline.attributes["subject"] = sid morphologist_pipeline.create_completion() # Create morphologist expected tree # ToDo: use ImportT1 from axon subjectdir = os.path.join(outdir, study, sid) if os.path.isdir(subjectdir): raise Exception("Folder '{0}' already created.".format(subjectdir)) os.makedirs(os.path.join( subjectdir, "t1mri", "default_acquisition", "default_analysis", "folds", "3.1", "default_session_auto")) os.makedirs(os.path.join( subjectdir, "t1mri", "default_acquisition", "registration")) os.makedirs(os.path.join( subjectdir, "t1mri", "default_acquisition", "segmentation", "mesh")) os.makedirs(os.path.join( subjectdir, "t1mri", "default_acquisition", "tmp")) # Copy T1 file in the morphologist expected location destfile = os.path.join(subjectdir, "t1mri", "default_acquisition", sid + ".nii.gz") shutil.copy(t1file, destfile) # Create source_referential morphologist expected file source_referential = {"uuid": str(soma.uuid.Uuid())} referential_file = os.path.join( subjectdir, "t1mri", "default_acquisition", "registration", "RawT1-{0}_default_acquisition.referential".format(sid)) attributes = "attributes = {0}".format(json.dumps(source_referential)) with open(referential_file, "w") as openfile: openfile.write(attributes) # Create a worflow from the morphologist pipeline workflow = Workflow(name="{0} {1}".format(study, sid), jobs=[]) workflow.root_group = [] # Create the workflow wf = pipeline_workflow.workflow_from_pipeline( morphologist_pipeline.process, study_config=study_config) workflow.add_workflow(wf, as_group="{0}_{1}".format(study, sid)) wffile = os.path.join(subjectdir, "{0}.wf".format(study)) pickle.dump(workflow, open(wffile, "w")) # Execute the workflow with somaworkflow if somaworkflow: controller = WorkflowController() wfid = controller.submit_workflow( workflow=workflow, name="{0}_{1}".format(study, sid)) # Return the worflow status after execution while True: time.sleep(waittime) wfstatus = controller.workflow_status(wfid) if wfstatus not in [ "worklflow_not_started", "workflow_in_progress"]: break # Execute the workflow with subprocess else: # -> construct the ordered list of commands to be executed workflow_repr = workflow.to_dict() graph = Graph() for job in workflow_repr["jobs"]: graph.add_node(GraphNode(job, None)) for link in workflow_repr["dependencies"]: graph.add_link(link[0], link[1]) ordered_nodes = [str(node[0]) for node in graph.topological_sort()] commands = [] jobs = workflow_repr["serialized_jobs"] temporaries = workflow_repr["serialized_temporary_paths"] barriers = workflow_repr["serialized_barriers"] for index in ordered_nodes: if index in jobs: commands.append(jobs[index]["command"]) elif index in barriers: continue else: raise Exception("Unexpected node in workflow.") # -> Go through all commands tmpmap = {} for cmd in commands: # -> deal with temporary files for index, item in enumerate(cmd): if not isinstance(item, basestring): if str(item) not in tmpmap: if str(item) in temporaries: struct = temporaries[str(item)] name = cmd[2].split(";")[1].split()[-1] tmppath = os.path.join( subjectdir, "t1mri", "default_acquisition", "tmp", str(item) + name + struct["suffix"]) tmpmap[str(item)] = tmppath else: raise MorphologistError( "Can't complete command '{0}'.".format( cmd)) cmd[index] = tmpmap[str(item)] # -> execute the command worker = MorphologistWrapper(cmd) worker() if worker.exitcode != 0: raise MorphologistRuntimeError( " ".join(worker.cmd), worker.stderr) wfstatus = "Done" wfid = "subprocess" return wffile, wfid, wfstatus
from soma_workflow.client import Job, Workflow, WorkflowController, SharedResourcePath, FileTransfer # SharedResourcePath creation for the input file. # The input file is read direclty in the data directory located on the # the computing resource side. myfile = SharedResourcePath(relative_path="myfile", namespace="MyApp", uuid="my_example_dir") # FileTransfer creation for the output file. # That way the output file will not be written in the data directory # located on the computing resource file system. copy_of_myfile = FileTransfer( is_input=False, client_path="/tmp/soma_workflow_examples/copy_of_myfile", name="copy of my file") # Job and Workflow creation copy_job = Job(command=["cp", myfile, copy_of_myfile], name="copy", referenced_input_files=[], referenced_output_files=[copy_of_myfile]) workflow = Workflow(jobs=[copy_job], dependencies=[]) # workflow submission controller = WorkflowController("DSV_cluster", login, password) controller.submit_workflow(workflow=workflow, name="shared resource path example")
class SomaWorkflowRunner(Runner): WORKFLOW_NAME_SUFFIX = "Morphologist user friendly analysis" def __init__(self, study): super(SomaWorkflowRunner, self).__init__(study) self._workflow_controller = None self._init_internal_parameters() def get_soma_workflow_credentials(self): resource_id = self._study.somaworkflow_computing_resource config_file_path = swconf.Configuration.search_config_path() resource_list = swconf.Configuration.get_configured_resources( config_file_path) login_list = swconf.Configuration.get_logins(config_file_path) login = None if resource_id in login_list: login = login_list[resource_id] password = None rsa_key_pass = None return resource_id, login, password, rsa_key_pass def _init_internal_parameters(self): self._workflow_id = None self._jobid_to_step = {} # subjectid -> (job_id -> step) self._cached_jobs_status = None def resource_id(self): if self._workflow_controller is None: resource_id = None else: resource_id = self._workflow_controller._resource_id return resource_id def update_controller(self): resource_id = self._study.somaworkflow_computing_resource if resource_id != self.resource_id(): self._setup_soma_workflow_controller(create_new=True) def set_study(self, study): super(SomaWorkflowRunner, self).set_study(study) self.update_controller() def _setup_soma_workflow_controller(self, create_new=False): resource_id, login, password, rsa_key_pass \ = self.get_soma_workflow_credentials() config_file_path = swconf.Configuration.search_config_path() try: sw_config = swconf.Configuration.load_from_file( resource_id, config_file_path) except swconf.ConfigurationError: sw_config = None resource_id = None if self._workflow_controller is None or create_new: self._workflow_controller = WorkflowController( resource_id, login, password=None, config=sw_config, rsa_key_pass=None) self._delete_old_workflows() def _delete_old_workflows(self): for (workflow_id, (name, _)) \ in six.iteritems(self._workflow_controller.workflows()): if name is not None and name.endswith(self.WORKFLOW_NAME_SUFFIX): self._workflow_controller.delete_workflow(workflow_id) def run(self, subject_ids=ALL_SUBJECTS): self._setup_soma_workflow_controller() self._init_internal_parameters() if self._workflow_controller.scheduler_config: # in local mode only cpus_number = self._cpus_number() self._workflow_controller.scheduler_config.set_proc_nb(cpus_number) if subject_ids == ALL_SUBJECTS: subject_ids = self._study.subjects # setup shared path in study_config study_config = self._study swf_resource = study_config.somaworkflow_computing_resource if not self._workflow_controller.scheduler_config: # remote config only # FIXME: must check if brainvisa shared dir is known in translation # config in soma-workflow if not study_config.somaworkflow_computing_resources_config.trait( swf_resource): setattr(study_config.somaworkflow_computing_resources_config, swf_resource, {}) resource_conf = getattr( study_config.somaworkflow_computing_resources_config, swf_resource) path_translations = resource_conf.path_translations setattr(path_translations, study_config.shared_directory, ['brainvisa', 'de25977f-abf5-9f1c-4384-2585338cd7af']) #self._check_input_files(subject_ids) workflow = self._create_workflow(subject_ids) jobs = [j for j in workflow.jobs if isinstance(j, Job)] if self._workflow_id is not None: self._workflow_controller.delete_workflow(self._workflow_id) if len(jobs) == 0: # empty workflow: nothing to do self._workflow_id = None return self._workflow_id = self._workflow_controller.submit_workflow( workflow, name=workflow.name) self._build_jobid_to_step() # run transfers, if any Helper.transfer_input_files(self._workflow_id, self._workflow_controller) # the status does not change immediately after run, # so we wait for the status WORKFLOW_IN_PROGRESS or timeout status = self._workflow_controller.workflow_status(self._workflow_id) try_count = 8 while ((status != sw.constants.WORKFLOW_IN_PROGRESS) and \ (try_count > 0)): time.sleep(0.25) status = self._workflow_controller.workflow_status( self._workflow_id) try_count -= 1 def _cpus_number(self): cpus_count = multiprocessing.cpu_count() cpus_settings = settings.runner.selected_processing_units_n if cpus_settings > cpus_count: print("Warning: bad setting value:\n" + " (selected_processing_units_n=%d) " % cpus_settings + "> number of available processing units: %d" % cpus_count) cpus_number = min(cpus_settings, cpus_count) else: cpus_number = cpus_settings return cpus_number def _create_workflow(self, subject_ids): study_config = self._study workflow = Workflow( name='Morphologist UI - %s' % study_config.study_name, jobs=[]) workflow.root_group = [] initial_vol_format = study_config.volumes_format priority = (len(subject_ids) - 1) * 100 for subject_id in subject_ids: analysis = self._study.analyses[subject_id] subject = self._study.subjects[subject_id] analysis.set_parameters(subject) #analysis.propagate_parameters() pipeline = analysis.pipeline pipeline.enable_all_pipeline_steps() # force highest priority normalization method # FIXME: specific knowledge of Morphologist should not be used here. pipeline.Normalization_select_Normalization_pipeline \ = 'NormalizeSPM' pipeline_tools.disable_runtime_steps_with_existing_outputs( pipeline) missing = pipeline_tools.nodes_with_missing_inputs(pipeline) if missing: self.check_missing_models(pipeline, missing) print('MISSING INPUTS IN NODES:', missing) raise MissingInputFileError("subject: %s" % subject_id) wf = pipeline_workflow.workflow_from_pipeline( pipeline, study_config=study_config, jobs_priority=priority) njobs = len([j for j in wf.jobs if isinstance(j, Job)]) if njobs != 0: priority -= 100 workflow.jobs += wf.jobs workflow.dependencies += wf.dependencies group = Group(wf.root_group, name='Morphologist %s' % str(subject)) group.user_storage = subject_id workflow.root_group.append(group) # += wf.root_group workflow.groups += [group] + wf.groups return workflow def check_missing_models(self, pipeline, missing): if 'SulciRecognition.SPAM_recognition09.global_recognition' in missing: node = missing[ 'SulciRecognition.SPAM_recognition09.global_recognition'] model = [m[0] for m in node if m[0] == 'model'] if model: raise MissingModelsError( "SPAM recognition models are not installed.") def _build_jobid_to_step(self): self._jobid_to_step = {} workflow = self._workflow_controller.workflow(self._workflow_id) for group in workflow.groups: subjectid = group.user_storage if subjectid: self._jobid_to_step[subjectid] = BidiMap( 'job_id', 'step_id') job_list = list(group.elements) while job_list: job = job_list.pop(0) if isinstance(job, Group): job_list += job.elements else: job_att = workflow.job_mapping.get(job) if job_att: job_id = job_att.job_id step_id = job.user_storage or job.name self._jobid_to_step[subjectid][job_id] = step_id else: print('job without mapping, subject: %s, job: %s' % (subjectid, job.name)) def _define_workflow_name(self): return self._study.name + " " + self.WORKFLOW_NAME_SUFFIX def is_running(self, subject_id=None, step_id=None, update_status=True): status = self.get_status(subject_id, step_id, update_status) return status == Runner.RUNNING def get_running_step_ids(self, subject_id, update_status=True): if update_status: self._update_jobs_status() running_step_ids = self._get_subject_filtered_step_ids( subject_id, Runner.RUNNING) return running_step_ids def wait(self, subject_id=None, step_id=None): if subject_id is None and step_id is None: Helper.wait_workflow( self._workflow_id, self._workflow_controller) elif subject_id is not None: if step_id is None: raise NotImplementedError else: self._step_wait(subject_id, step_id) else: raise NotImplementedError # transfer back files, if any Helper.transfer_output_files(self._workflow_id, self._workflow_controller) def _step_wait(self, subject_id, step_id): job_id = self._jobid_to_step[subject_id][step_id, 'step_id'] self._workflow_controller.wait_job([job_id]) def has_failed(self, subject_id=None, step_id=None, update_status=True): status = self.get_status(subject_id, step_id, update_status) return (status & Runner.FAILED) or (status & Runner.ABORTED_NOTRUN) def get_failed_step_ids(self, subject_id, update_status=True): if update_status: self._update_jobs_status() failed_step_ids = self._get_subject_filtered_step_ids( subject_id, Runner.FAILED) return failed_step_ids def stop(self, subject_id=None, step_id=None): if not self.is_running(): raise RuntimeError("Runner is not running.") if subject_id is None and step_id is None: self._workflow_stop() elif subject_id is not None: if step_id is None: raise NotImplementedError else: raise NotImplementedError else: raise NotImplementedError def _workflow_stop(self): self._workflow_controller.stop_workflow(self._workflow_id) # transfer back files, if any Helper.transfer_output_files(self._workflow_id, self._workflow_controller) interrupted_step_ids = self._get_interrupted_step_ids() for subject_id, step_ids in six.iteritems(interrupted_step_ids): if step_ids: analysis = self._study.analyses[subject_id] analysis.clear_results(step_ids) def _get_interrupted_step_ids(self, update_status = True): """ Interrupted steps are either steps with an interrupted job (killed or failed), or steps with both run and not-run jobs (all jobs have not been performed, but some of them have) """ if update_status: self._update_jobs_status() filtered_step_ids_by_subject_id = {} for subject_id in self._jobid_to_step: subject_jobs = self._get_subject_jobs(subject_id) jobs_status = self._get_jobs_status(update_status=False) interrupted_step_ids = set() started_step_ids = set() notrun_step_ids = set() for job_id, step in six.iteritems(subject_jobs): if step in interrupted_step_ids: continue # this one is already in the list job_status = jobs_status[job_id] if job_status & Runner.INTERRUPTED: interrupted_step_ids.add(step) else: if job_status & Runner.SUCCESS: started_step_ids.add(step) if step in notrun_step_ids: # both started and unfinished step interrupted_step_ids.add(step) elif job_status & Runner.ABORTED_NOTRUN: notrun_step_ids.add(step) if step in started_step_ids: # both started and unfinished step interrupted_step_ids.add(step) filtered_step_ids_by_subject_id[subject_id] = interrupted_step_ids return filtered_step_ids_by_subject_id def _get_filtered_step_ids(self, status, update_status = True): if update_status: self._update_jobs_status() filtered_step_ids_by_subject_id = {} for subject_id in self._jobid_to_step: filtered_step_ids = self._get_subject_filtered_step_ids( subject_id, status) filtered_step_ids_by_subject_id[subject_id] = filtered_step_ids return filtered_step_ids_by_subject_id def _get_subject_filtered_step_ids(self, subject_id, status): step_ids = set() subject_jobs = self._get_subject_jobs(subject_id) jobs_status = self._get_jobs_status(update_status=False) for job_id in subject_jobs: job_status = jobs_status[job_id] if job_status & status: step_ids.add(subject_jobs[job_id]) return list(step_ids) def get_status(self, subject_id=None, step_id=None, update_status=True): if self._workflow_id is None: status = Runner.NOT_STARTED elif subject_id is None and step_id is None: if update_status: self._update_jobs_status() status = self._get_workflow_status() elif subject_id is not None and step_id is None: status = self._get_subject_status(subject_id, update_status) else: status = self._get_step_status(subject_id, step_id, update_status) return status def _get_workflow_status(self): sw_status \ = self._workflow_controller.workflow_status(self._workflow_id) if (sw_status in [sw.constants.WORKFLOW_IN_PROGRESS, sw.constants.WORKFLOW_NOT_STARTED]): status = Runner.RUNNING else: has_failed = (len(Helper.list_failed_jobs( self._workflow_id, self._workflow_controller, include_aborted_jobs=True, include_user_killed_jobs=True)) != 0) if has_failed: status = Runner.FAILED else: status = Runner.SUCCESS return status def _get_subject_status(self, subject_id, update_status=True): status = Runner.NOT_STARTED subject_jobs = self._get_subject_jobs(subject_id) if subject_jobs: jobs_status=self._get_jobs_status(update_status) status = Runner.SUCCESS for job_id in subject_jobs: job_status = jobs_status[job_id] # XXX hypothesis: the workflow is linear for a subject (no branch) if job_status & (Runner.RUNNING | Runner.INTERRUPTED): status = job_status break elif job_status & Runner.UNKNOWN: status = job_status return status def _get_step_status(self, subject_id, step_id, update_status=True): status = Runner.NOT_STARTED subject_jobs = self._get_subject_jobs(subject_id) if subject_jobs: # WARNING: assumes only 1 job per step. FIXME. job_id = subject_jobs[step_id, "step_id"] jobs_status = self._get_jobs_status(update_status) status = jobs_status[job_id] return status def _get_subject_jobs(self, subject_id): return self._jobid_to_step.get(subject_id, []) def _get_jobs_status(self, update_status=True): if update_status or self._cached_jobs_status is None: self._update_jobs_status() return self._cached_jobs_status def _update_jobs_status(self): jobs_status = {} # job_id -> status job_info_seq = self._workflow_controller.workflow_elements_status( self._workflow_id)[0] for job_info in job_info_seq: job_id = job_info[0] sw_status = job_info[1] exit_info = job_info[3] exit_status, exit_value, _, _ = exit_info status = self._sw_status_to_runner_status(sw_status, exit_status, exit_value) jobs_status[job_id] = status self._cached_jobs_status = jobs_status def _sw_status_to_runner_status(self, sw_status, exit_status, exit_value): if sw_status in [sw.constants.FAILED, sw.constants.DELETE_PENDING, sw.constants.KILL_PENDING] or \ (exit_value is not None and exit_value != 0): if exit_status == sw.constants.USER_KILLED: status = Runner.STOPPED_BY_USER elif exit_status == sw.constants.EXIT_NOTRUN: status = Runner.ABORTED_NOTRUN else: status = Runner.FAILED elif sw_status == sw.constants.DONE: status = Runner.SUCCESS # XXX status UNDERTERMINED is supposed to be a transitory status # after or before the running status elif sw_status in [sw.constants.RUNNING, sw.constants.QUEUED_ACTIVE, sw.constants.SUBMISSION_PENDING, sw.constants.UNDETERMINED]: status = Runner.RUNNING elif sw_status == sw.constants.NOT_SUBMITTED: status = Runner.NOT_STARTED else: # WARNING, SYSTEM_ON_HOLD, USER_ON_HOLD, # USER_SYSTEM_ON_HOLD, SYSTEM_SUSPENDED, USER_SUSPENDED, # USER_SYSTEM_SUSPENDED status = Runner.UNKNOWN return status
s2 = LogisticRegression() p = [("scaler", s1), ("logit", s2)] est = Pipeline(p) # get the iris dataset X, y = iris.get_data() # jsonify the method and a cross-validation scheme method_conf = JSONify_estimator(est, out="./est.json") cv_conf = JSONify_cv(StratifiedKFold, cv_kwargs={"n_folds": 5}, score_func=f1_score, stratified=True, out="./cv.json") # build the dataset file dataset = build_dataset(X, y, method_conf, cv_conf, ".", compress=1) # create the workflow in the internal representation wfi = create_wf(dataset['folds'], cv_conf, method_conf, ".", verbose=True) # save to soma-workflow format wf = save_wf(wfi, "./workflow.json", mode="soma-workflow") # create a controler and submit controler = WorkflowController() wf_id = controler.submit_workflow(workflow=wf, name="first example") # wait for completion while controler.workflow_status(wf_id) != 'workflow_done': time.sleep(2) # read final result file print(joblib.load('./final_res.pkl'))
from soma_workflow.client import Job, Workflow, WorkflowController job_1 = Job(command=["sleep", "60"], name="job 1") job_2 = Job(command=["sleep", "60"], name="job 2") job_3 = Job(command=["sleep", "60"], name="job 3") job_4 = Job(command=["sleep", "60"], name="job 4") jobs = [job_1, job_2, job_3, job_4] dependencies = [(job_1, job_2), (job_1, job_3), (job_2, job_4), (job_3, job_4)] workflow = Workflow(jobs=jobs, dependencies=dependencies) controller = WorkflowController("DSV_cluster", login, password) controller.submit_workflow(workflow=workflow, name="simple example")
from __future__ import print_function import time import os from soma_workflow.client import Job, Workflow, WorkflowController, Helper, FileTransfer from soma_workflow.configuration import Configuration # from soma_workflow.connection import RemoteConnection user = '******' try: import pwd user = pwd.getpwuid(os.getuid()).pw_name except Exception: pass controller = WorkflowController("Gabriel", user) # FileTransfer creation for input files file1 = FileTransfer(is_input=True, client_path="%s/create_file.py" % Configuration.get_home_dir(), name="script") file2 = FileTransfer(is_input=True, client_path="%s/output_file" % Configuration.get_home_dir(), name="file created on the server") # Job and Workflow run_script = Job(command=["python", file1, file2], name="copy",
echo %s """ % test_bash_script fileout.write(filecontent) fileout.close() os.chdir(cur_work_dir) job1 = Job(command=[u"touch", test_filepath], name="epac_job_test", working_directory=tmp_work_dir_path) job2 = Job(command=["%s/readfile" % cur_file_dir, test_bash_script], name="epac_job_test", working_directory=tmp_work_dir_path) soma_workflow = Workflow(jobs=[job1, job2]) resource_id = socket.gethostname() controller = WorkflowController(resource_id, "", "") ## run soma-workflow ## ================= wf_id = controller.submit_workflow(workflow=soma_workflow, name="epac workflow") Helper.wait_workflow(wf_id, controller) nb_failed_jobs = len(Helper.list_failed_jobs(wf_id, controller)) if nb_failed_jobs > 0: raise ValueError("Soma-workflow error, cannot use working directory") if not os.path.isfile(os.path.join(tmp_work_dir_path, test_filepath)): raise ValueError("Soma-workflow cannot define working directory") else: print("OK for creating new file in working directory")
group_elements = [] first_job = Job(command=["sleep", "10"], name="first job") last_job = Job(command=["sleep", "10"], name="last job") jobs.append(first_job) jobs.append(last_job) for i in range(0, 30): job = Job(command=["sleep", "60"], name="job " + repr(i)) jobs.append(job) dependencies.append((first_job, job)) dependencies.append((job, last_job)) group_elements.append(job) thirty_jobs_group = Group(elements=group_elements, name="my 30 jobs") workflow = Workflow(jobs=jobs, dependencies=dependencies, root_group=[first_job, thirty_jobs_group, last_job]) controller = WorkflowController("DSV_cluster", login, password) controller.submit_workflow(workflow=workflow, name="Simple workflow with group")
#f = open("/tmp/soma_workflow_examples/myfile", "wb") #f.write("Content of my file \n") # f.close() # FileTransfer creation for input files myfile = FileTransfer(is_input=True, client_path="/tmp/soma_workflow_examples/myfile", name="myfile") # FileTransfer creation for output files copy_of_myfile = FileTransfer(is_input=False, client_path="/tmp/soma_workflow_examples/copy_of_myfile", name="copy of my file") # Job and Workflow copy_job = Job(command=["cp", myfile, copy_of_myfile], name="copy", referenced_input_files=[myfile], referenced_output_files=[copy_of_myfile]) workflow = Workflow(jobs=[copy_job], dependencies=[]) controller = WorkflowController("DSV_cluster", login, password) controller.submit_workflow(workflow=workflow, name="simple transfer")
def run_test(cls, debug=False, interactive=False, **kwargs): sys.stdout.write("********* soma-workflow tests: %s *********\n" % cls.__name__) config_file_path = Configuration.search_config_path() resource_ids = Configuration.get_configured_resources(config_file_path) enabled_resources = getattr(WorkflowTest, 'enabled_resources', None) enable_resources = [] if not hasattr(WorkflowTest, 'resource_pass'): WorkflowTest.resource_pass = {} for resource_id in resource_ids: sys.stdout.write("============ Resource : " + resource_id + " =================== \n") config = Configuration.load_from_file(resource_id, config_file_path) if not interactive \ and ((enabled_resources is None and config.get_mode() != LIGHT_MODE) or (enabled_resources is not None and resource_id not in enabled_resources)): sys.stdout.write('Resource %s is not tested in ' 'non-interactive mode\n' % resource_id) continue # skip login/password ask if interactive: if enabled_resources is None: sys.stdout.write("Do you want to test the resource " "%s (Y/n) ? " % resource_id) sys.stdout.flush() test_resource = sys.stdin.readline() if test_resource.strip() in ['no', 'n', 'N', 'No', 'NO']: # Skip the resource sys.stdout.write('Resource %s is not tested \n' % resource_id) sys.stdout.flush() continue enable_resources.append(resource_id) (login, password) = get_user_id(resource_id, config) WorkflowTest.resource_pass[resource_id] = (login, password) else: if resource_id not in enabled_resources: continue (login, password) = WorkflowTest.resource_pass[resource_id] else: (login, password) = get_user_id(resource_id, config, interactive=interactive) if config.get_mode() == LIGHT_MODE: # use a temporary sqlite database in soma-workflow to avoid # concurrent access problems tmpdb = tempfile.mkstemp('.db', prefix='swf_') os.close(tmpdb[0]) os.unlink(tmpdb[1]) # and so on for transfers / stdio files directory tmptrans = tempfile.mkdtemp(prefix='swf_') config._database_file = tmpdb[1] config._transfered_file_dir = tmptrans wf_controller = None try: with suppress_stdout(debug): wf_controller = WorkflowController(resource_id, login, password, config=config) cls.setup_wf_controller(wf_controller) allowed_config = cls.allowed_config[:] for configuration in cls.allowed_config: if config.get_mode() != configuration[0]: allowed_config.remove(configuration) if len(allowed_config) == 0: sys.stdout.write( "No tests available for the resource %s \n" % resource_id) for configuration in allowed_config: (mode, file_system) = configuration sys.stdout.write( "\n---------------------------------------\n") sys.stdout.write("Mode : " + mode + '\n') sys.stdout.write("File system : " + file_system + '\n') cls.setup_path_management(file_system) if file_system in (cls.SHARED_RESOURCE_PATH, cls.SHARED_TRANSFER) \ and not config.get_path_translation(): sys.stdout.write( "Paths translation unavailable - not testing " "this case\n") sys.stdout.flush() continue suite_list = [] list_tests = [] for test in dir(cls): prefix = "test_" if len(test) < len(prefix): continue if test[0:len(prefix)] == prefix: list_tests.append(test) suite_list.append( unittest.TestSuite(list(map(cls, list_tests)))) alltests = unittest.TestSuite(suite_list) with suppress_stdout(debug): res = unittest.TextTestRunner( verbosity=2).run(alltests) sys.stdout.flush() sys.stdout.write("after test\n") if len(res.errors) != 0 or len(res.failures) != 0: raise RuntimeError("tests failed.") finally: sys.stdout.write("del wf_controller") if wf_controller: wf_controller.stop_engine() del wf_controller cls.setup_wf_controller(None) # del WorkflowController sys.stdout.write("deleted.") if config.get_mode() == LIGHT_MODE: if not kwargs.get('keep_temporary', False): if os.path.exists(config._database_file): os.unlink(config._database_file) if os.path.exists(config._database_file + '-journal'): os.unlink(config._database_file + '-journal') shutil.rmtree(config._transfered_file_dir) else: print('temporary files kept:') print('databse file:', config._database_file) print('transfers:', config._transfered_file_dir) if interactive and enabled_resources is None: print('set enabled_resources') WorkflowTest.enabled_resources = enable_resources
], name="Merging all the r2 and distribution respectively together.", working_directory=scripts_path) jobs.append(job_merge) # Plotting the maps job_final = Job(command=[ "python", "create_maps.py", "--input", derivatives_path, "--parameters", parameters_path, "--subject", args.subject, "--fmri_data", fmri_path ], name="Creating the maps.", working_directory=scripts_path) jobs.append(job_final) dependencies.append((job_merge, job_final)) workflow = Workflow(jobs=jobs, dependencies=dependencies) Helper.serialize(os.path.join(inputs_path, 'delete.somawf'), workflow) ### Submit the workflow to computing resource (configured in the client-server mode) controller = WorkflowController( "DSV_cluster_{}".format(login), login, password) #"DSV_cluster_ap259944", login, password workflow_id = controller.submit_workflow(workflow=workflow, name="Ridge - LPP") print("Finished !!!")
p = [("scaler", s1), ("logit", s2)] est = Pipeline(p) # get the iris dataset X, y = iris.get_data() # jsonify the method and a cross-validation scheme method_conf = JSONify_estimator(est, out="./est.json") cv_conf = JSONify_cv(StratifiedKFold, cv_kwargs={"n_folds": 5}, score_func=f1_score, stratified=True, out="./cv.json") # build the dataset file dataset = build_dataset(X, y, method_conf, cv_conf, ".", compress=1) # create the workflow in the internal representation wfi = create_wf(dataset['folds'], cv_conf, method_conf, ".", verbose=True) # save to soma-workflow format wf = save_wf(wfi, "./workflow.json", mode="soma-workflow") # create a controler and submit controler = WorkflowController() wf_id = controler.submit_workflow(workflow=wf, name="first example") # wait for completion while controler.workflow_status(wf_id) != 'workflow_done': time.sleep(2) # read final result file print(joblib.load('./final_res.pkl'))
def run_soma_workflow(treatments, exec_cmd, tmp_local_dirs, server_id, remote_host, remote_user, remote_pathes, local_result_pathes, label_for_cluster, wait_ending=False): """Dispatch treatments using soma-workflow. Parameters ---------- treatments it is a dict mapping a treatment name to a treatment object exec_cmd it is the command to run on each ROI data. tmp_local_dirs it is a dict mapping a treatment name to a local tmp dir (used to store a temporary configuration file) server_id it is the server ID as expected by WorkflowController remote_host it is the remote machine where treatments are treated in parallel remote_user it is used to log in remote_host remote_pathes it is a dict mapping a treatment name to an existing remote dir which will be used to store ROI data and result files local_result_pathes it is a dict mapping a treatment name to a local path where final results will be sorted (host will send it there by scp) label_for_cluster it is the base name used to label workflows and sub jobs """ import getpass from socket import gethostname local_user = getpass.getuser() local_host = gethostname() all_nodes = [] all_deps = [] all_groups = [] split_jobs = [] for t_id, treatment in treatments.iteritems(): tmp_local_dir = tmp_local_dirs[t_id] remote_path = remote_pathes[t_id] local_result_path = local_result_pathes[t_id] sj, n, d, g = prepare_treatment_jobs( treatment, tmp_local_dir, local_result_path, local_user, local_host, remote_host, remote_user, remote_path, label_for_cluster + '-' + str(t_id)) all_nodes.extend(n) all_deps.extend(d) all_groups.append(g) split_jobs.append(sj) # Jobs for data splitting should be done sequentially. # If they're done in parallel, they may flood the remote file system for isj in xrange(len(split_jobs)): if isj + 1 < len(split_jobs): all_deps.append((split_jobs[isj], split_jobs[isj + 1])) # # Be sure that all splitting jobs are done first: # # Is there a better way ? # for n in all_nodes: # for sjob in split_jobs: # all_deps.append((sjob,n)) # Does not seem to work well -> maybe to many deps ? workflow = Workflow(all_nodes + split_jobs, all_deps, root_group=all_groups) # f = open('/tmp/workflow.pck','w') # cPickle.dump(workflow, f) # f.close() logger.info('Open connection ...') connection = WorkflowController(server_id, remote_user) logger.info('Submit workflow ...') wf_id = connection.submit_workflow( workflow=workflow, # expiration_date="", # queue="run32", name=label_for_cluster + '-' + local_user) #wf = connection.workflow(wf_id) if wait_ending: # wait for result logger.info('Wait for workflow to end and make outputs ...') Helper.wait_workflow(wf_id, connection) for t_id, local_result_path in local_result_pathes.iteritems(): treatment = treatments[t_id] rfilename = treatment.result_dump_file if rfilename is None: rfilename = 'result.pck' local_result_file = op.join(local_result_path, op.basename(rfilename)) if not op.exists(local_result_file): raise Exception('Local result does not exist "%s"' % local_result_file) if treatment.analyser.outFile is not None: # return result only for last treatment ... print 'Load result from %s ...' % local_result_file if splitext(local_result_file)[1] == '.gz': import gzip fresult = gzip.open(local_result_file) else: fresult = open(local_result_file) results = cPickle.load(fresult) fresult.close() # print 'Make outputs ...' #treatment.output(results, dump=False) logger.info('Cleaning tmp dirs ...') for tmp_dir in tmp_local_dirs.itervalues(): shutil.rmtree(tmp_dir) return results else: logger.info('Cleaning tmp dirs ...') for tmp_dir in tmp_local_dirs.itervalues(): shutil.rmtree(tmp_dir) logger.info('Workflow sent, returning ...') return []
def run(self, **Xy): '''Run soma-workflow without gui Example ------- >>> from sklearn import datasets >>> from epac.map_reduce.engine import SomaWorkflowEngine >>> from epac.tests.wfexamples2test import WFExample2 >>> ## Build dataset >>> ## ============= >>> X, y = datasets.make_classification(n_samples=10, ... n_features=20, ... n_informative=5, ... random_state=1) >>> Xy = {'X':X, 'y':y} >>> ## Build epac tree >>> ## =============== >>> tree_root_node = WFExample2().get_workflow() >>> ## Build SomaWorkflowEngine and run function for each node >>> ## ======================================================= >>> sfw_engine = SomaWorkflowEngine(tree_root=tree_root_node, ... function_name="transform", ... num_processes=3, remove_finished_wf=False) >>> tree_root_node = sfw_engine.run(**Xy) light mode >>> ## Run reduce process >>> ## ================== >>> tree_root_node.reduce() ResultSet( [{'key': SelectKBest/SVC(C=1), 'y/test/score_f1': [ 0.6 0.6], 'y/test/score_recall_mean/pval': [ 0.5], 'y/test/score_recall/pval': [ 0. 0.5], 'y/test/score_accuracy/pval': [ 0.], 'y/test/score_f1/pval': [ 0. 0.5], 'y/test/score_precision/pval': [ 0.5 0. ], 'y/test/score_precision': [ 0.6 0.6], 'y/test/score_recall': [ 0.6 0.6], 'y/test/score_accuracy': 0.6, 'y/test/score_recall_mean': 0.6}, {'key': SelectKBest/SVC(C=3), 'y/test/score_f1': [ 0.6 0.6], 'y/test/score_recall_mean/pval': [ 0.5], 'y/test/score_recall/pval': [ 0. 0.5], 'y/test/score_accuracy/pval': [ 0.], 'y/test/score_f1/pval': [ 0. 0.5], 'y/test/score_precision/pval': [ 0.5 0. ], 'y/test/score_precision': [ 0.6 0.6], 'y/test/score_recall': [ 0.6 0.6], 'y/test/score_accuracy': 0.6, 'y/test/score_recall_mean': 0.6}]) ''' try: from soma_workflow.client import Job, Workflow from soma_workflow.client import Helper, FileTransfer from soma_workflow.client import WorkflowController except ImportError: errmsg = "No soma-workflow is found. "\ "Please verify your soma-worklow"\ "on your computer (e.g. PYTHONPATH) \n" sys.stderr.write(errmsg) sys.stdout.write(errmsg) raise NoSomaWFError tmp_work_dir_path = tempfile.mkdtemp() cur_work_dir = os.getcwd() os.chdir(tmp_work_dir_path) is_run_local = False if not self.resource_id or self.resource_id == "": self.resource_id = socket.gethostname() is_run_local = True # print "is_run_local=", is_run_local if not is_run_local: ft_working_directory = FileTransfer(is_input=True, client_path=tmp_work_dir_path, name="working directory") else: ft_working_directory = tmp_work_dir_path ## Save the database and tree to working directory ## =============================================== # np.savez(os.path.join(tmp_work_dir_path, # SomaWorkflowEngine.dataset_relative_path), **Xy) save_dataset(SomaWorkflowEngine.dataset_relative_path, **Xy) store = StoreFs(dirpath=os.path.join( tmp_work_dir_path, SomaWorkflowEngine.tree_root_relative_path)) self.tree_root.save_tree(store=store) ## Subtree job allocation on disk ## ============================== node_input = NodesInput(self.tree_root.get_key()) split_node_input = SplitNodesInput(self.tree_root, num_processes=self.num_processes) nodesinput_list = split_node_input.split(node_input) keysfile_list = save_job_list(tmp_work_dir_path, nodesinput_list) ## Build soma-workflow ## =================== jobs = self._create_jobs(keysfile_list, is_run_local, ft_working_directory) soma_workflow = Workflow(jobs=jobs) controller = WorkflowController(self.resource_id, self.login, self.pw) ## run soma-workflow ## ================= wf_id = controller.submit_workflow(workflow=soma_workflow, name="epac workflow", queue=self.queue) Helper.transfer_input_files(wf_id, controller) Helper.wait_workflow(wf_id, controller) Helper.transfer_output_files(wf_id, controller) self.engine_info = self.get_engine_info(controller, wf_id) if self.remove_finished_wf: controller.delete_workflow(wf_id) ## read result tree ## ================ self.tree_root = store.load() os.chdir(cur_work_dir) if os.path.isdir(tmp_work_dir_path) and self.remove_local_tree: shutil.rmtree(tmp_work_dir_path) return self.tree_root
def remote_map_marshal(func, largs=None, lkwargs=None, mode='local'): if largs is None: if lkwargs is not None: largs = [[]] * len(lkwargs) else: largs = [] if lkwargs is None: lkwargs = [{}] * len(largs) lkwargs = [merge_default_kwargs(func, kw) for kw in lkwargs] assert len(lkwargs) == len(largs) all_args = zip(largs, lkwargs) if mode == 'local': return [func(*args, **kwargs) for args, kwargs in all_args] elif mode == 'local_with_dumps': func_fn = './func.marshal' dump_func(func, func_fn) results = [] for i, params in enumerate(all_args): print 'params:', params params_fn = 'params_%d.pck' % i fparams = open(params_fn, 'wb') cPickle.dump(params, fparams) fparams.close() output_fn = 'output_%d.pck' % i print 'call subprocess ...' subprocess.call( ['python', '-c', cfunc_marshal, params_fn, func_fn, output_fn]) print 'Read outputs ...' fout = open(output_fn) results.append(cPickle.load(fout)) return results elif mode == 'remote_cluster': # FileTransfer creation for input files #data_dir = './rmap_data' data_dir = mkdtemp(prefix="sw_rmap") func_fn = op.join(data_dir, 'func.marshal') dump_func(func, func_fn) func_file = FileTransfer(is_input=True, client_path=func_fn, name="func_file") all_jobs = [] param_files = [] for i, params in enumerate(all_args): params_fn = op.join(data_dir, 'params_%d.pck' % i) fparams = open(params_fn, 'wb') cPickle.dump(params, fparams) fparams.close() param_file = FileTransfer(is_input=True, client_path=params_fn, name='params_file_%d' % i) param_files.append(param_file) output_fn = op.join(data_dir, 'output_%d.pck' % i) output_file = FileTransfer(is_input=False, client_path=output_fn, name='output_file_%d' % i) job = Job(command=[ 'python', '-c', cfunc, param_file, func_file, output_file ], name="rmap, item %d" % i, referenced_input_files=[func_file, param_file], referenced_output_files=[output_file]) all_jobs.append(job) workflow = Workflow(jobs=all_jobs, dependencies=[]) # submit the workflow cfg = pyhrf.cfg['parallel-cluster'] controller = WorkflowController(cfg['server_id'], cfg['user']) # controller.transfer_files(fids_to_transfer) wf_id = controller.submit_workflow(workflow=workflow, name="remote_map") Helper.transfer_input_files(wf_id, controller) Helper.wait_workflow(wf_id, controller) Helper.transfer_output_files(wf_id, controller) results = [] for i in xrange(len(all_args)): fout = open(op.join(data_dir, 'output_%d.pck' % i)) results.append(cPickle.load(fout)) fout.close() return results
# f.write("Content of my file \n") # f.close() # FileTransfer creation for input files myfile = FileTransfer(is_input=True, client_path="/tmp/soma_workflow_examples/myfile", name="myfile") # FileTransfer creation for output files copy_of_myfile = FileTransfer(is_input=False, client_path="/tmp/soma_workflow_examples/copy_of_myfile", name="copy of my file") # Job and Workflow copy_job = Job(command=["cp", myfile, copy_of_myfile], name="copy", referenced_input_files=[myfile], referenced_output_files=[copy_of_myfile]) workflow = Workflow(jobs=[copy_job], dependencies=[]) login = '******' password = '******' controller = WorkflowController("DSV_cluster", login, password) controller.submit_workflow(workflow=workflow, name="simple transfer")
def remote_map(func, largs=None, lkwargs=None, mode='serial'): """ Execute a function in parallel on a list of arguments. Args: *func* (function): function to apply on each item. **this function must be importable on the remote side** *largs* (list of tuple): each item in the list is a tuple containing all positional argument values of the function *lkwargs* (list of dict): each item in the list is a dict containing all named arguments of the function mapped to their value. *mode* (str): indicates how execution is distributed. Choices are: - "serial": single-thread loop on the local machine - "local" : use joblib to run tasks in parallel. The number of simultaneous jobs is defined in the configuration section ['parallel-local']['nb_procs'] see ~/.pyhrf/config.cfg - "remote_cluster: use somaworkflow to run tasks in parallel. The connection setup has to be defined in the configuration section ['parallel-cluster'] of ~/.pyhrf/config.cfg. - "local_with_dumps": testing purpose only, run each task serially as a subprocess. Returns: a list of results Raises: RemoteException if any remote task has failed Example: >>> from pyhrf.parallel import remote_map >>> def foo(a, b=2): \ return a + b >>> remote_map(foo, [(2,),(3,)], [{'b':5}, {'b':7}]) [7, 10] """ if largs is None: if lkwargs is not None: largs = [tuple()] * len(lkwargs) else: largs = [tuple()] if lkwargs is None: lkwargs = [{}] * len(largs) lkwargs = [merge_default_kwargs(func, kw) for kw in lkwargs] assert len(lkwargs) == len(largs) all_args = zip(largs, lkwargs) # print 'all_args:', all_args fmodule = func.__module__ fname = '.'.join([fmodule, func.__name__]) if mode == 'serial': return [func(*args, **kwargs) for args, kwargs in all_args] elif mode == 'local': try: from joblib import Parallel, delayed except ImportError: raise ImportError('Can not import joblib. It is ' 'required to enable parallel ' 'processing on a local machine.') if logger.getEffectiveLevel() == logging.DEBUG: parallel_verb = 10 else: parallel_verb = 0 if pyhrf.cfg['parallel-local']['nb_procs']: n_jobs = pyhrf.cfg['parallel-local']['nb_procs'] else: n_jobs = available_cpu_count() p = Parallel(n_jobs=n_jobs, verbose=parallel_verb) return p(delayed(func)(*args, **kwargs) for args, kwargs in all_args) elif mode == 'local_with_dumps': results = [] for i, params in enumerate(all_args): # print 'params:', params params_fn = 'params_%d.pck' % i fparams = open(params_fn, 'wb') cPickle.dump(params, fparams) fparams.close() output_fn = 'output_%d.pck' % i # print 'call subprocess ...' subprocess.call([ 'python', '-c', cfunc % (fmodule, fname), params_fn, output_fn ]) # print 'Read outputs ...' fout = open(output_fn) results.append(cPickle.load(fout)) return results elif mode == 'remote_cluster': # FileTransfer creation for input files #data_dir = './rmap_data' data_dir = mkdtemp(prefix="sw_rmap") all_jobs = [] param_files = [] for i, params in enumerate(all_args): params_fn = op.join(data_dir, 'params_%d.pck' % i) fparams = open(params_fn, 'wb') cPickle.dump(params, fparams) fparams.close() param_file = FileTransfer(is_input=True, client_path=params_fn, name='params_file_%d' % i) param_files.append(param_file) output_fn = op.join(data_dir, 'output_%d.pck' % i) output_file = FileTransfer(is_input=False, client_path=output_fn, name='output_file_%d' % i) job = Job(command=[ 'pyhrf_exec_pyfunc', fmodule, fname, param_file, output_file ], name="rmap, item %d" % i, referenced_input_files=[param_file], referenced_output_files=[output_file]) all_jobs.append(job) workflow = Workflow(jobs=all_jobs, dependencies=[]) # submit the workflow cfg = pyhrf.cfg['parallel-cluster'] controller = WorkflowController(cfg['server_id'], cfg['user']) # controller.transfer_files(fids_to_transfer) wf_id = controller.submit_workflow(workflow=workflow, name="remote_map") Helper.transfer_input_files(wf_id, controller) Helper.wait_workflow(wf_id, controller) Helper.transfer_output_files(wf_id, controller) results = [] for i in xrange(len(all_args)): fnout = op.join(data_dir, 'output_%d.pck' % i) fout = open(fnout) o = cPickle.load(fout) print 'file cPickle loaded:', o fout.close() os.remove(fnout) if isinstance(o, Exception): raise RemoteException('Task %d failed' % i, o) if o.errno != 17: raise RemoteException('Task %d failed' % i, o) results.append(o) return results
f.close() # Creation of the FileTransfer object to transfer the working directory my_working_directory = FileTransfer(is_input=True, client_path="/tmp/my_working_directory", name="working directory") # Jobs and Workflow job1 = Job(command=["cp", "myfile1", "copy_of_myfile1"], name="job1", referenced_input_files=[my_working_directory], referenced_output_files=[my_working_directory], working_directory=my_working_directory) job2 = Job(command=["cp", "myfile2", "copy_of_myfile2"], name="job2", referenced_input_files=[my_working_directory], referenced_output_files=[my_working_directory], working_directory=my_working_directory) workflow = Workflow(jobs=[job1, job2], dependencies=[]) # Submit the workflow print("password? ") password = getpass.getpass() controller = WorkflowController("DSV_cluster", login, password) controller.submit_workflow(workflow=workflow, name="working directory transfer example")
jobs += group_score + group_significativity + group_merge jobs.append(job_final) scores = Group(elements=group_score, name="group where test scores are calculated") significativity = Group( elements=group_significativity, name="group where distributions are calculated for significance") merge = Group(elements=group_merge, name="group where we merge results") workflow = Workflow(jobs=jobs, dependencies=dependencies, root_group=[scores, significativity, merge, job_final]) Helper.serialize( os.path.join(inputs_path, 'optimized_cluster_part_2.somawf'), workflow) ### Submit the workflow to computing resource (configured in the client-server mode) controller = WorkflowController( "DSV_cluster_{}".format(login), login, password) #"DSV_cluster_ap259944", login, password workflow_id = controller.submit_workflow(workflow=workflow, name="Cluster optimized part 2") print("Finished !!!")
def run_soma_workflow(treatments, exec_cmd, tmp_local_dirs, server_id, remote_host, remote_user, remote_pathes, local_result_pathes, label_for_cluster, wait_ending=False): """Dispatch treatments using soma-workflow. Parameters ---------- treatments it is a dict mapping a treatment name to a treatment object exec_cmd it is the command to run on each ROI data. tmp_local_dirs it is a dict mapping a treatment name to a local tmp dir (used to store a temporary configuration file) server_id it is the server ID as expected by WorkflowController remote_host it is the remote machine where treatments are treated in parallel remote_user it is used to log in remote_host remote_pathes it is a dict mapping a treatment name to an existing remote dir which will be used to store ROI data and result files local_result_pathes it is a dict mapping a treatment name to a local path where final results will be sorted (host will send it there by scp) label_for_cluster it is the base name used to label workflows and sub jobs """ import getpass from socket import gethostname local_user = getpass.getuser() local_host = gethostname() all_nodes = [] all_deps = [] all_groups = [] split_jobs = [] for t_id, treatment in treatments.iteritems(): tmp_local_dir = tmp_local_dirs[t_id] remote_path = remote_pathes[t_id] local_result_path = local_result_pathes[t_id] sj, n, d, g = prepare_treatment_jobs(treatment, tmp_local_dir, local_result_path, local_user, local_host, remote_host, remote_user, remote_path, label_for_cluster + '-' + str(t_id)) all_nodes.extend(n) all_deps.extend(d) all_groups.append(g) split_jobs.append(sj) # Jobs for data splitting should be done sequentially. # If they're done in parallel, they may flood the remote file system for isj in xrange(len(split_jobs)): if isj + 1 < len(split_jobs): all_deps.append((split_jobs[isj], split_jobs[isj + 1])) # # Be sure that all splitting jobs are done first: # # Is there a better way ? # for n in all_nodes: # for sjob in split_jobs: # all_deps.append((sjob,n)) # Does not seem to work well -> maybe to many deps ? workflow = Workflow( all_nodes + split_jobs, all_deps, root_group=all_groups) # f = open('/tmp/workflow.pck','w') # cPickle.dump(workflow, f) # f.close() logger.info('Open connection ...') connection = WorkflowController(server_id, remote_user) logger.info('Submit workflow ...') wf_id = connection.submit_workflow(workflow=workflow, # expiration_date="", # queue="run32", name=label_for_cluster + '-' + local_user) #wf = connection.workflow(wf_id) if wait_ending: # wait for result logger.info('Wait for workflow to end and make outputs ...') Helper.wait_workflow(wf_id, connection) for t_id, local_result_path in local_result_pathes.iteritems(): treatment = treatments[t_id] rfilename = treatment.result_dump_file if rfilename is None: rfilename = 'result.pck' local_result_file = op.join(local_result_path, op.basename(rfilename)) if not op.exists(local_result_file): raise Exception('Local result does not exist "%s"' % local_result_file) if treatment.analyser.outFile is not None: # return result only for last treatment ... print 'Load result from %s ...' % local_result_file if splitext(local_result_file)[1] == '.gz': import gzip fresult = gzip.open(local_result_file) else: fresult = open(local_result_file) results = cPickle.load(fresult) fresult.close() # print 'Make outputs ...' #treatment.output(results, dump=False) logger.info('Cleaning tmp dirs ...') for tmp_dir in tmp_local_dirs.itervalues(): shutil.rmtree(tmp_dir) return results else: logger.info('Cleaning tmp dirs ...') for tmp_dir in tmp_local_dirs.itervalues(): shutil.rmtree(tmp_dir) logger.info('Workflow sent, returning ...') return []
echo %s """ % test_bash_script fileout.write(filecontent) fileout.close() os.chdir(cur_work_dir) job1 = Job(command=[u"touch", test_filepath], name="epac_job_test", working_directory=tmp_work_dir_path) job2 = Job(command=["%s/readfile" % cur_file_dir, test_bash_script], name="epac_job_test", working_directory=tmp_work_dir_path) soma_workflow = Workflow(jobs=[job1, job2]) resource_id = socket.gethostname() controller = WorkflowController(resource_id, "", "") ## run soma-workflow ## ================= wf_id = controller.submit_workflow(workflow=soma_workflow, name="epac workflow") Helper.wait_workflow(wf_id, controller) nb_failed_jobs = len(Helper.list_failed_jobs(wf_id, controller)) if nb_failed_jobs > 0: raise ValueError("Soma-workflow error, cannot use working directory") if not os.path.isfile(os.path.join(tmp_work_dir_path, test_filepath)): raise ValueError("Soma-workflow cannot define working directory") else: print "OK for creating new file in working directory"
def remote_map(func, largs=None, lkwargs=None, mode='serial'): """ Execute a function in parallel on a list of arguments. Args: *func* (function): function to apply on each item. **this function must be importable on the remote side** *largs* (list of tuple): each item in the list is a tuple containing all positional argument values of the function *lkwargs* (list of dict): each item in the list is a dict containing all named arguments of the function mapped to their value. *mode* (str): indicates how execution is distributed. Choices are: - "serial": single-thread loop on the local machine - "local" : use joblib to run tasks in parallel. The number of simultaneous jobs is defined in the configuration section ['parallel-local']['nb_procs'] see ~/.pyhrf/config.cfg - "remote_cluster: use somaworkflow to run tasks in parallel. The connection setup has to be defined in the configuration section ['parallel-cluster'] of ~/.pyhrf/config.cfg. - "local_with_dumps": testing purpose only, run each task serially as a subprocess. Returns: a list of results Raises: RemoteException if any remote task has failed Example: >>> from pyhrf.parallel import remote_map >>> def foo(a, b=2): \ return a + b >>> remote_map(foo, [(2,),(3,)], [{'b':5}, {'b':7}]) [7, 10] """ if largs is None: if lkwargs is not None: largs = [tuple()] * len(lkwargs) else: largs = [tuple()] if lkwargs is None: lkwargs = [{}] * len(largs) lkwargs = [merge_default_kwargs(func, kw) for kw in lkwargs] assert len(lkwargs) == len(largs) all_args = zip(largs, lkwargs) # print 'all_args:', all_args fmodule = func.__module__ fname = '.'.join([fmodule, func.__name__]) if mode == 'serial': return [func(*args, **kwargs) for args, kwargs in all_args] elif mode == 'local': try: from joblib import Parallel, delayed except ImportError: raise ImportError('Can not import joblib. It is ' 'required to enable parallel ' 'processing on a local machine.') if logger.getEffectiveLevel() == logging.DEBUG: parallel_verb = 10 else: parallel_verb = 0 if pyhrf.cfg['parallel-local']['nb_procs']: n_jobs = pyhrf.cfg['parallel-local']['nb_procs'] else: n_jobs = available_cpu_count() p = Parallel(n_jobs=n_jobs, verbose=parallel_verb) return p(delayed(func)(*args, **kwargs) for args, kwargs in all_args) elif mode == 'local_with_dumps': results = [] for i, params in enumerate(all_args): # print 'params:', params params_fn = 'params_%d.pck' % i fparams = open(params_fn, 'wb') cPickle.dump(params, fparams) fparams.close() output_fn = 'output_%d.pck' % i # print 'call subprocess ...' subprocess.call(['python', '-c', cfunc % (fmodule, fname), params_fn, output_fn]) # print 'Read outputs ...' fout = open(output_fn) results.append(cPickle.load(fout)) return results elif mode == 'remote_cluster': # FileTransfer creation for input files #data_dir = './rmap_data' data_dir = mkdtemp(prefix="sw_rmap") all_jobs = [] param_files = [] for i, params in enumerate(all_args): params_fn = op.join(data_dir, 'params_%d.pck' % i) fparams = open(params_fn, 'wb') cPickle.dump(params, fparams) fparams.close() param_file = FileTransfer(is_input=True, client_path=params_fn, name='params_file_%d' % i) param_files.append(param_file) output_fn = op.join(data_dir, 'output_%d.pck' % i) output_file = FileTransfer(is_input=False, client_path=output_fn, name='output_file_%d' % i) job = Job(command=['pyhrf_exec_pyfunc', fmodule, fname, param_file, output_file], name="rmap, item %d" % i, referenced_input_files=[param_file], referenced_output_files=[output_file]) all_jobs.append(job) workflow = Workflow(jobs=all_jobs, dependencies=[]) # submit the workflow cfg = pyhrf.cfg['parallel-cluster'] controller = WorkflowController(cfg['server_id'], cfg['user']) # controller.transfer_files(fids_to_transfer) wf_id = controller.submit_workflow( workflow=workflow, name="remote_map") Helper.transfer_input_files(wf_id, controller) Helper.wait_workflow(wf_id, controller) Helper.transfer_output_files(wf_id, controller) results = [] for i in xrange(len(all_args)): fnout = op.join(data_dir, 'output_%d.pck' % i) fout = open(fnout) o = cPickle.load(fout) print 'file cPickle loaded:', o fout.close() os.remove(fnout) if isinstance(o, Exception): raise RemoteException('Task %d failed' % i, o) if o.errno != 17: raise RemoteException('Task %d failed' % i, o) results.append(o) return results
from soma_workflow.client import Job, Workflow, WorkflowController, SharedResourcePath, FileTransfer # SharedResourcePath creation for the input file. # The input file is read direclty in the data directory located on the # the computing resource side. myfile = SharedResourcePath(relative_path="myfile", namespace="MyApp", uuid="my_example_dir") # FileTransfer creation for the output file. # That way the output file will not be written in the data directory # located on the computing resource file system. copy_of_myfile = FileTransfer(is_input=False, client_path="/tmp/soma_workflow_examples/copy_of_myfile", name="copy of my file") # Job and Workflow creation copy_job = Job(command=["cp", myfile, copy_of_myfile], name="copy", referenced_input_files=[], referenced_output_files=[copy_of_myfile]) workflow = Workflow(jobs=[copy_job], dependencies=[]) # workflow submission controller = WorkflowController("DSV_cluster", login, password) controller.submit_workflow(workflow=workflow, name="shared resource path example")
def run(self, **Xy): """Run soma-workflow without gui Example ------- >>> from sklearn import datasets >>> from epac.map_reduce.engine import SomaWorkflowEngine >>> from epac.tests.wfexamples2test import WFExample2 >>> ## Build dataset >>> ## ============= >>> X, y = datasets.make_classification(n_samples=10, ... n_features=20, ... n_informative=5, ... random_state=1) >>> Xy = {'X':X, 'y':y} >>> ## Build epac tree >>> ## =============== >>> tree_root_node = WFExample2().get_workflow() >>> ## Build SomaWorkflowEngine and run function for each node >>> ## ======================================================= >>> sfw_engine = SomaWorkflowEngine(tree_root=tree_root_node, ... function_name="transform", ... num_processes=3, remove_finished_wf=False) >>> tree_root_node = sfw_engine.run(**Xy) light mode >>> ## Run reduce process >>> ## ================== >>> tree_root_node.reduce() ResultSet( [{'key': SelectKBest/SVC(C=1), 'y/test/score_f1': [ 0.6 0.6], 'y/test/score_recall_mean/pval': [ 0.5], 'y/test/score_recall/pval': [ 0. 0.5], 'y/test/score_accuracy/pval': [ 0.], 'y/test/score_f1/pval': [ 0. 0.5], 'y/test/score_precision/pval': [ 0.5 0. ], 'y/test/score_precision': [ 0.6 0.6], 'y/test/score_recall': [ 0.6 0.6], 'y/test/score_accuracy': 0.6, 'y/test/score_recall_mean': 0.6}, {'key': SelectKBest/SVC(C=3), 'y/test/score_f1': [ 0.6 0.6], 'y/test/score_recall_mean/pval': [ 0.5], 'y/test/score_recall/pval': [ 0. 0.5], 'y/test/score_accuracy/pval': [ 0.], 'y/test/score_f1/pval': [ 0. 0.5], 'y/test/score_precision/pval': [ 0.5 0. ], 'y/test/score_precision': [ 0.6 0.6], 'y/test/score_recall': [ 0.6 0.6], 'y/test/score_accuracy': 0.6, 'y/test/score_recall_mean': 0.6}]) """ try: from soma_workflow.client import Job, Workflow from soma_workflow.client import Helper, FileTransfer from soma_workflow.client import WorkflowController except ImportError: errmsg = ( "No soma-workflow is found. " "Please verify your soma-worklow" "on your computer (e.g. PYTHONPATH) \n" ) sys.stderr.write(errmsg) sys.stdout.write(errmsg) raise NoSomaWFError tmp_work_dir_path = tempfile.mkdtemp() cur_work_dir = os.getcwd() os.chdir(tmp_work_dir_path) is_run_local = False if not self.resource_id or self.resource_id == "": self.resource_id = socket.gethostname() is_run_local = True # print "is_run_local=", is_run_local if not is_run_local: ft_working_directory = FileTransfer(is_input=True, client_path=tmp_work_dir_path, name="working directory") else: ft_working_directory = tmp_work_dir_path ## Save the database and tree to working directory ## =============================================== # np.savez(os.path.join(tmp_work_dir_path, # SomaWorkflowEngine.dataset_relative_path), **Xy) save_dataset(SomaWorkflowEngine.dataset_relative_path, **Xy) store = StoreFs(dirpath=os.path.join(tmp_work_dir_path, SomaWorkflowEngine.tree_root_relative_path)) self.tree_root.save_tree(store=store) ## Subtree job allocation on disk ## ============================== node_input = NodesInput(self.tree_root.get_key()) split_node_input = SplitNodesInput(self.tree_root, num_processes=self.num_processes) nodesinput_list = split_node_input.split(node_input) keysfile_list = save_job_list(tmp_work_dir_path, nodesinput_list) ## Build soma-workflow ## =================== jobs = self._create_jobs(keysfile_list, is_run_local, ft_working_directory) soma_workflow = Workflow(jobs=jobs) controller = WorkflowController(self.resource_id, self.login, self.pw) ## run soma-workflow ## ================= wf_id = controller.submit_workflow(workflow=soma_workflow, name="epac workflow", queue=self.queue) Helper.transfer_input_files(wf_id, controller) Helper.wait_workflow(wf_id, controller) Helper.transfer_output_files(wf_id, controller) self.engine_info = self.get_engine_info(controller, wf_id) if self.remove_finished_wf: controller.delete_workflow(wf_id) ## read result tree ## ================ self.tree_root = store.load() os.chdir(cur_work_dir) if os.path.isdir(tmp_work_dir_path) and self.remove_local_tree: shutil.rmtree(tmp_work_dir_path) return self.tree_root
"--output_r2", os.path.join(args.output, 'r2'), "--output_distribution", os.path.join(args.output, 'distribution'), "--x", args.x, "--y", args.y, "--shuffling", shuffling, "--n_permutations", args.nb_permutations, "--alpha_percentile", args.alpha_percentile ], name="job {} - alpha {}".format(run, alpha), working_directory=scripts_path) group_significativity.append(job) jobs.append(job) distribution_voxels = Group(elements=group_significativity, name="Voxel wise fitting of the models") workflow2 = Workflow(jobs=jobs, root_group=[distribution_voxels]) ### Submit the workflow to computing resource (configured in the client-server mode) controller2 = WorkflowController( "DSV_cluster_ap259944", args.login, args.password) #"DSV_cluster_ap259944", args.login, args.password workflow_id2 = controller2.submit_workflow(workflow=workflow2, name="Voxel-wise computations") # You may use the gui or manually transfer the files: manual = True if manual: Helper.transfer_input_files(workflow_id2, controller2) Helper.wait_workflow(workflow_id2, controller2) Helper.transfer_output_files(workflow_id2, controller2)