def _workflow_stop(self): self._workflow_controller.stop_workflow(self._workflow_id) # transfer back files, if any Helper.transfer_output_files(self._workflow_id, self._workflow_controller) interrupted_step_ids = self._get_interrupted_step_ids() for subject_id, step_ids in six.iteritems(interrupted_step_ids): if step_ids: analysis = self._study.analyses[subject_id] analysis.clear_results(step_ids)
def wait(self, subject_id=None, step_id=None): if subject_id is None and step_id is None: Helper.wait_workflow( self._workflow_id, self._workflow_controller) elif subject_id is not None: if step_id is None: raise NotImplementedError else: self._step_wait(subject_id, step_id) else: raise NotImplementedError # transfer back files, if any Helper.transfer_output_files(self._workflow_id, self._workflow_controller)
def test_result(self): workflow = self.wf_examples.example_wrong_native_spec_pbs() self.wf_id = self.wf_ctrl.submit_workflow( workflow=workflow, name=self.__class__.__name__) # Transfer input files if file transfer if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: Helper.transfer_input_files(self.wf_id, self.wf_ctrl) # Wait for the workflow to finish Helper.wait_workflow(self.wf_id, self.wf_ctrl) # Transfer output files if file transfer if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: Helper.transfer_output_files(self.wf_id, self.wf_ctrl) status = self.wf_ctrl.workflow_status(self.wf_id) self.assertTrue(status == constants.WORKFLOW_DONE, "workflow status : %s. Expected : %s" % (status, constants.WORKFLOW_DONE)) nb_failed_jobs = len(Helper.list_failed_jobs( self.wf_id, self.wf_ctrl)) self.assertTrue(nb_failed_jobs == 0, "nb failed jobs : %i. Expected : %i" % (nb_failed_jobs, 0)) nb_failed_aborted_jobs = len(Helper.list_failed_jobs( self.wf_id, self.wf_ctrl, include_aborted_jobs=True)) if self.path_management == self.LOCAL_PATH: self.assertTrue(nb_failed_aborted_jobs == 0, "nb failed jobs including aborted : %i. " "Expected : %i" % (nb_failed_aborted_jobs, 0)) else: self.assertTrue(nb_failed_aborted_jobs == 1, "nb failed jobs including aborted : %i. " "Expected : %i" % (nb_failed_aborted_jobs, 1))
def test_result(self): workflow = self.wf_examples.example_wrong_native_spec_pbs() self.wf_id = self.wf_ctrl.submit_workflow(workflow=workflow, name=self.__class__.__name__) # Transfer input files if file transfer if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: Helper.transfer_input_files(self.wf_id, self.wf_ctrl) # Wait for the workflow to finish Helper.wait_workflow(self.wf_id, self.wf_ctrl) # Transfer output files if file transfer if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: Helper.transfer_output_files(self.wf_id, self.wf_ctrl) status = self.wf_ctrl.workflow_status(self.wf_id) self.assertTrue( status == constants.WORKFLOW_DONE, "workflow status : %s. Expected : %s" % (status, constants.WORKFLOW_DONE)) nb_failed_jobs = len(Helper.list_failed_jobs(self.wf_id, self.wf_ctrl)) self.assertTrue( nb_failed_jobs == 0, "nb failed jobs : %i. Expected : %i" % (nb_failed_jobs, 0)) nb_failed_aborted_jobs = len( Helper.list_failed_jobs(self.wf_id, self.wf_ctrl, include_aborted_jobs=True)) if self.path_management == self.LOCAL_PATH \ or self.wf_ctrl.get_scheduler_type() != 'pbs': self.assertTrue( nb_failed_aborted_jobs == 0, "nb failed jobs including aborted : %i. " "Expected : %i" % (nb_failed_aborted_jobs, 0)) else: self.assertTrue( nb_failed_aborted_jobs == 1, "nb failed jobs including aborted : %i. " "Expected : %i" % (nb_failed_aborted_jobs, 1))
def remote_map_marshal(func, largs=None, lkwargs=None, mode='local'): if largs is None: if lkwargs is not None: largs = [[]] * len(lkwargs) else: largs = [] if lkwargs is None: lkwargs = [{}] * len(largs) lkwargs = [merge_default_kwargs(func, kw) for kw in lkwargs] assert len(lkwargs) == len(largs) all_args = zip(largs, lkwargs) if mode == 'local': return [func(*args, **kwargs) for args, kwargs in all_args] elif mode == 'local_with_dumps': func_fn = './func.marshal' dump_func(func, func_fn) results = [] for i, params in enumerate(all_args): print 'params:', params params_fn = 'params_%d.pck' % i fparams = open(params_fn, 'wb') cPickle.dump(params, fparams) fparams.close() output_fn = 'output_%d.pck' % i print 'call subprocess ...' subprocess.call(['python', '-c', cfunc_marshal, params_fn, func_fn, output_fn]) print 'Read outputs ...' fout = open(output_fn) results.append(cPickle.load(fout)) return results elif mode == 'remote_cluster': # FileTransfer creation for input files #data_dir = './rmap_data' data_dir = mkdtemp(prefix="sw_rmap") func_fn = op.join(data_dir, 'func.marshal') dump_func(func, func_fn) func_file = FileTransfer(is_input=True, client_path=func_fn, name="func_file") all_jobs = [] param_files = [] for i, params in enumerate(all_args): params_fn = op.join(data_dir, 'params_%d.pck' % i) fparams = open(params_fn, 'wb') cPickle.dump(params, fparams) fparams.close() param_file = FileTransfer(is_input=True, client_path=params_fn, name='params_file_%d' % i) param_files.append(param_file) output_fn = op.join(data_dir, 'output_%d.pck' % i) output_file = FileTransfer(is_input=False, client_path=output_fn, name='output_file_%d' % i) job = Job(command=['python', '-c', cfunc, param_file, func_file, output_file], name="rmap, item %d" % i, referenced_input_files=[func_file, param_file], referenced_output_files=[output_file]) all_jobs.append(job) workflow = Workflow(jobs=all_jobs, dependencies=[]) # submit the workflow cfg = pyhrf.cfg['parallel-cluster'] controller = WorkflowController(cfg['server_id'], cfg['user']) # controller.transfer_files(fids_to_transfer) wf_id = controller.submit_workflow( workflow=workflow, name="remote_map") Helper.transfer_input_files(wf_id, controller) Helper.wait_workflow(wf_id, controller) Helper.transfer_output_files(wf_id, controller) results = [] for i in xrange(len(all_args)): fout = open(op.join(data_dir, 'output_%d.pck' % i)) results.append(cPickle.load(fout)) fout.close() return results
def test_result(self): workflow = self.wf_examples.example_native_spec_pbs() self.wf_id = self.wf_ctrl.submit_workflow(workflow=workflow, name=self.__class__.__name__) # Transfer input files if file transfer if self.path_management == self.FILE_TRANSFER or self.path_management == self.SHARED_TRANSFER: Helper.transfer_input_files(self.wf_id, self.wf_ctrl) # Wait for the workflow to finish Helper.wait_workflow(self.wf_id, self.wf_ctrl) # Transfer output files if file transfer if self.path_management == self.FILE_TRANSFER or self.path_management == self.SHARED_TRANSFER: Helper.transfer_output_files(self.wf_id, self.wf_ctrl) status = self.wf_ctrl.workflow_status(self.wf_id) self.assertTrue( status == constants.WORKFLOW_DONE, "workflow status : %s. Expected : %s" % (status, constants.WORKFLOW_DONE) ) nb_failed_jobs = len(Helper.list_failed_jobs(self.wf_id, self.wf_ctrl)) self.assertTrue(nb_failed_jobs == 0, "nb failed jobs : %i. Expected : %i" % (nb_failed_jobs, 0)) nb_failed_aborted_jobs = len(Helper.list_failed_jobs(self.wf_id, self.wf_ctrl, include_aborted_jobs=True)) self.assertTrue( nb_failed_aborted_jobs == 0, "nb failed jobs including aborted : %i. Expected : %i" % (nb_failed_aborted_jobs, 0), ) (jobs_info, transfers_info, workflow_status, workflow_queue, tmp_files) = self.wf_ctrl.workflow_elements_status( self.wf_id ) for (job_id, tmp_status, queue, exit_info, dates) in jobs_info: job_list = self.wf_ctrl.jobs([job_id]) job_name, job_command, job_submission_date = job_list[job_id] self.tested_job = job_id if exit_info[0] == constants.FINISHED_REGULARLY: # To check job standard out and standard err job_stdout_file = tempfile.NamedTemporaryFile( prefix="job_soma_out_log_", suffix=repr(job_id), delete=False ) job_stdout_file = job_stdout_file.name job_stderr_file = tempfile.NamedTemporaryFile( prefix="job_soma_outerr_log_", suffix=repr(job_id), delete=False ) job_stderr_file = job_stderr_file.name try: self.wf_ctrl.retrieve_job_stdouterr(job_id, job_stdout_file, job_stderr_file) # Test stdout isSame, msg = identical_files(job_stdout_file, self.wf_examples.lo_stdout[1]) self.assertTrue(isSame, msg) # Test no stderr self.assertTrue( os.stat(job_stderr_file).st_size == 0, "job stderr not empty : cf %s" % job_stderr_file ) # Test output files if self.path_management == self.LOCAL_PATH: isSame, msg = identical_files( self.wf_examples.lo_out_model_file[11], self.wf_examples.lo_file[11] ) self.assertTrue(isSame, msg) isSame, msg = identical_files( self.wf_examples.lo_out_model_file[12], self.wf_examples.lo_file[12] ) self.assertTrue(isSame, msg) if self.path_management == self.FILE_TRANSFER or self.path_management == self.SHARED_TRANSFER: isSame, msg = identical_files( self.wf_examples.lo_out_model_file[11], self.wf_examples.tr_file[11].client_path ) self.assertTrue(isSame, msg) isSame, msg = identical_files( self.wf_examples.lo_out_model_file[12], self.wf_examples.tr_file[12].client_path ) self.assertTrue(isSame, msg) finally: os.unlink(job_stdout_file) os.unlink(job_stderr_file) del self.tested_job
def test_result(self): workflow = self.wf_examples.example_simple_exception1() self.wf_id = self.wf_ctrl.submit_workflow( workflow=workflow, name=self.__class__.__name__) # Transfer input files if file transfer if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: Helper.transfer_input_files(self.wf_id, self.wf_ctrl) # Wait for the workflow to finish Helper.wait_workflow(self.wf_id, self.wf_ctrl) # Transfer output files if file transfer if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: Helper.transfer_output_files(self.wf_id, self.wf_ctrl) status = self.wf_ctrl.workflow_status(self.wf_id) self.assertTrue(status == constants.WORKFLOW_DONE, "workflow status : %s. Expected : %s" % (status, constants.WORKFLOW_DONE)) nb_failed_jobs = len(Helper.list_failed_jobs(self.wf_id, self.wf_ctrl)) self.assertTrue(nb_failed_jobs == 1, "nb failed jobs : %i. Expected : %i" % (nb_failed_jobs, 1)) nb_failed_aborted_jobs = len(Helper.list_failed_jobs( self.wf_id, self.wf_ctrl, include_aborted_jobs=True)) self.assertTrue(nb_failed_aborted_jobs == 4, "nb failed jobs including aborted : %i. Expected : %i" % (nb_failed_aborted_jobs, 4)) (jobs_info, transfers_info, workflow_status, workflow_queue, tmp_files) = self.wf_ctrl.workflow_elements_status(self.wf_id) for (job_id, tmp_status, queue, exit_info, dates) in jobs_info: job_list = self.wf_ctrl.jobs([job_id]) job_name, job_command, job_submission_date = job_list[job_id] self.tested_job = job_id if exit_info[0] == constants.FINISHED_REGULARLY: # To check job standard out and standard err job_stdout_file = tempfile.NamedTemporaryFile( prefix="job_soma_out_log_", suffix=repr(job_id), delete=False) job_stdout_file = job_stdout_file.name job_stderr_file = tempfile.NamedTemporaryFile( prefix="job_soma_outerr_log_", suffix=repr(job_id), delete=False) job_stderr_file = job_stderr_file.name try: self.wf_ctrl.retrieve_job_stdouterr(job_id, job_stdout_file, job_stderr_file) if job_name == 'job1 with exception': # Test stdout isSame, msg = identical_files( job_stdout_file, self.wf_examples.lo_stdout_exception_model) self.assertTrue(isSame, msg) # Test the last line of stderr with open(job_stderr_file) as f: lines = f.readlines() expected_error = 'Exception: Paf Boum Boum Bada Boum !!!\n' isSame = (lines[-1] == expected_error) self.assertTrue(isSame, "Job exception : %s. Expected : %s" % (lines[-1], expected_error)) finally: os.unlink(job_stdout_file) os.unlink(job_stderr_file) del self.tested_job
def run(self, **Xy): '''Run soma-workflow without gui Example ------- >>> from sklearn import datasets >>> from epac.map_reduce.engine import SomaWorkflowEngine >>> from epac.tests.wfexamples2test import WFExample2 >>> ## Build dataset >>> ## ============= >>> X, y = datasets.make_classification(n_samples=10, ... n_features=20, ... n_informative=5, ... random_state=1) >>> Xy = {'X':X, 'y':y} >>> ## Build epac tree >>> ## =============== >>> tree_root_node = WFExample2().get_workflow() >>> ## Build SomaWorkflowEngine and run function for each node >>> ## ======================================================= >>> sfw_engine = SomaWorkflowEngine(tree_root=tree_root_node, ... function_name="transform", ... num_processes=3, remove_finished_wf=False) >>> tree_root_node = sfw_engine.run(**Xy) light mode >>> ## Run reduce process >>> ## ================== >>> tree_root_node.reduce() ResultSet( [{'key': SelectKBest/SVC(C=1), 'y/test/score_f1': [ 0.6 0.6], 'y/test/score_recall_mean/pval': [ 0.5], 'y/test/score_recall/pval': [ 0. 0.5], 'y/test/score_accuracy/pval': [ 0.], 'y/test/score_f1/pval': [ 0. 0.5], 'y/test/score_precision/pval': [ 0.5 0. ], 'y/test/score_precision': [ 0.6 0.6], 'y/test/score_recall': [ 0.6 0.6], 'y/test/score_accuracy': 0.6, 'y/test/score_recall_mean': 0.6}, {'key': SelectKBest/SVC(C=3), 'y/test/score_f1': [ 0.6 0.6], 'y/test/score_recall_mean/pval': [ 0.5], 'y/test/score_recall/pval': [ 0. 0.5], 'y/test/score_accuracy/pval': [ 0.], 'y/test/score_f1/pval': [ 0. 0.5], 'y/test/score_precision/pval': [ 0.5 0. ], 'y/test/score_precision': [ 0.6 0.6], 'y/test/score_recall': [ 0.6 0.6], 'y/test/score_accuracy': 0.6, 'y/test/score_recall_mean': 0.6}]) ''' try: from soma_workflow.client import Job, Workflow from soma_workflow.client import Helper, FileTransfer from soma_workflow.client import WorkflowController except ImportError: errmsg = "No soma-workflow is found. "\ "Please verify your soma-worklow"\ "on your computer (e.g. PYTHONPATH) \n" sys.stderr.write(errmsg) sys.stdout.write(errmsg) raise NoSomaWFError tmp_work_dir_path = tempfile.mkdtemp() cur_work_dir = os.getcwd() os.chdir(tmp_work_dir_path) is_run_local = False if not self.resource_id or self.resource_id == "": self.resource_id = socket.gethostname() is_run_local = True # print "is_run_local=", is_run_local if not is_run_local: ft_working_directory = FileTransfer(is_input=True, client_path=tmp_work_dir_path, name="working directory") else: ft_working_directory = tmp_work_dir_path ## Save the database and tree to working directory ## =============================================== # np.savez(os.path.join(tmp_work_dir_path, # SomaWorkflowEngine.dataset_relative_path), **Xy) save_dataset(SomaWorkflowEngine.dataset_relative_path, **Xy) store = StoreFs(dirpath=os.path.join( tmp_work_dir_path, SomaWorkflowEngine.tree_root_relative_path)) self.tree_root.save_tree(store=store) ## Subtree job allocation on disk ## ============================== node_input = NodesInput(self.tree_root.get_key()) split_node_input = SplitNodesInput(self.tree_root, num_processes=self.num_processes) nodesinput_list = split_node_input.split(node_input) keysfile_list = save_job_list(tmp_work_dir_path, nodesinput_list) ## Build soma-workflow ## =================== jobs = self._create_jobs(keysfile_list, is_run_local, ft_working_directory) soma_workflow = Workflow(jobs=jobs) controller = WorkflowController(self.resource_id, self.login, self.pw) ## run soma-workflow ## ================= wf_id = controller.submit_workflow(workflow=soma_workflow, name="epac workflow", queue=self.queue) Helper.transfer_input_files(wf_id, controller) Helper.wait_workflow(wf_id, controller) Helper.transfer_output_files(wf_id, controller) self.engine_info = self.get_engine_info(controller, wf_id) if self.remove_finished_wf: controller.delete_workflow(wf_id) ## read result tree ## ================ self.tree_root = store.load() os.chdir(cur_work_dir) if os.path.isdir(tmp_work_dir_path) and self.remove_local_tree: shutil.rmtree(tmp_work_dir_path) return self.tree_root
def remote_map_marshal(func, largs=None, lkwargs=None, mode='local'): if largs is None: if lkwargs is not None: largs = [[]] * len(lkwargs) else: largs = [] if lkwargs is None: lkwargs = [{}] * len(largs) lkwargs = [merge_default_kwargs(func, kw) for kw in lkwargs] assert len(lkwargs) == len(largs) all_args = zip(largs, lkwargs) if mode == 'local': return [func(*args, **kwargs) for args, kwargs in all_args] elif mode == 'local_with_dumps': func_fn = './func.marshal' dump_func(func, func_fn) results = [] for i, params in enumerate(all_args): print 'params:', params params_fn = 'params_%d.pck' % i fparams = open(params_fn, 'wb') cPickle.dump(params, fparams) fparams.close() output_fn = 'output_%d.pck' % i print 'call subprocess ...' subprocess.call( ['python', '-c', cfunc_marshal, params_fn, func_fn, output_fn]) print 'Read outputs ...' fout = open(output_fn) results.append(cPickle.load(fout)) return results elif mode == 'remote_cluster': # FileTransfer creation for input files #data_dir = './rmap_data' data_dir = mkdtemp(prefix="sw_rmap") func_fn = op.join(data_dir, 'func.marshal') dump_func(func, func_fn) func_file = FileTransfer(is_input=True, client_path=func_fn, name="func_file") all_jobs = [] param_files = [] for i, params in enumerate(all_args): params_fn = op.join(data_dir, 'params_%d.pck' % i) fparams = open(params_fn, 'wb') cPickle.dump(params, fparams) fparams.close() param_file = FileTransfer(is_input=True, client_path=params_fn, name='params_file_%d' % i) param_files.append(param_file) output_fn = op.join(data_dir, 'output_%d.pck' % i) output_file = FileTransfer(is_input=False, client_path=output_fn, name='output_file_%d' % i) job = Job(command=[ 'python', '-c', cfunc, param_file, func_file, output_file ], name="rmap, item %d" % i, referenced_input_files=[func_file, param_file], referenced_output_files=[output_file]) all_jobs.append(job) workflow = Workflow(jobs=all_jobs, dependencies=[]) # submit the workflow cfg = pyhrf.cfg['parallel-cluster'] controller = WorkflowController(cfg['server_id'], cfg['user']) # controller.transfer_files(fids_to_transfer) wf_id = controller.submit_workflow(workflow=workflow, name="remote_map") Helper.transfer_input_files(wf_id, controller) Helper.wait_workflow(wf_id, controller) Helper.transfer_output_files(wf_id, controller) results = [] for i in xrange(len(all_args)): fout = open(op.join(data_dir, 'output_%d.pck' % i)) results.append(cPickle.load(fout)) fout.close() return results
def test_result(self): workflow = self.wf_examples.example_simple_exception2() self.wf_id = self.wf_ctrl.submit_workflow(workflow=workflow, name=self.__class__.__name__, queue='Cati_run4') # Transfer input files if file transfer if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: Helper.transfer_input_files(self.wf_id, self.wf_ctrl) # Wait for the workflow to finish Helper.wait_workflow(self.wf_id, self.wf_ctrl) # Transfer output files if file transfer if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: Helper.transfer_output_files(self.wf_id, self.wf_ctrl) status = self.wf_ctrl.workflow_status(self.wf_id) self.assertTrue( status == constants.WORKFLOW_DONE, "workflow status : %s. Expected : %s" % (status, constants.WORKFLOW_DONE)) nb_failed_jobs = len(Helper.list_failed_jobs(self.wf_id, self.wf_ctrl)) self.assertTrue( nb_failed_jobs == 1, "nb failed jobs : %i. Expected : %i" % (nb_failed_jobs, 1)) nb_failed_aborted_jobs = len( Helper.list_failed_jobs(self.wf_id, self.wf_ctrl, include_aborted_jobs=True)) self.assertTrue( nb_failed_aborted_jobs == 2, "nb failed jobs including aborted : %i. Expected : %i" % (nb_failed_aborted_jobs, 2)) (jobs_info, transfers_info, workflow_status, workflow_queue, tmp_files) = self.wf_ctrl.workflow_elements_status(self.wf_id) for (job_id, tmp_status, queue, exit_info, dates, drmaa_id) \ in jobs_info: job_list = self.wf_ctrl.jobs([job_id]) job_name, job_command, job_submission_date = job_list[job_id] self.tested_job = job_id if exit_info[0] == constants.FINISHED_REGULARLY: # To check job standard out and standard err job_stdout_file = tempfile.NamedTemporaryFile( prefix="job_soma_out_log_", suffix=repr(job_id), delete=False) job_stdout_file = job_stdout_file.name job_stderr_file = tempfile.NamedTemporaryFile( prefix="job_soma_outerr_log_", suffix=repr(job_id), delete=False) job_stderr_file = job_stderr_file.name try: self.wf_ctrl.retrieve_job_stdouterr( job_id, job_stdout_file, job_stderr_file) if job_name == 'job1': # Test stdout isSame, msg = identical_files( job_stdout_file, self.wf_examples.lo_stdout[1]) self.assertTrue(isSame, msg) # Test no stderr self.assertTrue( os.stat(job_stderr_file).st_size == 0, "job stderr not empty : cf %s" % job_stderr_file) # Test output files if self.path_management == self.LOCAL_PATH: isSame, msg = identical_files( self.wf_examples.lo_out_model_file[11], self.wf_examples.lo_file[11]) self.assertTrue(isSame, msg) isSame, msg = identical_files( self.wf_examples.lo_out_model_file[12], self.wf_examples.lo_file[12]) self.assertTrue(isSame, msg) if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: isSame, msg = identical_files( self.wf_examples.lo_out_model_file[11], self.wf_examples.tr_file[11].client_path) self.assertTrue(isSame, msg) isSame, msg = identical_files( self.wf_examples.lo_out_model_file[12], self.wf_examples.tr_file[12].client_path) self.assertTrue(isSame, msg) if job_name == 'job2': # Test stdout isSame, msg = identical_files( job_stdout_file, self.wf_examples.lo_stdout[2]) self.assertTrue(isSame, msg) # Test no stderr self.assertTrue( os.stat(job_stderr_file).st_size == 0, "job stderr not empty : cf %s" % job_stderr_file) # Test output files if self.path_management == self.LOCAL_PATH: isSame, msg = identical_files( self.wf_examples.lo_out_model_file[2], self.wf_examples.lo_file[2]) self.assertTrue(isSame, msg) if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: isSame, msg = identical_files( self.wf_examples.lo_out_model_file[2], self.wf_examples.tr_file[2].client_path) self.assertTrue(isSame, msg) if job_name == 'job3 with exception': # Test stdout isSame, msg = identical_files( job_stdout_file, self.wf_examples.lo_stdout_exception_model) self.assertTrue(isSame, msg) # Test the last line of stderr with open(job_stderr_file) as f: lines = f.readlines() expected_error = 'Exception: Paf Boum Boum Bada Boum !!!\n' isSame = (lines[-1] == expected_error) self.assertTrue( isSame, "Job exception : %s. Expected : %s" % (lines[-1], expected_error)) finally: os.unlink(job_stdout_file) os.unlink(job_stderr_file) del self.tested_job
def test_result(self): workflow = self.wf_examples.example_special_transfer() self.wf_id = self.wf_ctrl.submit_workflow(workflow=workflow, name=self.__class__.__name__) # Transfer input files Helper.transfer_input_files(self.wf_id, self.wf_ctrl) # Wait for the worklow to finish Helper.wait_workflow(self.wf_id, self.wf_ctrl) status = self.wf_ctrl.workflow_status(self.wf_id) # Transfer output files Helper.transfer_output_files(self.wf_id, self.wf_ctrl) status = self.wf_ctrl.workflow_status(self.wf_id) self.assertTrue( status == constants.WORKFLOW_DONE, "workflow status : %s. Expected : %s" % (status, constants.WORKFLOW_DONE) ) nb_failed_jobs = len(Helper.list_failed_jobs(self.wf_id, self.wf_ctrl)) self.assertTrue(nb_failed_jobs == 0, "nb failed jobs : %i. Expected : %i" % (nb_failed_jobs, 0)) nb_failed_aborted_jobs = len(Helper.list_failed_jobs(self.wf_id, self.wf_ctrl, include_aborted_jobs=True)) self.assertTrue( nb_failed_aborted_jobs == 0, "nb failed jobs including aborted : %i. Expected : %i" % (nb_failed_aborted_jobs, 0), ) (jobs_info, transfers_info, workflow_status, workflow_queue, tmp_files) = self.wf_ctrl.workflow_elements_status( self.wf_id ) for (job_id, tmp_status, queue, exit_info, dates) in jobs_info: job_list = self.wf_ctrl.jobs([job_id]) job_name, job_command, job_submission_date = job_list[job_id] self.tested_job = job_id if exit_info[0] == constants.FINISHED_REGULARLY: # To check job standard out and standard err job_stdout_file = tempfile.NamedTemporaryFile( prefix="job_soma_out_log_", suffix=repr(job_id), delete=False ) job_stdout_file = job_stdout_file.name job_stderr_file = tempfile.NamedTemporaryFile( prefix="job_soma_outerr_log_", suffix=repr(job_id), delete=False ) job_stderr_file = job_stderr_file.name try: self.wf_ctrl.retrieve_job_stdouterr(job_id, job_stdout_file, job_stderr_file) if job_name == "dir_contents": # Test job standard out with open(job_stdout_file, "r+") as f: dir_contents = f.readlines() dir_path_in = self.wf_examples.lo_in_dir full_path_list = [] for element in os.listdir(dir_path_in): full_path_list.append(os.path.join(dir_path_in, element)) dir_contents_model = list_contents(full_path_list, []) self.assertTrue(sorted(dir_contents) == sorted(dir_contents_model)) # Test no stderr self.assertTrue( os.stat(job_stderr_file).st_size == 0, "job stderr not empty : cf %s" % job_stderr_file ) if job_name == "multi file format test": # Test job standard out isSame, msg = identical_files(job_stdout_file, self.wf_examples.lo_mff_stdout) self.assertTrue(isSame, msg) # Test no stderr self.assertTrue( os.stat(job_stderr_file).st_size == 0, "job stderr not empty : cf %s" % job_stderr_file ) finally: os.unlink(job_stdout_file) os.unlink(job_stderr_file) del self.tested_job
def test_result(self): workflow = self.wf_examples.example_special_transfer() self.wf_id = self.wf_ctrl.submit_workflow( workflow=workflow, name=self.__class__.__name__) # Transfer input files Helper.transfer_input_files(self.wf_id, self.wf_ctrl) # Wait for the worklow to finish Helper.wait_workflow(self.wf_id, self.wf_ctrl) status = self.wf_ctrl.workflow_status(self.wf_id) # Transfer output files Helper.transfer_output_files(self.wf_id, self.wf_ctrl) status = self.wf_ctrl.workflow_status(self.wf_id) self.assertTrue(status == constants.WORKFLOW_DONE, "workflow status : %s. Expected : %s" % (status, constants.WORKFLOW_DONE)) nb_failed_jobs = len(Helper.list_failed_jobs( self.wf_id, self.wf_ctrl)) self.assertTrue(nb_failed_jobs == 0, "nb failed jobs : %i. Expected : %i" % (nb_failed_jobs, 0)) nb_failed_aborted_jobs = len(Helper.list_failed_jobs( self.wf_id, self.wf_ctrl, include_aborted_jobs=True)) self.assertTrue(nb_failed_aborted_jobs == 0, "nb failed jobs including aborted : %i. Expected : %i" % (nb_failed_aborted_jobs, 0)) (jobs_info, transfers_info, workflow_status, workflow_queue, tmp_files) = self.wf_ctrl.workflow_elements_status(self.wf_id) for (job_id, tmp_status, queue, exit_info, dates) in jobs_info: job_list = self.wf_ctrl.jobs([job_id]) job_name, job_command, job_submission_date = job_list[job_id] self.tested_job = job_id if exit_info[0] == constants.FINISHED_REGULARLY: # To check job standard out and standard err job_stdout_file = tempfile.NamedTemporaryFile( prefix="job_soma_out_log_", suffix=repr(job_id), delete=False) job_stdout_file = job_stdout_file.name job_stderr_file = tempfile.NamedTemporaryFile( prefix="job_soma_outerr_log_", suffix=repr(job_id), delete=False) job_stderr_file = job_stderr_file.name try: self.wf_ctrl.retrieve_job_stdouterr(job_id, job_stdout_file, job_stderr_file) if job_name == 'dir_contents': # Test job standard out with open(job_stdout_file, 'r+') as f: dir_contents = f.readlines() dir_path_in = self.wf_examples.lo_in_dir full_path_list = [] for element in os.listdir(dir_path_in): full_path_list.append(os.path.join(dir_path_in, element)) dir_contents_model = list_contents(full_path_list, []) self.assertTrue( sorted(dir_contents) == sorted(dir_contents_model)) # Test no stderr self.assertTrue(os.stat(job_stderr_file).st_size == 0, "job stderr not empty : cf %s" % job_stderr_file) if job_name == 'multi file format test': # Test job standard out isSame, msg = identical_files( job_stdout_file, self.wf_examples.lo_mff_stdout) self.assertTrue(isSame, msg) # Test no stderr self.assertTrue(os.stat(job_stderr_file).st_size == 0, "job stderr not empty : cf %s" % job_stderr_file) finally: os.unlink(job_stdout_file) os.unlink(job_stderr_file) del self.tested_job
def test_result(self): if hasattr(self.wf_ctrl.scheduler_config, 'get_proc_nb'): n_iter = 5 * self.wf_ctrl.scheduler_config.get_proc_nb() else: n_iter = 5 workflow = self.wf_examples.example_fake_pipelineT1(n_iter) self.wf_id = self.wf_ctrl.submit_workflow(workflow=workflow, name=self.__class__.__name__) # Transfer input files if file transfer if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: Helper.transfer_input_files(self.wf_id, self.wf_ctrl) # Wait for the workflow to finish Helper.wait_workflow(self.wf_id, self.wf_ctrl) # Transfer output files if file transfer if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: Helper.transfer_output_files(self.wf_id, self.wf_ctrl) status = self.wf_ctrl.workflow_status(self.wf_id) self.assertTrue(status == constants.WORKFLOW_DONE) self.assertTrue( len(Helper.list_failed_jobs(self.wf_id, self.wf_ctrl)) == 0) self.assertTrue( len( Helper.list_failed_jobs( self.wf_id, self.wf_ctrl, include_aborted_jobs=True)) == 0) (jobs_info, transfers_info, workflow_status, workflow_queue, tmp_files) = self.wf_ctrl.workflow_elements_status(self.wf_id) for (job_id, tmp_status, queue, exit_info, dates, dmraa_id) \ in jobs_info: job_list = self.wf_ctrl.jobs([job_id]) job_name, job_command, job_submission_date = job_list[job_id] self.tested_job = job_id if exit_info[0] == constants.FINISHED_REGULARLY: # To check job standard out and standard err job_stdout_file = tempfile.NamedTemporaryFile( prefix="job_soma_out_log_", suffix=repr(job_id), delete=False) job_stdout_file = job_stdout_file.name job_stderr_file = tempfile.NamedTemporaryFile( prefix="job_soma_outerr_log_", suffix=repr(job_id), delete=False) job_stderr_file = job_stderr_file.name try: self.wf_ctrl.retrieve_job_stdouterr( job_id, job_stdout_file, job_stderr_file) # Test stdout self.assertTrue( os.stat(job_stdout_file).st_size == 0, "job stdout not empty : cf %s" % job_stdout_file) # Test no stderr self.assertTrue( os.stat(job_stderr_file).st_size == 0, "job stderr not empty : cf %s" % job_stderr_file) finally: if os.path.exists(job_stdout_file): os.unlink(job_stdout_file) if os.path.exists(job_stderr_file): os.unlink(job_stderr_file) del self.tested_job
def run_workflow(self, workflow, test_files=[], test_dyn_files={}): self.wf_id = self.wf_ctrl.submit_workflow( workflow=workflow, name=self.__class__.__name__) # Transfer input files if file transfer if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: Helper.transfer_input_files(self.wf_id, self.wf_ctrl) # Wait for the workflow to finish Helper.wait_workflow(self.wf_id, self.wf_ctrl) # Transfer output files if file transfer if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: Helper.transfer_output_files(self.wf_id, self.wf_ctrl) status = self.wf_ctrl.workflow_status(self.wf_id) self.assertTrue(status == constants.WORKFLOW_DONE, "workflow status : %s. Expected : %s" % (status, constants.WORKFLOW_DONE)) failed_jobs = Helper.list_failed_jobs(self.wf_id, self.wf_ctrl) nb_failed_jobs = len(failed_jobs) if nb_failed_jobs != 0: self.print_jobs(failed_jobs, 'Failed jobs') self.assertTrue(nb_failed_jobs == 0, "nb failed jobs : %i. Expected : %i" % (nb_failed_jobs, 0)) failed_aborted_jobs = Helper.list_failed_jobs( self.wf_id, self.wf_ctrl, include_aborted_jobs=True) nb_failed_aborted_jobs = len(failed_aborted_jobs) if nb_failed_aborted_jobs != 0: self.print_jobs(failed_aborted_jobs, 'Aborted jobs') self.assertTrue(nb_failed_aborted_jobs == 0, "nb failed jobs including aborted : %i. Expected : %i" % (nb_failed_aborted_jobs, 0)) (jobs_info, transfers_info, workflow_status, workflow_queue, tmp_files) = self.wf_ctrl.workflow_elements_status(self.wf_id) dyn_out_params = {1: 'filePathOut2', 2: 'filePathOut'} dyn_out_params = {} for (job_id, tmp_status, queue, exit_info, dates, drmaa_id) \ in jobs_info: job_list = self.wf_ctrl.jobs([job_id]) job_name, job_command, job_submission_date = job_list[job_id] self.tested_job = job_id if exit_info[0] == constants.FINISHED_REGULARLY: # To check job standard out and standard err job_stdout_file = tempfile.NamedTemporaryFile( prefix="job_soma_out_log_", suffix=repr(job_id), delete=False) job_stdout_file = job_stdout_file.name job_stderr_file = tempfile.NamedTemporaryFile( prefix="job_soma_outerr_log_", suffix=repr(job_id), delete=False) job_stderr_file = job_stderr_file.name try: self.wf_ctrl.retrieve_job_stdouterr(job_id, job_stdout_file, job_stderr_file) if job_name.startswith('job1'): # Test stdout isSame, msg = identical_files( job_stdout_file, self.wf_examples.lo_stdout[1]) self.assertTrue(isSame, msg) # Test no stderr with open(job_stderr_file) as f: msg = "job stderr not empty : cf %s\n" \ "stderr:\n---\n%s\n---" \ % (job_stderr_file, f.read()) self.assertTrue(os.stat(job_stderr_file).st_size == 0, msg) if job_name in test_dyn_files: out_params = self.wf_ctrl.get_job_output_params(job_id) dyn_out_params[job_name] = out_params # For unknown reason, it raises some errors # http://stackoverflow.com/questions/10496758/unexpected-end-of-file-and-error-importing-function-definition-error-running # isSame, msg = identical_files(job_stderr_file,self.wf_examples.lo_stderr[1]) # self.failUnless(isSame == True) finally: os.unlink(job_stdout_file) os.unlink(job_stderr_file) for out_file_num in test_files: # Test output files if self.path_management == self.LOCAL_PATH: out_file = self.wf_examples.lo_file[out_file_num] elif self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: out_file = self.wf_examples.tr_file[out_file_num].client_path isSame, msg = identical_files( self.wf_examples.lo_out_model_file[out_file_num], out_file) self.assertTrue(isSame, msg) for job_name, ref_out_params in six.iteritems(test_dyn_files): out_params = dyn_out_params[job_name] for param, file_num in six.iteritems(ref_out_params): isSame, msg = identical_files( self.wf_examples.lo_out_model_file[file_num], out_params[param]) self.assertTrue(isSame, msg) del self.tested_job
def remote_map(func, largs=None, lkwargs=None, mode='serial'): """ Execute a function in parallel on a list of arguments. Args: *func* (function): function to apply on each item. **this function must be importable on the remote side** *largs* (list of tuple): each item in the list is a tuple containing all positional argument values of the function *lkwargs* (list of dict): each item in the list is a dict containing all named arguments of the function mapped to their value. *mode* (str): indicates how execution is distributed. Choices are: - "serial": single-thread loop on the local machine - "local" : use joblib to run tasks in parallel. The number of simultaneous jobs is defined in the configuration section ['parallel-local']['nb_procs'] see ~/.pyhrf/config.cfg - "remote_cluster: use somaworkflow to run tasks in parallel. The connection setup has to be defined in the configuration section ['parallel-cluster'] of ~/.pyhrf/config.cfg. - "local_with_dumps": testing purpose only, run each task serially as a subprocess. Returns: a list of results Raises: RemoteException if any remote task has failed Example: >>> from pyhrf.parallel import remote_map >>> def foo(a, b=2): \ return a + b >>> remote_map(foo, [(2,),(3,)], [{'b':5}, {'b':7}]) [7, 10] """ if largs is None: if lkwargs is not None: largs = [tuple()] * len(lkwargs) else: largs = [tuple()] if lkwargs is None: lkwargs = [{}] * len(largs) lkwargs = [merge_default_kwargs(func, kw) for kw in lkwargs] assert len(lkwargs) == len(largs) all_args = zip(largs, lkwargs) # print 'all_args:', all_args fmodule = func.__module__ fname = '.'.join([fmodule, func.__name__]) if mode == 'serial': return [func(*args, **kwargs) for args, kwargs in all_args] elif mode == 'local': try: from joblib import Parallel, delayed except ImportError: raise ImportError('Can not import joblib. It is ' 'required to enable parallel ' 'processing on a local machine.') if logger.getEffectiveLevel() == logging.DEBUG: parallel_verb = 10 else: parallel_verb = 0 if pyhrf.cfg['parallel-local']['nb_procs']: n_jobs = pyhrf.cfg['parallel-local']['nb_procs'] else: n_jobs = available_cpu_count() p = Parallel(n_jobs=n_jobs, verbose=parallel_verb) return p(delayed(func)(*args, **kwargs) for args, kwargs in all_args) elif mode == 'local_with_dumps': results = [] for i, params in enumerate(all_args): # print 'params:', params params_fn = 'params_%d.pck' % i fparams = open(params_fn, 'wb') cPickle.dump(params, fparams) fparams.close() output_fn = 'output_%d.pck' % i # print 'call subprocess ...' subprocess.call([ 'python', '-c', cfunc % (fmodule, fname), params_fn, output_fn ]) # print 'Read outputs ...' fout = open(output_fn) results.append(cPickle.load(fout)) return results elif mode == 'remote_cluster': # FileTransfer creation for input files #data_dir = './rmap_data' data_dir = mkdtemp(prefix="sw_rmap") all_jobs = [] param_files = [] for i, params in enumerate(all_args): params_fn = op.join(data_dir, 'params_%d.pck' % i) fparams = open(params_fn, 'wb') cPickle.dump(params, fparams) fparams.close() param_file = FileTransfer(is_input=True, client_path=params_fn, name='params_file_%d' % i) param_files.append(param_file) output_fn = op.join(data_dir, 'output_%d.pck' % i) output_file = FileTransfer(is_input=False, client_path=output_fn, name='output_file_%d' % i) job = Job(command=[ 'pyhrf_exec_pyfunc', fmodule, fname, param_file, output_file ], name="rmap, item %d" % i, referenced_input_files=[param_file], referenced_output_files=[output_file]) all_jobs.append(job) workflow = Workflow(jobs=all_jobs, dependencies=[]) # submit the workflow cfg = pyhrf.cfg['parallel-cluster'] controller = WorkflowController(cfg['server_id'], cfg['user']) # controller.transfer_files(fids_to_transfer) wf_id = controller.submit_workflow(workflow=workflow, name="remote_map") Helper.transfer_input_files(wf_id, controller) Helper.wait_workflow(wf_id, controller) Helper.transfer_output_files(wf_id, controller) results = [] for i in xrange(len(all_args)): fnout = op.join(data_dir, 'output_%d.pck' % i) fout = open(fnout) o = cPickle.load(fout) print 'file cPickle loaded:', o fout.close() os.remove(fnout) if isinstance(o, Exception): raise RemoteException('Task %d failed' % i, o) if o.errno != 17: raise RemoteException('Task %d failed' % i, o) results.append(o) return results
def remote_map(func, largs=None, lkwargs=None, mode='serial'): """ Execute a function in parallel on a list of arguments. Args: *func* (function): function to apply on each item. **this function must be importable on the remote side** *largs* (list of tuple): each item in the list is a tuple containing all positional argument values of the function *lkwargs* (list of dict): each item in the list is a dict containing all named arguments of the function mapped to their value. *mode* (str): indicates how execution is distributed. Choices are: - "serial": single-thread loop on the local machine - "local" : use joblib to run tasks in parallel. The number of simultaneous jobs is defined in the configuration section ['parallel-local']['nb_procs'] see ~/.pyhrf/config.cfg - "remote_cluster: use somaworkflow to run tasks in parallel. The connection setup has to be defined in the configuration section ['parallel-cluster'] of ~/.pyhrf/config.cfg. - "local_with_dumps": testing purpose only, run each task serially as a subprocess. Returns: a list of results Raises: RemoteException if any remote task has failed Example: >>> from pyhrf.parallel import remote_map >>> def foo(a, b=2): \ return a + b >>> remote_map(foo, [(2,),(3,)], [{'b':5}, {'b':7}]) [7, 10] """ if largs is None: if lkwargs is not None: largs = [tuple()] * len(lkwargs) else: largs = [tuple()] if lkwargs is None: lkwargs = [{}] * len(largs) lkwargs = [merge_default_kwargs(func, kw) for kw in lkwargs] assert len(lkwargs) == len(largs) all_args = zip(largs, lkwargs) # print 'all_args:', all_args fmodule = func.__module__ fname = '.'.join([fmodule, func.__name__]) if mode == 'serial': return [func(*args, **kwargs) for args, kwargs in all_args] elif mode == 'local': try: from joblib import Parallel, delayed except ImportError: raise ImportError('Can not import joblib. It is ' 'required to enable parallel ' 'processing on a local machine.') if logger.getEffectiveLevel() == logging.DEBUG: parallel_verb = 10 else: parallel_verb = 0 if pyhrf.cfg['parallel-local']['nb_procs']: n_jobs = pyhrf.cfg['parallel-local']['nb_procs'] else: n_jobs = available_cpu_count() p = Parallel(n_jobs=n_jobs, verbose=parallel_verb) return p(delayed(func)(*args, **kwargs) for args, kwargs in all_args) elif mode == 'local_with_dumps': results = [] for i, params in enumerate(all_args): # print 'params:', params params_fn = 'params_%d.pck' % i fparams = open(params_fn, 'wb') cPickle.dump(params, fparams) fparams.close() output_fn = 'output_%d.pck' % i # print 'call subprocess ...' subprocess.call(['python', '-c', cfunc % (fmodule, fname), params_fn, output_fn]) # print 'Read outputs ...' fout = open(output_fn) results.append(cPickle.load(fout)) return results elif mode == 'remote_cluster': # FileTransfer creation for input files #data_dir = './rmap_data' data_dir = mkdtemp(prefix="sw_rmap") all_jobs = [] param_files = [] for i, params in enumerate(all_args): params_fn = op.join(data_dir, 'params_%d.pck' % i) fparams = open(params_fn, 'wb') cPickle.dump(params, fparams) fparams.close() param_file = FileTransfer(is_input=True, client_path=params_fn, name='params_file_%d' % i) param_files.append(param_file) output_fn = op.join(data_dir, 'output_%d.pck' % i) output_file = FileTransfer(is_input=False, client_path=output_fn, name='output_file_%d' % i) job = Job(command=['pyhrf_exec_pyfunc', fmodule, fname, param_file, output_file], name="rmap, item %d" % i, referenced_input_files=[param_file], referenced_output_files=[output_file]) all_jobs.append(job) workflow = Workflow(jobs=all_jobs, dependencies=[]) # submit the workflow cfg = pyhrf.cfg['parallel-cluster'] controller = WorkflowController(cfg['server_id'], cfg['user']) # controller.transfer_files(fids_to_transfer) wf_id = controller.submit_workflow( workflow=workflow, name="remote_map") Helper.transfer_input_files(wf_id, controller) Helper.wait_workflow(wf_id, controller) Helper.transfer_output_files(wf_id, controller) results = [] for i in xrange(len(all_args)): fnout = op.join(data_dir, 'output_%d.pck' % i) fout = open(fnout) o = cPickle.load(fout) print 'file cPickle loaded:', o fout.close() os.remove(fnout) if isinstance(o, Exception): raise RemoteException('Task %d failed' % i, o) if o.errno != 17: raise RemoteException('Task %d failed' % i, o) results.append(o) return results
def run(self, **Xy): """Run soma-workflow without gui Example ------- >>> from sklearn import datasets >>> from epac.map_reduce.engine import SomaWorkflowEngine >>> from epac.tests.wfexamples2test import WFExample2 >>> ## Build dataset >>> ## ============= >>> X, y = datasets.make_classification(n_samples=10, ... n_features=20, ... n_informative=5, ... random_state=1) >>> Xy = {'X':X, 'y':y} >>> ## Build epac tree >>> ## =============== >>> tree_root_node = WFExample2().get_workflow() >>> ## Build SomaWorkflowEngine and run function for each node >>> ## ======================================================= >>> sfw_engine = SomaWorkflowEngine(tree_root=tree_root_node, ... function_name="transform", ... num_processes=3, remove_finished_wf=False) >>> tree_root_node = sfw_engine.run(**Xy) light mode >>> ## Run reduce process >>> ## ================== >>> tree_root_node.reduce() ResultSet( [{'key': SelectKBest/SVC(C=1), 'y/test/score_f1': [ 0.6 0.6], 'y/test/score_recall_mean/pval': [ 0.5], 'y/test/score_recall/pval': [ 0. 0.5], 'y/test/score_accuracy/pval': [ 0.], 'y/test/score_f1/pval': [ 0. 0.5], 'y/test/score_precision/pval': [ 0.5 0. ], 'y/test/score_precision': [ 0.6 0.6], 'y/test/score_recall': [ 0.6 0.6], 'y/test/score_accuracy': 0.6, 'y/test/score_recall_mean': 0.6}, {'key': SelectKBest/SVC(C=3), 'y/test/score_f1': [ 0.6 0.6], 'y/test/score_recall_mean/pval': [ 0.5], 'y/test/score_recall/pval': [ 0. 0.5], 'y/test/score_accuracy/pval': [ 0.], 'y/test/score_f1/pval': [ 0. 0.5], 'y/test/score_precision/pval': [ 0.5 0. ], 'y/test/score_precision': [ 0.6 0.6], 'y/test/score_recall': [ 0.6 0.6], 'y/test/score_accuracy': 0.6, 'y/test/score_recall_mean': 0.6}]) """ try: from soma_workflow.client import Job, Workflow from soma_workflow.client import Helper, FileTransfer from soma_workflow.client import WorkflowController except ImportError: errmsg = ( "No soma-workflow is found. " "Please verify your soma-worklow" "on your computer (e.g. PYTHONPATH) \n" ) sys.stderr.write(errmsg) sys.stdout.write(errmsg) raise NoSomaWFError tmp_work_dir_path = tempfile.mkdtemp() cur_work_dir = os.getcwd() os.chdir(tmp_work_dir_path) is_run_local = False if not self.resource_id or self.resource_id == "": self.resource_id = socket.gethostname() is_run_local = True # print "is_run_local=", is_run_local if not is_run_local: ft_working_directory = FileTransfer(is_input=True, client_path=tmp_work_dir_path, name="working directory") else: ft_working_directory = tmp_work_dir_path ## Save the database and tree to working directory ## =============================================== # np.savez(os.path.join(tmp_work_dir_path, # SomaWorkflowEngine.dataset_relative_path), **Xy) save_dataset(SomaWorkflowEngine.dataset_relative_path, **Xy) store = StoreFs(dirpath=os.path.join(tmp_work_dir_path, SomaWorkflowEngine.tree_root_relative_path)) self.tree_root.save_tree(store=store) ## Subtree job allocation on disk ## ============================== node_input = NodesInput(self.tree_root.get_key()) split_node_input = SplitNodesInput(self.tree_root, num_processes=self.num_processes) nodesinput_list = split_node_input.split(node_input) keysfile_list = save_job_list(tmp_work_dir_path, nodesinput_list) ## Build soma-workflow ## =================== jobs = self._create_jobs(keysfile_list, is_run_local, ft_working_directory) soma_workflow = Workflow(jobs=jobs) controller = WorkflowController(self.resource_id, self.login, self.pw) ## run soma-workflow ## ================= wf_id = controller.submit_workflow(workflow=soma_workflow, name="epac workflow", queue=self.queue) Helper.transfer_input_files(wf_id, controller) Helper.wait_workflow(wf_id, controller) Helper.transfer_output_files(wf_id, controller) self.engine_info = self.get_engine_info(controller, wf_id) if self.remove_finished_wf: controller.delete_workflow(wf_id) ## read result tree ## ================ self.tree_root = store.load() os.chdir(cur_work_dir) if os.path.isdir(tmp_work_dir_path) and self.remove_local_tree: shutil.rmtree(tmp_work_dir_path) return self.tree_root
def test_result(self): workflow = self.wf_examples.example_simple() self.wf_id = self.wf_ctrl.submit_workflow( workflow=workflow, name=self.__class__.__name__) # Transfer input files if file transfer if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: Helper.transfer_input_files(self.wf_id, self.wf_ctrl) # Wait for the workflow to finish Helper.wait_workflow(self.wf_id, self.wf_ctrl) # Transfer output files if file transfer if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: Helper.transfer_output_files(self.wf_id, self.wf_ctrl) status = self.wf_ctrl.workflow_status(self.wf_id) self.assertTrue(status == constants.WORKFLOW_DONE, "workflow status : %s. Expected : %s" % (status, constants.WORKFLOW_DONE)) nb_failed_jobs = len(Helper.list_failed_jobs(self.wf_id, self.wf_ctrl)) self.assertTrue(nb_failed_jobs == 0, "nb failed jobs : %i. Expected : %i" % (nb_failed_jobs, 0)) nb_failed_aborted_jobs = len(Helper.list_failed_jobs( self.wf_id, self.wf_ctrl, include_aborted_jobs=True)) self.assertTrue(nb_failed_aborted_jobs == 0, "nb failed jobs including aborted : %i. Expected : %i" % (nb_failed_aborted_jobs, 0)) (jobs_info, transfers_info, workflow_status, workflow_queue, tmp_files) = self.wf_ctrl.workflow_elements_status(self.wf_id) for (job_id, tmp_status, queue, exit_info, dates) in jobs_info: job_list = self.wf_ctrl.jobs([job_id]) job_name, job_command, job_submission_date = job_list[job_id] self.tested_job = job_id if exit_info[0] == constants.FINISHED_REGULARLY: # To check job standard out and standard err job_stdout_file = tempfile.NamedTemporaryFile( prefix="job_soma_out_log_", suffix=repr(job_id), delete=False) job_stdout_file = job_stdout_file.name job_stderr_file = tempfile.NamedTemporaryFile( prefix="job_soma_outerr_log_", suffix=repr(job_id), delete=False) job_stderr_file = job_stderr_file.name try: self.wf_ctrl.retrieve_job_stdouterr(job_id, job_stdout_file, job_stderr_file) if job_name == 'job1': # Test stdout isSame, msg = identical_files( job_stdout_file, self.wf_examples.lo_stdout[1]) self.assertTrue(isSame, msg) # Test no stderr msg = "job stderr not empty : cf %s\n" \ "stderr:\n---\n%s\n---" \ % (job_stderr_file, open(job_stderr_file).read()) self.assertTrue(os.stat(job_stderr_file).st_size == 0, msg) # Test output files if self.path_management == self.LOCAL_PATH: isSame, msg = identical_files( self.wf_examples.lo_out_model_file[11], self.wf_examples.lo_file[11]) self.assertTrue(isSame, msg) isSame, msg = identical_files( self.wf_examples.lo_out_model_file[12], self.wf_examples.lo_file[12]) self.assertTrue(isSame, msg) if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: isSame, msg = identical_files( self.wf_examples.lo_out_model_file[11], self.wf_examples.tr_file[11].client_path) self.assertTrue(isSame, msg) isSame, msg = identical_files( self.wf_examples.lo_out_model_file[12], self.wf_examples.tr_file[12].client_path) self.assertTrue(isSame, msg) # For unknown reason, it raises some errors # http://stackoverflow.com/questions/10496758/unexpected-end-of-file-and-error-importing-function-definition-error-running # isSame, msg = identical_files(job_stderr_file,self.wf_examples.lo_stderr[1]) # self.failUnless(isSame == True) if job_name in ['job2', 'job3', 'job4']: job_nb = int(job_name[3]) # Test stdout isSame, msg = identical_files( job_stdout_file, self.wf_examples.lo_stdout[job_nb]) self.assertTrue(isSame, msg) # Test no stderr self.assertTrue(os.stat(job_stderr_file).st_size == 0, "job stderr not empty : cf %s" % job_stderr_file) # Test output files if self.path_management == self.LOCAL_PATH: isSame, msg = identical_files( self.wf_examples.lo_out_model_file[job_nb], self.wf_examples.lo_file[job_nb]) self.assertTrue(isSame, msg) if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: isSame, msg = identical_files( self.wf_examples.lo_out_model_file[job_nb], self.wf_examples.tr_file[job_nb].client_path) self.assertTrue(isSame, msg) finally: os.unlink(job_stdout_file) os.unlink(job_stderr_file) del self.tested_job
def test_result(self): if hasattr(self.wf_ctrl.scheduler_config, 'get_proc_nb'): n_iter = 10 * self.wf_ctrl.scheduler_config.get_proc_nb() else: n_iter = 100 workflow = self.wf_examples.example_fake_pipelineT1(n_iter) self.wf_id = self.wf_ctrl.submit_workflow( workflow=workflow, name=self.__class__.__name__) # Transfer input files if file transfer if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: Helper.transfer_input_files(self.wf_id, self.wf_ctrl) # Wait for the workflow to finish Helper.wait_workflow(self.wf_id, self.wf_ctrl) # Transfer output files if file transfer if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: Helper.transfer_output_files(self.wf_id, self.wf_ctrl) status = self.wf_ctrl.workflow_status(self.wf_id) self.assertTrue(status == constants.WORKFLOW_DONE) self.assertTrue(len(Helper.list_failed_jobs( self.wf_id, self.wf_ctrl)) == 0) self.assertTrue(len(Helper.list_failed_jobs( self.wf_id, self.wf_ctrl, include_aborted_jobs=True)) == 0) (jobs_info, transfers_info, workflow_status, workflow_queue, tmp_files) = self.wf_ctrl.workflow_elements_status(self.wf_id) for (job_id, tmp_status, queue, exit_info, dates) in jobs_info: job_list = self.wf_ctrl.jobs([job_id]) job_name, job_command, job_submission_date = job_list[job_id] self.tested_job = job_id if exit_info[0] == constants.FINISHED_REGULARLY: # To check job standard out and standard err job_stdout_file = tempfile.NamedTemporaryFile( prefix="job_soma_out_log_", suffix=repr(job_id), delete=False) job_stdout_file = job_stdout_file.name job_stderr_file = tempfile.NamedTemporaryFile( prefix="job_soma_outerr_log_", suffix=repr(job_id), delete=False) job_stderr_file = job_stderr_file.name try: self.wf_ctrl.retrieve_job_stdouterr(job_id, job_stdout_file, job_stderr_file) # Test stdout self.assertTrue(os.stat(job_stdout_file).st_size == 0, "job stdout not empty : cf %s" % job_stdout_file) # Test no stderr self.assertTrue(os.stat(job_stderr_file).st_size == 0, "job stderr not empty : cf %s" % job_stderr_file) finally: if os.path.exists(job_stdout_file): os.unlink(job_stdout_file) if os.path.exists(job_stderr_file): os.unlink(job_stderr_file) del self.tested_job
def test_result(self): nb = 20 time_sleep = 1 workflow = self.wf_examples.example_n_jobs(nb=nb, time=time_sleep) self.wf_id = self.wf_ctrl.submit_workflow( workflow=workflow, name=self.__class__.__name__) # Transfer input files if file transfer if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: Helper.transfer_input_files(self.wf_id, self.wf_ctrl) # Wait for the workflow to finish Helper.wait_workflow(self.wf_id, self.wf_ctrl) # Transfer output files if file transfer if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: Helper.transfer_output_files(self.wf_id, self.wf_ctrl) status = self.wf_ctrl.workflow_status(self.wf_id) self.assertTrue(status == constants.WORKFLOW_DONE, "workflow status : %s. Expected : %s" % (status, constants.WORKFLOW_DONE)) nb_failed_jobs = len(Helper.list_failed_jobs( self.wf_id, self.wf_ctrl)) self.assertTrue(nb_failed_jobs == 0, "nb failed jobs : %i. Expected : %i" % (nb_failed_jobs, 0)) nb_failed_aborted_jobs = len(Helper.list_failed_jobs( self.wf_id, self.wf_ctrl, include_aborted_jobs=True)) self.assertTrue(nb_failed_aborted_jobs == 0, "nb failed jobs including aborted : %i. Expected : %i" % (nb_failed_aborted_jobs, 0)) (jobs_info, transfers_info, workflow_status, workflow_queue, tmp_files) = self.wf_ctrl.workflow_elements_status(self.wf_id) for (job_id, tmp_status, queue, exit_info, dates) in jobs_info: job_list = self.wf_ctrl.jobs([job_id]) job_name, job_command, job_submission_date = job_list[job_id] self.tested_job = job_id if exit_info[0] == constants.FINISHED_REGULARLY: # To check job standard out and standard err job_stdout_file = tempfile.NamedTemporaryFile( prefix="job_soma_out_log_", suffix=repr(job_id), delete=False) job_stdout_file = job_stdout_file.name job_stderr_file = tempfile.NamedTemporaryFile( prefix="job_soma_outerr_log_", suffix=repr(job_id), delete=False) job_stderr_file = job_stderr_file.name try: self.wf_ctrl.retrieve_job_stdouterr(job_id, job_stdout_file, job_stderr_file) # Test stdout self.assertTrue(os.stat(job_stdout_file).st_size == 0, "job stdout not empty : file: %s, " "contents:\n%s" % (job_stdout_file, open(job_stdout_file).read())) # Test no stderr self.assertTrue(os.stat(job_stderr_file).st_size == 0, "job stderr not empty : file %s, " "contents:\n%s" % (job_stderr_file, open(job_stderr_file).read())) finally: os.unlink(job_stdout_file) os.unlink(job_stderr_file) del self.tested_job
def test_result(self): workflow = self.wf_examples.example_multiple() self.wf_id = self.wf_ctrl.submit_workflow(workflow=workflow, name=self.__class__.__name__) # Transfer input files if file transfer if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: Helper.transfer_input_files(self.wf_id, self.wf_ctrl) # Wait for the workflow to finish Helper.wait_workflow(self.wf_id, self.wf_ctrl) # Transfer output files if file transfer if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: Helper.transfer_output_files(self.wf_id, self.wf_ctrl) status = self.wf_ctrl.workflow_status(self.wf_id) self.assertTrue( status == constants.WORKFLOW_DONE, "workflow status : %s. Expected : %s" % (status, constants.WORKFLOW_DONE)) nb_failed_jobs = len(Helper.list_failed_jobs(self.wf_id, self.wf_ctrl)) self.assertTrue( nb_failed_jobs == 2, "nb failed jobs : %i. Expected : %i" % (nb_failed_jobs, 2)) nb_failed_aborted_jobs = len( Helper.list_failed_jobs(self.wf_id, self.wf_ctrl, include_aborted_jobs=True)) self.assertTrue( nb_failed_aborted_jobs == 6, "nb failed jobs including aborted : %i. Expected : %i" % (nb_failed_aborted_jobs, 6)) (jobs_info, transfers_info, workflow_status, workflow_queue, tmp_files) = self.wf_ctrl.workflow_elements_status(self.wf_id) for (job_id, tmp_status, queue, exit_info, dates) in jobs_info: job_list = self.wf_ctrl.jobs([job_id]) job_name, job_command, job_submission_date = job_list[job_id] self.tested_job = job_id if exit_info[0] == constants.FINISHED_REGULARLY: # To check job standard out and standard err job_stdout_file = tempfile.NamedTemporaryFile( prefix="job_soma_out_log_", suffix=repr(job_id), delete=False) job_stdout_file = job_stdout_file.name job_stderr_file = tempfile.NamedTemporaryFile( prefix="job_soma_outerr_log_", suffix=repr(job_id), delete=False) job_stderr_file = job_stderr_file.name try: self.wf_ctrl.retrieve_job_stdouterr( job_id, job_stdout_file, job_stderr_file) if job_name == 'job1': # Test stdout isSame, msg = identical_files( job_stdout_file, self.wf_examples.lo_stdout[1]) self.assertTrue(isSame, msg) # Test no stderr self.assertTrue( os.stat(job_stderr_file).st_size == 0, "job stderr not empty : cf %s" % job_stderr_file) # Test output files if self.path_management == self.LOCAL_PATH: isSame, msg = identical_files( self.wf_examples.lo_out_model_file[11], self.wf_examples.lo_file[11]) self.assertTrue(isSame, msg) isSame, msg = identical_files( self.wf_examples.lo_out_model_file[12], self.wf_examples.lo_file[12]) self.assertTrue(isSame, msg) if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: isSame, msg = identical_files( self.wf_examples.lo_out_model_file[11], self.wf_examples.tr_file[11].client_path) self.assertTrue(isSame, msg) isSame, msg = identical_files( self.wf_examples.lo_out_model_file[12], self.wf_examples.tr_file[12].client_path) self.assertTrue(isSame, msg) # For unknown reason, it raises some errors # http://stackoverflow.com/questions/10496758/unexpected-end-of-file-and-error-importing-function-definition-error-running # isSame, msg = identical_files(job_stderr_file,self.wf_examples.lo_stderr[1]) # self.failUnless(isSame == True) if job_name in ['job2', 'job3', 'job4']: job_nb = int(job_name[3]) # Test stdout isSame, msg = identical_files( job_stdout_file, self.wf_examples.lo_stdout[job_nb]) self.assertTrue(isSame, msg) # Test no stderr self.assertTrue( os.stat(job_stderr_file).st_size == 0, "job stderr not empty : cf %s" % job_stderr_file) # Test output files if self.path_management == self.LOCAL_PATH: isSame, msg = identical_files( self.wf_examples.lo_out_model_file[job_nb], self.wf_examples.lo_file[job_nb]) self.assertTrue(isSame, msg) if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: isSame, msg = identical_files( self.wf_examples.lo_out_model_file[job_nb], self.wf_examples.tr_file[job_nb].client_path) self.assertTrue(isSame, msg) if job_name in [ 'job1 with exception', 'job3 with exception' ]: # Test stdout isSame, msg = identical_files( job_stdout_file, self.wf_examples.lo_stdout_exception_model) self.assertTrue(isSame) # Test stderr with open(job_stderr_file) as f: lines = f.readlines() print(lines) isSame = (lines[-1] == 'Exception: Paf Boum ' 'Boum Bada Boum !!!\n') self.assertTrue(isSame, "Job Exception: %s" % lines[-1]) finally: if os.path.exists(job_stdout_file): os.unlink(job_stdout_file) if os.path.exists(job_stderr_file): os.unlink(job_stderr_file) del self.tested_job
def test_result(self): nb = 20 time_sleep = 1 workflow = self.wf_examples.example_n_jobs(nb=nb, time=time_sleep) self.wf_id = self.wf_ctrl.submit_workflow(workflow=workflow, name=self.__class__.__name__) # Transfer input files if file transfer if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: Helper.transfer_input_files(self.wf_id, self.wf_ctrl) # Wait for the workflow to finish Helper.wait_workflow(self.wf_id, self.wf_ctrl) # Transfer output files if file transfer if self.path_management == self.FILE_TRANSFER or \ self.path_management == self.SHARED_TRANSFER: Helper.transfer_output_files(self.wf_id, self.wf_ctrl) status = self.wf_ctrl.workflow_status(self.wf_id) self.assertTrue( status == constants.WORKFLOW_DONE, "workflow status : %s. Expected : %s" % (status, constants.WORKFLOW_DONE)) nb_failed_jobs = len(Helper.list_failed_jobs(self.wf_id, self.wf_ctrl)) self.assertTrue( nb_failed_jobs == 0, "nb failed jobs : %i. Expected : %i" % (nb_failed_jobs, 0)) nb_failed_aborted_jobs = len( Helper.list_failed_jobs(self.wf_id, self.wf_ctrl, include_aborted_jobs=True)) self.assertTrue( nb_failed_aborted_jobs == 0, "nb failed jobs including aborted : %i. Expected : %i" % (nb_failed_aborted_jobs, 0)) (jobs_info, transfers_info, workflow_status, workflow_queue, tmp_files) = self.wf_ctrl.workflow_elements_status(self.wf_id) for (job_id, tmp_status, queue, exit_info, dates, drmaa_id) \ in jobs_info: job_list = self.wf_ctrl.jobs([job_id]) job_name, job_command, job_submission_date = job_list[job_id] self.tested_job = job_id if exit_info[0] == constants.FINISHED_REGULARLY: # To check job standard out and standard err job_stdout_file = tempfile.NamedTemporaryFile( prefix="job_soma_out_log_", suffix=repr(job_id), delete=False) job_stdout_file = job_stdout_file.name job_stderr_file = tempfile.NamedTemporaryFile( prefix="job_soma_outerr_log_", suffix=repr(job_id), delete=False) job_stderr_file = job_stderr_file.name try: self.wf_ctrl.retrieve_job_stdouterr( job_id, job_stdout_file, job_stderr_file) # Test stdout self.assertTrue( os.stat(job_stdout_file).st_size == 0, "job stdout not empty : file: %s, " "contents:\n%s" % (job_stdout_file, open(job_stdout_file).read())) # Test no stderr self.assertTrue( os.stat(job_stderr_file).st_size == 0, "job stderr not empty : file %s, " "contents:\n%s" % (job_stderr_file, open(job_stderr_file).read())) finally: os.unlink(job_stdout_file) os.unlink(job_stderr_file) del self.tested_job
file1 = FileTransfer(is_input=True, client_path="%s/create_file.py" % Configuration.get_home_dir(), name="script") file2 = FileTransfer(is_input=True, client_path="%s/output_file" % Configuration.get_home_dir(), name="file created on the server") # Job and Workflow run_script = Job(command=["python", file1, file2], name="copy", referenced_input_files=[file1], referenced_output_files=[file2]) workflow = Workflow(jobs=[run_script], dependencies=[]) workflow_id = controller.submit_workflow(workflow=workflow, name="Simple transfer") # You may use the gui or manually transfer the files: manual = True if manual: Helper.transfer_input_files(workflow_id, controller) Helper.wait_workflow(workflow_id, controller) Helper.transfer_output_files(workflow_id, controller) # RemoteConnection.kill_remote_servers("Gabriel") print("Finished !!!")