def _create_jobs(self, keysfile_list, is_run_local, ft_working_directory): from soma_workflow.client import Job jobs = [] for nodesfile in keysfile_list: command = [] command.append("epac_mapper") command.append("--datasets") command.append('"' + SomaWorkflowEngine.dataset_relative_path + '"') command.append("--keysfile") command.append('"' + (nodesfile) + '"') if self.mmap_mode: command.append("--mmap_mode") command.append(self.mmap_mode) if not is_run_local: job = Job(command, referenced_input_files=[ft_working_directory], referenced_output_files=[ft_working_directory], name="epac_job_key=%s" % (nodesfile), working_directory=ft_working_directory) else: job = Job(command, name="epac_job_key=%s" % (nodesfile), working_directory=ft_working_directory) jobs.append(job) return jobs
def SimpleJobExample(configuration_item_name, userid, userpw=None): """ Dummy workflow to test the install Parameters ---------- configuration_item_name: str the name of the configuration item (ex. "Gabriel") userid: str user name on the server side userpw: str (optional) user password to login the server using ssh. If you want to use "id_rsa.pub", just leave userpw to None To copy the public key on the server use ssh-copy-id -i name@server. """ job_1 = Job(command=["sleep", "5"], name="job 1") job_2 = Job(command=["sleep", "5"], name="job 2") job_3 = Job(command=["sleep", "5"], name="job 3") job_4 = Job(command=["sleep", "5"], name="job 4") jobs = [job_1, job_2, job_3, job_4] dependencies = [(job_1, job_2), (job_1, job_3), (job_2, job_4), (job_3, job_4)] workflow = Workflow(jobs=jobs, dependencies=dependencies) controller = WorkflowController(configuration_item_name, userid, userpw) controller.submit_workflow(workflow=workflow, name="TestConnectionExample")
def job_list_with_outputs2(self): job_name = 'copy_files2' job = Job( [self.python, '%(script)s'], self.shared_list([ self.sh_script[7], self.sh_file[0], self.sh_file[1], self.sh_file[5] ]), None, None, False, 168, job_name, param_dict={ 'script': self.sh_script[7], 'inputs': [ self.sh_file[0], self.sh_file[1], self.sh_file[5], self.sh_file[0], self.sh_file[1], self.sh_file[5] ], 'output_dir': os.path.join(self.output_dir, 'intermediate_results') }, has_outputs=True, use_input_params_file=True) return job
def job_test_dir_contents(self): job = Job(["python", self.tr_dir_contents_script, self.tr_in_dir, self.tr_out_dir], [self.tr_dir_contents_script, self.tr_in_dir], [self.tr_out_dir], None, False, 168, "dir_contents") return job
def submit_job4(self, time=10): self.file4_tr = self.wf_ctrl.register_transfer( FileTransfer(False, os.path.join(self.output_dir, "file4"), self.tr_timeout)) script4_tr = self.wf_ctrl.register_transfer( FileTransfer(True, os.path.join(self.complete_path, "job4.py"), self.tr_timeout)) stdin4_tr = self.wf_ctrl.register_transfer( FileTransfer(True, os.path.join(self.complete_path, "stdin4"), self.tr_timeout)) self.wf_ctrl.transfer_files(script4_tr.engine_path) self.wf_ctrl.transfer_files(stdin4_tr.engine_path) job4_id = self.wf_ctrl.submit_job(Job( command=[self.python, script4_tr, self.file2_tr, self.file3_tr, self.file4_tr, repr(time)], referenced_input_files=[self.file2_tr, self.file3_tr, script4_tr, stdin4_tr], referenced_output_files=[self.file4_tr], stdin=stdin4_tr, join_stderrout=False, disposal_timeout=self.jobs_timeout, name="job4 with transfers")) return (job4_id, [self.file4_tr.engine_path], None)
def job_test_command_1(self): test_command = Job([ "python", self.sh_cmd_check_script, "[13.5, 14.5, 15.0]", '[13.5, 14.5, 15.0]', "['un', 'deux', 'trois']", '["un", "deux", "trois"]' ], None, None, None, False, 168, "test_command_1") return test_command
def job2_with_outputs1(self): time_to_wait = 2 job_name = "job2_with_outputs" job2 = Job( [self.python, '%(script)s'], self.shared_list([ self.sh_script[2], self.sh_script[6], self.tr_file[11], self.sh_file[0] ]), self.shared_list([self.sh_file[2]]), self.sh_stdin[2], False, 168, job_name, param_dict={ 'script': self.sh_script[6], 'script_job2': self.sh_script[2], 'filePathIn1': self.tr_file[11], 'filePathIn2': self.sh_file[0], 'timeToSleep': str(time_to_wait), 'filePathOut': self.sh_file[2] }, has_outputs=True, use_input_params_file=True) return job2
def job1_with_outputs1(self): time_to_wait = 2 job_name = "job1_with_outputs" job1 = Job( [ self.python, '%(script)s', '%(script_job1)s', '%(filePathIn)s', '%(filePathOut1)s', '%(timeToSleep)s' ], self.shared_list( [self.sh_script[1], self.sh_script[5], self.sh_file[0]]), self.shared_list([self.tr_file[11], self.tr_file[12]]), self.sh_stdin[1], False, 168, job_name, param_dict={ 'script': self.sh_script[5], 'script_job1': self.sh_script[1], 'filePathIn': self.sh_file[0], 'filePathOut1': self.tr_file[11], 'timeToSleep': str(time_to_wait), 'filePathOut2': self.tr_file[12], 'output_directory': self.output_dir }, has_outputs=True) return job1
def job_test_multi_file_format(self): job = Job(["python", self.tr_mff_script, (self.tr_img_file, "example.img"), (self.tr_img_out_file, "example.img")], [self.tr_mff_script, self.tr_img_file], [self.tr_img_out_file], None, False, 168, "multi file format test") return job
def job3(self): time_to_wait = 2 job_name = "job3" job3 = Job([ "python", self.sh_script[3], self.sh_file[12], self.sh_file[3], repr(time_to_wait) ], None, None, self.sh_stdin[3], False, 168, job_name) return job3
def job_test_multi_file_format(self): job = Job([ self.python, self.tr_mff_script, (self.tr_img_file, 'example.img'), (self.tr_img_out_file, 'exampleout.img') ], self.shared_list([self.tr_mff_script, self.tr_img_file]), self.shared_list([self.tr_img_out_file]), None, False, 168, "multi file format test") return job
def job_test_dir_contents(self): job = Job([ self.python, self.tr_dir_contents_script, self.tr_in_dir, self.tr_out_dir ], self.shared_list([self.tr_dir_contents_script, self.tr_in_dir]), self.shared_list([self.tr_out_dir]), None, False, 168, "dir_contents") return job
def job1_exception(self): job = Job(["python", self.tr_exceptionJobScript], [self.tr_exceptionJobScript, self.tr_file[0], self.tr_script[1], self.tr_stdin[1]], [self.tr_file[11], self.tr_file[12]], self.tr_stdin[1], False, 168, "job1 with exception") return job
def job3_exception(self): job = Job(["python", self.tr_exceptionJobScript], [self.tr_exceptionJobScript, self.tr_file[12], self.tr_script[3], self.tr_stdin[3]], [self.tr_file[3]], self.tr_stdin[3], False, 168, "job3 with exception") return job
def job_sleep(self, period): complete_path = os.path.join(self.examples_dir, "complete") transfer = FileTransfer( True, os.path.join(complete_path, "file0"), 168, "file0") job = Job(["python", self.tr_sleep_script, repr(period)], [self.tr_sleep_script, transfer], [], None, False, 168, "sleep " + repr(period) + " s") return job
def job4(self): time_to_wait = 10 job_name = "job4" job4 = Job([ "python", self.sh_script[4], self.sh_file[2], self.sh_file[3], self.sh_file[4], repr(time_to_wait) ], None, None, self.sh_stdin[4], False, 168, job_name) return job4
def job2(self): time_to_wait = 2 job_name = "job2" job2 = Job([ "python", self.sh_script[2], self.sh_file[11], self.sh_file[0], self.sh_file[2], repr(time_to_wait) ], None, None, self.sh_stdin[2], False, 168, job_name) return job2
def job_test_command_1(self): test_command = Job([ self.python, self.tr_cmd_check_script, "[13.5, 14.5, 15.0]", '[13.5, 14.5, 15.0]', "['un', 'deux', 'trois']", '["un", "deux", "trois"]' ], self.shared_list([ self.tr_cmd_check_script, self.tr_script[1], self.tr_script[2], self.tr_script[3] ]), None, None, False, 168, "test_command_1") return test_command
def job1(self, option=None): time_to_wait = 2 job_name = "job1" job1 = Job(["python", self.lo_script[1], self.lo_file[0], self.lo_file[11], self.lo_file[12], repr(time_to_wait)], None, None, self.lo_stdin[1], False, 168, job_name, native_specification=option) return job1
def create_workflow(inp, out, names=None, verbose=False): if not osp.isfile(inp): raise Exception('File not found %s' % inp) commands = [e.rstrip('\n').split(' ') for e in open(inp).readlines()] if verbose: print commands names = ['job_%s' % i for i in xrange(len(commands))] if names is None else names jobs = [Job(command=cmd, name=name) for cmd, name in zip(commands, names)] workflow = Workflow(jobs=jobs, dependencies=[]) Helper.serialize(out, workflow)
def create_somaWF(liste_python_files): jobs = [] for file_python in liste_python_files: file_name = os.path.basename(file_python) job_1 = Job(command=["python", file_python], name=file_name) jobs.append(job_1) #jobs = [job_1] dependencies = [] workflow = Workflow(jobs=jobs, dependencies=dependencies) # save the workflow into a file somaWF_name = os.path.join(path_script, "soma_WF_JOBS") Helper.serialize(somaWF_name, workflow)
def local_submission(self): file11 = os.path.join(self.output_dir, "file11") file12 = os.path.join(self.output_dir, "file12") job_id = self.wf_ctrl.submit_job(Job( command=[self.python, os.path.join(self.complete_path, "job1.py"), os.path.join(self.complete_path, "file0"), file11, file12, "2"], stdin=os.path.join(self.complete_path, "stdin1"), join_stderrout=False, disposal_timeout=self.jobs_timeout, name="job1 local submission")) return (job_id, [file11, file12], None)
def job1(self, option=None): time_to_wait = 2 job_name = "job1" job1 = Job([ self.python, self.sh_script[1], self.sh_file[0], self.tr_file[11], self.tr_file[12], repr(time_to_wait) ], self.shared_list([self.sh_script[1], self.sh_file[0]]), self.shared_list([self.tr_file[11], self.tr_file[12]]), self.sh_stdin[1], False, 168, job_name, native_specification=option) return job1
def submit_exception_job(self): script_tr = self.wf_ctrl.register_transfer( FileTransfer(True, os.path.join(self.complete_path, "exception_job.py"), self.tr_timeout)) self.wf_ctrl.transfer_files(script_tr.engine_path) job_id = self.wf_ctrl.submit_job(Job( command=[self.python, script_tr], referenced_input_files=[script_tr], referenced_output_files=[], stdin=None, join_stderrout=False, disposal_timeout=self.jobs_timeout, name="job with exception")) return (job_id, None, None)
def job8_with_output(self): time_to_wait = 1 job_name = 'job8_with_output' job = Job([self.python, '%(script)s'], self.shared_list([self.sh_script[8]]), self.shared_list([]), None, False, 168, job_name, param_dict={ 'script': self.sh_script[8], 'input': 'nothing' }, has_outputs=True, use_input_params_file=True) return job
def job_reduce_cat(self, file_num=13): job_name = 'cat_files' job = Job( [self.python, '%(script)s'], self.shared_list([self.sh_script[9]]), self.shared_list([self.sh_file[file_num]]), None, False, 168, job_name, param_dict={ 'script': self.sh_script[9], 'inputs': [], 'output': self.sh_file[file_num] }, use_input_params_file=True) return job
def job3(self): time_to_wait = 2 job_name = "job3" job3 = Job( [ self.python, '%(script)s', '%(filePathIn)s', '%(filePathOut)s', '%(timeToSleep)s' ], self.shared_list([self.sh_script[3], self.tr_file[12]]), self.shared_list([self.tr_file[3]]), self.sh_stdin[3], False, 168, job_name, param_dict={ 'script': self.sh_script[3], 'filePathIn': self.tr_file[12], 'filePathOut': self.tr_file[3], 'timeToSleep': str(time_to_wait) }) return job3
def local_custom_submission(self): stdout = os.path.join(self.output_dir, "stdout_local_custom_submission") stderr = os.path.join(self.output_dir, "stderr_local_custom_submission") file11 = os.path.join(self.output_dir, "file11") file12 = os.path.join(self.output_dir, "file12") job_id = self.wf_ctrl.submit_job(Job( command=[self.python, os.path.join(self.complete_path, "job1.py"), os.path.join(self.complete_path, "file0"), file11, file12, "2"], stdin=os.path.join(self.complete_path, "stdin1"), join_stderrout=False, disposal_timeout=self.jobs_timeout, name="job1 local custom submission", stdout_file=stdout, stderr_file=stderr, working_directory=self.output_dir)) return (job_id, [file11, file12], [stdout, stderr])
def submit_job1(self, time=2): self.file11_tr = self.wf_ctrl.register_transfer( FileTransfer(is_input=False, client_path=os.path.join(self.output_dir, "file11"), disposal_timeout=self.tr_timeout)) self.file12_tr = self.wf_ctrl.register_transfer( FileTransfer(is_input=False, client_path=os.path.join(self.output_dir, "file12"), disposal_timeout=self.tr_timeout)) self.file0_tr = self.wf_ctrl.register_transfer( FileTransfer(True, os.path.join(self.complete_path, "file0"), self.tr_timeout)) script1_tr = self.wf_ctrl.register_transfer( FileTransfer(True, os.path.join(self.complete_path, "job1.py"), self.tr_timeout)) stdin1_tr = self.wf_ctrl.register_transfer( FileTransfer(True, os.path.join(self.complete_path, "stdin1"), self.tr_timeout)) self.wf_ctrl.transfer_files(self.file0_tr.engine_path) self.wf_ctrl.transfer_files(script1_tr.engine_path) self.wf_ctrl.transfer_files(stdin1_tr.engine_path) sys.stdout.write("files transfered \n") job1_id = self.wf_ctrl.submit_job(Job( command=[self.python, script1_tr, self.file0_tr, self.file11_tr, self.file12_tr, repr(time)], referenced_input_files=[self.file0_tr, script1_tr, stdin1_tr], referenced_output_files=[self.file11_tr, self.file12_tr], stdin=stdin1_tr, join_stderrout=False, disposal_timeout=self.jobs_timeout, name="job1 with transfers")) return ((job1_id, [self.file11_tr.engine_path, self.file12_tr.engine_path], None))
def job2(self): time_to_wait = 2 job_name = "job2" job2 = Job( [ self.python, '%(script)s', '%(file1)s', '%(file2)s', '%(file3)s', repr(time_to_wait) ], self.shared_list([self.sh_script[2], self.tr_file[11]]), self.shared_list([self.sh_file[2]]), self.sh_stdin[2], False, 168, job_name, param_dict={ 'script': self.sh_script[2], 'file1': self.tr_file[11], 'file2': self.sh_file[0], 'file3': self.sh_file[2] }) return job2