def _create_jobs(self, keysfile_list, is_run_local, ft_working_directory):
from soma_workflow.client import Job
jobs = []
for nodesfile in keysfile_list:
command = []
command.append("epac_mapper")
command.append("--datasets")
command.append('"' + SomaWorkflowEngine.dataset_relative_path +
'"')
command.append("--keysfile")
command.append('"' + (nodesfile) + '"')
if self.mmap_mode:
command.append("--mmap_mode")
command.append(self.mmap_mode)
if not is_run_local:
job = Job(command,
referenced_input_files=[ft_working_directory],
referenced_output_files=[ft_working_directory],
name="epac_job_key=%s" % (nodesfile),
working_directory=ft_working_directory)
else:
job = Job(command,
name="epac_job_key=%s" % (nodesfile),
working_directory=ft_working_directory)
jobs.append(job)
return jobs
def SimpleJobExample(configuration_item_name, userid, userpw=None):
""" Dummy workflow to test the install
Parameters
----------
configuration_item_name: str
the name of the configuration item (ex. "Gabriel")
userid: str
user name on the server side
userpw: str (optional)
user password to login the server using ssh.
If you want to use "id_rsa.pub", just leave userpw to None
To copy the public key on the server use ssh-copy-id -i name@server.
"""
job_1 = Job(command=["sleep", "5"], name="job 1")
job_2 = Job(command=["sleep", "5"], name="job 2")
job_3 = Job(command=["sleep", "5"], name="job 3")
job_4 = Job(command=["sleep", "5"], name="job 4")
jobs = [job_1, job_2, job_3, job_4]
dependencies = [(job_1, job_2), (job_1, job_3), (job_2, job_4),
(job_3, job_4)]
workflow = Workflow(jobs=jobs, dependencies=dependencies)
controller = WorkflowController(configuration_item_name, userid, userpw)
controller.submit_workflow(workflow=workflow, name="TestConnectionExample")
def job_list_with_outputs2(self):
job_name = 'copy_files2'
job = Job(
[self.python, '%(script)s'],
self.shared_list([
self.sh_script[7], self.sh_file[0], self.sh_file[1],
self.sh_file[5]
]),
None,
None,
False,
168,
job_name,
param_dict={
'script':
self.sh_script[7],
'inputs': [
self.sh_file[0], self.sh_file[1], self.sh_file[5],
self.sh_file[0], self.sh_file[1], self.sh_file[5]
],
'output_dir':
os.path.join(self.output_dir, 'intermediate_results')
},
has_outputs=True,
use_input_params_file=True)
return job
def job_test_dir_contents(self):
job = Job(["python", self.tr_dir_contents_script,
self.tr_in_dir, self.tr_out_dir],
[self.tr_dir_contents_script, self.tr_in_dir],
[self.tr_out_dir],
None, False, 168, "dir_contents")
return job
def submit_job4(self, time=10):
self.file4_tr = self.wf_ctrl.register_transfer(
FileTransfer(False,
os.path.join(self.output_dir, "file4"),
self.tr_timeout))
script4_tr = self.wf_ctrl.register_transfer(
FileTransfer(True,
os.path.join(self.complete_path, "job4.py"),
self.tr_timeout))
stdin4_tr = self.wf_ctrl.register_transfer(
FileTransfer(True,
os.path.join(self.complete_path, "stdin4"),
self.tr_timeout))
self.wf_ctrl.transfer_files(script4_tr.engine_path)
self.wf_ctrl.transfer_files(stdin4_tr.engine_path)
job4_id = self.wf_ctrl.submit_job(Job(
command=[self.python, script4_tr, self.file2_tr,
self.file3_tr, self.file4_tr, repr(time)],
referenced_input_files=[self.file2_tr, self.file3_tr,
script4_tr, stdin4_tr],
referenced_output_files=[self.file4_tr],
stdin=stdin4_tr,
join_stderrout=False,
disposal_timeout=self.jobs_timeout,
name="job4 with transfers"))
return (job4_id, [self.file4_tr.engine_path], None)
def job_test_command_1(self):
test_command = Job([
"python", self.sh_cmd_check_script, "[13.5, 14.5, 15.0]",
'[13.5, 14.5, 15.0]', "['un', 'deux', 'trois']",
'["un", "deux", "trois"]'
], None, None, None, False, 168, "test_command_1")
return test_command
def job2_with_outputs1(self):
time_to_wait = 2
job_name = "job2_with_outputs"
job2 = Job(
[self.python, '%(script)s'],
self.shared_list([
self.sh_script[2], self.sh_script[6], self.tr_file[11],
self.sh_file[0]
]),
self.shared_list([self.sh_file[2]]),
self.sh_stdin[2],
False,
168,
job_name,
param_dict={
'script': self.sh_script[6],
'script_job2': self.sh_script[2],
'filePathIn1': self.tr_file[11],
'filePathIn2': self.sh_file[0],
'timeToSleep': str(time_to_wait),
'filePathOut': self.sh_file[2]
},
has_outputs=True,
use_input_params_file=True)
return job2
def job1_with_outputs1(self):
time_to_wait = 2
job_name = "job1_with_outputs"
job1 = Job(
[
self.python, '%(script)s', '%(script_job1)s', '%(filePathIn)s',
'%(filePathOut1)s', '%(timeToSleep)s'
],
self.shared_list(
[self.sh_script[1], self.sh_script[5], self.sh_file[0]]),
self.shared_list([self.tr_file[11], self.tr_file[12]]),
self.sh_stdin[1],
False,
168,
job_name,
param_dict={
'script': self.sh_script[5],
'script_job1': self.sh_script[1],
'filePathIn': self.sh_file[0],
'filePathOut1': self.tr_file[11],
'timeToSleep': str(time_to_wait),
'filePathOut2': self.tr_file[12],
'output_directory': self.output_dir
},
has_outputs=True)
return job1
def job_test_multi_file_format(self):
job = Job(["python", self.tr_mff_script,
(self.tr_img_file, "example.img"),
(self.tr_img_out_file, "example.img")],
[self.tr_mff_script, self.tr_img_file],
[self.tr_img_out_file],
None, False, 168, "multi file format test")
return job
def job3(self):
time_to_wait = 2
job_name = "job3"
job3 = Job([
"python", self.sh_script[3], self.sh_file[12], self.sh_file[3],
repr(time_to_wait)
], None, None, self.sh_stdin[3], False, 168, job_name)
return job3
def job_test_multi_file_format(self):
job = Job([
self.python, self.tr_mff_script, (self.tr_img_file, 'example.img'),
(self.tr_img_out_file, 'exampleout.img')
], self.shared_list([self.tr_mff_script, self.tr_img_file]),
self.shared_list([self.tr_img_out_file]), None, False, 168,
"multi file format test")
return job
def job_test_dir_contents(self):
job = Job([
self.python, self.tr_dir_contents_script, self.tr_in_dir,
self.tr_out_dir
], self.shared_list([self.tr_dir_contents_script, self.tr_in_dir]),
self.shared_list([self.tr_out_dir]), None, False, 168,
"dir_contents")
return job
def job1_exception(self):
job = Job(["python", self.tr_exceptionJobScript],
[self.tr_exceptionJobScript,
self.tr_file[0],
self.tr_script[1],
self.tr_stdin[1]],
[self.tr_file[11], self.tr_file[12]],
self.tr_stdin[1], False, 168, "job1 with exception")
return job
def job3_exception(self):
job = Job(["python", self.tr_exceptionJobScript],
[self.tr_exceptionJobScript,
self.tr_file[12],
self.tr_script[3],
self.tr_stdin[3]],
[self.tr_file[3]],
self.tr_stdin[3], False, 168, "job3 with exception")
return job
def job_sleep(self, period):
complete_path = os.path.join(self.examples_dir, "complete")
transfer = FileTransfer(
True, os.path.join(complete_path, "file0"),
168, "file0")
job = Job(["python", self.tr_sleep_script, repr(period)],
[self.tr_sleep_script, transfer], [],
None, False, 168, "sleep " + repr(period) + " s")
return job
def job4(self):
time_to_wait = 10
job_name = "job4"
job4 = Job([
"python", self.sh_script[4], self.sh_file[2], self.sh_file[3],
self.sh_file[4],
repr(time_to_wait)
], None, None, self.sh_stdin[4], False, 168, job_name)
return job4
def job2(self):
time_to_wait = 2
job_name = "job2"
job2 = Job([
"python", self.sh_script[2], self.sh_file[11], self.sh_file[0],
self.sh_file[2],
repr(time_to_wait)
], None, None, self.sh_stdin[2], False, 168, job_name)
return job2
def job_test_command_1(self):
test_command = Job([
self.python, self.tr_cmd_check_script, "[13.5, 14.5, 15.0]",
'[13.5, 14.5, 15.0]', "['un', 'deux', 'trois']",
'["un", "deux", "trois"]'
],
self.shared_list([
self.tr_cmd_check_script, self.tr_script[1],
self.tr_script[2], self.tr_script[3]
]), None, None, False, 168, "test_command_1")
return test_command
def job1(self, option=None):
time_to_wait = 2
job_name = "job1"
job1 = Job(["python",
self.lo_script[1], self.lo_file[0],
self.lo_file[11], self.lo_file[12],
repr(time_to_wait)],
None, None,
self.lo_stdin[1], False, 168, job_name,
native_specification=option)
return job1
def create_workflow(inp, out, names=None, verbose=False):
if not osp.isfile(inp):
raise Exception('File not found %s' % inp)
commands = [e.rstrip('\n').split(' ') for e in open(inp).readlines()]
if verbose:
print commands
names = ['job_%s' % i
for i in xrange(len(commands))] if names is None else names
jobs = [Job(command=cmd, name=name) for cmd, name in zip(commands, names)]
workflow = Workflow(jobs=jobs, dependencies=[])
Helper.serialize(out, workflow)
def create_somaWF(liste_python_files):
jobs = []
for file_python in liste_python_files:
file_name = os.path.basename(file_python)
job_1 = Job(command=["python", file_python], name=file_name)
jobs.append(job_1)
#jobs = [job_1]
dependencies = []
workflow = Workflow(jobs=jobs, dependencies=dependencies)
# save the workflow into a file
somaWF_name = os.path.join(path_script, "soma_WF_JOBS")
Helper.serialize(somaWF_name, workflow)
def local_submission(self):
file11 = os.path.join(self.output_dir, "file11")
file12 = os.path.join(self.output_dir, "file12")
job_id = self.wf_ctrl.submit_job(Job(
command=[self.python,
os.path.join(self.complete_path, "job1.py"),
os.path.join(self.complete_path, "file0"),
file11, file12, "2"],
stdin=os.path.join(self.complete_path, "stdin1"),
join_stderrout=False,
disposal_timeout=self.jobs_timeout,
name="job1 local submission"))
return (job_id, [file11, file12], None)
def job1(self, option=None):
time_to_wait = 2
job_name = "job1"
job1 = Job([
self.python, self.sh_script[1], self.sh_file[0], self.tr_file[11],
self.tr_file[12],
repr(time_to_wait)
],
self.shared_list([self.sh_script[1], self.sh_file[0]]),
self.shared_list([self.tr_file[11], self.tr_file[12]]),
self.sh_stdin[1],
False,
168,
job_name,
native_specification=option)
return job1
def submit_exception_job(self):
script_tr = self.wf_ctrl.register_transfer(
FileTransfer(True,
os.path.join(self.complete_path, "exception_job.py"),
self.tr_timeout))
self.wf_ctrl.transfer_files(script_tr.engine_path)
job_id = self.wf_ctrl.submit_job(Job(
command=[self.python, script_tr],
referenced_input_files=[script_tr],
referenced_output_files=[],
stdin=None,
join_stderrout=False,
disposal_timeout=self.jobs_timeout,
name="job with exception"))
return (job_id, None, None)
def job8_with_output(self):
time_to_wait = 1
job_name = 'job8_with_output'
job = Job([self.python, '%(script)s'],
self.shared_list([self.sh_script[8]]),
self.shared_list([]),
None,
False,
168,
job_name,
param_dict={
'script': self.sh_script[8],
'input': 'nothing'
},
has_outputs=True,
use_input_params_file=True)
return job
def job_reduce_cat(self, file_num=13):
job_name = 'cat_files'
job = Job(
[self.python, '%(script)s'],
self.shared_list([self.sh_script[9]]),
self.shared_list([self.sh_file[file_num]]),
None,
False,
168,
job_name,
param_dict={
'script': self.sh_script[9],
'inputs': [],
'output': self.sh_file[file_num]
},
use_input_params_file=True)
return job
def job3(self):
time_to_wait = 2
job_name = "job3"
job3 = Job(
[
self.python, '%(script)s', '%(filePathIn)s', '%(filePathOut)s',
'%(timeToSleep)s'
],
self.shared_list([self.sh_script[3], self.tr_file[12]]),
self.shared_list([self.tr_file[3]]),
self.sh_stdin[3],
False,
168,
job_name,
param_dict={
'script': self.sh_script[3],
'filePathIn': self.tr_file[12],
'filePathOut': self.tr_file[3],
'timeToSleep': str(time_to_wait)
})
return job3
def local_custom_submission(self):
stdout = os.path.join(self.output_dir,
"stdout_local_custom_submission")
stderr = os.path.join(self.output_dir,
"stderr_local_custom_submission")
file11 = os.path.join(self.output_dir, "file11")
file12 = os.path.join(self.output_dir, "file12")
job_id = self.wf_ctrl.submit_job(Job(
command=[self.python,
os.path.join(self.complete_path, "job1.py"),
os.path.join(self.complete_path, "file0"),
file11, file12, "2"],
stdin=os.path.join(self.complete_path, "stdin1"),
join_stderrout=False,
disposal_timeout=self.jobs_timeout,
name="job1 local custom submission",
stdout_file=stdout,
stderr_file=stderr,
working_directory=self.output_dir))
return (job_id, [file11, file12], [stdout, stderr])
def submit_job1(self, time=2):
self.file11_tr = self.wf_ctrl.register_transfer(
FileTransfer(is_input=False,
client_path=os.path.join(self.output_dir, "file11"),
disposal_timeout=self.tr_timeout))
self.file12_tr = self.wf_ctrl.register_transfer(
FileTransfer(is_input=False,
client_path=os.path.join(self.output_dir, "file12"),
disposal_timeout=self.tr_timeout))
self.file0_tr = self.wf_ctrl.register_transfer(
FileTransfer(True,
os.path.join(self.complete_path, "file0"),
self.tr_timeout))
script1_tr = self.wf_ctrl.register_transfer(
FileTransfer(True,
os.path.join(self.complete_path, "job1.py"),
self.tr_timeout))
stdin1_tr = self.wf_ctrl.register_transfer(
FileTransfer(True,
os.path.join(self.complete_path, "stdin1"),
self.tr_timeout))
self.wf_ctrl.transfer_files(self.file0_tr.engine_path)
self.wf_ctrl.transfer_files(script1_tr.engine_path)
self.wf_ctrl.transfer_files(stdin1_tr.engine_path)
sys.stdout.write("files transfered \n")
job1_id = self.wf_ctrl.submit_job(Job(
command=[self.python, script1_tr, self.file0_tr,
self.file11_tr, self.file12_tr, repr(time)],
referenced_input_files=[self.file0_tr, script1_tr,
stdin1_tr],
referenced_output_files=[self.file11_tr, self.file12_tr],
stdin=stdin1_tr,
join_stderrout=False,
disposal_timeout=self.jobs_timeout,
name="job1 with transfers"))
return ((job1_id,
[self.file11_tr.engine_path,
self.file12_tr.engine_path],
None))
def job2(self):
time_to_wait = 2
job_name = "job2"
job2 = Job(
[
self.python, '%(script)s', '%(file1)s', '%(file2)s',
'%(file3)s',
repr(time_to_wait)
],
self.shared_list([self.sh_script[2], self.tr_file[11]]),
self.shared_list([self.sh_file[2]]),
self.sh_stdin[2],
False,
168,
job_name,
param_dict={
'script': self.sh_script[2],
'file1': self.tr_file[11],
'file2': self.sh_file[0],
'file3': self.sh_file[2]
})
return job2