def wait(self, subject_id=None, step_id=None):
if subject_id is None and step_id is None:
Helper.wait_workflow(
self._workflow_id, self._workflow_controller)
elif subject_id is not None:
if step_id is None:
raise NotImplementedError
else:
self._step_wait(subject_id, step_id)
else:
raise NotImplementedError
# transfer back files, if any
Helper.transfer_output_files(self._workflow_id,
self._workflow_controller)
def test_result(self):
workflow = self.wf_examples.example_wrong_native_spec_pbs()
self.wf_id = self.wf_ctrl.submit_workflow(
workflow=workflow,
name=self.__class__.__name__)
# Transfer input files if file transfer
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
Helper.transfer_input_files(self.wf_id,
self.wf_ctrl)
# Wait for the workflow to finish
Helper.wait_workflow(self.wf_id, self.wf_ctrl)
# Transfer output files if file transfer
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
Helper.transfer_output_files(self.wf_id,
self.wf_ctrl)
status = self.wf_ctrl.workflow_status(self.wf_id)
self.assertTrue(status == constants.WORKFLOW_DONE,
"workflow status : %s. Expected : %s" %
(status, constants.WORKFLOW_DONE))
nb_failed_jobs = len(Helper.list_failed_jobs(
self.wf_id,
self.wf_ctrl))
self.assertTrue(nb_failed_jobs == 0,
"nb failed jobs : %i. Expected : %i" %
(nb_failed_jobs, 0))
nb_failed_aborted_jobs = len(Helper.list_failed_jobs(
self.wf_id,
self.wf_ctrl,
include_aborted_jobs=True))
if self.path_management == self.LOCAL_PATH:
self.assertTrue(nb_failed_aborted_jobs == 0,
"nb failed jobs including aborted : %i. "
"Expected : %i" % (nb_failed_aborted_jobs, 0))
else:
self.assertTrue(nb_failed_aborted_jobs == 1,
"nb failed jobs including aborted : %i. "
"Expected : %i" % (nb_failed_aborted_jobs, 1))
def test_result(self):
workflow = self.wf_examples.example_wrong_native_spec_pbs()
self.wf_id = self.wf_ctrl.submit_workflow(workflow=workflow,
name=self.__class__.__name__)
# Transfer input files if file transfer
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
Helper.transfer_input_files(self.wf_id, self.wf_ctrl)
# Wait for the workflow to finish
Helper.wait_workflow(self.wf_id, self.wf_ctrl)
# Transfer output files if file transfer
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
Helper.transfer_output_files(self.wf_id, self.wf_ctrl)
status = self.wf_ctrl.workflow_status(self.wf_id)
self.assertTrue(
status == constants.WORKFLOW_DONE,
"workflow status : %s. Expected : %s" %
(status, constants.WORKFLOW_DONE))
nb_failed_jobs = len(Helper.list_failed_jobs(self.wf_id, self.wf_ctrl))
self.assertTrue(
nb_failed_jobs == 0,
"nb failed jobs : %i. Expected : %i" % (nb_failed_jobs, 0))
nb_failed_aborted_jobs = len(
Helper.list_failed_jobs(self.wf_id,
self.wf_ctrl,
include_aborted_jobs=True))
if self.path_management == self.LOCAL_PATH \
or self.wf_ctrl.get_scheduler_type() != 'pbs':
self.assertTrue(
nb_failed_aborted_jobs == 0,
"nb failed jobs including aborted : %i. "
"Expected : %i" % (nb_failed_aborted_jobs, 0))
else:
self.assertTrue(
nb_failed_aborted_jobs == 1,
"nb failed jobs including aborted : %i. "
"Expected : %i" % (nb_failed_aborted_jobs, 1))
echo %s
""" % test_bash_script
fileout.write(filecontent)
fileout.close()
os.chdir(cur_work_dir)
job1 = Job(command=[u"touch", test_filepath],
name="epac_job_test",
working_directory=tmp_work_dir_path)
job2 = Job(command=["%s/readfile" % cur_file_dir, test_bash_script],
name="epac_job_test",
working_directory=tmp_work_dir_path)
soma_workflow = Workflow(jobs=[job1, job2])
resource_id = socket.gethostname()
controller = WorkflowController(resource_id, "", "")
## run soma-workflow
## =================
wf_id = controller.submit_workflow(workflow=soma_workflow,
name="epac workflow")
Helper.wait_workflow(wf_id, controller)
nb_failed_jobs = len(Helper.list_failed_jobs(wf_id, controller))
if nb_failed_jobs > 0:
raise ValueError("Soma-workflow error, cannot use working directory")
if not os.path.isfile(os.path.join(tmp_work_dir_path, test_filepath)):
raise ValueError("Soma-workflow cannot define working directory")
else:
print "OK for creating new file in working directory"
def run(self, **Xy):
'''Run soma-workflow without gui
Example
-------
>>> from sklearn import datasets
>>> from epac.map_reduce.engine import SomaWorkflowEngine
>>> from epac.tests.wfexamples2test import WFExample2
>>> ## Build dataset
>>> ## =============
>>> X, y = datasets.make_classification(n_samples=10,
... n_features=20,
... n_informative=5,
... random_state=1)
>>> Xy = {'X':X, 'y':y}
>>> ## Build epac tree
>>> ## ===============
>>> tree_root_node = WFExample2().get_workflow()
>>> ## Build SomaWorkflowEngine and run function for each node
>>> ## =======================================================
>>> sfw_engine = SomaWorkflowEngine(tree_root=tree_root_node,
... function_name="transform",
... num_processes=3,
remove_finished_wf=False)
>>> tree_root_node = sfw_engine.run(**Xy)
light mode
>>> ## Run reduce process
>>> ## ==================
>>> tree_root_node.reduce()
ResultSet(
[{'key': SelectKBest/SVC(C=1), 'y/test/score_f1': [ 0.6 0.6], 'y/test/score_recall_mean/pval': [ 0.5], 'y/test/score_recall/pval': [ 0. 0.5], 'y/test/score_accuracy/pval': [ 0.], 'y/test/score_f1/pval': [ 0. 0.5], 'y/test/score_precision/pval': [ 0.5 0. ], 'y/test/score_precision': [ 0.6 0.6], 'y/test/score_recall': [ 0.6 0.6], 'y/test/score_accuracy': 0.6, 'y/test/score_recall_mean': 0.6},
{'key': SelectKBest/SVC(C=3), 'y/test/score_f1': [ 0.6 0.6], 'y/test/score_recall_mean/pval': [ 0.5], 'y/test/score_recall/pval': [ 0. 0.5], 'y/test/score_accuracy/pval': [ 0.], 'y/test/score_f1/pval': [ 0. 0.5], 'y/test/score_precision/pval': [ 0.5 0. ], 'y/test/score_precision': [ 0.6 0.6], 'y/test/score_recall': [ 0.6 0.6], 'y/test/score_accuracy': 0.6, 'y/test/score_recall_mean': 0.6}])
'''
try:
from soma_workflow.client import Job, Workflow
from soma_workflow.client import Helper, FileTransfer
from soma_workflow.client import WorkflowController
except ImportError:
errmsg = "No soma-workflow is found. "\
"Please verify your soma-worklow"\
"on your computer (e.g. PYTHONPATH) \n"
sys.stderr.write(errmsg)
sys.stdout.write(errmsg)
raise NoSomaWFError
tmp_work_dir_path = tempfile.mkdtemp()
cur_work_dir = os.getcwd()
os.chdir(tmp_work_dir_path)
is_run_local = False
if not self.resource_id or self.resource_id == "":
self.resource_id = socket.gethostname()
is_run_local = True
# print "is_run_local=", is_run_local
if not is_run_local:
ft_working_directory = FileTransfer(is_input=True,
client_path=tmp_work_dir_path,
name="working directory")
else:
ft_working_directory = tmp_work_dir_path
## Save the database and tree to working directory
## ===============================================
# np.savez(os.path.join(tmp_work_dir_path,
# SomaWorkflowEngine.dataset_relative_path), **Xy)
save_dataset(SomaWorkflowEngine.dataset_relative_path, **Xy)
store = StoreFs(dirpath=os.path.join(
tmp_work_dir_path, SomaWorkflowEngine.tree_root_relative_path))
self.tree_root.save_tree(store=store)
## Subtree job allocation on disk
## ==============================
node_input = NodesInput(self.tree_root.get_key())
split_node_input = SplitNodesInput(self.tree_root,
num_processes=self.num_processes)
nodesinput_list = split_node_input.split(node_input)
keysfile_list = save_job_list(tmp_work_dir_path, nodesinput_list)
## Build soma-workflow
## ===================
jobs = self._create_jobs(keysfile_list, is_run_local,
ft_working_directory)
soma_workflow = Workflow(jobs=jobs)
controller = WorkflowController(self.resource_id, self.login, self.pw)
## run soma-workflow
## =================
wf_id = controller.submit_workflow(workflow=soma_workflow,
name="epac workflow",
queue=self.queue)
Helper.transfer_input_files(wf_id, controller)
Helper.wait_workflow(wf_id, controller)
Helper.transfer_output_files(wf_id, controller)
self.engine_info = self.get_engine_info(controller, wf_id)
if self.remove_finished_wf:
controller.delete_workflow(wf_id)
## read result tree
## ================
self.tree_root = store.load()
os.chdir(cur_work_dir)
if os.path.isdir(tmp_work_dir_path) and self.remove_local_tree:
shutil.rmtree(tmp_work_dir_path)
return self.tree_root
def test_result(self):
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
with warnings.catch_warnings(record=True) as w:
# Trigger a warning.
workflow = self.wf_examples.example_special_command()
# Verify some things
self.assertTrue(len(w) == 1)
self.assertTrue(issubclass(w[-1].category, UserWarning))
self.assertTrue("contains single quote. It could fail using DRMAA"
in str(w[-1].message))
self.wf_id = self.wf_ctrl.submit_workflow(workflow=workflow,
name=self.__class__.__name__)
# Transfer input files if file transfer
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
Helper.transfer_input_files(self.wf_id, self.wf_ctrl)
# Wait for the worklow to finish
Helper.wait_workflow(self.wf_id, self.wf_ctrl)
status = self.wf_ctrl.workflow_status(self.wf_id)
self.assertTrue(
status == constants.WORKFLOW_DONE,
"workflow status : %s. Expected : %s" %
(status, constants.WORKFLOW_DONE))
# TODO : sometimes raises an error
# because status = "workflow_in_progress"
nb_failed_jobs = len(Helper.list_failed_jobs(self.wf_id, self.wf_ctrl))
self.assertTrue(
nb_failed_jobs == 0,
"nb failed jobs : %i. Expected : %i" % (nb_failed_jobs, 0))
nb_failed_aborted_jobs = len(
Helper.list_failed_jobs(self.wf_id,
self.wf_ctrl,
include_aborted_jobs=True))
self.assertTrue(
nb_failed_aborted_jobs == 0,
"nb failed jobs including aborted : %i. Expected : %i" %
(nb_failed_aborted_jobs, 0))
(jobs_info, transfers_info, workflow_status, workflow_queue,
tmp_files) = self.wf_ctrl.workflow_elements_status(self.wf_id)
for (job_id, tmp_status, queue, exit_info, dates) in jobs_info:
job_list = self.wf_ctrl.jobs([job_id])
job_name, job_command, job_submission_date = job_list[job_id]
self.tested_job = job_id
if exit_info[0] == constants.FINISHED_REGULARLY:
# To check job standard out and standard err
job_stdout_file = tempfile.NamedTemporaryFile(
prefix="job_soma_out_log_",
suffix=repr(job_id),
delete=False)
job_stdout_file = job_stdout_file.name
job_stderr_file = tempfile.NamedTemporaryFile(
prefix="job_soma_outerr_log_",
suffix=repr(job_id),
delete=False)
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(
job_id, job_stdout_file, job_stderr_file)
# Test job stdout
if self.path_management == self.LOCAL_PATH:
isSame, msg = identical_files(
job_stdout_file,
self.wf_examples.lo_stdout_command_local)
self.assertTrue(isSame, msg)
else:
isSame, msg = identical_files(
job_stdout_file,
self.wf_examples.lo_stdout_command_remote)
self.assertTrue(isSame, msg)
# Test no stderr
self.assertTrue(
os.stat(job_stderr_file).st_size == 0,
"job stderr not empty : cf %s" % job_stderr_file)
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
file1 = FileTransfer(is_input=True,
client_path="%s/create_file.py" %
Configuration.get_home_dir(),
name="script")
file2 = FileTransfer(is_input=True,
client_path="%s/output_file" %
Configuration.get_home_dir(),
name="file created on the server")
# Job and Workflow
run_script = Job(command=["python", file1, file2],
name="copy",
referenced_input_files=[file1],
referenced_output_files=[file2])
workflow = Workflow(jobs=[run_script], dependencies=[])
workflow_id = controller.submit_workflow(workflow=workflow,
name="Simple transfer")
# You may use the gui or manually transfer the files:
manual = True
if manual:
Helper.transfer_input_files(workflow_id, controller)
Helper.wait_workflow(workflow_id, controller)
Helper.transfer_output_files(workflow_id, controller)
# RemoteConnection.kill_remote_servers("Gabriel")
print("Finished !!!")
def test_result(self):
cmd = [
sys.executable, '-c',
'from __future__ import print_function; import sys;'
'from soma_workflow.test.workflow_tests.test_workflow_config import print_cmdline; '
'print_cmdline(sys.argv[1:])', 'conf=%(configuration_dict)s'
]
configuration = {'config1': 'value1', 'config2': 'value2'}
print(cmd)
job1 = Job(name='job1', command=cmd)
job2 = Job(name='job2',
command=cmd,
configuration=configuration,
param_dict={})
job3 = Job(name='job3',
command=cmd,
configuration=configuration,
use_input_params_file=True)
expected_outputs = {
'workflow': {
'job1':
'args: [\'conf=%(configuration_dict)s\']\n'
'config:\n'
'None',
'job2':
'''args: ['conf={"config1": "value1", "config2": "value2"}']
conf param
config:
{'config1': 'value1', 'config2': 'value2'}''',
'job3':
'''args: [\'conf=%(configuration_dict)s\']
with input file
params:
{'parameters': {}, 'configuration_dict': {'config1': 'value1', 'config2': 'value2'}}
config:
{'config1': 'value1', 'config2': 'value2'}''',
},
}
workflow1 = Workflow(name='workflow', jobs=[job1, job2, job3])
for workflow in (workflow1, ):
self.wf_id = self.wf_ctrl.submit_workflow(
workflow=workflow, name=self.__class__.__name__)
# Transfer input files if file transfer
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
Helper.transfer_input_files(self.wf_id, self.wf_ctrl)
# Wait for the workflow to finish
Helper.wait_workflow(self.wf_id, self.wf_ctrl)
status = self.wf_ctrl.workflow_status(self.wf_id)
self.assertTrue(
status == constants.WORKFLOW_DONE,
"workflow status : %s. Expected : %s" %
(status, constants.WORKFLOW_DONE))
nb_failed_jobs = len(
Helper.list_failed_jobs(self.wf_id, self.wf_ctrl))
try:
self.assertTrue(
nb_failed_jobs == 0,
"nb failed jobs : %i. Expected : %i" % (nb_failed_jobs, 0))
except: # noqa: E722
print('jobs failed:', file=sys.stderr)
print(Helper.list_failed_jobs(self.wf_id, self.wf_ctrl),
file=sys.stderr)
raise
nb_failed_aborted_jobs = len(
Helper.list_failed_jobs(self.wf_id,
self.wf_ctrl,
include_aborted_jobs=True))
try:
self.assertTrue(
nb_failed_aborted_jobs == 0,
"nb failed jobs including aborted : %i. Expected : %i" %
(nb_failed_aborted_jobs, 0))
except: # noqa: E722
print('aborted jobs:', file=sys.stderr)
print(Helper.list_failed_jobs(self.wf_id,
self.wf_ctrl,
include_aborted_jobs=True),
file=sys.stderr)
raise
(jobs_info, transfers_info, workflow_status, workflow_queue,
tmp_files) = self.wf_ctrl.workflow_elements_status(self.wf_id)
for (job_id, tmp_status, queue, exit_info, dates, drmaa_id) \
in jobs_info:
job_list = self.wf_ctrl.jobs([job_id])
job_name, job_command, job_submission_date = job_list[job_id]
self.tested_job = job_id
if exit_info[0] == constants.FINISHED_REGULARLY:
# To check job standard out and standard err
job_stdout_file = tempfile.NamedTemporaryFile(
prefix="job_soma_out_log_",
suffix=repr(job_id),
delete=False)
job_stdout_file = job_stdout_file.name
job_stderr_file = tempfile.NamedTemporaryFile(
prefix="job_soma_outerr_log_",
suffix=repr(job_id),
delete=False)
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(
job_id, job_stdout_file, job_stderr_file)
output \
= open(job_stdout_file).read().strip().split('\n')
exp_wf_outputs = expected_outputs[workflow.name]
if job_name in exp_wf_outputs:
exp = exp_wf_outputs[job_name].split('\n')
#print('### job', job_name, 'output:', output, file=sys.stderr)
#print('### expected:', exp, file=sys.stderr)
#print('### res:', [o in output for o in exp], file=sys.stderr)
self.assertTrue(all([o in output for o in exp]))
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
def test_result(self):
workflow = self.wf_examples.example_simple_exception2()
self.wf_id = self.wf_ctrl.submit_workflow(workflow=workflow,
name=self.__class__.__name__,
queue='Cati_run4')
# Transfer input files if file transfer
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
Helper.transfer_input_files(self.wf_id, self.wf_ctrl)
# Wait for the workflow to finish
Helper.wait_workflow(self.wf_id, self.wf_ctrl)
# Transfer output files if file transfer
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
Helper.transfer_output_files(self.wf_id, self.wf_ctrl)
status = self.wf_ctrl.workflow_status(self.wf_id)
self.assertTrue(
status == constants.WORKFLOW_DONE,
"workflow status : %s. Expected : %s" %
(status, constants.WORKFLOW_DONE))
nb_failed_jobs = len(Helper.list_failed_jobs(self.wf_id, self.wf_ctrl))
self.assertTrue(
nb_failed_jobs == 1,
"nb failed jobs : %i. Expected : %i" % (nb_failed_jobs, 1))
nb_failed_aborted_jobs = len(
Helper.list_failed_jobs(self.wf_id,
self.wf_ctrl,
include_aborted_jobs=True))
self.assertTrue(
nb_failed_aborted_jobs == 2,
"nb failed jobs including aborted : %i. Expected : %i" %
(nb_failed_aborted_jobs, 2))
(jobs_info, transfers_info, workflow_status, workflow_queue,
tmp_files) = self.wf_ctrl.workflow_elements_status(self.wf_id)
for (job_id, tmp_status, queue, exit_info, dates, drmaa_id) \
in jobs_info:
job_list = self.wf_ctrl.jobs([job_id])
job_name, job_command, job_submission_date = job_list[job_id]
self.tested_job = job_id
if exit_info[0] == constants.FINISHED_REGULARLY:
# To check job standard out and standard err
job_stdout_file = tempfile.NamedTemporaryFile(
prefix="job_soma_out_log_",
suffix=repr(job_id),
delete=False)
job_stdout_file = job_stdout_file.name
job_stderr_file = tempfile.NamedTemporaryFile(
prefix="job_soma_outerr_log_",
suffix=repr(job_id),
delete=False)
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(
job_id, job_stdout_file, job_stderr_file)
if job_name == 'job1':
# Test stdout
isSame, msg = identical_files(
job_stdout_file, self.wf_examples.lo_stdout[1])
self.assertTrue(isSame, msg)
# Test no stderr
self.assertTrue(
os.stat(job_stderr_file).st_size == 0,
"job stderr not empty : cf %s" % job_stderr_file)
# Test output files
if self.path_management == self.LOCAL_PATH:
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[11],
self.wf_examples.lo_file[11])
self.assertTrue(isSame, msg)
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[12],
self.wf_examples.lo_file[12])
self.assertTrue(isSame, msg)
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[11],
self.wf_examples.tr_file[11].client_path)
self.assertTrue(isSame, msg)
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[12],
self.wf_examples.tr_file[12].client_path)
self.assertTrue(isSame, msg)
if job_name == 'job2':
# Test stdout
isSame, msg = identical_files(
job_stdout_file, self.wf_examples.lo_stdout[2])
self.assertTrue(isSame, msg)
# Test no stderr
self.assertTrue(
os.stat(job_stderr_file).st_size == 0,
"job stderr not empty : cf %s" % job_stderr_file)
# Test output files
if self.path_management == self.LOCAL_PATH:
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[2],
self.wf_examples.lo_file[2])
self.assertTrue(isSame, msg)
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[2],
self.wf_examples.tr_file[2].client_path)
self.assertTrue(isSame, msg)
if job_name == 'job3 with exception':
# Test stdout
isSame, msg = identical_files(
job_stdout_file,
self.wf_examples.lo_stdout_exception_model)
self.assertTrue(isSame, msg)
# Test the last line of stderr
with open(job_stderr_file) as f:
lines = f.readlines()
expected_error = 'Exception: Paf Boum Boum Bada Boum !!!\n'
isSame = (lines[-1] == expected_error)
self.assertTrue(
isSame, "Job exception : %s. Expected : %s" %
(lines[-1], expected_error))
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
def run(self, **Xy):
"""Run soma-workflow without gui
Example
-------
>>> from sklearn import datasets
>>> from epac.map_reduce.engine import SomaWorkflowEngine
>>> from epac.tests.wfexamples2test import WFExample2
>>> ## Build dataset
>>> ## =============
>>> X, y = datasets.make_classification(n_samples=10,
... n_features=20,
... n_informative=5,
... random_state=1)
>>> Xy = {'X':X, 'y':y}
>>> ## Build epac tree
>>> ## ===============
>>> tree_root_node = WFExample2().get_workflow()
>>> ## Build SomaWorkflowEngine and run function for each node
>>> ## =======================================================
>>> sfw_engine = SomaWorkflowEngine(tree_root=tree_root_node,
... function_name="transform",
... num_processes=3,
remove_finished_wf=False)
>>> tree_root_node = sfw_engine.run(**Xy)
light mode
>>> ## Run reduce process
>>> ## ==================
>>> tree_root_node.reduce()
ResultSet(
[{'key': SelectKBest/SVC(C=1), 'y/test/score_f1': [ 0.6 0.6], 'y/test/score_recall_mean/pval': [ 0.5], 'y/test/score_recall/pval': [ 0. 0.5], 'y/test/score_accuracy/pval': [ 0.], 'y/test/score_f1/pval': [ 0. 0.5], 'y/test/score_precision/pval': [ 0.5 0. ], 'y/test/score_precision': [ 0.6 0.6], 'y/test/score_recall': [ 0.6 0.6], 'y/test/score_accuracy': 0.6, 'y/test/score_recall_mean': 0.6},
{'key': SelectKBest/SVC(C=3), 'y/test/score_f1': [ 0.6 0.6], 'y/test/score_recall_mean/pval': [ 0.5], 'y/test/score_recall/pval': [ 0. 0.5], 'y/test/score_accuracy/pval': [ 0.], 'y/test/score_f1/pval': [ 0. 0.5], 'y/test/score_precision/pval': [ 0.5 0. ], 'y/test/score_precision': [ 0.6 0.6], 'y/test/score_recall': [ 0.6 0.6], 'y/test/score_accuracy': 0.6, 'y/test/score_recall_mean': 0.6}])
"""
try:
from soma_workflow.client import Job, Workflow
from soma_workflow.client import Helper, FileTransfer
from soma_workflow.client import WorkflowController
except ImportError:
errmsg = (
"No soma-workflow is found. " "Please verify your soma-worklow" "on your computer (e.g. PYTHONPATH) \n"
)
sys.stderr.write(errmsg)
sys.stdout.write(errmsg)
raise NoSomaWFError
tmp_work_dir_path = tempfile.mkdtemp()
cur_work_dir = os.getcwd()
os.chdir(tmp_work_dir_path)
is_run_local = False
if not self.resource_id or self.resource_id == "":
self.resource_id = socket.gethostname()
is_run_local = True
# print "is_run_local=", is_run_local
if not is_run_local:
ft_working_directory = FileTransfer(is_input=True, client_path=tmp_work_dir_path, name="working directory")
else:
ft_working_directory = tmp_work_dir_path
## Save the database and tree to working directory
## ===============================================
# np.savez(os.path.join(tmp_work_dir_path,
# SomaWorkflowEngine.dataset_relative_path), **Xy)
save_dataset(SomaWorkflowEngine.dataset_relative_path, **Xy)
store = StoreFs(dirpath=os.path.join(tmp_work_dir_path, SomaWorkflowEngine.tree_root_relative_path))
self.tree_root.save_tree(store=store)
## Subtree job allocation on disk
## ==============================
node_input = NodesInput(self.tree_root.get_key())
split_node_input = SplitNodesInput(self.tree_root, num_processes=self.num_processes)
nodesinput_list = split_node_input.split(node_input)
keysfile_list = save_job_list(tmp_work_dir_path, nodesinput_list)
## Build soma-workflow
## ===================
jobs = self._create_jobs(keysfile_list, is_run_local, ft_working_directory)
soma_workflow = Workflow(jobs=jobs)
controller = WorkflowController(self.resource_id, self.login, self.pw)
## run soma-workflow
## =================
wf_id = controller.submit_workflow(workflow=soma_workflow, name="epac workflow", queue=self.queue)
Helper.transfer_input_files(wf_id, controller)
Helper.wait_workflow(wf_id, controller)
Helper.transfer_output_files(wf_id, controller)
self.engine_info = self.get_engine_info(controller, wf_id)
if self.remove_finished_wf:
controller.delete_workflow(wf_id)
## read result tree
## ================
self.tree_root = store.load()
os.chdir(cur_work_dir)
if os.path.isdir(tmp_work_dir_path) and self.remove_local_tree:
shutil.rmtree(tmp_work_dir_path)
return self.tree_root
def run_soma_workflow(treatments, exec_cmd, tmp_local_dirs, server_id,
remote_host, remote_user, remote_pathes,
local_result_pathes, label_for_cluster,
wait_ending=False):
"""Dispatch treatments using soma-workflow.
Parameters
----------
treatments
it is a dict mapping a treatment name to a treatment object
exec_cmd
it is the command to run on each ROI data.
tmp_local_dirs
it is a dict mapping a treatment name to a local tmp dir (used to store a temporary configuration file)
server_id
it is the server ID as expected by WorkflowController
remote_host
it is the remote machine where treatments are treated in parallel
remote_user
it is used to log in remote_host
remote_pathes
it is a dict mapping a treatment name to an existing remote dir which will be used to store ROI data and result
files
local_result_pathes
it is a dict mapping a treatment name to a local path where final results will be sorted (host will send it
there by scp)
label_for_cluster
it is the base name used to label workflows and sub jobs
"""
import getpass
from socket import gethostname
local_user = getpass.getuser()
local_host = gethostname()
all_nodes = []
all_deps = []
all_groups = []
split_jobs = []
for t_id, treatment in treatments.iteritems():
tmp_local_dir = tmp_local_dirs[t_id]
remote_path = remote_pathes[t_id]
local_result_path = local_result_pathes[t_id]
sj, n, d, g = prepare_treatment_jobs(treatment, tmp_local_dir,
local_result_path,
local_user, local_host,
remote_host,
remote_user, remote_path,
label_for_cluster + '-' + str(t_id))
all_nodes.extend(n)
all_deps.extend(d)
all_groups.append(g)
split_jobs.append(sj)
# Jobs for data splitting should be done sequentially.
# If they're done in parallel, they may flood the remote file system
for isj in xrange(len(split_jobs)):
if isj + 1 < len(split_jobs):
all_deps.append((split_jobs[isj], split_jobs[isj + 1]))
# # Be sure that all splitting jobs are done first:
# # Is there a better way ?
# for n in all_nodes:
# for sjob in split_jobs:
# all_deps.append((sjob,n))
# Does not seem to work well -> maybe to many deps ?
workflow = Workflow(
all_nodes + split_jobs, all_deps, root_group=all_groups)
# f = open('/tmp/workflow.pck','w')
# cPickle.dump(workflow, f)
# f.close()
logger.info('Open connection ...')
connection = WorkflowController(server_id, remote_user)
logger.info('Submit workflow ...')
wf_id = connection.submit_workflow(workflow=workflow,
# expiration_date="",
# queue="run32",
name=label_for_cluster + '-' +
local_user)
#wf = connection.workflow(wf_id)
if wait_ending: # wait for result
logger.info('Wait for workflow to end and make outputs ...')
Helper.wait_workflow(wf_id, connection)
for t_id, local_result_path in local_result_pathes.iteritems():
treatment = treatments[t_id]
rfilename = treatment.result_dump_file
if rfilename is None:
rfilename = 'result.pck'
local_result_file = op.join(local_result_path,
op.basename(rfilename))
if not op.exists(local_result_file):
raise Exception('Local result does not exist "%s"'
% local_result_file)
if treatment.analyser.outFile is not None:
# return result only for last treatment ...
print 'Load result from %s ...' % local_result_file
if splitext(local_result_file)[1] == '.gz':
import gzip
fresult = gzip.open(local_result_file)
else:
fresult = open(local_result_file)
results = cPickle.load(fresult)
fresult.close()
# print 'Make outputs ...'
#treatment.output(results, dump=False)
logger.info('Cleaning tmp dirs ...')
for tmp_dir in tmp_local_dirs.itervalues():
shutil.rmtree(tmp_dir)
return results
else:
logger.info('Cleaning tmp dirs ...')
for tmp_dir in tmp_local_dirs.itervalues():
shutil.rmtree(tmp_dir)
logger.info('Workflow sent, returning ...')
return []
def remote_map(func, largs=None, lkwargs=None, mode='serial'):
"""
Execute a function in parallel on a list of arguments.
Args:
*func* (function): function to apply on each item.
**this function must be importable on the remote side**
*largs* (list of tuple): each item in the list is a tuple
containing all positional argument values of the
function
*lkwargs* (list of dict): each item in the list is a dict
containing all named arguments of the
function mapped to their value.
*mode* (str): indicates how execution is distributed. Choices are:
- "serial": single-thread loop on the local machine
- "local" : use joblib to run tasks in parallel.
The number of simultaneous jobs is defined in
the configuration section ['parallel-local']['nb_procs']
see ~/.pyhrf/config.cfg
- "remote_cluster: use somaworkflow to run tasks in parallel.
The connection setup has to be defined
in the configuration section ['parallel-cluster']
of ~/.pyhrf/config.cfg.
- "local_with_dumps": testing purpose only, run each task serially as
a subprocess.
Returns:
a list of results
Raises:
RemoteException if any remote task has failed
Example:
>>> from pyhrf.parallel import remote_map
>>> def foo(a, b=2): \
return a + b
>>> remote_map(foo, [(2,),(3,)], [{'b':5}, {'b':7}])
[7, 10]
"""
if largs is None:
if lkwargs is not None:
largs = [tuple()] * len(lkwargs)
else:
largs = [tuple()]
if lkwargs is None:
lkwargs = [{}] * len(largs)
lkwargs = [merge_default_kwargs(func, kw) for kw in lkwargs]
assert len(lkwargs) == len(largs)
all_args = zip(largs, lkwargs)
# print 'all_args:', all_args
fmodule = func.__module__
fname = '.'.join([fmodule, func.__name__])
if mode == 'serial':
return [func(*args, **kwargs) for args, kwargs in all_args]
elif mode == 'local':
try:
from joblib import Parallel, delayed
except ImportError:
raise ImportError('Can not import joblib. It is '
'required to enable parallel '
'processing on a local machine.')
if logger.getEffectiveLevel() == logging.DEBUG:
parallel_verb = 10
else:
parallel_verb = 0
if pyhrf.cfg['parallel-local']['nb_procs']:
n_jobs = pyhrf.cfg['parallel-local']['nb_procs']
else:
n_jobs = available_cpu_count()
p = Parallel(n_jobs=n_jobs, verbose=parallel_verb)
return p(delayed(func)(*args, **kwargs)
for args, kwargs in all_args)
elif mode == 'local_with_dumps':
results = []
for i, params in enumerate(all_args):
# print 'params:', params
params_fn = 'params_%d.pck' % i
fparams = open(params_fn, 'wb')
cPickle.dump(params, fparams)
fparams.close()
output_fn = 'output_%d.pck' % i
# print 'call subprocess ...'
subprocess.call(['python', '-c', cfunc % (fmodule, fname),
params_fn, output_fn])
# print 'Read outputs ...'
fout = open(output_fn)
results.append(cPickle.load(fout))
return results
elif mode == 'remote_cluster':
# FileTransfer creation for input files
#data_dir = './rmap_data'
data_dir = mkdtemp(prefix="sw_rmap")
all_jobs = []
param_files = []
for i, params in enumerate(all_args):
params_fn = op.join(data_dir, 'params_%d.pck' % i)
fparams = open(params_fn, 'wb')
cPickle.dump(params, fparams)
fparams.close()
param_file = FileTransfer(is_input=True,
client_path=params_fn,
name='params_file_%d' % i)
param_files.append(param_file)
output_fn = op.join(data_dir, 'output_%d.pck' % i)
output_file = FileTransfer(is_input=False,
client_path=output_fn,
name='output_file_%d' % i)
job = Job(command=['pyhrf_exec_pyfunc', fmodule, fname,
param_file, output_file],
name="rmap, item %d" % i,
referenced_input_files=[param_file],
referenced_output_files=[output_file])
all_jobs.append(job)
workflow = Workflow(jobs=all_jobs, dependencies=[])
# submit the workflow
cfg = pyhrf.cfg['parallel-cluster']
controller = WorkflowController(cfg['server_id'], cfg['user'])
# controller.transfer_files(fids_to_transfer)
wf_id = controller.submit_workflow(
workflow=workflow, name="remote_map")
Helper.transfer_input_files(wf_id, controller)
Helper.wait_workflow(wf_id, controller)
Helper.transfer_output_files(wf_id, controller)
results = []
for i in xrange(len(all_args)):
fnout = op.join(data_dir, 'output_%d.pck' % i)
fout = open(fnout)
o = cPickle.load(fout)
print 'file cPickle loaded:', o
fout.close()
os.remove(fnout)
if isinstance(o, Exception):
raise RemoteException('Task %d failed' % i, o)
if o.errno != 17:
raise RemoteException('Task %d failed' % i, o)
results.append(o)
return results
def run_workflow(self, workflow, test_files=[], test_dyn_files={}):
self.wf_id = self.wf_ctrl.submit_workflow(
workflow=workflow,
name=self.__class__.__name__)
# Transfer input files if file transfer
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
Helper.transfer_input_files(self.wf_id, self.wf_ctrl)
# Wait for the workflow to finish
Helper.wait_workflow(self.wf_id, self.wf_ctrl)
# Transfer output files if file transfer
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
Helper.transfer_output_files(self.wf_id, self.wf_ctrl)
status = self.wf_ctrl.workflow_status(self.wf_id)
self.assertTrue(status == constants.WORKFLOW_DONE,
"workflow status : %s. Expected : %s" %
(status, constants.WORKFLOW_DONE))
failed_jobs = Helper.list_failed_jobs(self.wf_id, self.wf_ctrl)
nb_failed_jobs = len(failed_jobs)
if nb_failed_jobs != 0:
self.print_jobs(failed_jobs, 'Failed jobs')
self.assertTrue(nb_failed_jobs == 0,
"nb failed jobs : %i. Expected : %i" %
(nb_failed_jobs, 0))
failed_aborted_jobs = Helper.list_failed_jobs(
self.wf_id,
self.wf_ctrl,
include_aborted_jobs=True)
nb_failed_aborted_jobs = len(failed_aborted_jobs)
if nb_failed_aborted_jobs != 0:
self.print_jobs(failed_aborted_jobs, 'Aborted jobs')
self.assertTrue(nb_failed_aborted_jobs == 0,
"nb failed jobs including aborted : %i. Expected : %i"
% (nb_failed_aborted_jobs, 0))
(jobs_info, transfers_info, workflow_status, workflow_queue,
tmp_files) = self.wf_ctrl.workflow_elements_status(self.wf_id)
dyn_out_params = {1: 'filePathOut2',
2: 'filePathOut'}
dyn_out_params = {}
for (job_id, tmp_status, queue, exit_info, dates, drmaa_id) \
in jobs_info:
job_list = self.wf_ctrl.jobs([job_id])
job_name, job_command, job_submission_date = job_list[job_id]
self.tested_job = job_id
if exit_info[0] == constants.FINISHED_REGULARLY:
# To check job standard out and standard err
job_stdout_file = tempfile.NamedTemporaryFile(
prefix="job_soma_out_log_",
suffix=repr(job_id),
delete=False)
job_stdout_file = job_stdout_file.name
job_stderr_file = tempfile.NamedTemporaryFile(
prefix="job_soma_outerr_log_",
suffix=repr(job_id),
delete=False)
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(job_id,
job_stdout_file,
job_stderr_file)
if job_name.startswith('job1'):
# Test stdout
isSame, msg = identical_files(
job_stdout_file,
self.wf_examples.lo_stdout[1])
self.assertTrue(isSame, msg)
# Test no stderr
with open(job_stderr_file) as f:
msg = "job stderr not empty : cf %s\n" \
"stderr:\n---\n%s\n---" \
% (job_stderr_file, f.read())
self.assertTrue(os.stat(job_stderr_file).st_size == 0,
msg)
if job_name in test_dyn_files:
out_params = self.wf_ctrl.get_job_output_params(job_id)
dyn_out_params[job_name] = out_params
# For unknown reason, it raises some errors
# http://stackoverflow.com/questions/10496758/unexpected-end-of-file-and-error-importing-function-definition-error-running
# isSame, msg = identical_files(job_stderr_file,self.wf_examples.lo_stderr[1])
# self.failUnless(isSame == True)
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
for out_file_num in test_files:
# Test output files
if self.path_management == self.LOCAL_PATH:
out_file = self.wf_examples.lo_file[out_file_num]
elif self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
out_file = self.wf_examples.tr_file[out_file_num].client_path
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[out_file_num], out_file)
self.assertTrue(isSame, msg)
for job_name, ref_out_params in six.iteritems(test_dyn_files):
out_params = dyn_out_params[job_name]
for param, file_num in six.iteritems(ref_out_params):
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[file_num],
out_params[param])
self.assertTrue(isSame, msg)
del self.tested_job
def test_result(self):
workflow = self.wf_examples.example_simple_exception1()
self.wf_id = self.wf_ctrl.submit_workflow(
workflow=workflow,
name=self.__class__.__name__)
# Transfer input files if file transfer
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
Helper.transfer_input_files(self.wf_id, self.wf_ctrl)
# Wait for the workflow to finish
Helper.wait_workflow(self.wf_id, self.wf_ctrl)
# Transfer output files if file transfer
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
Helper.transfer_output_files(self.wf_id, self.wf_ctrl)
status = self.wf_ctrl.workflow_status(self.wf_id)
self.assertTrue(status == constants.WORKFLOW_DONE,
"workflow status : %s. Expected : %s" %
(status, constants.WORKFLOW_DONE))
nb_failed_jobs = len(Helper.list_failed_jobs(self.wf_id,
self.wf_ctrl))
self.assertTrue(nb_failed_jobs == 1,
"nb failed jobs : %i. Expected : %i" %
(nb_failed_jobs, 1))
nb_failed_aborted_jobs = len(Helper.list_failed_jobs(
self.wf_id,
self.wf_ctrl,
include_aborted_jobs=True))
self.assertTrue(nb_failed_aborted_jobs == 4,
"nb failed jobs including aborted : %i. Expected : %i"
% (nb_failed_aborted_jobs, 4))
(jobs_info, transfers_info, workflow_status, workflow_queue,
tmp_files) = self.wf_ctrl.workflow_elements_status(self.wf_id)
for (job_id, tmp_status, queue, exit_info, dates) in jobs_info:
job_list = self.wf_ctrl.jobs([job_id])
job_name, job_command, job_submission_date = job_list[job_id]
self.tested_job = job_id
if exit_info[0] == constants.FINISHED_REGULARLY:
# To check job standard out and standard err
job_stdout_file = tempfile.NamedTemporaryFile(
prefix="job_soma_out_log_",
suffix=repr(job_id),
delete=False)
job_stdout_file = job_stdout_file.name
job_stderr_file = tempfile.NamedTemporaryFile(
prefix="job_soma_outerr_log_",
suffix=repr(job_id),
delete=False)
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(job_id,
job_stdout_file,
job_stderr_file)
if job_name == 'job1 with exception':
# Test stdout
isSame, msg = identical_files(
job_stdout_file,
self.wf_examples.lo_stdout_exception_model)
self.assertTrue(isSame, msg)
# Test the last line of stderr
with open(job_stderr_file) as f:
lines = f.readlines()
expected_error = 'Exception: Paf Boum Boum Bada Boum !!!\n'
isSame = (lines[-1] == expected_error)
self.assertTrue(isSame,
"Job exception : %s. Expected : %s" %
(lines[-1], expected_error))
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
def remote_map(func, largs=None, lkwargs=None, mode='serial'):
"""
Execute a function in parallel on a list of arguments.
Args:
*func* (function): function to apply on each item.
**this function must be importable on the remote side**
*largs* (list of tuple): each item in the list is a tuple
containing all positional argument values of the
function
*lkwargs* (list of dict): each item in the list is a dict
containing all named arguments of the
function mapped to their value.
*mode* (str): indicates how execution is distributed. Choices are:
- "serial": single-thread loop on the local machine
- "local" : use joblib to run tasks in parallel.
The number of simultaneous jobs is defined in
the configuration section ['parallel-local']['nb_procs']
see ~/.pyhrf/config.cfg
- "remote_cluster: use somaworkflow to run tasks in parallel.
The connection setup has to be defined
in the configuration section ['parallel-cluster']
of ~/.pyhrf/config.cfg.
- "local_with_dumps": testing purpose only, run each task serially as
a subprocess.
Returns:
a list of results
Raises:
RemoteException if any remote task has failed
Example:
>>> from pyhrf.parallel import remote_map
>>> def foo(a, b=2): \
return a + b
>>> remote_map(foo, [(2,),(3,)], [{'b':5}, {'b':7}])
[7, 10]
"""
if largs is None:
if lkwargs is not None:
largs = [tuple()] * len(lkwargs)
else:
largs = [tuple()]
if lkwargs is None:
lkwargs = [{}] * len(largs)
lkwargs = [merge_default_kwargs(func, kw) for kw in lkwargs]
assert len(lkwargs) == len(largs)
all_args = zip(largs, lkwargs)
# print 'all_args:', all_args
fmodule = func.__module__
fname = '.'.join([fmodule, func.__name__])
if mode == 'serial':
return [func(*args, **kwargs) for args, kwargs in all_args]
elif mode == 'local':
try:
from joblib import Parallel, delayed
except ImportError:
raise ImportError('Can not import joblib. It is '
'required to enable parallel '
'processing on a local machine.')
if logger.getEffectiveLevel() == logging.DEBUG:
parallel_verb = 10
else:
parallel_verb = 0
if pyhrf.cfg['parallel-local']['nb_procs']:
n_jobs = pyhrf.cfg['parallel-local']['nb_procs']
else:
n_jobs = available_cpu_count()
p = Parallel(n_jobs=n_jobs, verbose=parallel_verb)
return p(delayed(func)(*args, **kwargs) for args, kwargs in all_args)
elif mode == 'local_with_dumps':
results = []
for i, params in enumerate(all_args):
# print 'params:', params
params_fn = 'params_%d.pck' % i
fparams = open(params_fn, 'wb')
cPickle.dump(params, fparams)
fparams.close()
output_fn = 'output_%d.pck' % i
# print 'call subprocess ...'
subprocess.call([
'python', '-c', cfunc % (fmodule, fname), params_fn, output_fn
])
# print 'Read outputs ...'
fout = open(output_fn)
results.append(cPickle.load(fout))
return results
elif mode == 'remote_cluster':
# FileTransfer creation for input files
#data_dir = './rmap_data'
data_dir = mkdtemp(prefix="sw_rmap")
all_jobs = []
param_files = []
for i, params in enumerate(all_args):
params_fn = op.join(data_dir, 'params_%d.pck' % i)
fparams = open(params_fn, 'wb')
cPickle.dump(params, fparams)
fparams.close()
param_file = FileTransfer(is_input=True,
client_path=params_fn,
name='params_file_%d' % i)
param_files.append(param_file)
output_fn = op.join(data_dir, 'output_%d.pck' % i)
output_file = FileTransfer(is_input=False,
client_path=output_fn,
name='output_file_%d' % i)
job = Job(command=[
'pyhrf_exec_pyfunc', fmodule, fname, param_file, output_file
],
name="rmap, item %d" % i,
referenced_input_files=[param_file],
referenced_output_files=[output_file])
all_jobs.append(job)
workflow = Workflow(jobs=all_jobs, dependencies=[])
# submit the workflow
cfg = pyhrf.cfg['parallel-cluster']
controller = WorkflowController(cfg['server_id'], cfg['user'])
# controller.transfer_files(fids_to_transfer)
wf_id = controller.submit_workflow(workflow=workflow,
name="remote_map")
Helper.transfer_input_files(wf_id, controller)
Helper.wait_workflow(wf_id, controller)
Helper.transfer_output_files(wf_id, controller)
results = []
for i in xrange(len(all_args)):
fnout = op.join(data_dir, 'output_%d.pck' % i)
fout = open(fnout)
o = cPickle.load(fout)
print 'file cPickle loaded:', o
fout.close()
os.remove(fnout)
if isinstance(o, Exception):
raise RemoteException('Task %d failed' % i, o)
if o.errno != 17:
raise RemoteException('Task %d failed' % i, o)
results.append(o)
return results
def remote_map_marshal(func, largs=None, lkwargs=None, mode='local'):
if largs is None:
if lkwargs is not None:
largs = [[]] * len(lkwargs)
else:
largs = []
if lkwargs is None:
lkwargs = [{}] * len(largs)
lkwargs = [merge_default_kwargs(func, kw) for kw in lkwargs]
assert len(lkwargs) == len(largs)
all_args = zip(largs, lkwargs)
if mode == 'local':
return [func(*args, **kwargs) for args, kwargs in all_args]
elif mode == 'local_with_dumps':
func_fn = './func.marshal'
dump_func(func, func_fn)
results = []
for i, params in enumerate(all_args):
print 'params:', params
params_fn = 'params_%d.pck' % i
fparams = open(params_fn, 'wb')
cPickle.dump(params, fparams)
fparams.close()
output_fn = 'output_%d.pck' % i
print 'call subprocess ...'
subprocess.call(
['python', '-c', cfunc_marshal, params_fn, func_fn, output_fn])
print 'Read outputs ...'
fout = open(output_fn)
results.append(cPickle.load(fout))
return results
elif mode == 'remote_cluster':
# FileTransfer creation for input files
#data_dir = './rmap_data'
data_dir = mkdtemp(prefix="sw_rmap")
func_fn = op.join(data_dir, 'func.marshal')
dump_func(func, func_fn)
func_file = FileTransfer(is_input=True,
client_path=func_fn,
name="func_file")
all_jobs = []
param_files = []
for i, params in enumerate(all_args):
params_fn = op.join(data_dir, 'params_%d.pck' % i)
fparams = open(params_fn, 'wb')
cPickle.dump(params, fparams)
fparams.close()
param_file = FileTransfer(is_input=True,
client_path=params_fn,
name='params_file_%d' % i)
param_files.append(param_file)
output_fn = op.join(data_dir, 'output_%d.pck' % i)
output_file = FileTransfer(is_input=False,
client_path=output_fn,
name='output_file_%d' % i)
job = Job(command=[
'python', '-c', cfunc, param_file, func_file, output_file
],
name="rmap, item %d" % i,
referenced_input_files=[func_file, param_file],
referenced_output_files=[output_file])
all_jobs.append(job)
workflow = Workflow(jobs=all_jobs, dependencies=[])
# submit the workflow
cfg = pyhrf.cfg['parallel-cluster']
controller = WorkflowController(cfg['server_id'], cfg['user'])
# controller.transfer_files(fids_to_transfer)
wf_id = controller.submit_workflow(workflow=workflow,
name="remote_map")
Helper.transfer_input_files(wf_id, controller)
Helper.wait_workflow(wf_id, controller)
Helper.transfer_output_files(wf_id, controller)
results = []
for i in xrange(len(all_args)):
fout = open(op.join(data_dir, 'output_%d.pck' % i))
results.append(cPickle.load(fout))
fout.close()
return results
def run_soma_workflow(treatments,
exec_cmd,
tmp_local_dirs,
server_id,
remote_host,
remote_user,
remote_pathes,
local_result_pathes,
label_for_cluster,
wait_ending=False):
"""Dispatch treatments using soma-workflow.
Parameters
----------
treatments
it is a dict mapping a treatment name to a treatment object
exec_cmd
it is the command to run on each ROI data.
tmp_local_dirs
it is a dict mapping a treatment name to a local tmp dir (used to store a temporary configuration file)
server_id
it is the server ID as expected by WorkflowController
remote_host
it is the remote machine where treatments are treated in parallel
remote_user
it is used to log in remote_host
remote_pathes
it is a dict mapping a treatment name to an existing remote dir which will be used to store ROI data and result
files
local_result_pathes
it is a dict mapping a treatment name to a local path where final results will be sorted (host will send it
there by scp)
label_for_cluster
it is the base name used to label workflows and sub jobs
"""
import getpass
from socket import gethostname
local_user = getpass.getuser()
local_host = gethostname()
all_nodes = []
all_deps = []
all_groups = []
split_jobs = []
for t_id, treatment in treatments.iteritems():
tmp_local_dir = tmp_local_dirs[t_id]
remote_path = remote_pathes[t_id]
local_result_path = local_result_pathes[t_id]
sj, n, d, g = prepare_treatment_jobs(
treatment, tmp_local_dir, local_result_path, local_user,
local_host, remote_host, remote_user, remote_path,
label_for_cluster + '-' + str(t_id))
all_nodes.extend(n)
all_deps.extend(d)
all_groups.append(g)
split_jobs.append(sj)
# Jobs for data splitting should be done sequentially.
# If they're done in parallel, they may flood the remote file system
for isj in xrange(len(split_jobs)):
if isj + 1 < len(split_jobs):
all_deps.append((split_jobs[isj], split_jobs[isj + 1]))
# # Be sure that all splitting jobs are done first:
# # Is there a better way ?
# for n in all_nodes:
# for sjob in split_jobs:
# all_deps.append((sjob,n))
# Does not seem to work well -> maybe to many deps ?
workflow = Workflow(all_nodes + split_jobs,
all_deps,
root_group=all_groups)
# f = open('/tmp/workflow.pck','w')
# cPickle.dump(workflow, f)
# f.close()
logger.info('Open connection ...')
connection = WorkflowController(server_id, remote_user)
logger.info('Submit workflow ...')
wf_id = connection.submit_workflow(
workflow=workflow,
# expiration_date="",
# queue="run32",
name=label_for_cluster + '-' + local_user)
#wf = connection.workflow(wf_id)
if wait_ending: # wait for result
logger.info('Wait for workflow to end and make outputs ...')
Helper.wait_workflow(wf_id, connection)
for t_id, local_result_path in local_result_pathes.iteritems():
treatment = treatments[t_id]
rfilename = treatment.result_dump_file
if rfilename is None:
rfilename = 'result.pck'
local_result_file = op.join(local_result_path,
op.basename(rfilename))
if not op.exists(local_result_file):
raise Exception('Local result does not exist "%s"' %
local_result_file)
if treatment.analyser.outFile is not None:
# return result only for last treatment ...
print 'Load result from %s ...' % local_result_file
if splitext(local_result_file)[1] == '.gz':
import gzip
fresult = gzip.open(local_result_file)
else:
fresult = open(local_result_file)
results = cPickle.load(fresult)
fresult.close()
# print 'Make outputs ...'
#treatment.output(results, dump=False)
logger.info('Cleaning tmp dirs ...')
for tmp_dir in tmp_local_dirs.itervalues():
shutil.rmtree(tmp_dir)
return results
else:
logger.info('Cleaning tmp dirs ...')
for tmp_dir in tmp_local_dirs.itervalues():
shutil.rmtree(tmp_dir)
logger.info('Workflow sent, returning ...')
return []
def test_result(self):
workflow = self.wf_examples.example_multiple()
self.wf_id = self.wf_ctrl.submit_workflow(workflow=workflow,
name=self.__class__.__name__)
# Transfer input files if file transfer
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
Helper.transfer_input_files(self.wf_id, self.wf_ctrl)
# Wait for the workflow to finish
Helper.wait_workflow(self.wf_id, self.wf_ctrl)
# Transfer output files if file transfer
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
Helper.transfer_output_files(self.wf_id, self.wf_ctrl)
status = self.wf_ctrl.workflow_status(self.wf_id)
self.assertTrue(
status == constants.WORKFLOW_DONE,
"workflow status : %s. Expected : %s" %
(status, constants.WORKFLOW_DONE))
nb_failed_jobs = len(Helper.list_failed_jobs(self.wf_id, self.wf_ctrl))
self.assertTrue(
nb_failed_jobs == 2,
"nb failed jobs : %i. Expected : %i" % (nb_failed_jobs, 2))
nb_failed_aborted_jobs = len(
Helper.list_failed_jobs(self.wf_id,
self.wf_ctrl,
include_aborted_jobs=True))
self.assertTrue(
nb_failed_aborted_jobs == 6,
"nb failed jobs including aborted : %i. Expected : %i" %
(nb_failed_aborted_jobs, 6))
(jobs_info, transfers_info, workflow_status, workflow_queue,
tmp_files) = self.wf_ctrl.workflow_elements_status(self.wf_id)
for (job_id, tmp_status, queue, exit_info, dates) in jobs_info:
job_list = self.wf_ctrl.jobs([job_id])
job_name, job_command, job_submission_date = job_list[job_id]
self.tested_job = job_id
if exit_info[0] == constants.FINISHED_REGULARLY:
# To check job standard out and standard err
job_stdout_file = tempfile.NamedTemporaryFile(
prefix="job_soma_out_log_",
suffix=repr(job_id),
delete=False)
job_stdout_file = job_stdout_file.name
job_stderr_file = tempfile.NamedTemporaryFile(
prefix="job_soma_outerr_log_",
suffix=repr(job_id),
delete=False)
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(
job_id, job_stdout_file, job_stderr_file)
if job_name == 'job1':
# Test stdout
isSame, msg = identical_files(
job_stdout_file, self.wf_examples.lo_stdout[1])
self.assertTrue(isSame, msg)
# Test no stderr
self.assertTrue(
os.stat(job_stderr_file).st_size == 0,
"job stderr not empty : cf %s" % job_stderr_file)
# Test output files
if self.path_management == self.LOCAL_PATH:
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[11],
self.wf_examples.lo_file[11])
self.assertTrue(isSame, msg)
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[12],
self.wf_examples.lo_file[12])
self.assertTrue(isSame, msg)
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[11],
self.wf_examples.tr_file[11].client_path)
self.assertTrue(isSame, msg)
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[12],
self.wf_examples.tr_file[12].client_path)
self.assertTrue(isSame, msg)
# For unknown reason, it raises some errors
# http://stackoverflow.com/questions/10496758/unexpected-end-of-file-and-error-importing-function-definition-error-running
# isSame, msg = identical_files(job_stderr_file,self.wf_examples.lo_stderr[1])
# self.failUnless(isSame == True)
if job_name in ['job2', 'job3', 'job4']:
job_nb = int(job_name[3])
# Test stdout
isSame, msg = identical_files(
job_stdout_file,
self.wf_examples.lo_stdout[job_nb])
self.assertTrue(isSame, msg)
# Test no stderr
self.assertTrue(
os.stat(job_stderr_file).st_size == 0,
"job stderr not empty : cf %s" % job_stderr_file)
# Test output files
if self.path_management == self.LOCAL_PATH:
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[job_nb],
self.wf_examples.lo_file[job_nb])
self.assertTrue(isSame, msg)
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[job_nb],
self.wf_examples.tr_file[job_nb].client_path)
self.assertTrue(isSame, msg)
if job_name in [
'job1 with exception', 'job3 with exception'
]:
# Test stdout
isSame, msg = identical_files(
job_stdout_file,
self.wf_examples.lo_stdout_exception_model)
self.assertTrue(isSame)
# Test stderr
with open(job_stderr_file) as f:
lines = f.readlines()
print(lines)
isSame = (lines[-1] == 'Exception: Paf Boum '
'Boum Bada Boum !!!\n')
self.assertTrue(isSame,
"Job Exception: %s" % lines[-1])
finally:
if os.path.exists(job_stdout_file):
os.unlink(job_stdout_file)
if os.path.exists(job_stderr_file):
os.unlink(job_stderr_file)
del self.tested_job
def test_result(self):
if hasattr(self.wf_ctrl.scheduler_config, 'get_proc_nb'):
n_iter = 10 * self.wf_ctrl.scheduler_config.get_proc_nb()
else:
n_iter = 100
workflow = self.wf_examples.example_fake_pipelineT1(n_iter)
self.wf_id = self.wf_ctrl.submit_workflow(
workflow=workflow,
name=self.__class__.__name__)
# Transfer input files if file transfer
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
Helper.transfer_input_files(self.wf_id,
self.wf_ctrl)
# Wait for the workflow to finish
Helper.wait_workflow(self.wf_id, self.wf_ctrl)
# Transfer output files if file transfer
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
Helper.transfer_output_files(self.wf_id,
self.wf_ctrl)
status = self.wf_ctrl.workflow_status(self.wf_id)
self.assertTrue(status == constants.WORKFLOW_DONE)
self.assertTrue(len(Helper.list_failed_jobs(
self.wf_id,
self.wf_ctrl)) == 0)
self.assertTrue(len(Helper.list_failed_jobs(
self.wf_id,
self.wf_ctrl,
include_aborted_jobs=True)) == 0)
(jobs_info, transfers_info, workflow_status, workflow_queue,
tmp_files) = self.wf_ctrl.workflow_elements_status(self.wf_id)
for (job_id, tmp_status, queue, exit_info, dates) in jobs_info:
job_list = self.wf_ctrl.jobs([job_id])
job_name, job_command, job_submission_date = job_list[job_id]
self.tested_job = job_id
if exit_info[0] == constants.FINISHED_REGULARLY:
# To check job standard out and standard err
job_stdout_file = tempfile.NamedTemporaryFile(
prefix="job_soma_out_log_",
suffix=repr(job_id),
delete=False)
job_stdout_file = job_stdout_file.name
job_stderr_file = tempfile.NamedTemporaryFile(
prefix="job_soma_outerr_log_",
suffix=repr(job_id),
delete=False)
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(job_id,
job_stdout_file,
job_stderr_file)
# Test stdout
self.assertTrue(os.stat(job_stdout_file).st_size == 0,
"job stdout not empty : cf %s" %
job_stdout_file)
# Test no stderr
self.assertTrue(os.stat(job_stderr_file).st_size == 0,
"job stderr not empty : cf %s" %
job_stderr_file)
finally:
if os.path.exists(job_stdout_file):
os.unlink(job_stdout_file)
if os.path.exists(job_stderr_file):
os.unlink(job_stderr_file)
del self.tested_job
def test_result(self):
workflow = self.wf_examples.example_native_spec_pbs()
self.wf_id = self.wf_ctrl.submit_workflow(workflow=workflow, name=self.__class__.__name__)
# Transfer input files if file transfer
if self.path_management == self.FILE_TRANSFER or self.path_management == self.SHARED_TRANSFER:
Helper.transfer_input_files(self.wf_id, self.wf_ctrl)
# Wait for the workflow to finish
Helper.wait_workflow(self.wf_id, self.wf_ctrl)
# Transfer output files if file transfer
if self.path_management == self.FILE_TRANSFER or self.path_management == self.SHARED_TRANSFER:
Helper.transfer_output_files(self.wf_id, self.wf_ctrl)
status = self.wf_ctrl.workflow_status(self.wf_id)
self.assertTrue(
status == constants.WORKFLOW_DONE, "workflow status : %s. Expected : %s" % (status, constants.WORKFLOW_DONE)
)
nb_failed_jobs = len(Helper.list_failed_jobs(self.wf_id, self.wf_ctrl))
self.assertTrue(nb_failed_jobs == 0, "nb failed jobs : %i. Expected : %i" % (nb_failed_jobs, 0))
nb_failed_aborted_jobs = len(Helper.list_failed_jobs(self.wf_id, self.wf_ctrl, include_aborted_jobs=True))
self.assertTrue(
nb_failed_aborted_jobs == 0,
"nb failed jobs including aborted : %i. Expected : %i" % (nb_failed_aborted_jobs, 0),
)
(jobs_info, transfers_info, workflow_status, workflow_queue, tmp_files) = self.wf_ctrl.workflow_elements_status(
self.wf_id
)
for (job_id, tmp_status, queue, exit_info, dates) in jobs_info:
job_list = self.wf_ctrl.jobs([job_id])
job_name, job_command, job_submission_date = job_list[job_id]
self.tested_job = job_id
if exit_info[0] == constants.FINISHED_REGULARLY:
# To check job standard out and standard err
job_stdout_file = tempfile.NamedTemporaryFile(
prefix="job_soma_out_log_", suffix=repr(job_id), delete=False
)
job_stdout_file = job_stdout_file.name
job_stderr_file = tempfile.NamedTemporaryFile(
prefix="job_soma_outerr_log_", suffix=repr(job_id), delete=False
)
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(job_id, job_stdout_file, job_stderr_file)
# Test stdout
isSame, msg = identical_files(job_stdout_file, self.wf_examples.lo_stdout[1])
self.assertTrue(isSame, msg)
# Test no stderr
self.assertTrue(
os.stat(job_stderr_file).st_size == 0, "job stderr not empty : cf %s" % job_stderr_file
)
# Test output files
if self.path_management == self.LOCAL_PATH:
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[11], self.wf_examples.lo_file[11]
)
self.assertTrue(isSame, msg)
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[12], self.wf_examples.lo_file[12]
)
self.assertTrue(isSame, msg)
if self.path_management == self.FILE_TRANSFER or self.path_management == self.SHARED_TRANSFER:
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[11], self.wf_examples.tr_file[11].client_path
)
self.assertTrue(isSame, msg)
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[12], self.wf_examples.tr_file[12].client_path
)
self.assertTrue(isSame, msg)
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
def test_result(self):
if hasattr(self.wf_ctrl.scheduler_config, 'get_proc_nb'):
n_iter = 5 * self.wf_ctrl.scheduler_config.get_proc_nb()
else:
n_iter = 5
workflow = self.wf_examples.example_fake_pipelineT1(n_iter)
self.wf_id = self.wf_ctrl.submit_workflow(workflow=workflow,
name=self.__class__.__name__)
# Transfer input files if file transfer
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
Helper.transfer_input_files(self.wf_id, self.wf_ctrl)
# Wait for the workflow to finish
Helper.wait_workflow(self.wf_id, self.wf_ctrl)
# Transfer output files if file transfer
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
Helper.transfer_output_files(self.wf_id, self.wf_ctrl)
status = self.wf_ctrl.workflow_status(self.wf_id)
self.assertTrue(status == constants.WORKFLOW_DONE)
self.assertTrue(
len(Helper.list_failed_jobs(self.wf_id, self.wf_ctrl)) == 0)
self.assertTrue(
len(
Helper.list_failed_jobs(
self.wf_id, self.wf_ctrl, include_aborted_jobs=True)) == 0)
(jobs_info, transfers_info, workflow_status, workflow_queue,
tmp_files) = self.wf_ctrl.workflow_elements_status(self.wf_id)
for (job_id, tmp_status, queue, exit_info, dates, dmraa_id) \
in jobs_info:
job_list = self.wf_ctrl.jobs([job_id])
job_name, job_command, job_submission_date = job_list[job_id]
self.tested_job = job_id
if exit_info[0] == constants.FINISHED_REGULARLY:
# To check job standard out and standard err
job_stdout_file = tempfile.NamedTemporaryFile(
prefix="job_soma_out_log_",
suffix=repr(job_id),
delete=False)
job_stdout_file = job_stdout_file.name
job_stderr_file = tempfile.NamedTemporaryFile(
prefix="job_soma_outerr_log_",
suffix=repr(job_id),
delete=False)
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(
job_id, job_stdout_file, job_stderr_file)
# Test stdout
self.assertTrue(
os.stat(job_stdout_file).st_size == 0,
"job stdout not empty : cf %s" % job_stdout_file)
# Test no stderr
self.assertTrue(
os.stat(job_stderr_file).st_size == 0,
"job stderr not empty : cf %s" % job_stderr_file)
finally:
if os.path.exists(job_stdout_file):
os.unlink(job_stdout_file)
if os.path.exists(job_stderr_file):
os.unlink(job_stderr_file)
del self.tested_job
def remote_map_marshal(func, largs=None, lkwargs=None, mode='local'):
if largs is None:
if lkwargs is not None:
largs = [[]] * len(lkwargs)
else:
largs = []
if lkwargs is None:
lkwargs = [{}] * len(largs)
lkwargs = [merge_default_kwargs(func, kw) for kw in lkwargs]
assert len(lkwargs) == len(largs)
all_args = zip(largs, lkwargs)
if mode == 'local':
return [func(*args, **kwargs) for args, kwargs in all_args]
elif mode == 'local_with_dumps':
func_fn = './func.marshal'
dump_func(func, func_fn)
results = []
for i, params in enumerate(all_args):
print 'params:', params
params_fn = 'params_%d.pck' % i
fparams = open(params_fn, 'wb')
cPickle.dump(params, fparams)
fparams.close()
output_fn = 'output_%d.pck' % i
print 'call subprocess ...'
subprocess.call(['python', '-c', cfunc_marshal, params_fn,
func_fn, output_fn])
print 'Read outputs ...'
fout = open(output_fn)
results.append(cPickle.load(fout))
return results
elif mode == 'remote_cluster':
# FileTransfer creation for input files
#data_dir = './rmap_data'
data_dir = mkdtemp(prefix="sw_rmap")
func_fn = op.join(data_dir, 'func.marshal')
dump_func(func, func_fn)
func_file = FileTransfer(is_input=True,
client_path=func_fn,
name="func_file")
all_jobs = []
param_files = []
for i, params in enumerate(all_args):
params_fn = op.join(data_dir, 'params_%d.pck' % i)
fparams = open(params_fn, 'wb')
cPickle.dump(params, fparams)
fparams.close()
param_file = FileTransfer(is_input=True,
client_path=params_fn,
name='params_file_%d' % i)
param_files.append(param_file)
output_fn = op.join(data_dir, 'output_%d.pck' % i)
output_file = FileTransfer(is_input=False,
client_path=output_fn,
name='output_file_%d' % i)
job = Job(command=['python', '-c', cfunc, param_file, func_file,
output_file],
name="rmap, item %d" % i,
referenced_input_files=[func_file, param_file],
referenced_output_files=[output_file])
all_jobs.append(job)
workflow = Workflow(jobs=all_jobs, dependencies=[])
# submit the workflow
cfg = pyhrf.cfg['parallel-cluster']
controller = WorkflowController(cfg['server_id'], cfg['user'])
# controller.transfer_files(fids_to_transfer)
wf_id = controller.submit_workflow(
workflow=workflow, name="remote_map")
Helper.transfer_input_files(wf_id, controller)
Helper.wait_workflow(wf_id, controller)
Helper.transfer_output_files(wf_id, controller)
results = []
for i in xrange(len(all_args)):
fout = open(op.join(data_dir, 'output_%d.pck' % i))
results.append(cPickle.load(fout))
fout.close()
return results
def test_result(self):
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
with warnings.catch_warnings(record=True) as w:
# Trigger a warning.
workflow = self.wf_examples.example_special_command()
# Verify some things
self.assertTrue(len(w) == 1)
self.assertTrue(issubclass(w[-1].category, UserWarning))
self.assertTrue("contains single quote. It could fail using DRMAA"
in str(w[-1].message))
self.wf_id = self.wf_ctrl.submit_workflow(
workflow=workflow,
name=self.__class__.__name__)
# Transfer input files if file transfer
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
Helper.transfer_input_files(self.wf_id, self.wf_ctrl)
# Wait for the worklow to finish
Helper.wait_workflow(self.wf_id, self.wf_ctrl)
status = self.wf_ctrl.workflow_status(self.wf_id)
self.assertTrue(status == constants.WORKFLOW_DONE,
"workflow status : %s. Expected : %s" %
(status, constants.WORKFLOW_DONE))
# TODO : sometimes raises an error
# because status = "workflow_in_progress"
nb_failed_jobs = len(Helper.list_failed_jobs(
self.wf_id,
self.wf_ctrl))
self.assertTrue(nb_failed_jobs == 0,
"nb failed jobs : %i. Expected : %i" %
(nb_failed_jobs, 0))
nb_failed_aborted_jobs = len(Helper.list_failed_jobs(
self.wf_id,
self.wf_ctrl,
include_aborted_jobs=True))
self.assertTrue(nb_failed_aborted_jobs == 0,
"nb failed jobs including aborted : %i. Expected : %i"
% (nb_failed_aborted_jobs, 0))
(jobs_info, transfers_info, workflow_status, workflow_queue,
tmp_files) = self.wf_ctrl.workflow_elements_status(self.wf_id)
for (job_id, tmp_status, queue, exit_info, dates) in jobs_info:
job_list = self.wf_ctrl.jobs([job_id])
job_name, job_command, job_submission_date = job_list[job_id]
self.tested_job = job_id
if exit_info[0] == constants.FINISHED_REGULARLY:
# To check job standard out and standard err
job_stdout_file = tempfile.NamedTemporaryFile(
prefix="job_soma_out_log_",
suffix=repr(job_id),
delete=False)
job_stdout_file = job_stdout_file.name
job_stderr_file = tempfile.NamedTemporaryFile(
prefix="job_soma_outerr_log_",
suffix=repr(job_id),
delete=False)
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(job_id,
job_stdout_file,
job_stderr_file)
# Test job stdout
if self.path_management == self.LOCAL_PATH:
isSame, msg = identical_files(
job_stdout_file,
self.wf_examples.lo_stdout_command_local)
self.assertTrue(isSame, msg)
else:
isSame, msg = identical_files(
job_stdout_file,
self.wf_examples.lo_stdout_command_remote)
self.assertTrue(isSame, msg)
# Test no stderr
self.assertTrue(os.stat(job_stderr_file).st_size == 0,
"job stderr not empty : cf %s" %
job_stderr_file)
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
def test_result(self):
workflow = self.wf_examples.example_simple()
self.wf_id = self.wf_ctrl.submit_workflow(
workflow=workflow,
name=self.__class__.__name__)
# Transfer input files if file transfer
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
Helper.transfer_input_files(self.wf_id, self.wf_ctrl)
# Wait for the workflow to finish
Helper.wait_workflow(self.wf_id, self.wf_ctrl)
# Transfer output files if file transfer
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
Helper.transfer_output_files(self.wf_id, self.wf_ctrl)
status = self.wf_ctrl.workflow_status(self.wf_id)
self.assertTrue(status == constants.WORKFLOW_DONE,
"workflow status : %s. Expected : %s" %
(status, constants.WORKFLOW_DONE))
nb_failed_jobs = len(Helper.list_failed_jobs(self.wf_id,
self.wf_ctrl))
self.assertTrue(nb_failed_jobs == 0,
"nb failed jobs : %i. Expected : %i" %
(nb_failed_jobs, 0))
nb_failed_aborted_jobs = len(Helper.list_failed_jobs(
self.wf_id,
self.wf_ctrl,
include_aborted_jobs=True))
self.assertTrue(nb_failed_aborted_jobs == 0,
"nb failed jobs including aborted : %i. Expected : %i"
% (nb_failed_aborted_jobs, 0))
(jobs_info, transfers_info, workflow_status, workflow_queue,
tmp_files) = self.wf_ctrl.workflow_elements_status(self.wf_id)
for (job_id, tmp_status, queue, exit_info, dates) in jobs_info:
job_list = self.wf_ctrl.jobs([job_id])
job_name, job_command, job_submission_date = job_list[job_id]
self.tested_job = job_id
if exit_info[0] == constants.FINISHED_REGULARLY:
# To check job standard out and standard err
job_stdout_file = tempfile.NamedTemporaryFile(
prefix="job_soma_out_log_",
suffix=repr(job_id),
delete=False)
job_stdout_file = job_stdout_file.name
job_stderr_file = tempfile.NamedTemporaryFile(
prefix="job_soma_outerr_log_",
suffix=repr(job_id),
delete=False)
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(job_id,
job_stdout_file,
job_stderr_file)
if job_name == 'job1':
# Test stdout
isSame, msg = identical_files(
job_stdout_file,
self.wf_examples.lo_stdout[1])
self.assertTrue(isSame, msg)
# Test no stderr
msg = "job stderr not empty : cf %s\n" \
"stderr:\n---\n%s\n---" \
% (job_stderr_file, open(job_stderr_file).read())
self.assertTrue(os.stat(job_stderr_file).st_size == 0,
msg)
# Test output files
if self.path_management == self.LOCAL_PATH:
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[11],
self.wf_examples.lo_file[11])
self.assertTrue(isSame, msg)
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[12],
self.wf_examples.lo_file[12])
self.assertTrue(isSame, msg)
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[11],
self.wf_examples.tr_file[11].client_path)
self.assertTrue(isSame, msg)
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[12],
self.wf_examples.tr_file[12].client_path)
self.assertTrue(isSame, msg)
# For unknown reason, it raises some errors
# http://stackoverflow.com/questions/10496758/unexpected-end-of-file-and-error-importing-function-definition-error-running
# isSame, msg = identical_files(job_stderr_file,self.wf_examples.lo_stderr[1])
# self.failUnless(isSame == True)
if job_name in ['job2', 'job3', 'job4']:
job_nb = int(job_name[3])
# Test stdout
isSame, msg = identical_files(
job_stdout_file,
self.wf_examples.lo_stdout[job_nb])
self.assertTrue(isSame, msg)
# Test no stderr
self.assertTrue(os.stat(job_stderr_file).st_size == 0,
"job stderr not empty : cf %s" %
job_stderr_file)
# Test output files
if self.path_management == self.LOCAL_PATH:
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[job_nb],
self.wf_examples.lo_file[job_nb])
self.assertTrue(isSame, msg)
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[job_nb],
self.wf_examples.tr_file[job_nb].client_path)
self.assertTrue(isSame, msg)
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
def test_result(self):
workflow = self.wf_examples.example_special_transfer()
self.wf_id = self.wf_ctrl.submit_workflow(
workflow=workflow,
name=self.__class__.__name__)
# Transfer input files
Helper.transfer_input_files(self.wf_id, self.wf_ctrl)
# Wait for the worklow to finish
Helper.wait_workflow(self.wf_id, self.wf_ctrl)
status = self.wf_ctrl.workflow_status(self.wf_id)
# Transfer output files
Helper.transfer_output_files(self.wf_id, self.wf_ctrl)
status = self.wf_ctrl.workflow_status(self.wf_id)
self.assertTrue(status == constants.WORKFLOW_DONE,
"workflow status : %s. Expected : %s" %
(status, constants.WORKFLOW_DONE))
nb_failed_jobs = len(Helper.list_failed_jobs(
self.wf_id,
self.wf_ctrl))
self.assertTrue(nb_failed_jobs == 0,
"nb failed jobs : %i. Expected : %i" %
(nb_failed_jobs, 0))
nb_failed_aborted_jobs = len(Helper.list_failed_jobs(
self.wf_id,
self.wf_ctrl,
include_aborted_jobs=True))
self.assertTrue(nb_failed_aborted_jobs == 0,
"nb failed jobs including aborted : %i. Expected : %i"
% (nb_failed_aborted_jobs, 0))
(jobs_info, transfers_info, workflow_status, workflow_queue,
tmp_files) = self.wf_ctrl.workflow_elements_status(self.wf_id)
for (job_id, tmp_status, queue, exit_info, dates) in jobs_info:
job_list = self.wf_ctrl.jobs([job_id])
job_name, job_command, job_submission_date = job_list[job_id]
self.tested_job = job_id
if exit_info[0] == constants.FINISHED_REGULARLY:
# To check job standard out and standard err
job_stdout_file = tempfile.NamedTemporaryFile(
prefix="job_soma_out_log_",
suffix=repr(job_id),
delete=False)
job_stdout_file = job_stdout_file.name
job_stderr_file = tempfile.NamedTemporaryFile(
prefix="job_soma_outerr_log_",
suffix=repr(job_id),
delete=False)
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(job_id,
job_stdout_file,
job_stderr_file)
if job_name == 'dir_contents':
# Test job standard out
with open(job_stdout_file, 'r+') as f:
dir_contents = f.readlines()
dir_path_in = self.wf_examples.lo_in_dir
full_path_list = []
for element in os.listdir(dir_path_in):
full_path_list.append(os.path.join(dir_path_in,
element))
dir_contents_model = list_contents(full_path_list, [])
self.assertTrue(
sorted(dir_contents) == sorted(dir_contents_model))
# Test no stderr
self.assertTrue(os.stat(job_stderr_file).st_size == 0,
"job stderr not empty : cf %s" %
job_stderr_file)
if job_name == 'multi file format test':
# Test job standard out
isSame, msg = identical_files(
job_stdout_file,
self.wf_examples.lo_mff_stdout)
self.assertTrue(isSame, msg)
# Test no stderr
self.assertTrue(os.stat(job_stderr_file).st_size == 0,
"job stderr not empty : cf %s" %
job_stderr_file)
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
def test_result(self):
nb = 20
time_sleep = 1
workflow = self.wf_examples.example_n_jobs(nb=nb, time=time_sleep)
self.wf_id = self.wf_ctrl.submit_workflow(
workflow=workflow,
name=self.__class__.__name__)
# Transfer input files if file transfer
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
Helper.transfer_input_files(self.wf_id, self.wf_ctrl)
# Wait for the workflow to finish
Helper.wait_workflow(self.wf_id, self.wf_ctrl)
# Transfer output files if file transfer
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
Helper.transfer_output_files(self.wf_id, self.wf_ctrl)
status = self.wf_ctrl.workflow_status(self.wf_id)
self.assertTrue(status == constants.WORKFLOW_DONE,
"workflow status : %s. Expected : %s" %
(status, constants.WORKFLOW_DONE))
nb_failed_jobs = len(Helper.list_failed_jobs(
self.wf_id,
self.wf_ctrl))
self.assertTrue(nb_failed_jobs == 0,
"nb failed jobs : %i. Expected : %i" %
(nb_failed_jobs, 0))
nb_failed_aborted_jobs = len(Helper.list_failed_jobs(
self.wf_id,
self.wf_ctrl,
include_aborted_jobs=True))
self.assertTrue(nb_failed_aborted_jobs == 0,
"nb failed jobs including aborted : %i. Expected : %i"
% (nb_failed_aborted_jobs, 0))
(jobs_info, transfers_info, workflow_status, workflow_queue,
tmp_files) = self.wf_ctrl.workflow_elements_status(self.wf_id)
for (job_id, tmp_status, queue, exit_info, dates) in jobs_info:
job_list = self.wf_ctrl.jobs([job_id])
job_name, job_command, job_submission_date = job_list[job_id]
self.tested_job = job_id
if exit_info[0] == constants.FINISHED_REGULARLY:
# To check job standard out and standard err
job_stdout_file = tempfile.NamedTemporaryFile(
prefix="job_soma_out_log_",
suffix=repr(job_id),
delete=False)
job_stdout_file = job_stdout_file.name
job_stderr_file = tempfile.NamedTemporaryFile(
prefix="job_soma_outerr_log_",
suffix=repr(job_id),
delete=False)
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(job_id,
job_stdout_file,
job_stderr_file)
# Test stdout
self.assertTrue(os.stat(job_stdout_file).st_size == 0,
"job stdout not empty : file: %s, "
"contents:\n%s" %
(job_stdout_file,
open(job_stdout_file).read()))
# Test no stderr
self.assertTrue(os.stat(job_stderr_file).st_size == 0,
"job stderr not empty : file %s, "
"contents:\n%s" %
(job_stderr_file,
open(job_stderr_file).read()))
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
def test_result(self):
workflow = self.wf_examples.example_special_transfer()
self.wf_id = self.wf_ctrl.submit_workflow(workflow=workflow, name=self.__class__.__name__)
# Transfer input files
Helper.transfer_input_files(self.wf_id, self.wf_ctrl)
# Wait for the worklow to finish
Helper.wait_workflow(self.wf_id, self.wf_ctrl)
status = self.wf_ctrl.workflow_status(self.wf_id)
# Transfer output files
Helper.transfer_output_files(self.wf_id, self.wf_ctrl)
status = self.wf_ctrl.workflow_status(self.wf_id)
self.assertTrue(
status == constants.WORKFLOW_DONE, "workflow status : %s. Expected : %s" % (status, constants.WORKFLOW_DONE)
)
nb_failed_jobs = len(Helper.list_failed_jobs(self.wf_id, self.wf_ctrl))
self.assertTrue(nb_failed_jobs == 0, "nb failed jobs : %i. Expected : %i" % (nb_failed_jobs, 0))
nb_failed_aborted_jobs = len(Helper.list_failed_jobs(self.wf_id, self.wf_ctrl, include_aborted_jobs=True))
self.assertTrue(
nb_failed_aborted_jobs == 0,
"nb failed jobs including aborted : %i. Expected : %i" % (nb_failed_aborted_jobs, 0),
)
(jobs_info, transfers_info, workflow_status, workflow_queue, tmp_files) = self.wf_ctrl.workflow_elements_status(
self.wf_id
)
for (job_id, tmp_status, queue, exit_info, dates) in jobs_info:
job_list = self.wf_ctrl.jobs([job_id])
job_name, job_command, job_submission_date = job_list[job_id]
self.tested_job = job_id
if exit_info[0] == constants.FINISHED_REGULARLY:
# To check job standard out and standard err
job_stdout_file = tempfile.NamedTemporaryFile(
prefix="job_soma_out_log_", suffix=repr(job_id), delete=False
)
job_stdout_file = job_stdout_file.name
job_stderr_file = tempfile.NamedTemporaryFile(
prefix="job_soma_outerr_log_", suffix=repr(job_id), delete=False
)
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(job_id, job_stdout_file, job_stderr_file)
if job_name == "dir_contents":
# Test job standard out
with open(job_stdout_file, "r+") as f:
dir_contents = f.readlines()
dir_path_in = self.wf_examples.lo_in_dir
full_path_list = []
for element in os.listdir(dir_path_in):
full_path_list.append(os.path.join(dir_path_in, element))
dir_contents_model = list_contents(full_path_list, [])
self.assertTrue(sorted(dir_contents) == sorted(dir_contents_model))
# Test no stderr
self.assertTrue(
os.stat(job_stderr_file).st_size == 0, "job stderr not empty : cf %s" % job_stderr_file
)
if job_name == "multi file format test":
# Test job standard out
isSame, msg = identical_files(job_stdout_file, self.wf_examples.lo_mff_stdout)
self.assertTrue(isSame, msg)
# Test no stderr
self.assertTrue(
os.stat(job_stderr_file).st_size == 0, "job stderr not empty : cf %s" % job_stderr_file
)
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
def test_result(self):
nb = 20
time_sleep = 1
workflow = self.wf_examples.example_n_jobs(nb=nb, time=time_sleep)
self.wf_id = self.wf_ctrl.submit_workflow(workflow=workflow,
name=self.__class__.__name__)
# Transfer input files if file transfer
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
Helper.transfer_input_files(self.wf_id, self.wf_ctrl)
# Wait for the workflow to finish
Helper.wait_workflow(self.wf_id, self.wf_ctrl)
# Transfer output files if file transfer
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
Helper.transfer_output_files(self.wf_id, self.wf_ctrl)
status = self.wf_ctrl.workflow_status(self.wf_id)
self.assertTrue(
status == constants.WORKFLOW_DONE,
"workflow status : %s. Expected : %s" %
(status, constants.WORKFLOW_DONE))
nb_failed_jobs = len(Helper.list_failed_jobs(self.wf_id, self.wf_ctrl))
self.assertTrue(
nb_failed_jobs == 0,
"nb failed jobs : %i. Expected : %i" % (nb_failed_jobs, 0))
nb_failed_aborted_jobs = len(
Helper.list_failed_jobs(self.wf_id,
self.wf_ctrl,
include_aborted_jobs=True))
self.assertTrue(
nb_failed_aborted_jobs == 0,
"nb failed jobs including aborted : %i. Expected : %i" %
(nb_failed_aborted_jobs, 0))
(jobs_info, transfers_info, workflow_status, workflow_queue,
tmp_files) = self.wf_ctrl.workflow_elements_status(self.wf_id)
for (job_id, tmp_status, queue, exit_info, dates, drmaa_id) \
in jobs_info:
job_list = self.wf_ctrl.jobs([job_id])
job_name, job_command, job_submission_date = job_list[job_id]
self.tested_job = job_id
if exit_info[0] == constants.FINISHED_REGULARLY:
# To check job standard out and standard err
job_stdout_file = tempfile.NamedTemporaryFile(
prefix="job_soma_out_log_",
suffix=repr(job_id),
delete=False)
job_stdout_file = job_stdout_file.name
job_stderr_file = tempfile.NamedTemporaryFile(
prefix="job_soma_outerr_log_",
suffix=repr(job_id),
delete=False)
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(
job_id, job_stdout_file, job_stderr_file)
# Test stdout
self.assertTrue(
os.stat(job_stdout_file).st_size == 0,
"job stdout not empty : file: %s, "
"contents:\n%s" %
(job_stdout_file, open(job_stdout_file).read()))
# Test no stderr
self.assertTrue(
os.stat(job_stderr_file).st_size == 0,
"job stderr not empty : file %s, "
"contents:\n%s" %
(job_stderr_file, open(job_stderr_file).read()))
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
"--output_r2",
os.path.join(args.output, 'r2'), "--output_distribution",
os.path.join(args.output, 'distribution'), "--x", args.x, "--y",
args.y, "--shuffling", shuffling, "--n_permutations",
args.nb_permutations, "--alpha_percentile", args.alpha_percentile
],
name="job {} - alpha {}".format(run, alpha),
working_directory=scripts_path)
group_significativity.append(job)
jobs.append(job)
distribution_voxels = Group(elements=group_significativity,
name="Voxel wise fitting of the models")
workflow2 = Workflow(jobs=jobs, root_group=[distribution_voxels])
### Submit the workflow to computing resource (configured in the client-server mode)
controller2 = WorkflowController(
"DSV_cluster_ap259944", args.login,
args.password) #"DSV_cluster_ap259944", args.login, args.password
workflow_id2 = controller2.submit_workflow(workflow=workflow2,
name="Voxel-wise computations")
# You may use the gui or manually transfer the files:
manual = True
if manual:
Helper.transfer_input_files(workflow_id2, controller2)
Helper.wait_workflow(workflow_id2, controller2)
Helper.transfer_output_files(workflow_id2, controller2)
