os.stat(job_stderr_file).st_size == 0, "job stderr not empty : cf %s" % job_stderr_file
)
# Test output files
if self.path_management == self.LOCAL_PATH:
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[11], self.wf_examples.lo_file[11]
)
self.assertTrue(isSame, msg)
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[12], self.wf_examples.lo_file[12]
)
self.assertTrue(isSame, msg)
if self.path_management == self.FILE_TRANSFER or self.path_management == self.SHARED_TRANSFER:
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[11], self.wf_examples.tr_file[11].client_path
)
self.assertTrue(isSame, msg)
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[12], self.wf_examples.tr_file[12].client_path
)
self.assertTrue(isSame, msg)
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
if __name__ == "__main__":
NativeSpecPbsTest.run_test(**WorkflowTest.parse_args(sys.argv))
def test():
return SimpleTest.run_test_function(**WorkflowTest.parse_args(sys.argv))
if exit_info[0] == constants.FINISHED_REGULARLY:
# To check job standard out and standard err
job_stdout_file = tempfile.NamedTemporaryFile(
prefix="job_soma_out_log_",
suffix=repr(job_id),
delete=False)
job_stdout_file = job_stdout_file.name
job_stderr_file = tempfile.NamedTemporaryFile(
prefix="job_soma_outerr_log_",
suffix=repr(job_id),
delete=False)
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(job_id,
job_stdout_file,
job_stderr_file)
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
def test():
return BarrierTest.run_test_function(**WorkflowTest.parse_args(sys.argv))
if __name__ == '__main__':
BarrierTest.run_test(**WorkflowTest.parse_args(sys.argv))
delete=False)
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(
job_id, job_stdout_file, job_stderr_file)
# Test job stdout
if self.path_management == self.LOCAL_PATH:
isSame, msg = identical_files(
job_stdout_file,
self.wf_examples.lo_stdout_command_local)
self.assertTrue(isSame, msg)
else:
isSame, msg = identical_files(
job_stdout_file,
self.wf_examples.lo_stdout_command_remote)
self.assertTrue(isSame, msg)
# Test no stderr
self.assertTrue(
os.stat(job_stderr_file).st_size == 0,
"job stderr not empty : cf %s" % job_stderr_file)
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
if __name__ == '__main__':
SpecialCommandTest.run_test(**WorkflowTest.parse_args(sys.argv))
def test():
return WrongNativeSpecPbsTest.run_test_function(
**WorkflowTest.parse_args(sys.argv))
with open(job_stdout_file, "r+") as f:
dir_contents = f.readlines()
dir_path_in = self.wf_examples.lo_in_dir
full_path_list = []
for element in os.listdir(dir_path_in):
full_path_list.append(os.path.join(dir_path_in, element))
dir_contents_model = list_contents(full_path_list, [])
self.assertTrue(sorted(dir_contents) == sorted(dir_contents_model))
# Test no stderr
self.assertTrue(
os.stat(job_stderr_file).st_size == 0, "job stderr not empty : cf %s" % job_stderr_file
)
if job_name == "multi file format test":
# Test job standard out
isSame, msg = identical_files(job_stdout_file, self.wf_examples.lo_mff_stdout)
self.assertTrue(isSame, msg)
# Test no stderr
self.assertTrue(
os.stat(job_stderr_file).st_size == 0, "job stderr not empty : cf %s" % job_stderr_file
)
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
if __name__ == "__main__":
SpecialTransferTest.run_test(**WorkflowTest.parse_args(sys.argv))
def test():
return SpecialCommandTest.run_test_function(
**WorkflowTest.parse_args(sys.argv))
def test():
return MultipleTest.run_test_function(**WorkflowTest.parse_args(sys.argv))
def test():
return NJobsWithDependenciesTest.run_test_function(
**WorkflowTest.parse_args(sys.argv))
def test():
return SpecialCommandTest.run_test_function(
**WorkflowTest.parse_args(sys.argv))
job_stdout_file = job_stdout_file.name
job_stderr_file = tempfile.NamedTemporaryFile(
prefix="job_soma_outerr_log_",
suffix=repr(job_id),
delete=False)
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(job_id,
job_stdout_file,
job_stderr_file)
# Test stdout
self.assertTrue(os.stat(job_stdout_file).st_size == 0,
"job stdout not empty : cf %s" %
job_stdout_file)
# Test no stderr
self.assertTrue(os.stat(job_stderr_file).st_size == 0,
"job stderr not empty : cf %s" %
job_stderr_file)
finally:
if os.path.exists(job_stdout_file):
os.unlink(job_stdout_file)
if os.path.exists(job_stderr_file):
os.unlink(job_stderr_file)
del self.tested_job
if __name__ == '__main__':
FakePipelineTest.run_test(**WorkflowTest.parse_args(sys.argv))
def test():
return SpecialTransferTest.run_test_function(
**WorkflowTest.parse_args(sys.argv))
suffix=repr(job_id),
delete=False)
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(
job_id, job_stdout_file, job_stderr_file)
output \
= open(job_stdout_file).read().strip().split('\n')
exp_wf_outputs = expected_outputs[workflow.name]
if job_name in exp_wf_outputs:
exp = exp_wf_outputs[job_name].split('\n')
#print('### job', job_name, 'output:', output, file=sys.stderr)
#print('### expected:', exp, file=sys.stderr)
#print('### res:', [o in output for o in exp], file=sys.stderr)
self.assertTrue(all([o in output for o in exp]))
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
def test():
return WorkflowConfTest.run_test_function(
**WorkflowTest.parse_args(sys.argv))
if __name__ == '__main__':
WorkflowConfTest.run_test(**WorkflowTest.parse_args(sys.argv))
def test():
return Exception2Test.run_test_function(
**WorkflowTest.parse_args(sys.argv))
def test():
return Exception1Test.run_test_function(
**WorkflowTest.parse_args(sys.argv))
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(
job_id, job_stdout_file, job_stderr_file)
# Test stdout
self.assertTrue(
os.stat(job_stdout_file).st_size == 0,
"job stdout not empty : file: %s, "
"contents:\n%s" %
(job_stdout_file, open(job_stdout_file).read()))
# Test no stderr
self.assertTrue(
os.stat(job_stderr_file).st_size == 0,
"job stderr not empty : file %s, "
"contents:\n%s" %
(job_stderr_file, open(job_stderr_file).read()))
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
def test():
return NJobsTest.run_test_function(**WorkflowTest.parse_args(sys.argv))
if __name__ == '__main__':
NJobsTest.run_test(**WorkflowTest.parse_args(sys.argv))
return self.run_workflow(
workflow, test_files=[14],
test_dyn_files={'test': {'output': 15}})
def test_workflow_cv(self):
workflow = self.wf_examples.example_dynamic_outputs_with_cv()
return self.run_workflow(workflow, test_files=[16, 17])
def test_workflow_mapreduce_with_jobs(self):
workflow \
= self.wf_examples.example_dynamic_outputs_with_mapreduce_jobs()
#Helper.serialize('/tmp/workflow.workflow', workflow)
return self.run_workflow(workflow, test_files=[13])
def test_workflow_loo_with_jobs(self):
workflow = self.wf_examples.example_dynamic_outputs_with_loo_jobs()
return self.run_workflow(
workflow, test_files=[14],
test_dyn_files={'test': {'output': 15}})
def test_workflow_cv_with_jobs(self):
workflow = self.wf_examples.example_dynamic_outputs_with_cv_jobs()
return self.run_workflow(workflow, test_files=[16, 17])
def test():
return SimpleTest.run_test_function(**WorkflowTest.parse_args(sys.argv))
if __name__ == '__main__':
SimpleTest.run_test(**WorkflowTest.parse_args(sys.argv))
for element in os.listdir(dir_path_in):
full_path_list.append(os.path.join(dir_path_in,
element))
dir_contents_model = list_contents(full_path_list, [])
self.assertTrue(
sorted(dir_contents) == sorted(dir_contents_model))
# Test no stderr
self.assertTrue(os.stat(job_stderr_file).st_size == 0,
"job stderr not empty : cf %s" %
job_stderr_file)
if job_name == 'multi file format test':
# Test job standard out
isSame, msg = identical_files(
job_stdout_file,
self.wf_examples.lo_mff_stdout)
self.assertTrue(isSame, msg)
# Test no stderr
self.assertTrue(os.stat(job_stderr_file).st_size == 0,
"job stderr not empty : cf %s" %
job_stderr_file)
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
if __name__ == '__main__':
SpecialTransferTest.run_test(**WorkflowTest.parse_args(sys.argv))
def test():
return NJobsWithDependenciesTest.run_test_function(
**WorkflowTest.parse_args(sys.argv))
"job stderr not empty : cf %s" % job_stderr_file)
# Test output files
if self.path_management == self.LOCAL_PATH:
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[11],
self.wf_examples.lo_file[11])
self.assertTrue(isSame, msg)
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[12],
self.wf_examples.lo_file[12])
self.assertTrue(isSame, msg)
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[11],
self.wf_examples.tr_file[11].client_path)
self.assertTrue(isSame, msg)
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[12],
self.wf_examples.tr_file[12].client_path)
self.assertTrue(isSame, msg)
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
if __name__ == '__main__':
NativeSpecPbsTest.run_test(**WorkflowTest.parse_args(sys.argv))
suffix=repr(job_id),
delete=False)
job_stdout_file = job_stdout_file.name
job_stderr_file = tempfile.NamedTemporaryFile(
prefix="job_soma_outerr_log_",
suffix=repr(job_id),
delete=False)
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(job_id,
job_stdout_file,
job_stderr_file)
# Test stdout
self.assertTrue(os.stat(job_stdout_file).st_size == 0,
"job stdout not empty : cf %s" %
job_stdout_file)
# Test no stderr
self.assertTrue(os.stat(job_stderr_file).st_size == 0,
"job stderr not empty : cf %s" %
job_stderr_file)
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
if __name__ == '__main__':
NJobsWithDependenciesTest.run_test(**WorkflowTest.parse_args(sys.argv))
prefix="job_soma_out_log_",
suffix=repr(job_id),
delete=False)
job_stdout_file = job_stdout_file.name
job_stderr_file = tempfile.NamedTemporaryFile(
prefix="job_soma_outerr_log_",
suffix=repr(job_id),
delete=False)
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(
job_id, job_stdout_file, job_stderr_file)
# Test stdout
self.assertTrue(
os.stat(job_stdout_file).st_size == 0,
"job stdout not empty : cf %s" % job_stdout_file)
# Test no stderr
self.assertTrue(
os.stat(job_stderr_file).st_size == 0,
"job stderr not empty : cf %s" % job_stderr_file)
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
if __name__ == '__main__':
NJobsWithDependenciesTest.run_test(**WorkflowTest.parse_args(sys.argv))
def test():
return SerialJobsTest.run_test_function(
**WorkflowTest.parse_args(sys.argv))
def test():
return BarrierTest.run_test_function(**WorkflowTest.parse_args(sys.argv))
def test():
return WrongNativeSpecPbsTest.run_test_function(
**WorkflowTest.parse_args(sys.argv))
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[2],
self.wf_examples.tr_file[2].client_path)
self.assertTrue(isSame, msg)
if job_name == 'job3 with exception':
# Test stdout
isSame, msg = identical_files(
job_stdout_file,
self.wf_examples.lo_stdout_exception_model)
self.assertTrue(isSame, msg)
# Test the last line of stderr
with open(job_stderr_file) as f:
lines = f.readlines()
expected_error = 'Exception: Paf Boum Boum Bada Boum !!!\n'
isSame = (lines[-1] == expected_error)
self.assertTrue(isSame,
"Job exception : %s. Expected : %s" %
(lines[-1], expected_error))
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
if __name__ == '__main__':
Exception2Test.run_test(**WorkflowTest.parse_args(sys.argv))
delete=False)
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(
job_id, job_stdout_file, job_stderr_file)
# Test stdout
self.assertTrue(
os.stat(job_stdout_file).st_size == 0,
"job stdout not empty : cf %s" % job_stdout_file)
# Test no stderr
self.assertTrue(
os.stat(job_stderr_file).st_size == 0,
"job stderr not empty : cf %s" % job_stderr_file)
finally:
if os.path.exists(job_stdout_file):
os.unlink(job_stdout_file)
if os.path.exists(job_stderr_file):
os.unlink(job_stderr_file)
del self.tested_job
def test():
return FakePipelineTest.run_test_function(
**WorkflowTest.parse_args(sys.argv))
if __name__ == '__main__':
FakePipelineTest.run_test(**WorkflowTest.parse_args(sys.argv))
self.wf_examples.lo_stdout[job_nb])
self.assertTrue(isSame, msg)
# Test no stderr
self.assertTrue(os.stat(job_stderr_file).st_size == 0,
"job stderr not empty : cf %s" %
job_stderr_file)
# Test output files
if self.path_management == self.LOCAL_PATH:
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[job_nb],
self.wf_examples.lo_file[job_nb])
self.assertTrue(isSame, msg)
if self.path_management == self.FILE_TRANSFER or \
self.path_management == self.SHARED_TRANSFER:
isSame, msg = identical_files(
self.wf_examples.lo_out_model_file[job_nb],
self.wf_examples.tr_file[job_nb].client_path)
self.assertTrue(isSame, msg)
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
def test():
return SimpleTest.run_test_function(**WorkflowTest.parse_args(sys.argv))
if __name__ == '__main__':
SimpleTest.run_test(**WorkflowTest.parse_args(sys.argv))
def test():
return FakePipelineTest.run_test_function(
**WorkflowTest.parse_args(sys.argv))
self.assertTrue(isSame, msg)
if job_name == 'job3 with exception':
# Test stdout
isSame, msg = identical_files(
job_stdout_file,
self.wf_examples.lo_stdout_exception_model)
self.assertTrue(isSame, msg)
# Test the last line of stderr
with open(job_stderr_file) as f:
lines = f.readlines()
expected_error = 'Exception: Paf Boum Boum Bada Boum !!!\n'
isSame = (lines[-1] == expected_error)
self.assertTrue(
isSame, "Job exception : %s. Expected : %s" %
(lines[-1], expected_error))
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
def test():
return Exception2Test.run_test_function(
**WorkflowTest.parse_args(sys.argv))
if __name__ == '__main__':
Exception2Test.run_test(**WorkflowTest.parse_args(sys.argv))
if exit_info[0] == constants.FINISHED_REGULARLY:
# To check job standard out and standard err
job_stdout_file = tempfile.NamedTemporaryFile(
prefix="job_soma_out_log_",
suffix=repr(job_id),
delete=False)
job_stdout_file = job_stdout_file.name
job_stderr_file = tempfile.NamedTemporaryFile(
prefix="job_soma_outerr_log_",
suffix=repr(job_id),
delete=False)
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(
job_id, job_stdout_file, job_stderr_file)
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
def test():
return BarrierTest.run_test_function(**WorkflowTest.parse_args(sys.argv))
if __name__ == '__main__':
BarrierTest.run_test(**WorkflowTest.parse_args(sys.argv))
prefix="job_soma_outerr_log_",
suffix=repr(job_id),
delete=False)
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(job_id,
job_stdout_file,
job_stderr_file)
# Test stdout
self.assertTrue(os.stat(job_stdout_file).st_size == 0,
"job stdout not empty : file: %s, "
"contents:\n%s" %
(job_stdout_file,
open(job_stdout_file).read()))
# Test no stderr
self.assertTrue(os.stat(job_stderr_file).st_size == 0,
"job stderr not empty : file %s, "
"contents:\n%s" %
(job_stderr_file,
open(job_stderr_file).read()))
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
if __name__ == '__main__':
NJobsTest.run_test(**WorkflowTest.parse_args(sys.argv))
job_stderr_file = job_stderr_file.name
try:
self.wf_ctrl.retrieve_job_stdouterr(job_id,
job_stdout_file,
job_stderr_file)
# Test job stdout
if self.path_management == self.LOCAL_PATH:
isSame, msg = identical_files(
job_stdout_file,
self.wf_examples.lo_stdout_command_local)
self.assertTrue(isSame, msg)
else:
isSame, msg = identical_files(
job_stdout_file,
self.wf_examples.lo_stdout_command_remote)
self.assertTrue(isSame, msg)
# Test no stderr
self.assertTrue(os.stat(job_stderr_file).st_size == 0,
"job stderr not empty : cf %s" %
job_stderr_file)
finally:
os.unlink(job_stdout_file)
os.unlink(job_stderr_file)
del self.tested_job
if __name__ == '__main__':
SpecialCommandTest.run_test(**WorkflowTest.parse_args(sys.argv))
def test():
return NJobsTest.run_test_function(**WorkflowTest.parse_args(sys.argv))
def test():
return FakePipelineTest.run_test_function(
**WorkflowTest.parse_args(sys.argv))
