def test_workflow_thread_cancel_external(self):
with TestAreaContext(
"python/job_queue/workflow_runner_external") as work_area:
WorkflowCommon.createWaitJob()
joblist = WorkflowJoblist()
self.assertTrue(joblist.addJobFromFile("WAIT",
"external_wait_job"))
self.assertTrue("WAIT" in joblist)
workflow = Workflow("wait_workflow", joblist)
self.assertEqual(len(workflow), 3)
workflow_runner = WorkflowRunner(workflow,
ert=None,
context=SubstitutionList())
self.assertFalse(workflow_runner.isRunning())
with workflow_runner:
wait_until(
lambda: self.assertTrue(workflow_runner.isRunning()))
wait_until(lambda: self.assertFileExists("wait_started_0"))
wait_until(lambda: self.assertFileExists("wait_finished_0"))
wait_until(lambda: self.assertFileExists("wait_started_1"))
workflow_runner.cancel()
self.assertTrue(workflow_runner.isCancelled())
self.assertFileDoesNotExist("wait_finished_1")
self.assertFileDoesNotExist("wait_started_2")
self.assertFileDoesNotExist("wait_cancelled_2")
self.assertFileDoesNotExist("wait_finished_2")
def test_workflow_success(self):
with TestAreaContext(
"python/job_queue/workflow_runner_fast") as work_area:
WorkflowCommon.createWaitJob()
joblist = WorkflowJoblist()
self.assertTrue(joblist.addJobFromFile("WAIT", "wait_job"))
self.assertTrue(
joblist.addJobFromFile("EXTERNAL_WAIT", "external_wait_job"))
workflow = Workflow("fast_wait_workflow", joblist)
self.assertEqual(len(workflow), 2)
workflow_runner = WorkflowRunner(workflow,
ert=None,
context=SubstitutionList())
self.assertFalse(workflow_runner.isRunning())
with workflow_runner:
workflow_runner.wait()
self.assertFileExists("wait_started_0")
self.assertFileDoesNotExist("wait_cancelled_0")
self.assertFileExists("wait_finished_0")
self.assertFileExists("wait_started_1")
self.assertFileDoesNotExist("wait_cancelled_1")
self.assertFileExists("wait_finished_1")
self.assertTrue(workflow_runner.workflowResult())
def test_failing_workflow_run(self):
with TestAreaContext("python/job_queue/workflow") as work_area:
WorkflowCommon.createExternalDumpJob()
joblist = WorkflowJoblist()
self.assertTrue(joblist.addJobFromFile("DUMP", "dump_job"))
self.assertTrue("DUMP" in joblist)
workflow = Workflow("undefined", joblist)
context = SubstitutionList()
self.assertFalse(workflow.run(None, verbose=True, context=context))
def test_workflow_joblist_creation(self):
joblist = WorkflowJoblist()
job = WorkflowJob("JOB1")
joblist.addJob(job)
self.assertTrue(job in joblist)
self.assertTrue("JOB1" in joblist)
job_ref = joblist["JOB1"]
self.assertEqual(job.name(), job_ref.name())
def __init__(self, subst_list=None, config_content=None, config_dict=None):
if subst_list is None:
raise ValueError('Failed to construct ErtWorkflowList with no substitution list')
if config_content is None and config_dict is None:
raise ValueError('Failed to construct ErtWorkflowList instance with no config object')
if config_content is not None and config_dict is not None:
raise ValueError('Failed to construct ErtWorkflowList instance with multiple config object')
c_ptr = None
if config_content is not None:
c_ptr = self._alloc(subst_list, config_content)
if config_dict is not None:
workflow_joblist = WorkflowJoblist()
parser = WorkflowJob.configParser( )
for job in config_dict.get(ConfigKeys.LOAD_WORKFLOW_JOB, []):
try:
new_job = WorkflowJob.fromFile(config_file=job['PATH'], name=job['NAME'], parser=parser)
except:
print("WARNING: Unable to create job from {}".format(job['PATH']))
continue
workflow_joblist.addJob(new_job)
new_job.convertToCReference(None)
for job_path in config_dict.get(ConfigKeys.WORKFLOW_JOB_DIRECTORY, []) :
if not os.path.isdir(job_path):
print("WARNING: Unable to open job directory {}".format(job_path))
continue
files = os.listdir(job_path)
for file_name in files:
full_path = os.path.join(job_path, file_name)
try:
new_job = WorkflowJob.fromFile(config_file=full_path, parser=parser)
workflow_joblist.addJob(new_job)
new_job.convertToCReference(None)
except:
print("WARNING: Unable to create job from {}".format(full_path))
continue
workflow_joblist.convertToCReference(None)
c_ptr = self._alloc_full(subst_list, workflow_joblist )
if c_ptr is None:
raise ValueError('Failed to construct ErtWorkflowList instance')
super(ErtWorkflowList, self).__init__(c_ptr)
if config_dict is not None:
for job in config_dict.get(ConfigKeys.LOAD_WORKFLOW, []):
self.addWorkflow(job['PATH'], job['NAME'])
def test_workflow_failed_job(self):
with TestAreaContext("python/job_queue/workflow_runner_fails"):
WorkflowCommon.createExternalDumpJob()
joblist = WorkflowJoblist()
self.assertTrue(joblist.addJobFromFile("DUMP", "dump_failing_job"))
workflow = Workflow("dump_workflow", joblist)
self.assertEqual(len(workflow), 2)
workflow_runner = WorkflowRunner(
workflow, ert=None, context=SubstitutionList()
)
self.assertFalse(workflow_runner.isRunning())
with patch.object(
Workflow, "run", side_effect=Exception("mocked workflow error")
), workflow_runner:
workflow_runner.wait()
self.assertNotEqual(workflow_runner.exception(), None)
def test_workflow_thread_cancel_ert_script(self):
with TestAreaContext(
"python/job_queue/workflow_runner_ert_script") as work_area:
WorkflowCommon.createWaitJob()
joblist = WorkflowJoblist()
self.assertTrue(joblist.addJobFromFile("WAIT", "wait_job"))
self.assertTrue("WAIT" in joblist)
workflow = Workflow("wait_workflow", joblist)
self.assertEqual(len(workflow), 3)
workflow_runner = WorkflowRunner(workflow)
self.assertFalse(workflow_runner.isRunning())
workflow_runner.run()
self.assertIsNone(workflow_runner.workflowResult())
time.sleep(1) # wait for workflow to start
self.assertTrue(workflow_runner.isRunning())
self.assertFileExists("wait_started_0")
time.sleep(1) # wait for first job to finish
workflow_runner.cancel()
time.sleep(1) # wait for cancel to take effect
self.assertFileExists("wait_finished_0")
self.assertFileExists("wait_started_1")
self.assertFileExists("wait_cancelled_1")
self.assertFileDoesNotExist("wait_finished_1")
self.assertTrue(workflow_runner.isCancelled())
workflow_runner.wait() # wait for runner to complete
self.assertFileDoesNotExist("wait_started_2")
self.assertFileDoesNotExist("wait_cancelled_2")
self.assertFileDoesNotExist("wait_finished_2")
def test_workflow_run(self):
with TestAreaContext("python/job_queue/workflow") as work_area:
WorkflowCommon.createExternalDumpJob()
joblist = WorkflowJoblist()
self.assertTrue(joblist.addJobFromFile("DUMP", "dump_job"))
self.assertTrue("DUMP" in joblist)
workflow = Workflow("dump_workflow", joblist)
self.assertTrue(len(workflow), 2)
context = SubstitutionList()
context.addItem("<PARAM>", "text")
self.assertTrue(workflow.run(None, verbose=True, context=context))
with open("dump1", "r") as f:
self.assertEqual(f.read(), "dump_text_1")
with open("dump2", "r") as f:
self.assertEqual(f.read(), "dump_text_2")
def test_workflow(self):
with TestAreaContext("python/job_queue/workflow") as work_area:
WorkflowCommon.createExternalDumpJob()
joblist = WorkflowJoblist()
self.assertTrue(joblist.addJobFromFile("DUMP", "dump_job"))
with self.assertRaises(UserWarning):
joblist.addJobFromFile("KNOCK", "knock_job")
self.assertTrue("DUMP" in joblist)
workflow = Workflow("dump_workflow", joblist)
self.assertEqual(len(workflow), 2)
job, args = workflow[0]
self.assertEqual(job, joblist["DUMP"])
self.assertEqual(args[0], "dump1")
self.assertEqual(args[1], "dump_text_1")
job, args = workflow[1]
self.assertEqual(job, joblist["DUMP"])
def test_workflow_thread_cancel_ert_script(self):
with TestAreaContext("python/job_queue/workflow_runner_ert_script"):
WorkflowCommon.createWaitJob()
joblist = WorkflowJoblist()
self.assertTrue(joblist.addJobFromFile("WAIT", "wait_job"))
self.assertTrue("WAIT" in joblist)
workflow = Workflow("wait_workflow", joblist)
self.assertEqual(len(workflow), 3)
workflow_runner = WorkflowRunner(workflow)
self.assertFalse(workflow_runner.isRunning())
with workflow_runner:
self.assertIsNone(workflow_runner.workflowResult())
wait_until(lambda: self.assertTrue(workflow_runner.isRunning()))
wait_until(lambda: self.assertFileExists("wait_started_0"))
wait_until(lambda: self.assertFileExists("wait_finished_0"))
wait_until(lambda: self.assertFileExists("wait_started_1"))
workflow_runner.cancel()
wait_until(lambda: self.assertFileExists("wait_cancelled_1"))
self.assertTrue(workflow_runner.isCancelled())
self.assertFileDoesNotExist("wait_finished_1")
self.assertFileDoesNotExist("wait_started_2")
self.assertFileDoesNotExist("wait_cancelled_2")
self.assertFileDoesNotExist("wait_finished_2")
def test_workflow_joblist_with_files(self):
with TestAreaContext("python/job_queue/workflow_joblist") as work_area:
WorkflowCommon.createErtScriptsJob()
WorkflowCommon.createExternalDumpJob()
WorkflowCommon.createInternalFunctionJob()
joblist = WorkflowJoblist()
joblist.addJobFromFile("DUMP_JOB", "dump_job")
joblist.addJobFromFile("SELECT_CASE_JOB", "select_case_job")
joblist.addJobFromFile("SUBTRACT_SCRIPT_JOB", "subtract_script_job")
self.assertTrue("DUMP_JOB" in joblist)
self.assertTrue("SELECT_CASE_JOB" in joblist)
self.assertTrue("SUBTRACT_SCRIPT_JOB" in joblist)
self.assertFalse((joblist["DUMP_JOB"]).isInternal())
self.assertTrue((joblist["SELECT_CASE_JOB"]).isInternal())
self.assertTrue((joblist["SUBTRACT_SCRIPT_JOB"]).isInternal())