def test_create(self):
with TestAreaContext("run_context"):
arg = None
sim_fs = EnkfFs.createFileSystem("sim_fs",
EnKFFSType.BLOCK_FS_DRIVER_ID,
arg)
target_fs = None
mask = BoolVector(initial_size=100, default_value=True)
mask[50] = False
runpath_fmt = PathFormat("path/to/sim%d")
subst_list = SubstitutionList()
itr = 0
jobname_fmt = "job%d"
run_context1 = ErtRunContext(EnkfRunType.ENSEMBLE_EXPERIMENT,
sim_fs, target_fs, mask, runpath_fmt,
jobname_fmt, subst_list, itr)
run_id1 = run_context1.get_id()
run_arg0 = run_context1[0]
with self.assertRaises(ValueError):
run_arg0.getQueueIndex()
self.assertEqual(run_arg0.iter_id, itr)
self.assertEqual(run_id1, run_arg0.get_run_id())
run_context2 = ErtRunContext(EnkfRunType.ENSEMBLE_EXPERIMENT,
sim_fs, target_fs, mask, runpath_fmt,
jobname_fmt, subst_list, itr)
run_id2 = run_context2.get_id()
self.assertFalse(run_id1 == run_id2)
self.assertTrue(run_context1.is_active(49))
self.assertFalse(run_context1.is_active(50))
def test_workflow_success(self):
with TestAreaContext(
"python/job_queue/workflow_runner_fast") as work_area:
WorkflowCommon.createWaitJob()
joblist = WorkflowJoblist()
self.assertTrue(joblist.addJobFromFile("WAIT", "wait_job"))
self.assertTrue(
joblist.addJobFromFile("EXTERNAL_WAIT", "external_wait_job"))
workflow = Workflow("fast_wait_workflow", joblist)
self.assertEqual(len(workflow), 2)
workflow_runner = WorkflowRunner(workflow,
ert=None,
context=SubstitutionList())
self.assertFalse(workflow_runner.isRunning())
with workflow_runner:
workflow_runner.wait()
self.assertFileExists("wait_started_0")
self.assertFileDoesNotExist("wait_cancelled_0")
self.assertFileExists("wait_finished_0")
self.assertFileExists("wait_started_1")
self.assertFileDoesNotExist("wait_cancelled_1")
self.assertFileExists("wait_finished_1")
self.assertTrue(workflow_runner.workflowResult())
def test_transfer_arg_types(self):
with TestAreaContext(
"python/job_queue/forward_model_transfer_arg_types"):
with open("FWD_MODEL", "w") as f:
f.write("EXECUTABLE ls\n")
f.write("MIN_ARG 2\n")
f.write("MAX_ARG 6\n")
f.write("ARG_TYPE 0 INT\n")
f.write("ARG_TYPE 1 FLOAT\n")
f.write("ARG_TYPE 2 STRING\n")
f.write("ARG_TYPE 3 BOOL\n")
f.write("ARG_TYPE 4 RUNTIME_FILE\n")
f.write("ARG_TYPE 5 RUNTIME_INT\n")
job = ExtJob("FWD_MODEL", True)
ext_joblist = ExtJoblist()
ext_joblist.add_job(job.name(), job)
forward_model = ForwardModel(ext_joblist)
forward_model.add_job("FWD_MODEL")
run_id = "test_no_jobs_id"
umask = 4
global_args = SubstitutionList()
forward_model.formatted_fprintf(run_id, os.getcwd(), "data_root",
global_args, umask,
EnvironmentVarlist())
config = load_configs(self.JOBS_JSON_FILE)
printed_job = config["jobList"][0]
self.assertEqual(printed_job["min_arg"], 2)
self.assertEqual(printed_job["max_arg"], 6)
self.assertEqual(printed_job["arg_types"], [
"INT", "FLOAT", "STRING", "BOOL", "RUNTIME_FILE", "RUNTIME_INT"
])
def test_env_varlist(self):
varlist_string = "global_environment"
update_string = "global_update_path"
first = "FIRST"
second = "SECOND"
third = "THIRD"
first_value = "TheFirstValue"
second_value = "TheSecondValue"
third_value = "$FIRST:$SECOND"
third_value_correct = "%s:%s" % (first_value, second_value)
varlist = EnvironmentVarlist()
varlist[first] = first_value
varlist[second] = second_value
varlist[third] = third_value
self.assertEqual(len(varlist), 3)
with TestAreaContext("python/job_queue/env_varlist"):
forward_model = self.set_up_forward_model([])
run_id = "test_no_jobs_id"
umask = 4
global_args = SubstitutionList()
forward_model.formatted_fprintf(run_id, os.getcwd(), "data_root",
global_args, umask, varlist)
config = load_configs(self.JOBS_JSON_FILE)
env_config = config[varlist_string]
self.assertEqual(first_value, env_config[first])
self.assertEqual(second_value, env_config[second])
self.assertEqual(third_value_correct, env_config[third])
update_config = config[update_string]
def test_assert_symlink_deleted(self):
with ErtTestContext("create_runpath2" , self.createTestPath("local/snake_oil_field/snake_oil.ert")) as tc:
ert = tc.getErt( )
runpath_list = ert.getRunpathList( )
ens_size = ert.getEnsembleSize( )
runner = ert.getEnkfSimulationRunner( )
mask = BoolVector( initial_size = ens_size , default_value = True )
fs_manager = ert.getEnkfFsManager( )
init_fs = fs_manager.getFileSystem("init_fs")
# create directory structure
runpath_fmt = ert.getModelConfig().getRunpathFormat( )
subst_list = SubstitutionList( )
itr = 0
run_context = ErtRunContext( EnkfRunType.INIT_ONLY , init_fs, None , mask , runpath_fmt, subst_list , itr )
runner.createRunPath( run_context )
# replace field file with symlink
linkpath = '%s/permx.grdcel' % str(runpath_list[0].runpath)
targetpath = '%s/permx.grdcel.target' % str(runpath_list[0].runpath)
open(targetpath, 'a').close()
remove(linkpath)
symlink(targetpath, linkpath)
# recreate directory structure
runner.createRunPath( run_context )
# ensure field symlink is replaced by file
self.assertFalse( path.islink(linkpath) )
def test_assert_export(self):
with ErtTestContext(
"create_runpath1",
self.createTestPath(
"local/snake_oil_no_data/snake_oil.ert")) as tc:
ert = tc.getErt()
runpath_list = ert.getRunpathList()
self.assertFalse(path.isfile(runpath_list.getExportFile()))
ens_size = ert.getEnsembleSize()
runner = ert.getEnkfSimulationRunner()
fs_manager = ert.getEnkfFsManager()
init_fs = fs_manager.getFileSystem("init_fs")
mask = BoolVector(initial_size=100, default_value=True)
runpath_fmt = ert.getModelConfig().getRunpathFormat()
subst_list = SubstitutionList()
itr = 0
run_context1 = ErtRunContext(EnkfRunType.INIT_ONLY, init_fs, None,
None, mask, runpath_fmt, subst_list,
itr)
runner.createRunPath(run_context1)
self.assertTrue(path.isfile(runpath_list.getExportFile()))
self.assertEqual("test_runpath_list.txt",
path.basename(runpath_list.getExportFile()))
def test_workflow_thread_cancel_external(self):
with TestAreaContext(
"python/job_queue/workflow_runner_external") as work_area:
WorkflowCommon.createWaitJob()
joblist = WorkflowJoblist()
self.assertTrue(joblist.addJobFromFile("WAIT",
"external_wait_job"))
self.assertTrue("WAIT" in joblist)
workflow = Workflow("wait_workflow", joblist)
self.assertEqual(len(workflow), 3)
workflow_runner = WorkflowRunner(workflow,
ert=None,
context=SubstitutionList())
self.assertFalse(workflow_runner.isRunning())
with workflow_runner:
wait_until(
lambda: self.assertTrue(workflow_runner.isRunning()))
wait_until(lambda: self.assertFileExists("wait_started_0"))
wait_until(lambda: self.assertFileExists("wait_finished_0"))
wait_until(lambda: self.assertFileExists("wait_started_1"))
workflow_runner.cancel()
self.assertTrue(workflow_runner.isCancelled())
self.assertFileDoesNotExist("wait_finished_1")
self.assertFileDoesNotExist("wait_started_2")
self.assertFileDoesNotExist("wait_cancelled_2")
self.assertFileDoesNotExist("wait_finished_2")
def run_all(self):
forward_model = self.set_up_forward_model(range(len(joblist)))
umask = 0
run_id = "run_all"
global_args = SubstitutionList()
forward_model.formatted_fprintf(run_id, os.getcwd(), "data_root",
global_args, umask)
self.verify_json_dump(range(len(joblist)), global_args, umask, run_id)
def test_no_jobs(self):
with TestAreaContext("python/job_queue/forward_model_no_jobs"):
forward_model = self.set_up_forward_model([])
run_id = "test_no_jobs_id"
umask = 4
global_args = SubstitutionList()
forward_model.formatted_fprintf(run_id, os.getcwd(), "data_root",
global_args, umask)
self.verify_json_dump([], global_args, umask, run_id)
def test_one_job(self):
with TestAreaContext("python/job_queue/forward_model_one_job"):
for i in range(len(joblist)):
forward_model = self.set_up_forward_model([i])
run_id = "test_one_job"
umask = 11
global_args = SubstitutionList()
forward_model.formatted_fprintf(run_id, os.getcwd(),
global_args, umask)
self.verify_json_dump([i], global_args, umask, run_id)
def test_failing_workflow_run(self):
with TestAreaContext("python/job_queue/workflow") as work_area:
WorkflowCommon.createExternalDumpJob()
joblist = WorkflowJoblist()
self.assertTrue(joblist.addJobFromFile("DUMP", "dump_job"))
self.assertTrue("DUMP" in joblist)
workflow = Workflow("undefined", joblist)
context = SubstitutionList()
self.assertFalse(workflow.run(None, verbose=True, context=context))
def test_workflow_run(self):
with TestAreaContext("python/job_queue/workflow") as work_area:
WorkflowCommon.createExternalDumpJob()
joblist = WorkflowJoblist()
self.assertTrue(joblist.addJobFromFile("DUMP", "dump_job"))
self.assertTrue("DUMP" in joblist)
workflow = Workflow("dump_workflow", joblist)
self.assertTrue(len(workflow), 2)
context = SubstitutionList()
context.addItem("<PARAM>", "text")
self.assertTrue(workflow.run(None, verbose=True, context=context))
with open("dump1", "r") as f:
self.assertEqual(f.read(), "dump_text_1")
with open("dump2", "r") as f:
self.assertEqual(f.read(), "dump_text_2")
def __init__(self, workflow, ert=None, context=None):
"""
@type workflow: Workflow
@type ert: res.enkf.EnKFMain
@type context: SubstitutionList
"""
super(WorkflowRunner, self).__init__()
self.__workflow = workflow
self.__ert = ert
if context is None:
context = SubstitutionList()
self.__context = context
self.__workflow_result = None
def __init__(self, workflow, ert=None, context=None):
"""
@type workflow: Workflow
@type ert: res.enkf.EnKFMain
@type context: SubstitutionList
"""
super(WorkflowRunner, self).__init__()
self.__workflow = workflow
self.__ert = ert
if context is None:
context = SubstitutionList()
self.__context = context
self.__workflow_result = None
self._workflow_executor = futures.ThreadPoolExecutor(max_workers=1)
self._workflow_job = None
def test_workflow_failed_job(self):
with TestAreaContext("python/job_queue/workflow_runner_fails"):
WorkflowCommon.createExternalDumpJob()
joblist = WorkflowJoblist()
self.assertTrue(joblist.addJobFromFile("DUMP", "dump_failing_job"))
workflow = Workflow("dump_workflow", joblist)
self.assertEqual(len(workflow), 2)
workflow_runner = WorkflowRunner(
workflow, ert=None, context=SubstitutionList()
)
self.assertFalse(workflow_runner.isRunning())
with patch.object(
Workflow, "run", side_effect=Exception("mocked workflow error")
), workflow_runner:
workflow_runner.wait()
self.assertNotEqual(workflow_runner.exception(), None)
def test_status_file(self):
with TestAreaContext("status_json"):
forward_model = self.set_up_forward_model()
run_id = "test_no_jobs_id"
umask = 4
global_args = SubstitutionList()
varlist = EnvironmentVarlist()
forward_model.formatted_fprintf(run_id, os.getcwd(), "data_root",
global_args, umask, varlist)
s = '{"start_time": null, "jobs": [{"status": "Success", "start_time": 1519653419.0, "end_time": 1519653419.0, "name": "SQUARE_PARAMS", "error": null, "current_memory_usage": 2000, "max_memory_usage": 3000}], "end_time": null, "run_id": ""}'
with open("status.json", "w") as f:
f.write(s)
status = ForwardModelStatus.try_load("")
for job in status.jobs:
self.assertTrue(isinstance(job.start_time, datetime.datetime))
self.assertTrue(isinstance(job.end_time, datetime.datetime))
def test_workflow_thread_cancel_external(self):
with TestAreaContext(
"python/job_queue/workflow_runner_external") as work_area:
WorkflowCommon.createWaitJob()
joblist = WorkflowJoblist()
self.assertTrue(joblist.addJobFromFile("WAIT",
"external_wait_job"))
self.assertTrue("WAIT" in joblist)
workflow = Workflow("wait_workflow", joblist)
self.assertEqual(len(workflow), 3)
workflow_runner = WorkflowRunner(workflow,
ert=None,
context=SubstitutionList())
self.assertFalse(workflow_runner.isRunning())
workflow_runner.run()
time.sleep(1) # wait for workflow to start
self.assertTrue(workflow_runner.isRunning())
self.assertFileExists("wait_started_0")
time.sleep(1) # wait for first job to finish
workflow_runner.cancel()
time.sleep(1) # wait for cancel to take effect
self.assertFileExists("wait_finished_0")
self.assertFileExists("wait_started_1")
self.assertFileDoesNotExist("wait_finished_1")
self.assertTrue(workflow_runner.isCancelled())
workflow_runner.wait() # wait for runner to complete
self.assertFileDoesNotExist("wait_started_2")
self.assertFileDoesNotExist("wait_cancelled_2")
self.assertFileDoesNotExist("wait_finished_2")
def test_create(self):
with TestAreaContext("run_context"):
arg = None
init_fs = EnkfFs.createFileSystem("init_fs", EnKFFSType.BLOCK_FS_DRIVER_ID, arg)
result_fs = None
target_fs = None
mask = BoolVector( initial_size = 100 , default_value = True )
runpath_fmt = PathFormat( "path/to/sim%d" )
subst_list = SubstitutionList( )
itr = 0
run_context1 = ErtRunContext( EnkfRunType.ENSEMBLE_EXPERIMENT , init_fs, result_fs, target_fs , mask , runpath_fmt, subst_list , itr )
run_id1 = run_context1.get_id( )
run_arg0 = run_context1[0]
self.assertEqual( run_id1 , run_arg0.get_run_id( ))
run_context2 = ErtRunContext( EnkfRunType.ENSEMBLE_EXPERIMENT , init_fs, result_fs , target_fs, mask , runpath_fmt, subst_list , itr )
run_id2 = run_context2.get_id( )
self.assertFalse( run_id1 == run_id2 )
def test_substitution_list(self):
subst_list = SubstitutionList()
subst_list.addItem("Key", "Value", "Doc String")
self.assertEqual(len(subst_list), 1)
with self.assertRaises(KeyError):
item = subst_list[2]
with self.assertRaises(KeyError):
item = subst_list["NoSuchKey"]
with self.assertRaises(KeyError):
item = subst_list.doc("NoSuchKey")
self.assertTrue("Key" in subst_list)
self.assertEqual(subst_list["Key"], "Value")
self.assertEqual(subst_list.doc("Key"), "Doc String")
subst_list.addItem("Key2", "Value2", "Doc String2")
self.assertEqual(len(subst_list), 2)
keys = subst_list.keys()
self.assertEqual(keys[0], "Key")
self.assertEqual(keys[1], "Key2")
self.assertIn("Key", str(subst_list))
self.assertIn("SubstitutionList", repr(subst_list))
self.assertIn("2", repr(subst_list))
self.assertEqual(1729, subst_list.get("nosuchkey", 1729))
self.assertIsNone(subst_list.get("nosuchkey"))
self.assertIsNone(subst_list.get(513))
for key in ("Key", "Key2"):
self.assertEqual(subst_list[key], subst_list.get(key))
