def test_load_forward_model(self):
with self.assertRaises(IOError):
job = ExtJob("CONFIG_FILE", True)
with TestAreaContext("python/job_queue/forward_model1"):
create_valid_config("CONFIG")
job = ExtJob("CONFIG", True)
self.assertEqual(job.name(), "CONFIG")
self.assertEqual(job.get_stdout_file(), None)
self.assertEqual(job.get_stderr_file(), None)
self.assertEqual(job.get_executable(),
os.path.join(os.getcwd(), "script.sh"))
self.assertTrue(os.access(job.get_executable(), os.X_OK))
self.assertEqual(job.min_arg, -1)
job = ExtJob("CONFIG", True, name="Job")
self.assertEqual(job.name(), "Job")
pfx = "ExtJob("
self.assertEqual(pfx, repr(job)[:len(pfx)])
with TestAreaContext("python/job_queue/forward_model1a"):
create_upgraded_valid_config("CONFIG")
job = ExtJob("CONFIG", True)
self.assertEqual(job.min_arg, 2)
self.assertEqual(job.max_arg, 7)
argTypes = job.arg_types
self.assertEqual(
argTypes,
[
ContentTypeEnum.CONFIG_INT,
ContentTypeEnum.CONFIG_FLOAT,
ContentTypeEnum.CONFIG_STRING,
ContentTypeEnum.CONFIG_BOOL,
ContentTypeEnum.CONFIG_RUNTIME_FILE,
ContentTypeEnum.CONFIG_RUNTIME_INT,
ContentTypeEnum.CONFIG_STRING,
],
)
with TestAreaContext("python/job_queue/forward_model2"):
create_config_missing_executable("CONFIG")
with self.assertRaises(ValueError):
job = ExtJob("CONFIG", True)
with TestAreaContext("python/job_queue/forward_model3"):
create_config_missing_EXECUTABLE("CONFIG")
with self.assertRaises(ValueError):
job = ExtJob("CONFIG", True)
with TestAreaContext("python/job_queue/forward_model4"):
create_config_executable_directory("CONFIG")
with self.assertRaises(ValueError):
job = ExtJob("CONFIG", True)
with TestAreaContext("python/job_queue/forward_model5"):
create_config_foreign_file("CONFIG")
with self.assertRaises(ValueError):
job = ExtJob("CONFIG", True)
def _generate_job(name, executable, target_file, error_file, start_file,
stdout, stderr, stdin, environment, arglist,
max_running_minutes, max_running, license_root_path,
private):
config_file = (DEFAULT_NAME if name is None else name)
environment = (None if environment is None else environment.keys() +
environment.values())
values = [
str_none_sensitive(max_running), stdin, stdout, stderr, executable,
target_file, error_file, start_file,
None if arglist is None else " ".join(arglist),
None if environment is None else " ".join(environment),
str_none_sensitive(max_running_minutes)
]
conf = open(config_file, "w")
for key, val in zip(ext_job_keywords, values):
if val is not None:
conf.write("%s %s\n" % (key, val))
conf.close()
exec_file = open(executable, "w")
exec_file.close()
ext_job = ExtJob(config_file, private, name, license_root_path)
os.unlink(config_file)
os.unlink(executable)
return ext_job
def test_transfer_arg_types(self):
with TestAreaContext(
"python/job_queue/forward_model_transfer_arg_types"):
with open("FWD_MODEL", "w") as f:
f.write("EXECUTABLE ls\n")
f.write("MIN_ARG 2\n")
f.write("MAX_ARG 6\n")
f.write("ARG_TYPE 0 INT\n")
f.write("ARG_TYPE 1 FLOAT\n")
f.write("ARG_TYPE 2 STRING\n")
f.write("ARG_TYPE 3 BOOL\n")
f.write("ARG_TYPE 4 RUNTIME_FILE\n")
f.write("ARG_TYPE 5 RUNTIME_INT\n")
job = ExtJob("FWD_MODEL", True)
ext_joblist = ExtJoblist()
ext_joblist.add_job(job.name(), job)
forward_model = ForwardModel(ext_joblist)
forward_model.add_job("FWD_MODEL")
run_id = "test_no_jobs_id"
umask = 4
global_args = SubstitutionList()
forward_model.formatted_fprintf(run_id, os.getcwd(), "data_root",
global_args, umask,
EnvironmentVarlist())
config = load_configs(self.JOBS_JSON_FILE)
printed_job = config["jobList"][0]
self.assertEqual(printed_job["min_arg"], 2)
self.assertEqual(printed_job["max_arg"], 6)
self.assertEqual(printed_job["arg_types"], [
"INT", "FLOAT", "STRING", "BOOL", "RUNTIME_FILE", "RUNTIME_INT"
])
def test_load_forward_model(self):
with self.assertRaises(IOError):
job = ExtJob("CONFIG_FILE", True)
with TestAreaContext("python/job_queue/forward_model1"):
create_valid_config("CONFIG")
job = ExtJob("CONFIG", True)
self.assertEqual(job.name(), "CONFIG")
self.assertEqual(job.get_executable(),
os.path.join(os.getcwd(), "script.sh"))
self.assertTrue(os.access(job.get_executable(), os.X_OK))
job = ExtJob("CONFIG", True, name="Job")
self.assertEqual(job.name(), "Job")
pfx = 'ExtJob('
self.assertEqual(pfx, repr(job)[:len(pfx)])
with TestAreaContext("python/job_queue/forward_model2"):
create_config_missing_executable("CONFIG")
with self.assertRaises(ValueError):
job = ExtJob("CONFIG", True)
with TestAreaContext("python/job_queue/forward_model3"):
create_config_missing_EXECUTABLE("CONFIG")
with self.assertRaises(ValueError):
job = ExtJob("CONFIG", True)
with TestAreaContext("python/job_queue/forward_model4"):
create_config_executable_directory("CONFIG")
with self.assertRaises(ValueError):
job = ExtJob("CONFIG", True)
with TestAreaContext("python/job_queue/forward_model5"):
create_config_foreign_file("CONFIG")
with self.assertRaises(ValueError):
job = ExtJob("CONFIG", True)
def _make_ensemble_builder(tmpdir, num_reals, num_jobs):
builder = create_ensemble_builder()
with tmpdir.as_cwd():
ext_job_list = []
for job_index in range(0, num_jobs):
ext_job_config = Path(tmpdir) / f"EXT_JOB_{job_index}"
with open(ext_job_config, "w") as f:
f.write(f"EXECUTABLE ext_{job_index}.py\n")
ext_job_exec = Path(tmpdir) / f"ext_{job_index}.py"
with open(ext_job_exec, "w") as f:
f.write(
"#!/usr/bin/env python\n"
'if __name__ == "__main__":\n'
f' print("stdout from {job_index}")\n'
)
ext_job_list.append(
ExtJob(str(ext_job_config), False, name=f"ext_job_{job_index}")
)
for iens in range(0, num_reals):
run_path = Path(tmpdir / f"real_{iens}")
os.mkdir(run_path)
with open(run_path / "jobs.json", "w") as f:
json.dump(
{
"jobList": [
_dump_ext_job(ext_job, index)
for index, ext_job in enumerate(ext_job_list)
],
"umask": "0022",
},
f,
)
step = (
create_step_builder()
.set_id("0")
.set_job_name("job dispatch")
.set_job_script("job_dispatch.py")
.set_max_runtime(10000)
.set_run_arg(Mock(iens=iens))
.set_done_callback(lambda _: True)
.set_exit_callback(lambda _: True)
# the first callback_argument is expected to be a run_arg
# from the run_arg, the queue wants to access the iens prop
.set_callback_arguments([])
.set_run_path(str(run_path))
.set_num_cpu(1)
.set_name("dummy step")
.set_dummy_io()
)
for index, job in enumerate(ext_job_list):
step.add_job(
create_legacy_job_builder()
.set_id(index)
.set_name(f"dummy job {index}")
.set_ext_job(job)
)
builder.add_realization(
create_realization_builder()
.active(True)
.set_iens(iens)
.add_step(step)
)
analysis_config = Mock()
analysis_config.get_stop_long_running = Mock(return_value=False)
ecl_config = Mock()
ecl_config.assert_restart = Mock()
builder.set_legacy_dependencies(
queue_config,
analysis_config,
)
return builder