def _init_res_config(self, config):
if isinstance(config, ResConfig):
return config
# The res_config argument can be None; the only reason to
# allow that possibility is to be able to test that the
# site-config loads correctly.
if config is None:
config = ResConfig(None)
config.convertToCReference(self)
return config
raise TypeError("Expected ResConfig, received: %r" % config)
def build(self, path=None):
if path is None:
path = Path.cwd()
self._build_ert(path)
self._build_job(path)
self._build_observations(path)
self._build_priors(path)
config = ResConfig(str(path / "test.ert"))
enkfmain = EnKFMain(config)
# The C code doesn't do resource counting correctly, so we need to hook
# ResConfig to EnKFMain because otherwise ResConfig will be deleted and
# EnKFMain will use a dangling pointer.
enkfmain.__config = config
return LibresFacade(enkfmain)
def test_num_cpu_subst(monkeypatch, tmp_path, append, numcpu):
"""
Make sure that <NUM_CPU> is substituted to the correct values
"""
monkeypatch.chdir(tmp_path)
(tmp_path / "test.ert").write_text("JOBNAME test_%d\n"
"QUEUE_SYSTEM LOCAL\n"
"QUEUE_OPTION LOCAL MAX_RUNNING 50\n"
"NUM_REALIZATIONS 1\n"
"RUNPATH test/realization-%d/iter-%d\n"
"INSTALL_JOB dump DUMP\n"
"FORWARD_MODEL dump\n" + append)
(tmp_path / "DATA").write_text("PARALLEL 8 /")
(tmp_path / "DUMP").write_text("EXECUTABLE echo\nARGLIST <NUM_CPU>\n")
config = ResConfig(str(tmp_path / "test.ert"))
enkf_main = EnKFMain(config)
run_context = _create_runpath(enkf_main)
_evaluate_ensemble(enkf_main, run_context)
with open("test/realization-0/iter-0/dump.stdout.0") as f:
assert f.read() == f"{numcpu}\n"
def test_tracking(
experiment_folder,
cmd_line_arguments,
num_successful,
num_iters,
tmpdir,
source_root,
):
shutil.copytree(
os.path.join(source_root, "test-data", "local",
f"{experiment_folder}"),
os.path.join(str(tmpdir), f"{experiment_folder}"),
)
config_lines = [
"INSTALL_JOB poly_eval2 POLY_EVAL\n"
"SIMULATION_JOB poly_eval2\n"
]
with tmpdir.as_cwd():
with open(f"{experiment_folder}/poly.ert", "a") as fh:
fh.writelines(config_lines)
parser = ArgumentParser(prog="test_main")
parsed = ert_parser(
parser,
cmd_line_arguments,
)
FeatureToggling.update_from_args(parsed)
res_config = ResConfig(parsed.config)
os.chdir(res_config.config_path)
ert = EnKFMain(res_config, strict=True, verbose=parsed.verbose)
notifier = ErtCliNotifier(ert, parsed.config)
ERT.adapt(notifier)
model, argument = create_model(parsed)
ee_config = None
if FeatureToggling.is_enabled("ensemble-evaluator"):
ee_config = EvaluatorServerConfig()
argument.update({"ee_config": ee_config})
thread = threading.Thread(
name="ert_cli_simulation_thread",
target=model.startSimulations,
args=(argument, ),
)
thread.start()
tracker = create_tracker(model,
general_interval=1,
detailed_interval=2,
ee_config=ee_config)
snapshots = {}
for event in tracker.track():
if isinstance(event, FullSnapshotEvent):
snapshots[event.iteration] = event.snapshot
if isinstance(event, SnapshotUpdateEvent):
if event.partial_snapshot is not None:
snapshots[event.iteration].merge(
event.partial_snapshot.data())
if isinstance(event, EndEvent):
break
assert tracker._progress() == 1.0
assert len(snapshots) == num_iters
for iter_, snapshot in snapshots.items():
successful_reals = list(
filter(
lambda item: item[1].status == REALIZATION_STATE_FINISHED,
snapshot.get_reals().items(),
))
assert len(successful_reals) == num_successful
for real_id, real in successful_reals:
assert (real.status == REALIZATION_STATE_FINISHED
), f"iter:{iter_} real:{real_id} was not finished"
poly = real.steps["0"].jobs["0"]
poly2 = real.steps["0"].jobs["1"]
assert poly.name == "poly_eval"
assert (poly.status == JOB_STATE_FINISHED
), f"real {real_id}/{poly['name']} was not finished"
assert poly2.name == "poly_eval2"
assert (poly2.status == JOB_STATE_FINISHED
), f"real {real_id}/{poly['name']} was not finished"
def test_tracking(
extra_config,
extra_poly_eval,
cmd_line_arguments,
num_successful,
num_iters,
progress,
assert_present_in_snapshot,
tmpdir,
source_root,
):
experiment_folder = "poly_example"
shutil.copytree(
os.path.join(source_root, "test-data", "local", f"{experiment_folder}"),
os.path.join(str(tmpdir), f"{experiment_folder}"),
)
config_lines = [
"INSTALL_JOB poly_eval2 POLY_EVAL\n" "SIMULATION_JOB poly_eval2\n",
extra_config,
]
with tmpdir.as_cwd():
with open(f"{experiment_folder}/poly.ert", "a") as fh:
fh.writelines(config_lines)
with fileinput.input(f"{experiment_folder}/poly_eval.py", inplace=True) as fin:
for line in fin:
if line.strip().startswith("coeffs"):
print(extra_poly_eval)
print(line, end="")
parser = ArgumentParser(prog="test_main")
parsed = ert_parser(
parser,
cmd_line_arguments,
)
FeatureToggling.update_from_args(parsed)
res_config = ResConfig(parsed.config)
os.chdir(res_config.config_path)
ert = EnKFMain(res_config, strict=True, verbose=parsed.verbose)
facade = LibresFacade(ert)
model = create_model(
ert,
facade.get_ensemble_size(),
facade.get_current_case_name(),
parsed,
)
evaluator_server_config = EvaluatorServerConfig(
custom_port_range=range(1024, 65535), custom_host="127.0.0.1"
)
thread = threading.Thread(
name="ert_cli_simulation_thread",
target=model.start_simulations_thread,
args=(evaluator_server_config,),
)
thread.start()
tracker = EvaluatorTracker(
model,
ee_con_info=evaluator_server_config.get_connection_info(),
)
snapshots = {}
for event in tracker.track():
if isinstance(event, FullSnapshotEvent):
snapshots[event.iteration] = event.snapshot
if isinstance(event, SnapshotUpdateEvent):
if event.partial_snapshot is not None:
snapshots[event.iteration].merge(event.partial_snapshot.data())
if isinstance(event, EndEvent):
pass
assert tracker._progress() == progress
assert len(snapshots) == num_iters
for iter_, snapshot in snapshots.items():
successful_reals = list(
filter(
lambda item: item[1].status == REALIZATION_STATE_FINISHED,
snapshot.reals.items(),
)
)
assert len(successful_reals) == num_successful
for (
iter_expression,
snapshot_expression,
expected,
) in assert_present_in_snapshot:
for i, snapshot in snapshots.items():
if re.match(iter_expression, str(i)):
check_expression(
snapshot.to_dict(),
snapshot_expression,
expected,
f"Snapshot {i} did not match:\n",
)
thread.join()
FeatureToggling.reset()
def test_tracking(
experiment_folder,
cmd_line_arguments,
num_successful,
num_iters,
assert_present_in_snapshot,
tmpdir,
source_root,
):
shutil.copytree(
os.path.join(source_root, "test-data", "local",
f"{experiment_folder}"),
os.path.join(str(tmpdir), f"{experiment_folder}"),
)
config_lines = [
"INSTALL_JOB poly_eval2 POLY_EVAL\n"
"SIMULATION_JOB poly_eval2\n"
]
with tmpdir.as_cwd():
with open(f"{experiment_folder}/poly.ert", "a") as fh:
fh.writelines(config_lines)
parser = ArgumentParser(prog="test_main")
parsed = ert_parser(
parser,
cmd_line_arguments,
)
FeatureToggling.update_from_args(parsed)
res_config = ResConfig(parsed.config)
os.chdir(res_config.config_path)
ert = EnKFMain(res_config, strict=True, verbose=parsed.verbose)
notifier = ErtCliNotifier(ert, parsed.config)
ERT.adapt(notifier)
model, argument = create_model(parsed)
ee_config = None
if FeatureToggling.is_enabled("ensemble-evaluator"):
ee_config = EvaluatorServerConfig()
argument.update({"ee_config": ee_config})
thread = threading.Thread(
name="ert_cli_simulation_thread",
target=model.start_simulations_thread,
args=(argument, ),
)
thread.start()
tracker = create_tracker(model,
general_interval=1,
detailed_interval=2,
ee_config=ee_config)
snapshots = {}
for event in tracker.track():
if isinstance(event, FullSnapshotEvent):
snapshots[event.iteration] = event.snapshot
if isinstance(event, SnapshotUpdateEvent):
if event.partial_snapshot is not None:
snapshots[event.iteration].merge(
event.partial_snapshot.data())
if isinstance(event, EndEvent):
pass
assert tracker._progress() == 1.0
assert len(snapshots) == num_iters
for iter_, snapshot in snapshots.items():
successful_reals = list(
filter(
lambda item: item[1].status == state.
REALIZATION_STATE_FINISHED,
snapshot.get_reals().items(),
))
assert len(successful_reals) == num_successful
for (
iter_expression,
snapshot_expression,
expected,
) in assert_present_in_snapshot:
for i, snapshot in snapshots.items():
if re.match(iter_expression, str(i)):
check_expression(
snapshot.to_dict(),
snapshot_expression,
expected,
f"Snapshot {i} did not match:\n",
)
thread.join()