def test_monitor_failing_evaluation(make_ee_config):
ee_config = make_ee_config(use_token=False, generate_cert=False)
ensemble = TestEnsemble(iter=1, reals=1, steps=1, jobs=1)
ensemble.with_failure()
ee = EnsembleEvaluator(
ensemble,
ee_config,
0,
ee_id="ee-0",
)
ee.run()
with NarrativeProxy(
monitor_failing_evaluation().on_uri(f"ws://localhost:{ee_config.port}")
).proxy(ee_config.url) as port:
with ee_monitor.create("localhost", port, "ws", None, None) as monitor:
for event in monitor.track():
if event["type"] == identifiers.EVTYPE_EE_SNAPSHOT:
ensemble.start()
if (
event.data
and event.data.get(identifiers.STATUS) == ENSEMBLE_STATE_FAILED
):
monitor.signal_done()
ensemble.join()
def test_verify_dispatch_failing_job(make_ee_config):
ee_config = make_ee_config(use_token=False, generate_cert=False)
mock_ensemble = MagicMock()
mock_ensemble.snapshot.to_dict.return_value = {}
ee = EnsembleEvaluator(
mock_ensemble,
ee_config,
0,
ee_id="0",
)
ee.run()
dispatch_failing_job.verify(ee_config.client_uri, on_connect=lambda: None)
ee.stop()
def test_verify_monitor_failing_ensemble(make_ee_config):
ee_config = make_ee_config(use_token=False, generate_cert=False)
ensemble = TestEnsemble(iter=1, reals=2, steps=1, jobs=2)
ensemble.addFailJob(real=1, step=0, job=1)
ee = EnsembleEvaluator(
ensemble,
ee_config,
0,
ee_id="0",
)
ee.run()
monitor_failing_ensemble.verify(ee_config.client_uri,
on_connect=ensemble.start)
ensemble.join()
def test_verify_monitor_failing_evaluation(make_ee_config):
ee_config = make_ee_config(use_token=False, generate_cert=False)
ensemble = TestEnsemble(iter=1, reals=2, steps=1, jobs=2)
ensemble.with_failure()
ee = EnsembleEvaluator(
ensemble,
ee_config,
0,
ee_id="ee-0",
)
ee.run()
monitor_failing_evaluation.verify(ee_config.client_uri,
on_connect=ensemble.start)
ensemble.join()
def test_run_prefect_ensemble_with_path(unused_tcp_port):
with tmp(os.path.join(SOURCE_DIR, "test-data/local/prefect_test_case")):
config = parse_config("config.yml")
config.update({"config_path": Path.cwd()})
config.update({"realizations": 2})
config.update({"executor": "local"})
config["config_path"] = Path(config["config_path"])
config["run_path"] = Path(config["run_path"])
config["storage"]["storage_path"] = Path(
config["storage"]["storage_path"])
service_config = EvaluatorServerConfig(unused_tcp_port)
ensemble = PrefectEnsemble(config)
evaluator = EnsembleEvaluator(ensemble, service_config, ee_id="1")
mon = evaluator.run()
for event in mon.track():
if event.data is not None and event.data.get("status") in [
"Failed",
"Stopped",
]:
mon.signal_done()
assert evaluator._snapshot.get_status() == "Stopped"
successful_realizations = evaluator._snapshot.get_successful_realizations(
)
assert successful_realizations == config["realizations"]
def test_run_prefect_ensemble_exception(unused_tcp_port):
with tmp(os.path.join(SOURCE_DIR, "test-data/local/prefect_test_case")):
config = parse_config("config.yml")
config.update({"config_path": Path.absolute(Path("."))})
config.update({"realizations": 2})
config.update({"executor": "local"})
service_config = EvaluatorServerConfig(unused_tcp_port)
ensemble = PrefectEnsemble(config)
evaluator = EnsembleEvaluator(ensemble, service_config, ee_id="1")
with patch.object(ensemble,
"_fetch_input_files",
side_effect=RuntimeError()):
mon = evaluator.run()
for event in mon.track():
if event["type"] in (
ids.EVTYPE_EE_SNAPSHOT_UPDATE,
ids.EVTYPE_EE_SNAPSHOT,
) and event.data.get("status") in [
"Stopped",
"Failed",
]:
mon.signal_done()
assert evaluator._snapshot.get_status() == "Failed"
def _run(
ensemble_evaluator: EnsembleEvaluator,
) -> Dict[int, Dict[str, ert.data.RecordTransmitter]]:
result: Dict[int, Dict[str, ert.data.RecordTransmitter]] = {}
with ensemble_evaluator.run() as monitor:
realization_ids = set()
realizations_completed = set()
for event in monitor.track():
if isinstance(event.data, dict) and event.data.get("status") in [
ENSEMBLE_STATE_STOPPED,
ENSEMBLE_STATE_FAILED,
]:
monitor.signal_done()
if event.data.get("status") == ENSEMBLE_STATE_FAILED:
raise RuntimeError("Ensemble evaluation failed")
if event["type"] == EVTYPE_EE_TERMINATED and isinstance(
event.data, bytes):
result = pickle.loads(event.data)
if isinstance(event.data, dict) and "reals" in event.data:
for real_id in event.data["reals"]:
realization_ids.add(real_id)
real_status = event.data["reals"][real_id].get("status")
if (real_status == REALIZATION_STATE_FINISHED
and real_id not in realizations_completed):
realizations_completed.add(real_id)
print(
f"Realization {real_id} completed successfully"
f" ({len(realizations_completed)}/{len(realization_ids)})"
)
return result
def test_run_prefect_for_function_defined_outside_py_environment(
evaluator_config,
coefficients,
function_ensemble_builder_factory,
ensemble_size,
external_sum_function,
):
"""Ensemble built from outside env. Assert state, realizations and result"""
# Build ensemble and run on server
ensemble = (function_ensemble_builder_factory(
external_sum_function).set_retry_delay(1).set_max_retries(0).build())
evaluator = EnsembleEvaluator(ensemble, evaluator_config, 0, ee_id="1")
with evaluator.run() as mon:
for event in mon.track():
if event["type"] == ids.EVTYPE_EE_TERMINATED:
results = pickle.loads(event.data)
wait_until_done(mon, event)
assert evaluator._ensemble.status == state.ENSEMBLE_STATE_STOPPED
successful_realizations = evaluator._ensemble.get_successful_realizations()
assert successful_realizations == ensemble_size
expected_results = [
pickle.loads(external_sum_function)(coeffs)["function_output"]
for coeffs in coefficients
]
transmitter_futures = [
res["function_output"].load() for res in results.values()
]
results = get_event_loop().run_until_complete(
asyncio.gather(*transmitter_futures))
assert expected_results == [res.data for res in results]
def test_prefect_no_retries(evaluator_config,
function_ensemble_builder_factory, tmpdir):
"""Evaluator tries and fails once. Asserts if job and step fails"""
cloudpickle.register_pickle_by_value(sys.modules[__name__])
pickle_func = cloudpickle.dumps(function_that_fails_once)
cloudpickle.unregister_pickle_by_value(sys.modules[__name__])
ensemble = (function_ensemble_builder_factory(pickle_func).set_retry_delay(
1).set_max_retries(0).build())
evaluator = EnsembleEvaluator(ensemble, evaluator_config, 0, ee_id="1")
with tmpdir.as_cwd():
# Get events
event_list = []
with evaluator.run() as mon:
for event in mon.track():
event_list.append(event)
wait_until_done(mon, event)
# Find if job and step failed
step_failed = False
job_failed = False
for real in ensemble.snapshot.reals.values():
for step in real.steps.values():
for job in step.jobs.values():
if job.status == state.JOB_STATE_FAILURE:
job_failed = True
assert job.error == "This is an expected ERROR"
if step.status == state.STEP_STATE_FAILURE:
step_failed = True
assert ensemble.status == state.ENSEMBLE_STATE_FAILED
assert job_failed, f"Events: {event_list}"
assert step_failed, f"Events: {event_list}"
def test_prefect_retries(evaluator_config, function_ensemble_builder_factory,
tmpdir, ensemble_size):
"""Evaluator fails once through pickled-fail-function. Asserts fail and retries"""
cloudpickle.register_pickle_by_value(sys.modules[__name__])
pickle_func = cloudpickle.dumps(function_that_fails_once)
cloudpickle.unregister_pickle_by_value(sys.modules[__name__])
ensemble = function_ensemble_builder_factory(pickle_func).set_retry_delay(
2).build()
evaluator = EnsembleEvaluator(ensemble, evaluator_config, 0, ee_id="1")
with tmpdir.as_cwd():
error_event_reals: Set[str] = set()
with evaluator.run() as mon:
# close_events_in_ensemble_run(monitor=mon) # more strict as above
for event in mon.track():
# Capture the job error messages
if event.data is not None and "This is an expected ERROR" in str(
event.data):
error_event_reals.update(event.data["reals"].keys())
wait_until_done(mon, event)
assert evaluator._ensemble.status == state.ENSEMBLE_STATE_STOPPED
successful_realizations = evaluator._ensemble.get_successful_realizations(
)
assert successful_realizations == ensemble_size
# Check we get only one job error message per realization
assert len(error_event_reals) == ensemble_size
assert "0" in error_event_reals
assert "1" in error_event_reals
def test_run_legacy_ensemble_exception(tmpdir, make_ensemble_builder):
num_reals = 2
custom_port_range = range(1024, 65535)
with tmpdir.as_cwd():
ensemble = make_ensemble_builder(tmpdir, num_reals, 2).build()
config = EvaluatorServerConfig(custom_port_range=custom_port_range,
custom_host="127.0.0.1")
evaluator = EnsembleEvaluator(ensemble, config, 0, ee_id="1")
with patch.object(ensemble._job_queue,
"submit_complete") as faulty_queue:
faulty_queue.side_effect = RuntimeError()
with evaluator.run() as monitor:
for e in monitor.track():
if e.data is not None and e.data.get(
identifiers.STATUS) in [
state.ENSEMBLE_STATE_FAILED,
state.ENSEMBLE_STATE_STOPPED,
]:
monitor.signal_done()
assert evaluator._ensemble.status == state.ENSEMBLE_STATE_FAILED
# realisations should not finish, thus not creating a status-file
for i in range(num_reals):
assert not os.path.isfile(f"real_{i}/status.txt")
def test_verify_monitor_successful_ensemble(make_ee_config, event_loop):
ensemble = TestEnsemble(iter=1, reals=2, steps=2, jobs=2).with_result(
b"\x80\x04\x95\x0f\x00\x00\x00\x00\x00\x00\x00\x8c\x0bhello world\x94.",
"application/octet-stream",
)
ee_config = make_ee_config(use_token=False, generate_cert=False)
ee = EnsembleEvaluator(
ensemble,
ee_config,
0,
ee_id="ee-0",
)
ee.run()
event_loop.run_until_complete(wait_for_evaluator(ee_config.url))
monitor_successful_ensemble().verify(ee_config.client_uri,
on_connect=ensemble.start)
ensemble.join()
def test_prefect_no_retries(unused_tcp_port, coefficients, tmpdir,
function_config):
def function_that_fails_once(coeffs):
run_path = Path("ran_once")
if not run_path.exists():
run_path.touch()
raise RuntimeError("This is an expected ERROR")
run_path.unlink()
return []
with tmpdir.as_cwd():
pickle_func = cloudpickle.dumps(function_that_fails_once)
config = function_config
coeffs_trans = coefficient_transmitters(
coefficients,
config.get(ids.STORAGE)["storage_path"])
service_config = EvaluatorServerConfig(unused_tcp_port)
config["realizations"] = len(coefficients)
config["executor"] = "local"
config["max_retries"] = 0
config["retry_delay"] = 1
config["steps"][0]["jobs"][0]["executable"] = pickle_func
config["inputs"] = {
iens: coeffs_trans[iens]
for iens in range(len(coefficients))
}
config["outputs"] = output_transmitters(config)
config["dispatch_uri"] = service_config.dispatch_uri
ensemble = PrefectEnsemble(config)
evaluator = EnsembleEvaluator(ensemble, service_config, 0, ee_id="1")
step_failed = False
job_failed = False
with evaluator.run() as mon:
for event in mon.track():
# Capture the job error messages
if event.data is not None and "This is an expected ERROR" in str(
event.data):
for real in event.data["reals"].values():
for step in real["steps"].values():
for job in step["jobs"].values():
if job["status"] == "Failed":
job_failed = True
if step["status"] == "Failed":
step_failed = True
if event.data is not None and event.data.get("status") in [
"Failed",
"Stopped",
]:
mon.signal_done()
assert evaluator._ensemble.get_status() == "Failed"
assert job_failed
assert step_failed
def test_prefect_no_retries(unused_tcp_port, coefficients, tmpdir, function_config):
def function_that_fails_once(coeffs):
run_path = Path("ran_once")
if not run_path.exists():
run_path.touch()
raise RuntimeError("This is an expected ERROR")
run_path.unlink()
return []
with tmpdir.as_cwd():
pickle_func = cloudpickle.dumps(function_that_fails_once)
config = function_config
coeffs_trans = coefficient_transmitters(
coefficients, config.get(ids.STORAGE)["storage_path"]
)
service_config = EvaluatorServerConfig(unused_tcp_port)
config["realizations"] = len(coefficients)
config["executor"] = "local"
config["max_retries"] = 0
config["retry_delay"] = 1
config["steps"][0]["jobs"][0]["executable"] = pickle_func
config["inputs"] = {
iens: coeffs_trans[iens] for iens in range(len(coefficients))
}
config["outputs"] = output_transmitters(config)
config["dispatch_uri"] = service_config.dispatch_uri
ensemble = PrefectEnsemble(config)
evaluator = EnsembleEvaluator(ensemble, service_config, 0, ee_id="1")
event_list = []
with evaluator.run() as mon:
for event in mon.track():
event_list.append(event)
if event.data is not None and event.data.get("status") in [
state.ENSEMBLE_STATE_FAILED,
state.ENSEMBLE_STATE_STOPPED,
]:
mon.signal_done()
step_failed = False
job_failed = False
for real in ensemble.snapshot.get_reals().values():
for step in real.steps.values():
for job in step.jobs.values():
if job.status == state.JOB_STATE_FAILURE:
job_failed = True
assert job.error == "This is an expected ERROR"
if step.status == state.STEP_STATE_FAILURE:
step_failed = True
assert ensemble.get_status() == state.ENSEMBLE_STATE_FAILED
assert job_failed, f"Events: {event_list}"
assert step_failed, f"Events: {event_list}"
def test_run_prefect_ensemble_exception(evaluator_config, poly_ensemble):
"""Test prefect on flow with runtime-error"""
poly_ensemble.get_flow = dummy_get_flow
evaluator = EnsembleEvaluator(poly_ensemble,
evaluator_config,
0,
ee_id="1")
with evaluator.run() as mon:
for event in mon.track():
wait_until_done(mon, event)
assert evaluator._ensemble.status == state.ENSEMBLE_STATE_FAILED
def test_run_prefect_ensemble(evaluator_config, poly_ensemble, ensemble_size):
"""Test successful realizations from prefect-run equals ensemble-size"""
evaluator = EnsembleEvaluator(poly_ensemble,
evaluator_config,
0,
ee_id="1")
with evaluator.run() as mon:
for event in mon.track():
wait_until_done(mon, event)
assert evaluator._ensemble.status == state.ENSEMBLE_STATE_STOPPED
successful_realizations = evaluator._ensemble.get_successful_realizations()
assert successful_realizations == ensemble_size
def test_prefect_reties(unused_tcp_port, coefficients, tmpdir,
function_config):
def function_that_fails_once(coeffs):
run_path = Path("ran_once")
if not run_path.exists():
run_path.touch()
raise RuntimeError("This is an expected ERROR")
run_path.unlink()
return []
with tmpdir.as_cwd():
pickle_func = cloudpickle.dumps(function_that_fails_once)
config = function_config
coeffs_trans = coefficient_transmitters(
coefficients,
config.get(ids.STORAGE)["storage_path"])
service_config = EvaluatorServerConfig(unused_tcp_port)
config["realizations"] = len(coefficients)
config["executor"] = "local"
config["max_retries"] = 2
config["retry_delay"] = 1
config["steps"][0]["jobs"][0]["executable"] = pickle_func
config["inputs"] = {
iens: coeffs_trans[iens]
for iens in range(len(coefficients))
}
config["outputs"] = output_transmitters(config)
config["dispatch_uri"] = service_config.dispatch_uri
ensemble = PrefectEnsemble(config)
evaluator = EnsembleEvaluator(ensemble, service_config, 0, ee_id="1")
error_event_reals = []
with evaluator.run() as mon:
for event in mon.track():
# Caputure the job error messages
if event.data is not None and "This is an expected ERROR" in str(
event.data):
error_event_reals.append(event.data["reals"])
if event.data is not None and event.data.get("status") in [
"Failed",
"Stopped",
]:
mon.signal_done()
assert evaluator._snapshot.get_status() == "Stopped"
successful_realizations = evaluator._snapshot.get_successful_realizations(
)
assert successful_realizations == config["realizations"]
# Check we get only one job error message per realization
assert len(error_event_reals) == config["realizations"]
for idx, reals in enumerate(error_event_reals):
assert len(reals) == 1
assert str(idx) in reals
def test_monitor_successful_ensemble(make_ee_config):
ensemble = TestEnsemble(iter=1, reals=2, steps=2, jobs=2)
ensemble.addFailJob(real=1, step=0, job=1)
ee_config = make_ee_config(use_token=False, generate_cert=False)
ee = EnsembleEvaluator(
ensemble,
ee_config,
0,
ee_id="ee-0",
)
ee.run()
with NarrativeProxy(monitor_successful_ensemble()).proxy(ee_config.url):
with ee_monitor.create(ee_config.get_connection_info()) as monitor:
for event in monitor.track():
if event["type"] == identifiers.EVTYPE_EE_SNAPSHOT:
ensemble.start()
if (event.data and event.data.get(identifiers.STATUS)
== ENSEMBLE_STATE_STOPPED):
monitor.signal_done()
ensemble.join()
def test_cancel_run_prefect_ensemble(evaluator_config, poly_ensemble):
"""Test cancellation of prefect-run"""
evaluator = EnsembleEvaluator(poly_ensemble,
evaluator_config,
0,
ee_id="1")
with evaluator.run() as mon:
cancel = True
for _ in mon.track():
if cancel:
mon.signal_cancel()
cancel = False
assert evaluator._ensemble.status == state.ENSEMBLE_STATE_CANCELLED
def _run(
ensemble_evaluator: EnsembleEvaluator,
) -> Dict[int, Dict[str, RecordTransmitter]]:
result = {}
with ensemble_evaluator.run() as monitor:
for event in monitor.track():
if event.data is not None and event.data.get("status") in [
_EVTYPE_SNAPSHOT_STOPPED,
_EVTYPE_SNAPSHOT_FAILED,
]:
if event.data.get("status") == _EVTYPE_SNAPSHOT_STOPPED:
result = monitor.get_result()
monitor.signal_done()
return result
def test_run_legacy_ensemble(tmpdir, unused_tcp_port, make_ensemble_builder):
num_reals = 2
with tmpdir.as_cwd():
ensemble = make_ensemble_builder(tmpdir, num_reals, 2).build()
config = EvaluatorServerConfig(unused_tcp_port)
evaluator = EnsembleEvaluator(ensemble, config, ee_id="1")
monitor = evaluator.run()
for e in monitor.track():
if (e["type"] in (
identifiers.EVTYPE_EE_SNAPSHOT_UPDATE,
identifiers.EVTYPE_EE_SNAPSHOT,
) and e.data.get("status") in ["Failed", "Stopped"]):
monitor.signal_done()
assert evaluator._snapshot.get_status() == "Stopped"
assert evaluator.get_successful_realizations() == num_reals
def _run(
ensemble_evaluator: EnsembleEvaluator,
) -> Dict[int, Dict[str, RecordTransmitter]]:
result = {}
with ensemble_evaluator.run() as monitor:
for event in monitor.track():
if isinstance(event.data, dict) and event.data.get("status") in [
_EVTYPE_SNAPSHOT_STOPPED,
_EVTYPE_SNAPSHOT_FAILED,
]:
monitor.signal_done()
if event["type"] == EVTYPE_EE_TERMINATED and isinstance(event.data, bytes):
result = pickle.loads(event.data)
return result
def test_run_legacy_ensemble_exception(tmpdir, unused_tcp_port, make_ensemble_builder):
num_reals = 2
with tmpdir.as_cwd():
ensemble = make_ensemble_builder(tmpdir, num_reals, 2).build()
config = EvaluatorServerConfig(unused_tcp_port)
evaluator = EnsembleEvaluator(ensemble, config, 0, ee_id="1")
with patch.object(ensemble, "get_active_reals", side_effect=RuntimeError()):
with evaluator.run() as monitor:
for e in monitor.track():
if e.data is not None and e.data.get(identifiers.STATUS) in [
state.ENSEMBLE_STATE_FAILED,
state.ENSEMBLE_STATE_STOPPED,
]:
monitor.signal_done()
assert evaluator._ensemble.get_status() == state.ENSEMBLE_STATE_FAILED
def test_run_and_cancel_legacy_ensemble(tmpdir, unused_tcp_port, make_ensemble_builder):
num_reals = 10
with tmpdir.as_cwd():
ensemble = make_ensemble_builder(tmpdir, num_reals, 2).build()
config = EvaluatorServerConfig(unused_tcp_port)
evaluator = EnsembleEvaluator(ensemble, config, 0, ee_id="1")
with evaluator.run() as mon:
cancel = True
for _ in mon.track():
if cancel:
mon.signal_cancel()
cancel = False
assert evaluator._ensemble.get_status() == state.ENSEMBLE_STATE_CANCELLED
def test_run_and_cancel_legacy_ensemble(tmpdir, unused_tcp_port,
make_ensemble_builder):
num_reals = 10
with tmpdir.as_cwd():
ensemble = make_ensemble_builder(tmpdir, num_reals, 2).build()
config = EvaluatorServerConfig(unused_tcp_port)
evaluator = EnsembleEvaluator(ensemble, config, ee_id="1")
mon = evaluator.run()
cancel = True
for _ in mon.track():
if cancel:
mon.signal_cancel()
cancel = False
assert evaluator._snapshot.get_status() == "Cancelled"
def test_run_prefect_ensemble_with_path(unused_tcp_port, coefficients):
with tmp(os.path.join(SOURCE_DIR, "test-data/local/prefect_test_case")):
config = parse_config("config.yml")
config.update(
{
"config_path": os.getcwd(),
"realizations": 2,
"executor": "local",
}
)
inputs = {}
coeffs_trans = coefficient_transmitters(
coefficients, config.get(ids.STORAGE)["storage_path"]
)
script_trans = script_transmitters(config)
for iens in range(2):
inputs[iens] = {**coeffs_trans[iens], **script_trans[iens]}
config.update(
{
"inputs": inputs,
"outputs": output_transmitters(config),
}
)
service_config = EvaluatorServerConfig(unused_tcp_port)
config["config_path"] = Path(config["config_path"])
config["run_path"] = Path(config["run_path"])
config["storage"]["storage_path"] = Path(config["storage"]["storage_path"])
config["dispatch_uri"] = service_config.dispatch_uri
ensemble = PrefectEnsemble(config)
evaluator = EnsembleEvaluator(ensemble, service_config, 0, ee_id="1")
with evaluator.run() as mon:
for event in mon.track():
if isinstance(event.data, dict) and event.data.get("status") in [
"Failed",
"Stopped",
]:
mon.signal_done()
assert evaluator._ensemble.get_status() == "Stopped"
successful_realizations = evaluator._ensemble.get_successful_realizations()
assert successful_realizations == config["realizations"]
def test_run_legacy_ensemble(tmpdir, unused_tcp_port, make_ensemble_builder):
num_reals = 2
with tmpdir.as_cwd():
ensemble = make_ensemble_builder(tmpdir, num_reals, 2).build()
config = EvaluatorServerConfig(unused_tcp_port)
evaluator = EnsembleEvaluator(ensemble, config, 0, ee_id="1")
with evaluator.run() as monitor:
for e in monitor.track():
if (e["type"] in (
identifiers.EVTYPE_EE_SNAPSHOT_UPDATE,
identifiers.EVTYPE_EE_SNAPSHOT,
) and e.data.get(identifiers.STATUS) in [
state.ENSEMBLE_STATE_FAILED,
state.ENSEMBLE_STATE_STOPPED
]):
monitor.signal_done()
assert evaluator._ensemble.get_status() == state.ENSEMBLE_STATE_STOPPED
assert evaluator._ensemble.get_successful_realizations() == num_reals
def test_run_legacy_ensemble(tmpdir, unused_tcp_port, make_ensemble_builder):
num_reals = 2
conf_file = Path(tmpdir / CONFIG_FILE)
with tmpdir.as_cwd():
with open(conf_file, "w") as f:
f.write(f'port: "{unused_tcp_port}"\n')
ensemble = make_ensemble_builder(tmpdir, num_reals, 2).build()
config = load_config(conf_file)
evaluator = EnsembleEvaluator(ensemble, config, ee_id="1")
monitor = evaluator.run()
for e in monitor.track():
if (e["type"] in (
identifiers.EVTYPE_EE_SNAPSHOT_UPDATE,
identifiers.EVTYPE_EE_SNAPSHOT,
) and e.data.get("status") == "Stopped"):
monitor.signal_done()
assert evaluator.get_successful_realizations() == num_reals
def test_run_legacy_ensemble_exception(tmpdir, unused_tcp_port,
make_ensemble_builder):
num_reals = 2
with tmpdir.as_cwd():
ensemble = make_ensemble_builder(tmpdir, num_reals, 2).build()
config = EvaluatorServerConfig(unused_tcp_port)
evaluator = EnsembleEvaluator(ensemble, config, ee_id="1")
with patch.object(ensemble,
"_run_path_list",
side_effect=RuntimeError()):
monitor = evaluator.run()
for e in monitor.track():
if (e["type"] in (
identifiers.EVTYPE_EE_SNAPSHOT_UPDATE,
identifiers.EVTYPE_EE_SNAPSHOT,
) and e.data.get("status") in ["Failed", "Stopped"]):
monitor.signal_done()
assert evaluator._snapshot.get_status() == "Failed"
