def test_run_prefect_ensemble_exception(unused_tcp_port):
with tmp(os.path.join(SOURCE_DIR, "test-data/local/prefect_test_case")):
config = parse_config("config.yml")
config.update({"config_path": Path.absolute(Path("."))})
config.update({"realizations": 2})
config.update({"executor": "local"})
service_config = EvaluatorServerConfig(unused_tcp_port)
ensemble = PrefectEnsemble(config)
evaluator = EnsembleEvaluator(ensemble, service_config, ee_id="1")
with patch.object(ensemble,
"_fetch_input_files",
side_effect=RuntimeError()):
mon = evaluator.run()
for event in mon.track():
if event["type"] in (
ids.EVTYPE_EE_SNAPSHOT_UPDATE,
ids.EVTYPE_EE_SNAPSHOT,
) and event.data.get("status") in [
"Stopped",
"Failed",
]:
mon.signal_done()
assert evaluator._snapshot.get_status() == "Failed"
def test_run_prefect_ensemble_with_path(unused_tcp_port):
with tmp(os.path.join(SOURCE_DIR, "test-data/local/prefect_test_case")):
config = parse_config("config.yml")
config.update({"config_path": Path.cwd()})
config.update({"realizations": 2})
config.update({"executor": "local"})
config["config_path"] = Path(config["config_path"])
config["run_path"] = Path(config["run_path"])
config["storage"]["storage_path"] = Path(
config["storage"]["storage_path"])
service_config = EvaluatorServerConfig(unused_tcp_port)
ensemble = PrefectEnsemble(config)
evaluator = EnsembleEvaluator(ensemble, service_config, ee_id="1")
mon = evaluator.run()
for event in mon.track():
if event.data is not None and event.data.get("status") in [
"Failed",
"Stopped",
]:
mon.signal_done()
assert evaluator._snapshot.get_status() == "Stopped"
successful_realizations = evaluator._snapshot.get_successful_realizations(
)
assert successful_realizations == config["realizations"]
def test_prefect_no_retries(evaluator_config,
function_ensemble_builder_factory, tmpdir):
"""Evaluator tries and fails once. Asserts if job and step fails"""
cloudpickle.register_pickle_by_value(sys.modules[__name__])
pickle_func = cloudpickle.dumps(function_that_fails_once)
cloudpickle.unregister_pickle_by_value(sys.modules[__name__])
ensemble = (function_ensemble_builder_factory(pickle_func).set_retry_delay(
1).set_max_retries(0).build())
evaluator = EnsembleEvaluator(ensemble, evaluator_config, 0, ee_id="1")
with tmpdir.as_cwd():
# Get events
event_list = []
with evaluator.run() as mon:
for event in mon.track():
event_list.append(event)
wait_until_done(mon, event)
# Find if job and step failed
step_failed = False
job_failed = False
for real in ensemble.snapshot.reals.values():
for step in real.steps.values():
for job in step.jobs.values():
if job.status == state.JOB_STATE_FAILURE:
job_failed = True
assert job.error == "This is an expected ERROR"
if step.status == state.STEP_STATE_FAILURE:
step_failed = True
assert ensemble.status == state.ENSEMBLE_STATE_FAILED
assert job_failed, f"Events: {event_list}"
assert step_failed, f"Events: {event_list}"
def test_monitor_failing_ensemble(make_ee_config, unused_tcp_port):
ensemble = TestEnsemble(iter=1, reals=2, steps=2, jobs=2)
ensemble.addFailJob(real=1, step=0, job=1)
ee_config = make_ee_config(use_token=False, generate_cert=False)
ee = EnsembleEvaluator(
ensemble,
ee_config,
0,
ee_id="ee-0",
)
with ee.run():
pass
with NarrativeProxy(
monitor_failing_ensemble.on_uri(
f"ws://localhost:{unused_tcp_port}")).proxy(
ee_config.url) as port:
with ee_monitor.create("localhost", port, "ws", None, None) as monitor:
for event in monitor.track():
if event["type"] == identifiers.EVTYPE_EE_SNAPSHOT:
ensemble.start()
if (event.data and event.data.get(identifiers.STATUS)
== ENSEMBLE_STATE_STOPPED):
monitor.signal_done()
ensemble.join()
def test_run_prefect_for_function_defined_outside_py_environment(
evaluator_config,
coefficients,
function_ensemble_builder_factory,
ensemble_size,
external_sum_function,
):
"""Ensemble built from outside env. Assert state, realizations and result"""
# Build ensemble and run on server
ensemble = (function_ensemble_builder_factory(
external_sum_function).set_retry_delay(1).set_max_retries(0).build())
evaluator = EnsembleEvaluator(ensemble, evaluator_config, 0, ee_id="1")
with evaluator.run() as mon:
for event in mon.track():
if event["type"] == ids.EVTYPE_EE_TERMINATED:
results = pickle.loads(event.data)
wait_until_done(mon, event)
assert evaluator._ensemble.status == state.ENSEMBLE_STATE_STOPPED
successful_realizations = evaluator._ensemble.get_successful_realizations()
assert successful_realizations == ensemble_size
expected_results = [
pickle.loads(external_sum_function)(coeffs)["function_output"]
for coeffs in coefficients
]
transmitter_futures = [
res["function_output"].load() for res in results.values()
]
results = get_event_loop().run_until_complete(
asyncio.gather(*transmitter_futures))
assert expected_results == [res.data for res in results]
def test_prefect_retries(evaluator_config, function_ensemble_builder_factory,
tmpdir, ensemble_size):
"""Evaluator fails once through pickled-fail-function. Asserts fail and retries"""
cloudpickle.register_pickle_by_value(sys.modules[__name__])
pickle_func = cloudpickle.dumps(function_that_fails_once)
cloudpickle.unregister_pickle_by_value(sys.modules[__name__])
ensemble = function_ensemble_builder_factory(pickle_func).set_retry_delay(
2).build()
evaluator = EnsembleEvaluator(ensemble, evaluator_config, 0, ee_id="1")
with tmpdir.as_cwd():
error_event_reals: Set[str] = set()
with evaluator.run() as mon:
# close_events_in_ensemble_run(monitor=mon) # more strict as above
for event in mon.track():
# Capture the job error messages
if event.data is not None and "This is an expected ERROR" in str(
event.data):
error_event_reals.update(event.data["reals"].keys())
wait_until_done(mon, event)
assert evaluator._ensemble.status == state.ENSEMBLE_STATE_STOPPED
successful_realizations = evaluator._ensemble.get_successful_realizations(
)
assert successful_realizations == ensemble_size
# Check we get only one job error message per realization
assert len(error_event_reals) == ensemble_size
assert "0" in error_event_reals
assert "1" in error_event_reals
def test_exhaust_retries_in_run_and_get_successful_realizations(
make_ee_config, num_realizations, num_failing):
ee_config = make_ee_config(use_token=False,
generate_cert=False,
custom_host="localhost")
ensemble = AutorunTestEnsemble(iter=1,
reals=num_realizations,
steps=1,
jobs=2)
for i in range(num_failing):
ensemble.addFailJob(real=i, step=0, job=1)
ee = EnsembleEvaluator(ensemble, ee_config, 0, ee_id="0")
with patch.object(
_Monitor,
"track",
side_effect=[get_connection_closed_exception()] * 2 +
[ConnectionRefusedError("Connection error")] +
[dummy_iterator("DUMMY TRACKING ITERATOR")] +
[get_connection_closed_exception()] * 2 +
[ConnectionRefusedError("Connection error")] * 3 +
["DUMMY TRACKING ITERATOR2"
], # This should not be reached, hence we assert call_count == 9
) as mock:
num_successful = ee.run_and_get_successful_realizations()
assert mock.call_count == 9
assert num_successful == num_realizations - num_failing
def test_run_legacy_ensemble_exception(tmpdir, make_ensemble_builder):
num_reals = 2
custom_port_range = range(1024, 65535)
with tmpdir.as_cwd():
ensemble = make_ensemble_builder(tmpdir, num_reals, 2).build()
config = EvaluatorServerConfig(custom_port_range=custom_port_range,
custom_host="127.0.0.1")
evaluator = EnsembleEvaluator(ensemble, config, 0, ee_id="1")
with patch.object(ensemble._job_queue,
"submit_complete") as faulty_queue:
faulty_queue.side_effect = RuntimeError()
with evaluator.run() as monitor:
for e in monitor.track():
if e.data is not None and e.data.get(
identifiers.STATUS) in [
state.ENSEMBLE_STATE_FAILED,
state.ENSEMBLE_STATE_STOPPED,
]:
monitor.signal_done()
assert evaluator._ensemble.status == state.ENSEMBLE_STATE_FAILED
# realisations should not finish, thus not creating a status-file
for i in range(num_reals):
assert not os.path.isfile(f"real_{i}/status.txt")
def test_prefect_no_retries(unused_tcp_port, coefficients, tmpdir,
function_config):
def function_that_fails_once(coeffs):
run_path = Path("ran_once")
if not run_path.exists():
run_path.touch()
raise RuntimeError("This is an expected ERROR")
run_path.unlink()
return []
with tmpdir.as_cwd():
pickle_func = cloudpickle.dumps(function_that_fails_once)
config = function_config
coeffs_trans = coefficient_transmitters(
coefficients,
config.get(ids.STORAGE)["storage_path"])
service_config = EvaluatorServerConfig(unused_tcp_port)
config["realizations"] = len(coefficients)
config["executor"] = "local"
config["max_retries"] = 0
config["retry_delay"] = 1
config["steps"][0]["jobs"][0]["executable"] = pickle_func
config["inputs"] = {
iens: coeffs_trans[iens]
for iens in range(len(coefficients))
}
config["outputs"] = output_transmitters(config)
config["dispatch_uri"] = service_config.dispatch_uri
ensemble = PrefectEnsemble(config)
evaluator = EnsembleEvaluator(ensemble, service_config, 0, ee_id="1")
step_failed = False
job_failed = False
with evaluator.run() as mon:
for event in mon.track():
# Capture the job error messages
if event.data is not None and "This is an expected ERROR" in str(
event.data):
for real in event.data["reals"].values():
for step in real["steps"].values():
for job in step["jobs"].values():
if job["status"] == "Failed":
job_failed = True
if step["status"] == "Failed":
step_failed = True
if event.data is not None and event.data.get("status") in [
"Failed",
"Stopped",
]:
mon.signal_done()
assert evaluator._ensemble.get_status() == "Failed"
assert job_failed
assert step_failed
def evaluator(make_ee_config):
ensemble = TestEnsemble(0, 2, 1, 2)
ee = EnsembleEvaluator(
ensemble,
make_ee_config(),
0,
ee_id="ee-0",
)
yield ee
ee.stop()
def test_prefect_no_retries(unused_tcp_port, coefficients, tmpdir, function_config):
def function_that_fails_once(coeffs):
run_path = Path("ran_once")
if not run_path.exists():
run_path.touch()
raise RuntimeError("This is an expected ERROR")
run_path.unlink()
return []
with tmpdir.as_cwd():
pickle_func = cloudpickle.dumps(function_that_fails_once)
config = function_config
coeffs_trans = coefficient_transmitters(
coefficients, config.get(ids.STORAGE)["storage_path"]
)
service_config = EvaluatorServerConfig(unused_tcp_port)
config["realizations"] = len(coefficients)
config["executor"] = "local"
config["max_retries"] = 0
config["retry_delay"] = 1
config["steps"][0]["jobs"][0]["executable"] = pickle_func
config["inputs"] = {
iens: coeffs_trans[iens] for iens in range(len(coefficients))
}
config["outputs"] = output_transmitters(config)
config["dispatch_uri"] = service_config.dispatch_uri
ensemble = PrefectEnsemble(config)
evaluator = EnsembleEvaluator(ensemble, service_config, 0, ee_id="1")
event_list = []
with evaluator.run() as mon:
for event in mon.track():
event_list.append(event)
if event.data is not None and event.data.get("status") in [
state.ENSEMBLE_STATE_FAILED,
state.ENSEMBLE_STATE_STOPPED,
]:
mon.signal_done()
step_failed = False
job_failed = False
for real in ensemble.snapshot.get_reals().values():
for step in real.steps.values():
for job in step.jobs.values():
if job.status == state.JOB_STATE_FAILURE:
job_failed = True
assert job.error == "This is an expected ERROR"
if step.status == state.STEP_STATE_FAILURE:
step_failed = True
assert ensemble.get_status() == state.ENSEMBLE_STATE_FAILED
assert job_failed, f"Events: {event_list}"
assert step_failed, f"Events: {event_list}"
def test_run_prefect_ensemble_exception(evaluator_config, poly_ensemble):
"""Test prefect on flow with runtime-error"""
poly_ensemble.get_flow = dummy_get_flow
evaluator = EnsembleEvaluator(poly_ensemble,
evaluator_config,
0,
ee_id="1")
with evaluator.run() as mon:
for event in mon.track():
wait_until_done(mon, event)
assert evaluator._ensemble.status == state.ENSEMBLE_STATE_FAILED
def test_run_prefect_ensemble(evaluator_config, poly_ensemble, ensemble_size):
"""Test successful realizations from prefect-run equals ensemble-size"""
evaluator = EnsembleEvaluator(poly_ensemble,
evaluator_config,
0,
ee_id="1")
with evaluator.run() as mon:
for event in mon.track():
wait_until_done(mon, event)
assert evaluator._ensemble.status == state.ENSEMBLE_STATE_STOPPED
successful_realizations = evaluator._ensemble.get_successful_realizations()
assert successful_realizations == ensemble_size
def test_prefect_reties(unused_tcp_port, coefficients, tmpdir,
function_config):
def function_that_fails_once(coeffs):
run_path = Path("ran_once")
if not run_path.exists():
run_path.touch()
raise RuntimeError("This is an expected ERROR")
run_path.unlink()
return []
with tmpdir.as_cwd():
pickle_func = cloudpickle.dumps(function_that_fails_once)
config = function_config
coeffs_trans = coefficient_transmitters(
coefficients,
config.get(ids.STORAGE)["storage_path"])
service_config = EvaluatorServerConfig(unused_tcp_port)
config["realizations"] = len(coefficients)
config["executor"] = "local"
config["max_retries"] = 2
config["retry_delay"] = 1
config["steps"][0]["jobs"][0]["executable"] = pickle_func
config["inputs"] = {
iens: coeffs_trans[iens]
for iens in range(len(coefficients))
}
config["outputs"] = output_transmitters(config)
config["dispatch_uri"] = service_config.dispatch_uri
ensemble = PrefectEnsemble(config)
evaluator = EnsembleEvaluator(ensemble, service_config, 0, ee_id="1")
error_event_reals = []
with evaluator.run() as mon:
for event in mon.track():
# Caputure the job error messages
if event.data is not None and "This is an expected ERROR" in str(
event.data):
error_event_reals.append(event.data["reals"])
if event.data is not None and event.data.get("status") in [
"Failed",
"Stopped",
]:
mon.signal_done()
assert evaluator._snapshot.get_status() == "Stopped"
successful_realizations = evaluator._snapshot.get_successful_realizations(
)
assert successful_realizations == config["realizations"]
# Check we get only one job error message per realization
assert len(error_event_reals) == config["realizations"]
for idx, reals in enumerate(error_event_reals):
assert len(reals) == 1
assert str(idx) in reals
def evaluator(ee_config):
ensemble = (create_ensemble_builder().add_realization(
real=create_realization_builder().active(True).set_iens(0).add_stage(
stage=create_stage_builder().add_step(
step=create_step_builder().set_id(0).add_job(
job=create_legacy_job_builder().set_id(0).set_name(
"cat").set_ext_job(Mock())).add_job(
job=create_legacy_job_builder().set_id(1).set_name(
"cat2").set_ext_job(Mock())).set_dummy_io()).
set_id(0).set_status("Unknown"))).set_ensemble_size(2).build())
ee = EnsembleEvaluator(ensemble=ensemble, config=ee_config)
yield ee
ee.stop()
def test_cancel_run_prefect_ensemble(evaluator_config, poly_ensemble):
"""Test cancellation of prefect-run"""
evaluator = EnsembleEvaluator(poly_ensemble,
evaluator_config,
0,
ee_id="1")
with evaluator.run() as mon:
cancel = True
for _ in mon.track():
if cancel:
mon.signal_cancel()
cancel = False
assert evaluator._ensemble.status == state.ENSEMBLE_STATE_CANCELLED
def test_verify_monitor_failing_ensemble(make_ee_config):
ee_config = make_ee_config(use_token=False, generate_cert=False)
ensemble = TestEnsemble(iter=1, reals=2, steps=1, jobs=2)
ensemble.addFailJob(real=1, step=0, job=1)
ee = EnsembleEvaluator(
ensemble,
ee_config,
0,
ee_id="0",
)
ee.run()
monitor_failing_ensemble.verify(ee_config.client_uri,
on_connect=ensemble.start)
ensemble.join()
def test_verify_monitor_failing_evaluation(make_ee_config):
ee_config = make_ee_config(use_token=False, generate_cert=False)
ensemble = TestEnsemble(iter=1, reals=2, steps=1, jobs=2)
ensemble.with_failure()
ee = EnsembleEvaluator(
ensemble,
ee_config,
0,
ee_id="ee-0",
)
ee.run()
monitor_failing_evaluation.verify(ee_config.client_uri,
on_connect=ensemble.start)
ensemble.join()
def test_run_legacy_ensemble(tmpdir, unused_tcp_port, make_ensemble_builder):
num_reals = 2
with tmpdir.as_cwd():
ensemble = make_ensemble_builder(tmpdir, num_reals, 2).build()
config = EvaluatorServerConfig(unused_tcp_port)
evaluator = EnsembleEvaluator(ensemble, config, ee_id="1")
monitor = evaluator.run()
for e in monitor.track():
if (e["type"] in (
identifiers.EVTYPE_EE_SNAPSHOT_UPDATE,
identifiers.EVTYPE_EE_SNAPSHOT,
) and e.data.get("status") in ["Failed", "Stopped"]):
monitor.signal_done()
assert evaluator._snapshot.get_status() == "Stopped"
assert evaluator.get_successful_realizations() == num_reals
def evaluate(
workspace_root: Path,
evaluation_name: str,
input_records: MultiEnsembleRecord,
ensemble_config: EnsembleConfig,
stages_config: StagesConfig,
) -> MultiEnsembleRecord:
evaluation_tmp_dir = _create_evaluator_tmp_dir(workspace_root,
evaluation_name)
config = EvaluatorServerConfig()
ee_config = _build_ee_config(
evaluation_tmp_dir,
ensemble_config,
stages_config,
input_records,
config.dispatch_uri,
)
ensemble = PrefectEnsemble(ee_config) # type: ignore
ee = EnsembleEvaluator(ensemble=ensemble, config=config, iter_=0)
result = _run(ee)
output_records = _prepare_output_records(result)
return output_records
def run_ensemble_evaluator(self, run_context: ErtRunContext,
ee_config: EvaluatorServerConfig) -> int:
if run_context.get_step():
self.ert().eclConfig().assert_restart()
ensemble = EnsembleBuilder.from_legacy(
run_context,
self.get_forward_model(),
self._queue_config,
self.ert().analysisConfig(),
self.ert().resConfig(),
).build()
self.ert().initRun(run_context)
totalOk = EnsembleEvaluator(
ensemble,
ee_config,
run_context.get_iter(),
ee_id=str(uuid.uuid1()).split("-", maxsplit=1)[0],
).run_and_get_successful_realizations()
for iens, run_arg in enumerate(run_context):
if run_context.is_active(iens):
if run_arg.run_status in (
RunStatusType.JOB_LOAD_FAILURE,
RunStatusType.JOB_RUN_FAILURE,
):
run_context.deactivate_realization(iens)
run_context.get_sim_fs().fsync()
return totalOk
def evaluator(ee_config):
ensemble = (create_ensemble_builder().add_realization(
real=create_realization_builder().active(True).set_iens(0).add_step(
step=create_step_builder().set_id("0").set_name("cats").add_job(
job=create_legacy_job_builder().set_id(0).set_name("cat").
set_ext_job(Mock())).add_job(job=create_legacy_job_builder(
).set_id(1).set_name("cat2").set_ext_job(
Mock())).set_dummy_io())).set_ensemble_size(2).build())
ee = EnsembleEvaluator(
ensemble,
ee_config,
0,
ee_id="ee-0",
)
yield ee
ee.stop()
def run_ensemble_evaluator(self, run_context, ee_config):
if run_context.get_step():
self.ert().eclConfig().assert_restart()
iactive = run_context.get_mask()
run_context.get_sim_fs().getStateMap().deselectMatching(
iactive,
RealizationStateEnum.STATE_LOAD_FAILURE
| RealizationStateEnum.STATE_PARENT_FAILURE,
)
ensemble = create_ensemble_builder_from_legacy(
run_context,
self.ert().resConfig().model_config.getForwardModel(),
self._queue_config,
self.ert().analysisConfig(),
self.ert().resConfig(),
).build()
self.ert().initRun(run_context)
return EnsembleEvaluator(ensemble,
ee_config,
ee_id=str(uuid.uuid1()).split("-")
[0]).run_and_get_successful_realizations()
async def test_run_and_cancel_legacy_ensemble(tmpdir, unused_tcp_port,
make_ensemble_builder):
num_reals = 10
conf_file = Path(tmpdir / CONFIG_FILE)
with tmpdir.as_cwd():
with open(conf_file, "w") as f:
f.write(f'port: "{unused_tcp_port}"\n')
ensemble = make_ensemble_builder(tmpdir, num_reals, 2).build()
config = load_config(conf_file)
evaluator = EnsembleEvaluator(ensemble, config, ee_id="1")
thread = threading.Thread(
name="test_eval",
target=evaluator.run_and_get_successful_realizations,
args=(),
)
thread.start()
# Wait for evaluator to start
await wait_for_ws(config["url"], 10)
# Send termination request to the evaluator
async with websockets.connect(config["client_url"]) as websocket:
out_cloudevent = CloudEvent({
"type": identifiers.EVTYPE_EE_USER_CANCEL,
"source": "/ert/test/0",
"id": "ID",
})
await websocket.send(to_json(out_cloudevent))
thread.join()
assert evaluator._snapshot.get_status() == "Cancelled"
def test_run_legacy_ensemble_exception(tmpdir, unused_tcp_port, make_ensemble_builder):
num_reals = 2
with tmpdir.as_cwd():
ensemble = make_ensemble_builder(tmpdir, num_reals, 2).build()
config = EvaluatorServerConfig(unused_tcp_port)
evaluator = EnsembleEvaluator(ensemble, config, 0, ee_id="1")
with patch.object(ensemble, "get_active_reals", side_effect=RuntimeError()):
with evaluator.run() as monitor:
for e in monitor.track():
if e.data is not None and e.data.get(identifiers.STATUS) in [
state.ENSEMBLE_STATE_FAILED,
state.ENSEMBLE_STATE_STOPPED,
]:
monitor.signal_done()
assert evaluator._ensemble.get_status() == state.ENSEMBLE_STATE_FAILED
def test_run_and_cancel_legacy_ensemble(tmpdir, unused_tcp_port, make_ensemble_builder):
num_reals = 10
with tmpdir.as_cwd():
ensemble = make_ensemble_builder(tmpdir, num_reals, 2).build()
config = EvaluatorServerConfig(unused_tcp_port)
evaluator = EnsembleEvaluator(ensemble, config, 0, ee_id="1")
with evaluator.run() as mon:
cancel = True
for _ in mon.track():
if cancel:
mon.signal_cancel()
cancel = False
assert evaluator._ensemble.get_status() == state.ENSEMBLE_STATE_CANCELLED
def _run(
ensemble_evaluator: EnsembleEvaluator,
) -> Dict[int, Dict[str, ert.data.RecordTransmitter]]:
result: Dict[int, Dict[str, ert.data.RecordTransmitter]] = {}
with ensemble_evaluator.run() as monitor:
realization_ids = set()
realizations_completed = set()
for event in monitor.track():
if isinstance(event.data, dict) and event.data.get("status") in [
ENSEMBLE_STATE_STOPPED,
ENSEMBLE_STATE_FAILED,
]:
monitor.signal_done()
if event.data.get("status") == ENSEMBLE_STATE_FAILED:
raise RuntimeError("Ensemble evaluation failed")
if event["type"] == EVTYPE_EE_TERMINATED and isinstance(
event.data, bytes):
result = pickle.loads(event.data)
if isinstance(event.data, dict) and "reals" in event.data:
for real_id in event.data["reals"]:
realization_ids.add(real_id)
real_status = event.data["reals"][real_id].get("status")
if (real_status == REALIZATION_STATE_FINISHED
and real_id not in realizations_completed):
realizations_completed.add(real_id)
print(
f"Realization {real_id} completed successfully"
f" ({len(realizations_completed)}/{len(realization_ids)})"
)
return result
def test_verify_monitor_successful_ensemble(make_ee_config, event_loop):
ensemble = TestEnsemble(iter=1, reals=2, steps=2, jobs=2).with_result(
b"\x80\x04\x95\x0f\x00\x00\x00\x00\x00\x00\x00\x8c\x0bhello world\x94.",
"application/octet-stream",
)
ee_config = make_ee_config(use_token=False, generate_cert=False)
ee = EnsembleEvaluator(
ensemble,
ee_config,
0,
ee_id="ee-0",
)
ee.run()
event_loop.run_until_complete(wait_for_evaluator(ee_config.url))
monitor_successful_ensemble().verify(ee_config.client_uri,
on_connect=ensemble.start)
ensemble.join()
def test_run_and_cancel_legacy_ensemble(tmpdir, unused_tcp_port,
make_ensemble_builder):
num_reals = 10
with tmpdir.as_cwd():
ensemble = make_ensemble_builder(tmpdir, num_reals, 2).build()
config = EvaluatorServerConfig(unused_tcp_port)
evaluator = EnsembleEvaluator(ensemble, config, ee_id="1")
mon = evaluator.run()
cancel = True
for _ in mon.track():
if cancel:
mon.signal_cancel()
cancel = False
assert evaluator._snapshot.get_status() == "Cancelled"
def test_run_prefect_ensemble_with_path(unused_tcp_port, coefficients):
with tmp(os.path.join(SOURCE_DIR, "test-data/local/prefect_test_case")):
config = parse_config("config.yml")
config.update(
{
"config_path": os.getcwd(),
"realizations": 2,
"executor": "local",
}
)
inputs = {}
coeffs_trans = coefficient_transmitters(
coefficients, config.get(ids.STORAGE)["storage_path"]
)
script_trans = script_transmitters(config)
for iens in range(2):
inputs[iens] = {**coeffs_trans[iens], **script_trans[iens]}
config.update(
{
"inputs": inputs,
"outputs": output_transmitters(config),
}
)
service_config = EvaluatorServerConfig(unused_tcp_port)
config["config_path"] = Path(config["config_path"])
config["run_path"] = Path(config["run_path"])
config["storage"]["storage_path"] = Path(config["storage"]["storage_path"])
config["dispatch_uri"] = service_config.dispatch_uri
ensemble = PrefectEnsemble(config)
evaluator = EnsembleEvaluator(ensemble, service_config, 0, ee_id="1")
with evaluator.run() as mon:
for event in mon.track():
if isinstance(event.data, dict) and event.data.get("status") in [
"Failed",
"Stopped",
]:
mon.signal_done()
assert evaluator._ensemble.get_status() == "Stopped"
successful_realizations = evaluator._ensemble.get_successful_realizations()
assert successful_realizations == config["realizations"]
