def test_ensure_multi_level_events_in_order(evaluator):
with evaluator.run() as monitor:
events = monitor.track()
token = evaluator._config.token
cert = evaluator._config.cert
url = evaluator._config.url
snapshot_event = next(events)
assert snapshot_event["type"] == identifiers.EVTYPE_EE_SNAPSHOT
with Client(url + "/dispatch", cert=cert, token=token) as dispatch1:
send_dispatch_event(
dispatch1,
identifiers.EVTYPE_ENSEMBLE_STARTED,
f"/ert/ee/{evaluator._ee_id}/ensemble",
"event0",
{},
)
send_dispatch_event(
dispatch1,
identifiers.EVTYPE_FM_STEP_SUCCESS,
f"/ert/ee/{evaluator._ee_id}/real/0/step/0",
"event1",
{},
)
send_dispatch_event(
dispatch1,
identifiers.EVTYPE_FM_STEP_SUCCESS,
f"/ert/ee/{evaluator._ee_id}/real/1/step/0",
"event2",
{},
)
send_dispatch_event(
dispatch1,
identifiers.EVTYPE_ENSEMBLE_STOPPED,
f"/ert/ee/{evaluator._ee_id}/ensemble",
"event3",
{},
)
monitor.signal_done()
events = list(events)
# Without making too many assumptions about what events to expect, it
# should be reasonable to expect that if an event contains information
# about realizations, the state of the ensemble up until that point
# should be not final (i.e. not cancelled, stopped, failed).
ensemble_state = snapshot_event.data.get("status")
for event in events:
if event.data:
if "reals" in event.data:
assert ensemble_state == ENSEMBLE_STATE_STARTED
ensemble_state = event.data.get("status", ensemble_state)
def test_dispatchers_can_connect_and_monitor_can_shut_down_evaluator(
evaluator):
with evaluator.run() as monitor:
events = monitor.track()
token = evaluator._config.token
cert = evaluator._config.cert
url = evaluator._config.url
# first snapshot before any event occurs
snapshot_event = next(events)
print(snapshot_event)
snapshot = Snapshot(snapshot_event.data)
assert snapshot.status == ENSEMBLE_STATE_UNKNOWN
# two dispatchers connect
with Client(
url + "/dispatch",
cert=cert,
token=token,
max_retries=1,
timeout_multiplier=1,
) as dispatch1, Client(
url + "/dispatch",
cert=cert,
token=token,
max_retries=1,
timeout_multiplier=1,
) as dispatch2:
# first dispatcher informs that job 0 is running
send_dispatch_event(
dispatch1,
identifiers.EVTYPE_FM_JOB_RUNNING,
f"/ert/ee/{evaluator._ee_id}/real/0/step/0/job/0",
"event1",
{"current_memory_usage": 1000},
)
# second dispatcher informs that job 0 is running
send_dispatch_event(
dispatch2,
identifiers.EVTYPE_FM_JOB_RUNNING,
f"/ert/ee/{evaluator._ee_id}/real/1/step/0/job/0",
"event1",
{"current_memory_usage": 1000},
)
# second dispatcher informs that job 0 is done
send_dispatch_event(
dispatch2,
identifiers.EVTYPE_FM_JOB_SUCCESS,
f"/ert/ee/{evaluator._ee_id}/real/1/step/0/job/0",
"event1",
{"current_memory_usage": 1000},
)
# second dispatcher informs that job 1 is failed
send_dispatch_event(
dispatch2,
identifiers.EVTYPE_FM_JOB_FAILURE,
f"/ert/ee/{evaluator._ee_id}/real/1/step/0/job/1",
"event_job_1_fail",
{identifiers.ERROR_MSG: "error"},
)
evt = next(events)
print(evt)
snapshot = Snapshot(evt.data)
assert snapshot.get_job("1", "0", "0").status == JOB_STATE_FINISHED
assert snapshot.get_job("0", "0", "0").status == JOB_STATE_RUNNING
assert snapshot.get_job("1", "0", "1").status == JOB_STATE_FAILURE
# a second monitor connects
with ee_monitor.create(
evaluator._config.get_connection_info()) as monitor2:
events2 = monitor2.track()
full_snapshot_event = next(events2)
assert full_snapshot_event[
"type"] == identifiers.EVTYPE_EE_SNAPSHOT
snapshot = Snapshot(full_snapshot_event.data)
assert snapshot.status == ENSEMBLE_STATE_UNKNOWN
assert snapshot.get_job("0", "0", "0").status == JOB_STATE_RUNNING
assert snapshot.get_job("1", "0", "0").status == JOB_STATE_FINISHED
# one monitor requests that server exit
monitor.signal_cancel()
# both monitors should get a terminated event
terminated = next(events)
terminated2 = next(events2)
assert terminated["type"] == identifiers.EVTYPE_EE_TERMINATED
assert terminated2["type"] == identifiers.EVTYPE_EE_TERMINATED
for e in [events, events2]:
for undexpected_event in e:
assert (
False
), f"got unexpected event {undexpected_event} from monitor"