def test_ensure_multi_level_events_in_order(evaluator): with evaluator.run() as monitor: events = monitor.track() token = evaluator._config.token cert = evaluator._config.cert url = evaluator._config.url snapshot_event = next(events) assert snapshot_event["type"] == identifiers.EVTYPE_EE_SNAPSHOT with Client(url + "/dispatch", cert=cert, token=token) as dispatch1: send_dispatch_event( dispatch1, identifiers.EVTYPE_ENSEMBLE_STARTED, f"/ert/ee/{evaluator._ee_id}/ensemble", "event0", {}, ) send_dispatch_event( dispatch1, identifiers.EVTYPE_FM_STEP_SUCCESS, f"/ert/ee/{evaluator._ee_id}/real/0/step/0", "event1", {}, ) send_dispatch_event( dispatch1, identifiers.EVTYPE_FM_STEP_SUCCESS, f"/ert/ee/{evaluator._ee_id}/real/1/step/0", "event2", {}, ) send_dispatch_event( dispatch1, identifiers.EVTYPE_ENSEMBLE_STOPPED, f"/ert/ee/{evaluator._ee_id}/ensemble", "event3", {}, ) monitor.signal_done() events = list(events) # Without making too many assumptions about what events to expect, it # should be reasonable to expect that if an event contains information # about realizations, the state of the ensemble up until that point # should be not final (i.e. not cancelled, stopped, failed). ensemble_state = snapshot_event.data.get("status") for event in events: if event.data: if "reals" in event.data: assert ensemble_state == ENSEMBLE_STATE_STARTED ensemble_state = event.data.get("status", ensemble_state)
def test_dispatchers_can_connect_and_monitor_can_shut_down_evaluator( evaluator): with evaluator.run() as monitor: events = monitor.track() token = evaluator._config.token cert = evaluator._config.cert url = evaluator._config.url # first snapshot before any event occurs snapshot_event = next(events) print(snapshot_event) snapshot = Snapshot(snapshot_event.data) assert snapshot.status == ENSEMBLE_STATE_UNKNOWN # two dispatchers connect with Client( url + "/dispatch", cert=cert, token=token, max_retries=1, timeout_multiplier=1, ) as dispatch1, Client( url + "/dispatch", cert=cert, token=token, max_retries=1, timeout_multiplier=1, ) as dispatch2: # first dispatcher informs that job 0 is running send_dispatch_event( dispatch1, identifiers.EVTYPE_FM_JOB_RUNNING, f"/ert/ee/{evaluator._ee_id}/real/0/step/0/job/0", "event1", {"current_memory_usage": 1000}, ) # second dispatcher informs that job 0 is running send_dispatch_event( dispatch2, identifiers.EVTYPE_FM_JOB_RUNNING, f"/ert/ee/{evaluator._ee_id}/real/1/step/0/job/0", "event1", {"current_memory_usage": 1000}, ) # second dispatcher informs that job 0 is done send_dispatch_event( dispatch2, identifiers.EVTYPE_FM_JOB_SUCCESS, f"/ert/ee/{evaluator._ee_id}/real/1/step/0/job/0", "event1", {"current_memory_usage": 1000}, ) # second dispatcher informs that job 1 is failed send_dispatch_event( dispatch2, identifiers.EVTYPE_FM_JOB_FAILURE, f"/ert/ee/{evaluator._ee_id}/real/1/step/0/job/1", "event_job_1_fail", {identifiers.ERROR_MSG: "error"}, ) evt = next(events) print(evt) snapshot = Snapshot(evt.data) assert snapshot.get_job("1", "0", "0").status == JOB_STATE_FINISHED assert snapshot.get_job("0", "0", "0").status == JOB_STATE_RUNNING assert snapshot.get_job("1", "0", "1").status == JOB_STATE_FAILURE # a second monitor connects with ee_monitor.create( evaluator._config.get_connection_info()) as monitor2: events2 = monitor2.track() full_snapshot_event = next(events2) assert full_snapshot_event[ "type"] == identifiers.EVTYPE_EE_SNAPSHOT snapshot = Snapshot(full_snapshot_event.data) assert snapshot.status == ENSEMBLE_STATE_UNKNOWN assert snapshot.get_job("0", "0", "0").status == JOB_STATE_RUNNING assert snapshot.get_job("1", "0", "0").status == JOB_STATE_FINISHED # one monitor requests that server exit monitor.signal_cancel() # both monitors should get a terminated event terminated = next(events) terminated2 = next(events2) assert terminated["type"] == identifiers.EVTYPE_EE_TERMINATED assert terminated2["type"] == identifiers.EVTYPE_EE_TERMINATED for e in [events, events2]: for undexpected_event in e: assert ( False ), f"got unexpected event {undexpected_event} from monitor"