def test_progression_start_iter_not_zero(full_snapshot):
source_model = SnapshotModel()
model = ProgressProxyModel(source_model, parent=None)
reporting_mode = qt_api.QtTest.QAbstractItemModelTester.FailureReportingMode.Warning
tester = qt_api.QtTest.QAbstractItemModelTester( # noqa, prevent GC
model, reporting_mode)
source_model._add_snapshot(SnapshotModel.prerender(full_snapshot), 1)
assert model.data(model.index(0, 0, QModelIndex()), ProgressRole) == {
"nr_reals": 100,
"status": {
REALIZATION_STATE_UNKNOWN: 100
},
}
partial = PartialSnapshot(full_snapshot)
partial.update_real("0", Realization(status=REALIZATION_STATE_FINISHED))
source_model._add_partial_snapshot(SnapshotModel.prerender(partial), 1)
assert model.data(model.index(0, 0, QModelIndex()), ProgressRole) == {
"nr_reals": 100,
"status": {
REALIZATION_STATE_UNKNOWN: 99,
REALIZATION_STATE_FINISHED: 1
},
}
def test_changes(full_snapshot):
source_model = SnapshotModel()
model = JobListProxyModel(None, 0, 0, 0, 0)
model.setSourceModel(source_model)
reporting_mode = qt_api.QtTest.QAbstractItemModelTester.FailureReportingMode.Warning
tester = qt_api.QtTest.QAbstractItemModelTester( # noqa, prevent GC
model, reporting_mode)
source_model._add_snapshot(SnapshotModel.prerender(full_snapshot), 0)
assert (model.index(0, _id_to_col(ids.STATUS),
QModelIndex()).data() == JOB_STATE_START)
partial = PartialSnapshot(full_snapshot)
start_time = datetime.datetime(year=2020, month=10, day=27, hour=12)
end_time = datetime.datetime(year=2020, month=10, day=28, hour=13)
partial.update_job(
"0",
"0",
"0",
job=Job(
status=JOB_STATE_FAILURE,
start_time=start_time,
end_time=end_time,
),
)
source_model._add_partial_snapshot(SnapshotModel.prerender(partial), 0)
assert (model.index(0, _id_to_col(DURATION),
QModelIndex()).data() == "1 day, 1:00:00")
assert (model.index(0, _id_to_col(ids.STATUS),
QModelIndex()).data() == JOB_STATE_FAILURE)
async def _ensemble_cancelled_handler(self, event):
if self._snapshot.get_status() != ENSEMBLE_STATE_FAILED:
snapshot_mutate_event = PartialSnapshot(self._snapshot).from_cloudevent(
event
)
await self._send_snapshot_update(snapshot_mutate_event)
self._stop()
def test_update_partial_from_multiple_cloudevents(snapshot):
partial = PartialSnapshot(snapshot)
partial.from_cloudevent(
CloudEvent({
"id": "0",
"type": ids.EVTYPE_FM_JOB_RUNNING,
"source": "/real/0/step/0/job/0",
}))
partial.from_cloudevent(
CloudEvent(
{
"id": "0",
"type": ids.EVTYPE_FM_JOB_FAILURE,
"source": "/real/0/step/0/job/0",
},
{ids.ERROR_MSG: "failed"},
))
partial.from_cloudevent(
CloudEvent({
"id": "1",
"type": ids.EVTYPE_FM_JOB_SUCCESS,
"source": "/real/0/step/0/job/1",
}))
jobs = partial.to_dict()["reals"]["0"]["steps"]["0"]["jobs"]
jobs["0"]["status"] == state.JOB_STATE_FAILURE
jobs["1"]["status"] == state.JOB_STATE_FINISHED
async def _ensemble_failed_handler(self, event):
if self._snapshot.get_status() not in [
ENSEMBLE_STATE_STOPPED,
ENSEMBLE_STATE_CANCELLED,
]:
snapshot_mutate_event = PartialSnapshot(self._snapshot).from_cloudevent(
event
)
await self._send_snapshot_update(snapshot_mutate_event)
def update_snapshot(self, events):
snapshot_mutate_event = PartialSnapshot(self._snapshot)
for event in events:
snapshot_mutate_event.from_cloudevent(event)
self._snapshot.merge_event(snapshot_mutate_event)
if self._status != self._snapshot.get_status():
self._status = self._status_tracker.update_state(
self._snapshot.get_status())
return snapshot_mutate_event
def track(self):
while True:
event = self._work_queue.get()
if isinstance(event, str):
try:
if event == EvaluatorTracker.DONE:
yield EndEvent(
failed=self._model.hasRunFailed(),
failed_msg=self._model.getFailMessage(),
)
elif event == EvaluatorTracker.CONNECTION_ERROR:
yield EndEvent(
failed=True,
failed_msg="Connection error",
)
except GeneratorExit:
# consumers may exit at this point, make sure the last
# task is marked as done
pass
self._work_queue.task_done()
break
elif event["type"] == ids.EVTYPE_EE_SNAPSHOT:
iter_ = event.data["iter"]
snapshot = Snapshot(event.data)
self._iter_snapshot[iter_] = snapshot
yield FullSnapshotEvent(
phase_name=self._model.getPhaseName(),
current_phase=self._model.currentPhase(),
total_phases=self._model.phaseCount(),
indeterminate=self._model.isIndeterminate(),
progress=self._progress(),
iteration=iter_,
snapshot=snapshot,
)
elif event["type"] == ids.EVTYPE_EE_SNAPSHOT_UPDATE:
iter_ = event.data["iter"]
if iter_ not in self._iter_snapshot:
raise OutOfOrderSnapshotUpdateException(
f"got {ids.EVTYPE_EE_SNAPSHOT_UPDATE} without having stored snapshot for iter {iter_}"
)
partial = PartialSnapshot(
self._iter_snapshot[iter_]).from_cloudevent(event)
self._iter_snapshot[iter_].merge_event(partial)
yield SnapshotUpdateEvent(
phase_name=self._model.getPhaseName(),
current_phase=self._model.currentPhase(),
total_phases=self._model.phaseCount(),
indeterminate=self._model.isIndeterminate(),
progress=self._progress(),
iteration=iter_,
partial_snapshot=partial,
)
self._work_queue.task_done()
def test_realization_job_hint(full_snapshot):
model = SnapshotModel()
model._add_snapshot(full_snapshot, 0)
partial = PartialSnapshot(full_snapshot)
partial.update_job("0", "0", "0", Job(status=JOB_STATE_RUNNING))
model._add_partial_snapshot(partial, 0)
first_real = model.index(0, 0, model.index(0, 0))
colors = model.data(first_real, RealJobColorHint)
assert colors[0].name() == QColor(*COLOR_RUNNING).name()
assert colors[1].name() == QColor(*COLOR_PENDING).name()
def test_multiple_cloud_events_trigger_non_communicated_change():
"""In other words, though we say all steps are finished, we don't
explicitly send an event that changes the realization status. It should
happen by virtue of the steps being completed."""
snapshot = (SnapshotBuilder().add_step(
step_id="0", status="Unknown").build(["0"], status="Unknown"))
partial = PartialSnapshot(snapshot)
partial.from_cloudevent(
CloudEvent({
"id": "0",
"type": ids.EVTYPE_FM_STEP_SUCCESS,
"source": "/real/0/step/0",
}))
assert partial.to_dict(
)["reals"]["0"]["status"] == state.REALIZATION_STATE_FINISHED
def _flush(self, batch: List[CloudEvent]) -> SnapshotUpdateEvent:
iter_: int = batch[0].data["iter"]
partial: PartialSnapshot = PartialSnapshot(self._iter_snapshot[iter_])
for event in batch:
partial.from_cloudevent(event)
self._iter_snapshot[iter_].merge_event(partial)
update_event = SnapshotUpdateEvent(
phase_name=self._model.getPhaseName(),
current_phase=self._model.currentPhase(),
total_phases=self._model.phaseCount(),
indeterminate=self._model.isIndeterminate(),
progress=self._progress(),
iteration=iter_,
partial_snapshot=partial,
)
for _ in range(len(batch)):
self._work_queue.task_done()
return update_event
def test_duration(mock_datetime, timezone, full_snapshot):
source_model = SnapshotModel()
model = JobListProxyModel(None, 0, 0, 0, 0)
model.setSourceModel(source_model)
reporting_mode = qt_api.QtTest.QAbstractItemModelTester.FailureReportingMode.Warning
tester = qt_api.QtTest.QAbstractItemModelTester( # noqa, prevent GC
model, reporting_mode)
source_model._add_snapshot(SnapshotModel.prerender(full_snapshot), 0)
assert (model.index(0, _id_to_col(ids.STATUS),
QModelIndex()).data() == JOB_STATE_START)
partial = PartialSnapshot(full_snapshot)
start_time = datetime.datetime(year=2020,
month=10,
day=27,
hour=12,
tzinfo=timezone)
# mock only datetime.datetime.now()
mock_datetime.datetime.now.return_value = datetime.datetime(
year=2020,
month=10,
day=28,
hour=13,
minute=12,
second=11,
microsecond=5, # Note that microseconds are intended to be removed
tzinfo=timezone,
)
partial.update_job(
"0",
"0",
"2",
job=Job(
status=JOB_STATE_RUNNING,
start_time=start_time,
),
)
source_model._add_partial_snapshot(SnapshotModel.prerender(partial), 0)
assert (model.index(2, _id_to_col(DURATION),
QModelIndex()).data() == "1 day, 1:12:11")
mock_datetime.datetime.now.assert_called_once_with(timezone)
def test_no_cross_talk(full_snapshot):
source_model = SnapshotModel()
model = JobListProxyModel(None, 0, 0, 0, 0)
model.setSourceModel(source_model)
reporting_mode = qt_api.QtTest.QAbstractItemModelTester.FailureReportingMode.Warning
qt_api.QtTest.QAbstractItemModelTester(model,
reporting_mode) # noqa, prevent GC
source_model._add_snapshot(SnapshotModel.prerender(full_snapshot), 0)
source_model._add_snapshot(SnapshotModel.prerender(full_snapshot), 1)
# Test that changes to iter=1 does not bleed into iter=0
partial = PartialSnapshot(full_snapshot)
partial.update_job("0", "0", "0", job=Job(status=JOB_STATE_FAILURE))
source_model._add_partial_snapshot(SnapshotModel.prerender(partial), 1)
assert (model.index(0, _id_to_col(ids.STATUS),
QModelIndex()).data() == JOB_STATE_START)
model.set_step(1, 0, 0, 0)
assert (model.index(0, _id_to_col(ids.STATUS),
QModelIndex()).data() == JOB_STATE_FAILURE)
def mock_ee_monitor(*args):
reals_ids = ["0", "1"]
snapshot = (SnapshotBuilder().add_stage(
stage_id="0", status="Running").add_step(
stage_id="0", step_id="0", status="Unknown").add_job(
stage_id="0",
step_id="0",
job_id="0",
name="job0",
data={},
status="Running",
).add_metadata("iter", 0).build(reals_ids, "Unknown"))
update = PartialSnapshot(snapshot)
update.update_step("0", "0", "0", "Finished")
update.update_step("1", "0", "0", "Finished")
events = [
MockCloudEvent(
{"type": ids.EVTYPE_EE_SNAPSHOT},
snapshot.to_dict(),
),
MockCloudEvent(
{"type": ids.EVTYPE_EE_SNAPSHOT_UPDATE},
update.to_dict(),
),
MockCloudEvent({"type": ids.EVTYPE_EE_TERMINATED}, {}),
]
def _track():
while True:
try:
yield events.pop(0)
except IndexError:
return
return MagicMock(track=MagicMock(side_effect=_track))
def test_snapshot_merge():
snapshot = _create_snapshot()
update_event = PartialSnapshot()
update_event.update_status(status="running")
snapshot.merge_event(update_event)
assert snapshot.get_status() == "running"
update_event = PartialSnapshot()
update_event.update_job(
real_id="1",
stage_id="0",
step_id="0",
job_id="0",
status="Finished",
start_time=datetime(year=2020, month=10, day=27).isoformat(),
end_time=datetime(year=2020, month=10, day=28).isoformat(),
data={"memory": 1000},
)
update_event.update_job(
real_id="1",
stage_id="0",
step_id="0",
job_id="1",
status="Running",
start_time=datetime(year=2020, month=10, day=27).isoformat(),
)
update_event.update_job(
real_id="9",
stage_id="0",
step_id="0",
job_id="0",
status="Running",
start_time=datetime(year=2020, month=10, day=27).isoformat(),
)
snapshot.merge_event(update_event)
assert snapshot.get_status() == "running"
assert _dict_equal(
snapshot.get_job(real_id="1", stage_id="0", step_id="0", job_id="0"),
{
"status": "Finished",
"start_time": "2020-10-27T00:00:00",
"end_time": "2020-10-28T00:00:00",
"data": {
"memory": 1000
},
"error": None,
"name": "job0",
"stderr": None,
"stdout": None,
},
)
assert snapshot.get_job(real_id="1", stage_id="0", step_id="0",
job_id="1") == {
"status": "Running",
"start_time": "2020-10-27T00:00:00",
"end_time": None,
"data": {},
"error": None,
"name": "job1",
"stderr": None,
"stdout": None,
}
assert (snapshot.get_job(real_id="9",
stage_id="0",
step_id="0",
job_id="0")["status"] == "Running")
assert snapshot.get_job(real_id="9", stage_id="0", step_id="0",
job_id="0") == {
"status": "Running",
"start_time": "2020-10-27T00:00:00",
"end_time": None,
"data": {},
"error": None,
"name": "job0",
"stderr": None,
"stdout": None,
}
def _create_partial_snapshot(
self,
run_context: ErtRunContext,
detailed_progress: typing.Tuple[typing.Dict, int],
iter_: int,
) -> typing.Optional[PartialSnapshot]:
"""Create a PartialSnapshot, or None if the sources of data were
destroyed or had not been created yet. Both run_context and
detailed_progress needs to be aligned with the stars if job status etc
is to be produced. If queue_snapshot is set, this means the the differ
will not be used to calculate changes."""
queue = self._iter_queue.get(iter_, None)
if queue is None:
logger.debug(f"no queue for {iter_}, no partial returned")
return None
queue_snapshot = queue.snapshot()
snapshot = self._iter_snapshot.get(iter_, None)
if snapshot is None:
logger.debug(f"no snapshot for {iter_}, no partial returned")
return None
partial = PartialSnapshot(snapshot)
if queue_snapshot is not None:
for iens, change in queue_snapshot.items():
change_enum = JobStatusType.from_string(change)
partial.update_real(
str(iens),
Realization(
status=queue_status_to_real_state(change_enum)),
)
iter_to_progress, progress_iter = detailed_progress
if not iter_to_progress:
logger.debug(f"partial: no detailed progress for iter:{iter_}")
return partial
if iter_ != progress_iter:
logger.debug(
f"partial: iter_to_progress iter ({progress_iter}) differed from run_context ({iter_})"
)
for iens, _ in _enumerate_run_context(run_context):
if not _is_iens_active(iens, run_context):
continue
progress = iter_to_progress[iter_].get(iens, None)
if not progress:
continue
jobs = progress[0]
for idx, fm in enumerate(jobs):
partial.update_job(
str(iens), # real_id
"0",
str(idx),
Job(
status=_map_job_state(fm.status),
start_time=fm.start_time,
end_time=fm.end_time,
data={
CURRENT_MEMORY_USAGE: fm.current_memory_usage,
MAX_MEMORY_USAGE: fm.max_memory_usage,
},
stdout=fm.std_out_file,
stderr=fm.std_err_file,
error=fm.error,
),
)
return partial
async def _ensemble_stopped_handler(self, event):
if self._snapshot.get_status() != "Failure":
snapshot_mutate_event = PartialSnapshot(
self._snapshot).from_cloudevent(event)
await self._send_snapshot_update(snapshot_mutate_event)
async def _fm_handler(self, event):
snapshot_mutate_event = PartialSnapshot(
self._snapshot).from_cloudevent(event)
await self._send_snapshot_update(snapshot_mutate_event)
async def _ensemble_failed_handler(self, event):
if self._snapshot.get_status() not in ["Stopped", "Cancelled"]:
snapshot_mutate_event = PartialSnapshot(
self._snapshot).from_cloudevent(event)
await self._send_snapshot_update(snapshot_mutate_event)
def partial_snapshot(snapshot) -> PartialSnapshot:
partial = PartialSnapshot(snapshot)
partial.update_real("0", Realization(status=JOB_STATE_FINISHED))
partial.update_job("0", "0", "0", Job(status=JOB_STATE_FINISHED))
return partial
def test_snapshot_merge(snapshot):
update_event = PartialSnapshot(snapshot)
update_event.update_status(status="running")
snapshot.merge_event(update_event)
assert snapshot.get_status() == "running"
update_event = PartialSnapshot(snapshot)
update_event.update_job(
real_id="1",
step_id="0",
job_id="0",
job=Job(
status="Finished",
start_time=datetime(year=2020, month=10, day=27),
end_time=datetime(year=2020, month=10, day=28),
data={"memory": 1000},
),
)
update_event.update_job(
real_id="1",
step_id="0",
job_id="1",
job=Job(
status="Running",
start_time=datetime(year=2020, month=10, day=27),
),
)
update_event.update_job(
real_id="9",
step_id="0",
job_id="0",
job=Job(
status="Running",
start_time=datetime(year=2020, month=10, day=27),
),
)
snapshot.merge_event(update_event)
assert snapshot.get_status() == "running"
assert snapshot.get_job(real_id="1", step_id="0", job_id="0") == Job(
status="Finished",
start_time=datetime(year=2020, month=10, day=27),
end_time=datetime(year=2020, month=10, day=28),
data={"memory": 1000},
error=None,
name="job0",
stderr=None,
stdout=None,
)
assert snapshot.get_job(real_id="1", step_id="0", job_id="1") == Job(
status="Running",
start_time=datetime(year=2020, month=10, day=27),
end_time=None,
data={},
error=None,
name="job1",
stderr=None,
stdout=None,
)
assert snapshot.get_job(real_id="9", step_id="0",
job_id="0").status == "Running"
assert snapshot.get_job(real_id="9", step_id="0", job_id="0") == Job(
status="Running",
start_time=datetime(year=2020, month=10, day=27),
end_time=None,
data={},
error=None,
name="job0",
stderr=None,
stdout=None,
)
data={},
status=state.JOB_STATE_START,
)
.build(["0"], state.REALIZATION_STATE_UNKNOWN)
),
phase_name="Foo",
current_phase=0,
total_phases=1,
progress=0.25,
iteration=0,
indeterminate=False,
),
SnapshotUpdateEvent(
partial_snapshot=PartialSnapshot(
SnapshotBuilder().build(
[], status=state.REALIZATION_STATE_FINISHED
)
),
phase_name="Foo",
current_phase=0,
total_phases=1,
progress=0.5,
iteration=0,
indeterminate=False,
),
EndEvent(failed=False, failed_msg=""),
],
1,
id="real_less_partial",
),
pytest.param(
def build_partial(real_list: List[str] = ["0"]):
partial = PartialSnapshot(build_snapshot(real_list))
return partial
