def test_print_progress(self):
out = StringIO()
monitor = Monitor(out=out)
sd = SnapshotDict(status="")
for i in range(0, 100):
status = REALIZATION_STATE_FINISHED if i < 50 else REALIZATION_STATE_WAITING
sd.reals[i] = Realization(status=status, active=True)
monitor._snapshots[0] = Snapshot(sd.dict())
monitor._start_time = datetime.now()
general_event = _UpdateEvent(
phase_name="Test Phase",
current_phase=0,
total_phases=2,
progress=0.5,
indeterminate=False,
iteration=0,
)
monitor._print_progress(general_event)
self.assertEqual(
"""\r
--> Test Phase
1/2 |███████████████ | 50% Running time: 0 seconds
Waiting 50/100
Pending 0/100
Running 0/100
Failed 0/100
Finished 50/100
Unknown 0/100
""",
out.getvalue(),
)
def _create_snapshot(self):
reals = {}
for real in self.get_active_reals():
reals[str(real.get_iens())] = Realization(
active=True,
status=state.REALIZATION_STATE_WAITING,
)
for step in real.get_steps():
reals[str(real.get_iens())].steps[str(step.get_id())] = Step(
status=state.STEP_STATE_UNKNOWN
)
for job in step.get_jobs():
reals[str(real.get_iens())].steps[str(step.get_id())].jobs[
str(job.get_id())
] = Job(
status=state.JOB_STATE_START,
data={},
name=job.get_name(),
)
top = SnapshotDict(
reals=reals,
status=state.ENSEMBLE_STATE_UNKNOWN,
metadata=self.get_metadata(),
)
return Snapshot(top.dict())
def full_snapshot() -> Snapshot:
real = Realization(
status=REALIZATION_STATE_UNKNOWN,
active=True,
steps={
"0":
Step(
status="",
jobs={
"0":
Job(
start_time=dt.now(),
end_time=dt.now(),
name="poly_eval",
status=JOB_STATE_START,
error="error",
stdout="std_out_file",
stderr="std_err_file",
data={
CURRENT_MEMORY_USAGE: "123",
MAX_MEMORY_USAGE: "312",
},
),
"1":
Job(
start_time=dt.now(),
end_time=dt.now(),
name="poly_postval",
status=JOB_STATE_START,
error="error",
stdout="std_out_file",
stderr="std_err_file",
data={
CURRENT_MEMORY_USAGE: "123",
MAX_MEMORY_USAGE: "312",
},
),
},
)
},
)
snapshot = SnapshotDict(
status=ENSEMBLE_STATE_STARTED,
reals={},
)
for i in range(0, 100):
snapshot.reals[str(i)] = copy.deepcopy(real)
return Snapshot(snapshot.dict())
def test_legends(self):
monitor = Monitor(out=StringIO())
sd = SnapshotDict(status="")
for i in range(0, 100):
status = REALIZATION_STATE_FINISHED if i < 10 else REALIZATION_STATE_RUNNING
sd.reals[i] = Realization(status=status, active=True)
monitor._snapshots[0] = Snapshot(sd.dict())
legends = monitor._get_legends()
self.assertEqual(
""" Waiting 0/100
Pending 0/100
Running 90/100
Failed 0/100
Finished 10/100
Unknown 0/100
""",
legends,
)
def test_print_progress(self):
out = StringIO()
monitor = Monitor(out=out)
sd = SnapshotDict(status="")
for i in range(0, 100):
status = REALIZATION_STATE_FINISHED if i < 50 else REALIZATION_STATE_WAITING
sd.reals[i] = Realization(status=status, active=True)
monitor._snapshots[0] = Snapshot(sd.dict())
monitor._start_time = datetime.now()
general_event = _UpdateEvent(
phase_name="Test Phase",
current_phase=0,
total_phases=2,
progress=0.5,
indeterminate=False,
iteration=0,
)
monitor._print_progress(general_event)
# For some reason, `tqdm` adds an extra line containing a progress-bar,
# even though this test only calls it once.
# I suspect this has something to do with the way `tqdm` does refresh,
# but do not know how to fix it.
# Seems not be a an issue when used normally.
expected = """ --> Test Phase
| | 0% it
1/2 |##############################5 | 50% Running time: 0 seconds
Waiting 50/100
Pending 0/100
Running 0/100
Failed 0/100
Finished 50/100
Unknown 0/100
"""
assert out.getvalue().replace("\r", "\n") == expected
def snapshot_to_tree(snapshot: Snapshot, iter_: int) -> Node:
iter_node = Node(iter_, {ids.STATUS: snapshot.get_status()}, NodeType.ITER)
snapshot_d = SnapshotDict(**snapshot.to_dict())
for real_id in sorted(snapshot_d.reals, key=int):
real = snapshot_d.reals[real_id]
real_node = Node(
real_id,
{
ids.STATUS: real.status,
ids.ACTIVE: real.active
},
NodeType.REAL,
)
iter_node.add_child(real_node)
for step_id, step in real.steps.items():
step_node = Node(step_id, {ids.STATUS: step.status}, NodeType.STEP)
real_node.add_child(step_node)
for job_id in sorted(step.jobs, key=int):
job = step.jobs[job_id]
job_dict = dict(job)
job_dict[ids.DATA] = job.data
job_node = Node(job_id, job_dict, NodeType.JOB)
step_node.add_child(job_node)
return iter_node
def _create_snapshot_dict(
self,
run_context: ErtRunContext,
detailed_progress: typing.Tuple[typing.Dict, int],
iter_: int,
) -> typing.Optional[SnapshotDict]:
"""create a snapshot of a run_context and detailed_progress.
detailed_progress is expected to be a tuple of a realization_progress
dict and iteration number. iter_ represents the current assimilation
cycle."""
self._set_iter_queue(iter_, self._model._job_queue)
snapshot = SnapshotDict(
status=ENSEMBLE_STATE_STARTED,
reals={},
metadata={"iter": iter_},
)
forward_model = self._model.get_forward_model()
iter_to_progress, progress_iter = detailed_progress
if progress_iter != iter_:
logger.debug(
f"run_context iter ({iter_}) and detailed_progress ({progress_iter} iter differed"
)
if iter_ in self._iter_queue and self._iter_queue[iter_] is not None:
queue_snapshot = self._iter_queue[iter_].snapshot()
else:
queue_snapshot = None
enumerated = 0
for iens, run_arg in _enumerate_run_context(run_context):
real_id = str(iens)
enumerated += 1
if not _is_iens_active(iens, run_context):
continue
status = JobStatusType.JOB_QUEUE_UNKNOWN
if queue_snapshot is not None and iens in queue_snapshot:
status = JobStatusType.from_string(queue_snapshot[iens])
snapshot.reals[real_id] = Realization(
status=queue_status_to_real_state(status),
active=True,
steps={})
step = Step(status="", jobs={})
snapshot.reals[real_id].steps["0"] = step
for index in range(0, len(forward_model)):
ext_job = forward_model.iget_job(index)
step.jobs[str(index)] = Job(name=ext_job.name(),
status=JOB_STATE_START,
data={})
progress = iter_to_progress[iter_].get(iens, None)
if not progress:
continue
jobs = progress[0]
for idx, fm in enumerate(jobs):
job = step.jobs[str(idx)]
job.start_time = fm.start_time
job.end_time = fm.end_time
job.name = fm.name
job.status = _map_job_state(fm.status)
job.error = fm.error
job.stdout = fm.std_out_file
job.stderr = fm.std_err_file
job.data = {
CURRENT_MEMORY_USAGE: fm.current_memory_usage,
MAX_MEMORY_USAGE: fm.max_memory_usage,
}
if enumerated == 0:
logger.debug("enumerated 0 items from run_context, it is gone")
return None
return snapshot
def _add_partial_snapshot(self, partial: PartialSnapshot, iter_: int):
partial_dict = partial.to_dict()
partial_s = SnapshotDict(**partial_dict)
if iter_ not in self.root.children:
logger.debug("no full snapshot yet, bailing")
return
iter_index = self.index(iter_, 0, QModelIndex())
iter_node = self.root.children[iter_]
if not partial_s.reals:
logger.debug(f"no realizations in partial for iter {iter_}")
return
for real_id in sorted(partial_s.reals, key=int):
real = partial_s.reals[real_id]
real_node = iter_node.children[real_id]
if real.status:
real_node.data[ids.STATUS] = real.status
real_index = self.index(real_node.row(), 0, iter_index)
real_index_bottom_right = self.index(
real_node.row(),
self.columnCount(iter_index) - 1, iter_index)
if not real.steps:
continue
for step_id, step in real.steps.items():
step_node = real_node.children[step_id]
if step.status:
step_node.data[ids.STATUS] = step.status
step_index = self.index(step_node.row(), 0, real_index)
step_index_bottom_right = self.index(
step_node.row(),
self.columnCount(real_index) - 1, real_index)
if not step.jobs:
continue
for job_id in sorted(step.jobs, key=int):
job = step.jobs[job_id]
job_node = step_node.children[job_id]
if job.status:
job_node.data[ids.STATUS] = job.status
if job.start_time:
job_node.data[ids.START_TIME] = job.start_time
if job.end_time:
job_node.data[ids.END_TIME] = job.end_time
if job.stdout:
job_node.data[ids.STDOUT] = job.stdout
if job.stderr:
job_node.data[ids.STDERR] = job.stderr
# Errors may be unset as the queue restarts the job
job_node.data[ids.ERROR] = job.error if job.error else ""
for attr in (ids.CURRENT_MEMORY_USAGE,
ids.MAX_MEMORY_USAGE):
if job.data and attr in job.data:
job_node.data[ids.DATA][attr] = job.data.get(attr)
job_index = self.index(job_node.row(), 0, step_index)
job_index_bottom_right = self.index(
job_node.row(),
self.columnCount() - 1, step_index)
self.dataChanged.emit(job_index, job_index_bottom_right)
self.dataChanged.emit(step_index, step_index_bottom_right)
self.dataChanged.emit(real_index, real_index_bottom_right)
# TODO: there is no check that any of the data *actually* changed
# https://github.com/equinor/ert/issues/1374
top_left = self.index(0, 0, iter_index)
bottom_right = self.index(0, 1, iter_index)
self.dataChanged.emit(top_left, bottom_right)