def test_no_cross_talk(full_snapshot):
source_model = SnapshotModel()
model = JobListProxyModel(None, 0, 0, 0, 0)
model.setSourceModel(source_model)
reporting_mode = qt_api.QtTest.QAbstractItemModelTester.FailureReportingMode.Warning
qt_api.QtTest.QAbstractItemModelTester(model, reporting_mode) # noqa, prevent GC
source_model._add_snapshot(SnapshotModel.prerender(full_snapshot), 0)
source_model._add_snapshot(SnapshotModel.prerender(full_snapshot), 1)
# Test that changes to iter=1 does not bleed into iter=0
partial = PartialSnapshot(full_snapshot)
partial.update_job("0", "0", "0", job=Job(status=JOB_STATE_FAILURE))
source_model._add_partial_snapshot(SnapshotModel.prerender(partial), 1)
assert (
model.index(0, _id_to_col(ids.STATUS), QModelIndex()).data() == JOB_STATE_START
)
model.set_step(1, 0, 0, 0)
assert (
model.index(0, _id_to_col(ids.STATUS), QModelIndex()).data()
== JOB_STATE_FAILURE
)
class RunDialog(QDialog):
simulation_done = Signal(bool, str)
simulation_termination_request = Signal()
def __init__(self,
config_file,
run_model,
simulation_arguments,
parent=None):
QDialog.__init__(self, parent)
self.setWindowFlags(Qt.Window)
self.setWindowFlags(self.windowFlags()
& ~Qt.WindowContextHelpButtonHint)
self.setModal(True)
self.setWindowModality(Qt.WindowModal)
self.setWindowTitle("Simulations - {}".format(config_file))
self._snapshot_model = SnapshotModel(self)
self._run_model = run_model
self._isDetailedDialog = False
self._minimum_width = 1200
ert = None
if isinstance(run_model, BaseRunModel):
ert = run_model.ert()
self._simulations_argments = simulation_arguments
self._ticker = QTimer(self)
self._ticker.timeout.connect(self._on_ticker)
progress_proxy_model = ProgressProxyModel(self._snapshot_model,
parent=self)
self._total_progress_label = QLabel(_TOTAL_PROGRESS_TEMPLATE.format(0),
self)
self._total_progress_bar = QProgressBar(self)
self._total_progress_bar.setRange(0, 100)
self._total_progress_bar.setTextVisible(False)
self._iteration_progress_label = QLabel(self)
self._progress_view = ProgressView(self)
self._progress_view.setModel(progress_proxy_model)
self._progress_view.setIndeterminate(True)
legend_view = LegendView(self)
legend_view.setModel(progress_proxy_model)
self._tab_widget = QTabWidget(self)
self._snapshot_model.rowsInserted.connect(self.on_new_iteration)
self._job_label = QLabel(self)
self._job_model = JobListProxyModel(self, 0, 0, 0, 0)
self._job_model.setSourceModel(self._snapshot_model)
self._job_view = QTableView(self)
self._job_view.clicked.connect(self._job_clicked)
self._open_files = {}
self._job_view.setModel(self._job_model)
self.running_time = QLabel("")
self.plot_tool = PlotTool(config_file)
self.plot_tool.setParent(self)
self.plot_button = QPushButton(self.plot_tool.getName())
self.plot_button.clicked.connect(self.plot_tool.trigger)
self.plot_button.setEnabled(ert is not None)
self.kill_button = QPushButton("Kill Simulations")
self.done_button = QPushButton("Done")
self.done_button.setHidden(True)
self.restart_button = QPushButton("Restart")
self.restart_button.setHidden(True)
self.show_details_button = QPushButton("Show Details")
self.show_details_button.setCheckable(True)
size = 20
spin_movie = resourceMovie("ide/loading.gif")
spin_movie.setSpeed(60)
spin_movie.setScaledSize(QSize(size, size))
spin_movie.start()
self.processing_animation = QLabel()
self.processing_animation.setMaximumSize(QSize(size, size))
self.processing_animation.setMinimumSize(QSize(size, size))
self.processing_animation.setMovie(spin_movie)
button_layout = QHBoxLayout()
button_layout.addWidget(self.processing_animation)
button_layout.addWidget(self.running_time)
button_layout.addStretch()
button_layout.addWidget(self.show_details_button)
button_layout.addWidget(self.plot_button)
button_layout.addWidget(self.kill_button)
button_layout.addWidget(self.done_button)
button_layout.addWidget(self.restart_button)
button_widget_container = QWidget()
button_widget_container.setLayout(button_layout)
layout = QVBoxLayout()
layout.addWidget(self._total_progress_label)
layout.addWidget(self._total_progress_bar)
layout.addWidget(self._iteration_progress_label)
layout.addWidget(self._progress_view)
layout.addWidget(legend_view)
layout.addWidget(self._tab_widget)
layout.addWidget(self._job_label)
layout.addWidget(self._job_view)
layout.addWidget(button_widget_container)
self.setLayout(layout)
self.kill_button.clicked.connect(self.killJobs)
self.done_button.clicked.connect(self.accept)
self.restart_button.clicked.connect(self.restart_failed_realizations)
self.show_details_button.clicked.connect(self.toggle_detailed_progress)
self.simulation_done.connect(self._on_simulation_done)
self.setMinimumWidth(self._minimum_width)
self._setSimpleDialog()
def _setSimpleDialog(self) -> None:
self._isDetailedDialog = False
self._tab_widget.setVisible(False)
self._job_label.setVisible(False)
self._job_view.setVisible(False)
self.show_details_button.setText("Show Details")
def _setDetailedDialog(self) -> None:
self._isDetailedDialog = True
self._tab_widget.setVisible(True)
self._job_label.setVisible(True)
self._job_view.setVisible(True)
self.show_details_button.setText("Hide Details")
@Slot(QModelIndex, int, int)
def on_new_iteration(self, parent: QModelIndex, start: int,
end: int) -> None:
if not parent.isValid():
iter = start
self._iteration_progress_label.setText(
f"Progress for iteration {iter}")
widget = RealizationWidget(iter)
widget.setSnapshotModel(self._snapshot_model)
widget.currentChanged.connect(self._select_real)
self._tab_widget.addTab(widget,
f"Realizations for iteration {iter}")
@Slot(QModelIndex)
def _job_clicked(self, index):
if not index.isValid():
return
selected_file = index.data(FileRole)
if selected_file and selected_file not in self._open_files:
job_name = index.siblingAtColumn(0).data()
viewer = FileDialog(
selected_file,
job_name,
index.row(),
index.model().get_real(),
index.model().get_iter(),
self,
)
self._open_files[selected_file] = viewer
viewer.finished.connect(
lambda _, f=selected_file: self._open_files.pop(f))
elif selected_file in self._open_files:
self._open_files[selected_file].raise_()
@Slot(QModelIndex)
def _select_real(self, index):
step = 0
stage = 0
real = index.row()
iter_ = index.model().get_iter()
self._job_model.set_step(iter_, real, stage, step)
self._job_label.setText(
f"Realization id {index.data(RealIens)} in iteration {iter_}")
self._job_view.horizontalHeader().setSectionResizeMode(
QHeaderView.Stretch)
# Clear the selection in the other tabs
for i in range(0, self._tab_widget.count()):
if i != self._tab_widget.currentIndex():
self._tab_widget.widget(i).clearSelection()
def reject(self):
return
def closeEvent(self, QCloseEvent):
if self._run_model.isFinished():
self.simulation_done.emit(self._run_model.hasRunFailed(),
self._run_model.getFailMessage())
else:
# Kill jobs if dialog is closed
if self.killJobs() != QMessageBox.Yes:
QCloseEvent.ignore()
def startSimulation(self):
self._run_model.reset()
self._snapshot_model.reset()
self._tab_widget.clear()
def run():
asyncio.set_event_loop(asyncio.new_event_loop())
self._run_model.startSimulations(self._simulations_argments)
simulation_thread = Thread(name="ert_gui_simulation_thread")
simulation_thread.setDaemon(True)
simulation_thread.run = run
simulation_thread.start()
self._ticker.start(1000)
tracker = create_tracker(
self._run_model,
num_realizations=self._simulations_argments["active_realizations"].
count(),
ee_config=self._simulations_argments.get("ee_config", None),
)
worker = TrackerWorker(tracker)
worker_thread = QThread()
worker.done.connect(worker_thread.quit)
worker.consumed_event.connect(self._on_tracker_event)
worker.moveToThread(worker_thread)
self.simulation_done.connect(worker.stop)
self._worker = worker
self._worker_thread = worker_thread
worker_thread.started.connect(worker.consume_and_emit)
self._worker_thread.start()
def killJobs(self):
msg = "Are you sure you want to kill the currently running simulations?"
if self._run_model.getQueueStatus().get(
JobStatusType.JOB_QUEUE_UNKNOWN, 0) > 0:
msg += "\n\nKilling a simulation with unknown status will not kill the realizations already submitted!"
kill_job = QMessageBox.question(self, "Kill simulations?", msg,
QMessageBox.Yes | QMessageBox.No)
if kill_job == QMessageBox.Yes:
# Normally this slot would be invoked by the signal/slot system,
# but the worker is busy tracking the evaluation.
self._worker.request_termination()
self.reject()
return kill_job
@Slot(bool, str)
def _on_simulation_done(self, failed, failed_msg):
self.processing_animation.hide()
self.kill_button.setHidden(True)
self.done_button.setHidden(False)
self.restart_button.setVisible(self.has_failed_realizations())
self.restart_button.setEnabled(self._run_model.support_restart)
self._total_progress_bar.setValue(100)
self._total_progress_label.setText(
_TOTAL_PROGRESS_TEMPLATE.format(100))
if failed:
QMessageBox.critical(
self,
"Simulations failed!",
f"The simulation failed with the following error:\n\n{failed_msg}",
)
@Slot()
def _on_ticker(self):
runtime = self._run_model.get_runtime()
self.running_time.setText(format_running_time(runtime))
@Slot(object)
def _on_tracker_event(self, event):
if isinstance(event, EndEvent):
self.simulation_done.emit(event.failed, event.failed_msg)
self._worker.stop()
self._ticker.stop()
elif isinstance(event, FullSnapshotEvent):
if event.snapshot is not None:
self._snapshot_model._add_snapshot(event.snapshot,
event.iteration)
self._progress_view.setIndeterminate(event.indeterminate)
progress = int(event.progress * 100)
self._total_progress_bar.setValue(progress)
self._total_progress_label.setText(
_TOTAL_PROGRESS_TEMPLATE.format(progress))
elif isinstance(event, SnapshotUpdateEvent):
if event.partial_snapshot is not None:
self._snapshot_model._add_partial_snapshot(
event.partial_snapshot, event.iteration)
self._progress_view.setIndeterminate(event.indeterminate)
progress = int(event.progress * 100)
self._total_progress_bar.setValue(progress)
self._total_progress_label.setText(
_TOTAL_PROGRESS_TEMPLATE.format(progress))
def has_failed_realizations(self):
completed = self._run_model.completed_realizations_mask
initial = self._run_model.initial_realizations_mask
for (index, successful) in enumerate(completed):
if initial[index] and not successful:
return True
return False
def count_successful_realizations(self):
"""
Counts the realizations completed in the prevoius ensemble run
:return:
"""
completed = self._run_model.completed_realizations_mask
return completed.count(True)
def create_mask_from_failed_realizations(self):
"""
Creates a BoolVector mask representing the failed realizations
:return: Type BoolVector
"""
completed = self._run_model.completed_realizations_mask
initial = self._run_model.initial_realizations_mask
inverted_mask = BoolVector(default_value=False)
for (index, successful) in enumerate(completed):
inverted_mask[index] = initial[index] and not successful
return inverted_mask
def restart_failed_realizations(self):
msg = QMessageBox(self)
msg.setIcon(QMessageBox.Information)
msg.setText(
"Note that workflows will only be executed on the restarted realizations and that this might have unexpected consequences."
)
msg.setWindowTitle("Restart Failed Realizations")
msg.setStandardButtons(QMessageBox.Ok | QMessageBox.Cancel)
result = msg.exec_()
if result == QMessageBox.Ok:
self.restart_button.setVisible(False)
self.kill_button.setVisible(True)
self.done_button.setVisible(False)
active_realizations = self.create_mask_from_failed_realizations()
self._simulations_argments[
"active_realizations"] = active_realizations
self._simulations_argments[
"prev_successful_realizations"] = self._simulations_argments.get(
"prev_successful_realizations", 0)
self._simulations_argments[
"prev_successful_realizations"] += self.count_successful_realizations(
)
self.startSimulation()
@Slot()
def toggle_detailed_progress(self):
if self._isDetailedDialog:
self._setSimpleDialog()
else:
self._setDetailedDialog()
self.adjustSize()
class RunDialog(QDialog):
simulation_done = Signal(bool, str)
simulation_termination_request = Signal()
def __init__(self, config_file, run_model, parent=None):
QDialog.__init__(self, parent)
self.setWindowFlags(Qt.Window)
self.setWindowFlags(self.windowFlags()
& ~Qt.WindowContextHelpButtonHint)
self.setModal(True)
self.setWindowModality(Qt.WindowModal)
self.setWindowTitle(f"Simulations - {config_file}")
self._snapshot_model = SnapshotModel(self)
self._run_model = run_model
self._isDetailedDialog = False
self._minimum_width = 1200
ert = None
if isinstance(run_model, BaseRunModel):
ert = run_model.ert()
self._ticker = QTimer(self)
self._ticker.timeout.connect(self._on_ticker)
progress_proxy_model = ProgressProxyModel(self._snapshot_model,
parent=self)
self._total_progress_label = QLabel(
_TOTAL_PROGRESS_TEMPLATE.format(
total_progress=0, phase_name=run_model.getPhaseName()),
self,
)
self._total_progress_bar = QProgressBar(self)
self._total_progress_bar.setRange(0, 100)
self._total_progress_bar.setTextVisible(False)
self._iteration_progress_label = QLabel(self)
self._progress_view = ProgressView(self)
self._progress_view.setModel(progress_proxy_model)
self._progress_view.setIndeterminate(True)
legend_view = LegendView(self)
legend_view.setModel(progress_proxy_model)
self._tab_widget = QTabWidget(self)
self._tab_widget.currentChanged.connect(self._current_tab_changed)
self._snapshot_model.rowsInserted.connect(self.on_new_iteration)
self._job_label = QLabel(self)
self._job_model = JobListProxyModel(self, 0, 0, 0, 0)
self._job_model.setSourceModel(self._snapshot_model)
self._job_view = QTableView(self)
self._job_view.setVerticalScrollMode(QAbstractItemView.ScrollPerItem)
self._job_view.setSelectionBehavior(QAbstractItemView.SelectRows)
self._job_view.setSelectionMode(QAbstractItemView.SingleSelection)
self._job_view.clicked.connect(self._job_clicked)
self._open_files = {}
self._job_view.setModel(self._job_model)
self.running_time = QLabel("")
self.plot_tool = PlotTool(ert, config_file)
self.plot_tool.setParent(self)
self.plot_button = QPushButton(self.plot_tool.getName())
self.plot_button.clicked.connect(self.plot_tool.trigger)
self.plot_button.setEnabled(ert is not None)
self.kill_button = QPushButton("Kill simulations")
self.done_button = QPushButton("Done")
self.done_button.setHidden(True)
self.restart_button = QPushButton("Restart")
self.restart_button.setHidden(True)
self.show_details_button = QPushButton("Show details")
self.show_details_button.setCheckable(True)
size = 20
spin_movie = resourceMovie("loading.gif")
spin_movie.setSpeed(60)
spin_movie.setScaledSize(QSize(size, size))
spin_movie.start()
self.processing_animation = QLabel()
self.processing_animation.setMaximumSize(QSize(size, size))
self.processing_animation.setMinimumSize(QSize(size, size))
self.processing_animation.setMovie(spin_movie)
button_layout = QHBoxLayout()
button_layout.addWidget(self.processing_animation)
button_layout.addWidget(self.running_time)
button_layout.addStretch()
button_layout.addWidget(self.show_details_button)
button_layout.addWidget(self.plot_button)
button_layout.addWidget(self.kill_button)
button_layout.addWidget(self.done_button)
button_layout.addWidget(self.restart_button)
button_widget_container = QWidget()
button_widget_container.setLayout(button_layout)
layout = QVBoxLayout()
layout.addWidget(self._total_progress_label)
layout.addWidget(self._total_progress_bar)
layout.addWidget(self._iteration_progress_label)
layout.addWidget(self._progress_view)
layout.addWidget(legend_view)
layout.addWidget(self._tab_widget)
layout.addWidget(self._job_label)
layout.addWidget(self._job_view)
layout.addWidget(button_widget_container)
self.setLayout(layout)
self.kill_button.clicked.connect(self.killJobs)
self.done_button.clicked.connect(self.accept)
self.restart_button.clicked.connect(self.restart_failed_realizations)
self.show_details_button.clicked.connect(self.toggle_detailed_progress)
self.simulation_done.connect(self._on_simulation_done)
self.setMinimumWidth(self._minimum_width)
self._setSimpleDialog()
def _current_tab_changed(self, index: int):
# Clear the selection in the other tabs
for i in range(0, self._tab_widget.count()):
if i != index:
self._tab_widget.widget(i).clearSelection()
def _setSimpleDialog(self) -> None:
self._isDetailedDialog = False
self._tab_widget.setVisible(False)
self._job_label.setVisible(False)
self._job_view.setVisible(False)
self.show_details_button.setText("Show details")
def _setDetailedDialog(self) -> None:
self._isDetailedDialog = True
self._tab_widget.setVisible(True)
self._job_label.setVisible(True)
self._job_view.setVisible(True)
self.show_details_button.setText("Hide details")
@Slot(QModelIndex, int, int)
def on_new_iteration(self, parent: QModelIndex, start: int,
end: int) -> None:
if not parent.isValid():
index = self._snapshot_model.index(start, 0, parent)
iter_row = start
self._iteration_progress_label.setText(
f"Progress for iteration {index.internalPointer().id}")
widget = RealizationWidget(iter_row)
widget.setSnapshotModel(self._snapshot_model)
widget.currentChanged.connect(self._select_real)
self._tab_widget.addTab(
widget,
f"Realizations for iteration {index.internalPointer().id}")
@Slot(QModelIndex)
def _job_clicked(self, index):
if not index.isValid():
return
selected_file = index.data(FileRole)
if selected_file and selected_file not in self._open_files:
job_name = index.siblingAtColumn(0).data()
viewer = FileDialog(
selected_file,
job_name,
index.row(),
index.model().get_real(),
index.model().get_iter(),
self,
)
self._open_files[selected_file] = viewer
def remove_file():
"""
We have sometimes seen this fail because the selected file is not
in open file, without being able to reproduce the exception.
"""
try:
self._open_files.pop(selected_file)
except KeyError:
logger = logging.getLogger(__name__)
logger.exception(
f"Failed to pop: {selected_file} from {self._open_files}"
)
viewer.finished.connect(remove_file)
elif selected_file in self._open_files:
self._open_files[selected_file].raise_()
@Slot(QModelIndex)
def _select_real(self, index):
step = 0
stage = 0
real = index.row()
iter_ = index.model().get_iter()
self._job_model.set_step(iter_, real, stage, step)
self._job_label.setText(
f"Realization id {index.data(RealIens)} in iteration {iter_}")
self._job_view.horizontalHeader().setSectionResizeMode(
QHeaderView.Stretch)
def reject(self):
return
def closeEvent(self, QCloseEvent):
if self._run_model.isFinished():
self.simulation_done.emit(self._run_model.hasRunFailed(),
self._run_model.getFailMessage())
else:
# Kill jobs if dialog is closed
if self.killJobs() != QMessageBox.Yes:
QCloseEvent.ignore()
def startSimulation(self):
self._run_model.reset()
self._snapshot_model.reset()
self._tab_widget.clear()
evaluator_server_config = EvaluatorServerConfig()
def run():
asyncio.set_event_loop(asyncio.new_event_loop())
self._run_model.startSimulations(
evaluator_server_config=evaluator_server_config, )
simulation_thread = Thread(name="ert_gui_simulation_thread")
simulation_thread.setDaemon(True)
simulation_thread.run = run
simulation_thread.start()
self._ticker.start(1000)
tracker = EvaluatorTracker(
self._run_model,
ee_con_info=evaluator_server_config.get_connection_info(),
)
worker = TrackerWorker(tracker)
worker_thread = QThread()
worker.done.connect(worker_thread.quit)
worker.consumed_event.connect(self._on_tracker_event)
worker.moveToThread(worker_thread)
self.simulation_done.connect(worker.stop)
self._worker = worker
self._worker_thread = worker_thread
worker_thread.started.connect(worker.consume_and_emit)
self._worker_thread.start()
def killJobs(self):
msg = "Are you sure you want to kill the currently running simulations?"
kill_job = QMessageBox.question(self, "Kill simulations?", msg,
QMessageBox.Yes | QMessageBox.No)
if kill_job == QMessageBox.Yes:
# Normally this slot would be invoked by the signal/slot system,
# but the worker is busy tracking the evaluation.
self._worker.request_termination()
self.reject()
return kill_job
@Slot(bool, str)
def _on_simulation_done(self, failed, failed_msg):
self.processing_animation.hide()
self.kill_button.setHidden(True)
self.done_button.setHidden(False)
self.restart_button.setVisible(
self._run_model.has_failed_realizations())
self.restart_button.setEnabled(self._run_model.support_restart)
self._total_progress_bar.setValue(100)
self._total_progress_label.setText(
_TOTAL_PROGRESS_TEMPLATE.format(
total_progress=100, phase_name=self._run_model.getPhaseName()))
if failed:
msg = QMessageBox()
msg.setIcon(QMessageBox.Critical)
msg.setText("Simulations failed!".center(100))
msg.setDetailedText(failed_msg)
msg.exec_()
@Slot()
def _on_ticker(self):
runtime = self._run_model.get_runtime()
self.running_time.setText(format_running_time(runtime))
@Slot(object)
def _on_tracker_event(self, event):
if isinstance(event, EndEvent):
self.simulation_done.emit(event.failed, event.failed_msg)
self._worker.stop()
self._ticker.stop()
elif isinstance(event, FullSnapshotEvent):
if event.snapshot is not None:
self._snapshot_model._add_snapshot(event.snapshot,
event.iteration)
self._progress_view.setIndeterminate(event.indeterminate)
progress = int(event.progress * 100)
self._total_progress_bar.setValue(progress)
self._total_progress_label.setText(
_TOTAL_PROGRESS_TEMPLATE.format(total_progress=progress,
phase_name=event.phase_name))
elif isinstance(event, SnapshotUpdateEvent):
if event.partial_snapshot is not None:
self._snapshot_model._add_partial_snapshot(
event.partial_snapshot, event.iteration)
self._progress_view.setIndeterminate(event.indeterminate)
progress = int(event.progress * 100)
self._total_progress_bar.setValue(progress)
self._total_progress_label.setText(
_TOTAL_PROGRESS_TEMPLATE.format(total_progress=progress,
phase_name=event.phase_name))
def restart_failed_realizations(self):
msg = QMessageBox(self)
msg.setIcon(QMessageBox.Information)
msg.setText(
"Note that workflows will only be executed on the restarted "
"realizations and that this might have unexpected consequences.")
msg.setWindowTitle("Restart failed realizations")
msg.setStandardButtons(QMessageBox.Ok | QMessageBox.Cancel)
result = msg.exec_()
if result == QMessageBox.Ok:
self.restart_button.setVisible(False)
self.kill_button.setVisible(True)
self.done_button.setVisible(False)
self._run_model.restart()
self.startSimulation()
@Slot()
def toggle_detailed_progress(self):
if self._isDetailedDialog:
self._setSimpleDialog()
else:
self._setDetailedDialog()
self.adjustSize()