def request_termination(self) -> None: logger = logging.getLogger("ert_shared.ensemble_evaluator.tracker") # There might be some situations where the # evaluation is finished or the evaluation # is yet to start when calling this function. # In these cases the monitor is not started # # To avoid waiting too long we exit if we are not # able to connect to the monitor after 2 tries # # See issue: https://github.com/equinor/ert/issues/1250 # try: get_event_loop().run_until_complete( wait_for_evaluator( base_url=self._ee_con_info.url, token=self._ee_con_info.token, cert=self._ee_con_info.cert, timeout=5, )) except ClientError as e: logger.warning(f"{__name__} - exception {e}") return with create_ee_monitor(self._ee_con_info) as monitor: monitor.signal_cancel() while self._drainer_thread.is_alive(): self._clear_work_queue() time.sleep(1)
def request_termination(self): logger = logging.getLogger("ert_shared.ensemble_evaluator.tracker") # There might be some situations where the # evaluation is finished or the evaluation # is yet to start when calling this function. # In these cases the monitor is not started # # To avoid waiting too long we exit if we are not # able to connect to the monitor after 2 tries # # See issue: https://github.com/equinor/ert/issues/1250 # try: wait_for_ws(self._monitor_url, 2) except ConnectionRefusedError as e: logger.warning(f"{__name__} - exception {e}") return with create_ee_monitor(self._monitor_host, self._monitor_port) as monitor: monitor.signal_cancel() while self._drainer_thread.is_alive(): self._clear_work_queue() time.sleep(1)
def _drain_monitor(self) -> None: asyncio.set_event_loop(asyncio.new_event_loop()) drainer_logger = logging.getLogger( "ert_shared.ensemble_evaluator.drainer") while not self._model.isFinished(): try: drainer_logger.debug("connecting to new monitor...") with create_ee_monitor(self._ee_con_info) as monitor: drainer_logger.debug("connected") for event in monitor.track(): if event["type"] in ( EVTYPE_EE_SNAPSHOT, EVTYPE_EE_SNAPSHOT_UPDATE, ): self._work_queue.put(event) if event.data.get(STATUS) in [ ENSEMBLE_STATE_STOPPED, ENSEMBLE_STATE_FAILED, ]: drainer_logger.debug( "observed evaluation stopped event, signal done" ) monitor.signal_done() if event.data.get( STATUS) == ENSEMBLE_STATE_CANCELLED: drainer_logger.debug( "observed evaluation cancelled event, exit drainer" ) # Allow track() to emit an EndEvent. self._work_queue.put(EvaluatorTracker.DONE) return elif event["type"] == EVTYPE_EE_TERMINATED: drainer_logger.debug("got terminator event") # This sleep needs to be there. Refer to issue #1250: `Authority # on information about evaluations/experiments` time.sleep(self._next_ensemble_evaluator_wait_time) except (ConnectionRefusedError, ClientError) as e: if not self._model.isFinished(): drainer_logger.debug(f"connection refused: {e}") except (ConnectionClosedError) as e: # The monitor connection closed unexpectedly drainer_logger.debug(f"connection closed error: {e}") except BaseException: # pylint: disable=broad-except drainer_logger.exception("unexpected error: ") # We really don't know what happened... shut down # the thread and get out of here. The monitor has # been stopped by the ctx-mgr self._work_queue.put(EvaluatorTracker.DONE) self._work_queue.join() return drainer_logger.debug( "observed that model was finished, waiting tasks completion...") # The model has finished, we indicate this by sending a DONE self._work_queue.put(EvaluatorTracker.DONE) self._work_queue.join() drainer_logger.debug("tasks complete") self._model.teardown_context()
def _drain_monitor(self): asyncio.set_event_loop(asyncio.new_event_loop()) drainer_logger = logging.getLogger("ert_shared.ensemble_evaluator.drainer") while not self._model.isFinished(): try: drainer_logger.debug("connecting to new monitor...") with create_ee_monitor( self._monitor_host, self._monitor_port, protocol=self._protocol, cert=self._cert, token=self._token, ) as monitor: drainer_logger.debug("connected") for event in monitor.track(): if event["type"] in ( ids.EVTYPE_EE_SNAPSHOT, ids.EVTYPE_EE_SNAPSHOT_UPDATE, ): self._work_queue.put(event) if event.data.get(ids.STATUS) in [ ENSEMBLE_STATE_STOPPED, ENSEMBLE_STATE_FAILED, ]: drainer_logger.debug( "observed evaluation stopped event, signal done" ) monitor.signal_done() if event.data.get(ids.STATUS) == ENSEMBLE_STATE_CANCELLED: drainer_logger.debug( "observed evaluation cancelled event, exit drainer" ) # Allow track() to emit an EndEvent. self._work_queue.put(EvaluatorTracker.DONE) return elif event["type"] == ids.EVTYPE_EE_TERMINATED: drainer_logger.debug("got terminator event") # This sleep needs to be there. Refer to issue #1250: `Authority # on information about evaluations/experiments` time.sleep(self._next_ensemble_evaluator_wait_time) except (ConnectionRefusedError, ClientError) as e: if not self._model.isFinished(): drainer_logger.debug(f"connection refused: {e}") except (ConnectionClosedError) as e: # The monitor connection closed unexpectedly drainer_logger.debug(f"connection closed error: {e}") drainer_logger.debug( "observed that model was finished, waiting tasks completion..." ) # The model has finished, we indicate this by sending a DONE self._work_queue.put(EvaluatorTracker.DONE) self._work_queue.join() drainer_logger.debug("tasks complete") self._model.teardown_context()
def _drain_monitor(self): drainer_logger = logging.getLogger( "ert_shared.ensemble_evaluator.drainer") failures = 0 while not self._model.isFinished(): try: drainer_logger.debug("connecting to new monitor...") with create_ee_monitor(self._monitor_host, self._monitor_port) as monitor: drainer_logger.debug("connected") for event in monitor.track(): if event["type"] in ( ids.EVTYPE_EE_SNAPSHOT, ids.EVTYPE_EE_SNAPSHOT_UPDATE, ): self._work_queue.put(event) if event.data.get(ids.STATUS) in [ ENSEMBLE_STATE_STOPPED, ENSEMBLE_STATE_FAILED, ]: drainer_logger.debug( "observed evaluation stopped event, signal done" ) monitor.signal_done() if event.data.get( ids.STATUS) == ENSEMBLE_STATE_CANCELLED: drainer_logger.debug( "observed evaluation cancelled event, exit drainer" ) return elif event["type"] == ids.EVTYPE_EE_TERMINATED: drainer_logger.debug("got terminator event") # This sleep needs to be there. Refer to issue #1250: `Authority # on information about evaluations/experiments` time.sleep(5) except ConnectionRefusedError as e: if not self._model.isFinished(): drainer_logger.debug(f"connection refused: {e}") failures += 1 if failures == 10: drainer_logger.debug(f"giving up.") raise e else: failures = 0 drainer_logger.debug( "observed that model was finished, waiting tasks completion...") # The model has finished, we indicate this by sending a None self._work_queue.put(None) self._work_queue.join() drainer_logger.debug("tasks complete")
def request_termination(self): logger = logging.getLogger("ert_shared.ensemble_evaluator.tracker") config = load_config() # There might be some situations where the # evaulation is finished or the evaluation # is yet to start when calling this function. # In these cases the monitor is not started # # To avoid waiting too long we exit if we are not # able to connect to the monitor after 2 tries # # See issue: https://github.com/equinor/ert/issues/1250 # try: wait_for_ws(config.get("url"), 2) except ConnectionRefusedError as e: logger.warning(f"{__name__} - exception {e}") return monitor = create_ee_monitor(self._monitor_host, self._monitor_port) monitor.signal_cancel()
def _drain_monitor(self): drainer_logger = logging.getLogger( "ert_shared.ensemble_evaluator.drainer") monitor = create_ee_monitor(self._monitor_host, self._monitor_port) while monitor: try: for event in monitor.track(): if event["type"] == ids.EVTYPE_EE_SNAPSHOT: iter_ = event.data["metadata"]["iter"] with self._state_mutex: self._realization_progress[ iter_] = self._snapshot_to_realization_progress( event.data) self._work_queue.put(None) if event.data.get( "status") == _EVTYPE_SNAPSHOT_STOPPED: drainer_logger.debug( "observed evaluation stopped event, signal done" ) monitor.signal_done() elif event["type"] == ids.EVTYPE_EE_SNAPSHOT_UPDATE: with self._state_mutex: self._updates.append(event.data) self._work_queue.put(None) if event.data.get( "status") == _EVTYPE_SNAPSHOT_CANCELLED: drainer_logger.debug( "observed evaluation cancelled event, return" ) return if event.data.get( "status") == _EVTYPE_SNAPSHOT_STOPPED: drainer_logger.debug( "observed evaluation stopped event, signal done" ) monitor.signal_done() elif event["type"] == ids.EVTYPE_EE_TERMINATED: drainer_logger.debug("got terminator event") while True: if self._model.isFinished(): drainer_logger.debug( "observed that model was finished, waiting tasks completion..." ) self._work_queue.join() drainer_logger.debug("tasks complete") return try: time.sleep(5) drainer_logger.debug( "connecting to new monitor...") monitor = create_ee_monitor( self._monitor_host, self._monitor_port) wait_for_ws(monitor.get_base_uri(), max_retries=2) drainer_logger.debug("connected") break except ConnectionRefusedError as e: drainer_logger.debug( f"connection refused: {e}") pass except ConnectionRefusedError as e: if self._model.isFinished(): return else: raise e