def test_ensemble_evaluator(tmpdir, source_root): shutil.copytree( os.path.join(source_root, "test-data", "local", "poly_example"), os.path.join(str(tmpdir), "poly_example"), ) with tmpdir.as_cwd(): parser = ArgumentParser(prog="test_main") parsed = ert_parser( parser, [ ENSEMBLE_SMOOTHER_MODE, "--target-case", "poly_runpath_file", "--realizations", "1,2,4,8,16,32,64", "--enable-ensemble-evaluator", "poly_example/poly.ert", ], ) FeatureToggling.update_from_args(parsed) assert FeatureToggling.is_enabled("ensemble-evaluator") is True run_cli(parsed) FeatureToggling.reset()
def _runAndPostProcess(self, run_context, arguments, update_id=None): phase_msg = "Running iteration %d of %d simulation iterations..." % ( run_context.get_iter(), self.phaseCount() - 1, ) self.setPhase(run_context.get_iter(), phase_msg, indeterminate=False) self.setPhaseName("Pre processing...", indeterminate=True) self.ert().getEnkfSimulationRunner().createRunPath(run_context) EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION, ert=ERT.ert) # create ensemble ensemble_id = self._post_ensemble_data(update_id=update_id) self.setPhaseName("Running forecast...", indeterminate=False) if FeatureToggling.is_enabled("ensemble-evaluator"): ee_config = arguments["ee_config"] num_successful_realizations = self.run_ensemble_evaluator( run_context, ee_config) else: self._job_queue = self._queue_config.create_job_queue() num_successful_realizations = ( self.ert().getEnkfSimulationRunner().runSimpleStep( self._job_queue, run_context)) self.checkHaveSufficientRealizations(num_successful_realizations) self.setPhaseName("Post processing...", indeterminate=True) EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION, ert=ERT.ert) self._post_ensemble_results(ensemble_id) return ensemble_id
def _simulateAndPostProcess(self, run_context, arguments): iteration = run_context.get_iter( ) phase_string = "Running simulation for iteration: %d" % iteration self.setPhaseName(phase_string, indeterminate=True) self.ert().getEnkfSimulationRunner().createRunPath(run_context) phase_string = "Pre processing for iteration: %d" % iteration self.setPhaseName(phase_string) EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION, ert=ERT.ert) phase_string = "Running forecast for iteration: %d" % iteration self.setPhaseName(phase_string, indeterminate=False) if FeatureToggling.is_enabled("ensemble-evaluator"): num_successful_realizations = self.run_ensemble_evaluator(run_context) else: self._job_queue = self._queue_config.create_job_queue() num_successful_realizations = self.ert().getEnkfSimulationRunner().runSimpleStep(self._job_queue, run_context) num_successful_realizations += arguments.get('prev_successful_realizations', 0) self.checkHaveSufficientRealizations(num_successful_realizations) phase_string = "Post processing for iteration: %d" % iteration self.setPhaseName(phase_string, indeterminate=True) EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION, ert=ERT.ert) return num_successful_realizations
def start_ert_server(mode: str): if mode in ("api", "vis") or not FeatureToggling.is_enabled("new-storage"): yield return with Storage.start_server(): yield
def runSimulations__(self, arguments, run_msg): run_context = self.create_context(arguments) self.setPhase(0, "Running simulations...", indeterminate=False) self.setPhaseName("Pre processing...", indeterminate=True) self.ert().getEnkfSimulationRunner().createRunPath(run_context) EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION, ERT.ert) self.setPhaseName(run_msg, indeterminate=False) if FeatureToggling.is_enabled("ensemble-evaluator"): num_successful_realizations = self.run_ensemble_evaluator( run_context) else: self._job_queue = self._queue_config.create_job_queue() num_successful_realizations = ( self.ert().getEnkfSimulationRunner().runEnsembleExperiment( self._job_queue, run_context)) num_successful_realizations += arguments.get( "prev_successful_realizations", 0) self.checkHaveSufficientRealizations(num_successful_realizations) self.setPhaseName("Post processing...", indeterminate=True) EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION, ERT.ert) self.setPhase(1, "Simulations completed.") # done... dump_to_new_storage() return run_context
def run_cli(args): logging.basicConfig(level=logging.INFO, format="%(message)s") res_config = ResConfig(args.config) os.chdir(res_config.config_path) ert = EnKFMain(res_config, strict=True, verbose=args.verbose) notifier = ErtCliNotifier(ert, args.config) ERT.adapt(notifier) if args.mode == WORKFLOW_MODE: execute_workflow(args.name) return model, argument = create_model(args) # Test run does not have a current_case if "current_case" in args and args.current_case: ERT.enkf_facade.select_or_create_new_case(args.current_case) if ( args.mode in [ENSEMBLE_SMOOTHER_MODE, ITERATIVE_ENSEMBLE_SMOOTHER_MODE, ES_MDA_MODE] and args.target_case == ERT.enkf_facade.get_current_case_name() ): msg = ( "ERROR: Target file system and source file system can not be the same. " "They were both: {}.".format(args.target_case) ) _clear_and_exit(msg) ee_config = None if FeatureToggling.is_enabled("ensemble-evaluator"): ee_config = EvaluatorServerConfig() argument.update({"ee_config": ee_config}) thread = threading.Thread( name="ert_cli_simulation_thread", target=model.startSimulations, args=(argument,), ) thread.start() tracker = create_tracker(model, detailed_interval=0, ee_config=ee_config) out = open(os.devnull, "w") if args.disable_monitoring else sys.stdout monitor = Monitor(out=out, color_always=args.color_always) try: monitor.monitor(tracker) except (SystemExit, KeyboardInterrupt): print("\nKilling simulations...") tracker.request_termination() if args.disable_monitoring: out.close() thread.join() if model.hasRunFailed(): _clear_and_exit(1) # the monitor has already reported the error message
def __init__(self, config_file): QWidget.__init__(self) self._config_file = config_file self._ee_config = None if FeatureToggling.is_enabled("ensemble-evaluator"): self._ee_config = EvaluatorServerConfig() self.setObjectName("Simulation_panel") layout = QVBoxLayout() self._simulation_mode_combo = QComboBox() self._simulation_mode_combo.setObjectName("Simulation_mode") addHelpToWidget(self._simulation_mode_combo, "run/simulation_mode") self._simulation_mode_combo.currentIndexChanged.connect( self.toggleSimulationMode) simulation_mode_layout = QHBoxLayout() simulation_mode_layout.addSpacing(10) simulation_mode_layout.addWidget(QLabel("Simulation mode:"), 0, Qt.AlignVCenter) simulation_mode_layout.addWidget(self._simulation_mode_combo, 0, Qt.AlignVCenter) simulation_mode_layout.addSpacing(20) self.run_button = QToolButton() self.run_button.setObjectName("start_simulation") self.run_button.setIconSize(QSize(32, 32)) self.run_button.setText("Start Simulation") self.run_button.setIcon(resourceIcon("ide/gear_in_play")) self.run_button.clicked.connect(self.runSimulation) self.run_button.setToolButtonStyle(Qt.ToolButtonTextBesideIcon) addHelpToWidget(self.run_button, "run/start_simulation") simulation_mode_layout.addWidget(self.run_button) simulation_mode_layout.addStretch(1) layout.addSpacing(5) layout.addLayout(simulation_mode_layout) layout.addSpacing(10) self._simulation_stack = QStackedWidget() self._simulation_stack.setLineWidth(1) self._simulation_stack.setFrameStyle(QFrame.StyledPanel) layout.addWidget(self._simulation_stack) self._simulation_widgets = OrderedDict() """ :type: OrderedDict[BaseRunModel,SimulationConfigPanel]""" self.addSimulationConfigPanel(SingleTestRunPanel()) self.addSimulationConfigPanel(EnsembleExperimentPanel()) if ERT.ert.have_observations(): self.addSimulationConfigPanel(EnsembleSmootherPanel()) self.addSimulationConfigPanel(MultipleDataAssimilationPanel()) self.addSimulationConfigPanel(IteratedEnsembleSmootherPanel()) self.setLayout(layout)
def create_tracker( model, general_interval=5, detailed_interval=10, qtimer_cls=None, event_handler=None, num_realizations=None, ): """Creates a tracker tracking a @model. The provided model is updated in two tiers: @general_interval, @detailed_interval. Setting any interval to <=0 disables update. Should a @qtimer_cls be defined, the Qt event loop will be used for tracking. @event_handler must then be defined. If @num_realizations is defined, then the intervals are scaled according to some affine transformation such that it is tractable to do tracking. If @ee_host_port_tuple then the factory will produce something that can track an ensemble evaluator. """ if num_realizations is not None: general_interval, detailed_interval = scale_intervals(num_realizations) ee_config = load_config() ee_monitor_connection_details = ( (ee_config.get("host"), ee_config.get("port")) if FeatureToggling.is_enabled("ensemble-evaluator") else None ) if qtimer_cls: if not event_handler: raise ValueError( "event_handler must be defined if" + "qtimer_cls is defined" ) return QTimerTracker( model, qtimer_cls, general_interval, detailed_interval, event_handler, ee_monitor_connection_details, ) else: return BlockingTracker( model, general_interval, detailed_interval, ee_monitor_connection_details, )
def start_ert_server(): monitor = None if FeatureToggling.is_enabled("new-storage"): from ert_shared.storage.server_monitor import ServerMonitor monitor = ServerMonitor.get_instance() monitor.start() try: yield finally: if monitor is not None: monitor.shutdown()
def _simulateAndPostProcess(self, run_context, arguments, update_id: int = None): iteration = run_context.get_iter() phase_string = "Running simulation for iteration: %d" % iteration self.setPhaseName(phase_string, indeterminate=True) self.ert().getEnkfSimulationRunner().createRunPath(run_context) phase_string = "Pre processing for iteration: %d" % iteration self.setPhaseName(phase_string) EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION, ert=ERT.ert) # Push ensemble, parameters, observations to new storage new_ensemble_id = post_ensemble_data(update_id=update_id, ensemble_size=self._ensemble_size) phase_string = "Running forecast for iteration: %d" % iteration self.setPhaseName(phase_string, indeterminate=False) if FeatureToggling.is_enabled("ensemble-evaluator"): ee_config = arguments["ee_config"] num_successful_realizations = self.run_ensemble_evaluator( run_context, ee_config) else: self._job_queue = self._queue_config.create_job_queue() num_successful_realizations = ( self.ert().getEnkfSimulationRunner().runSimpleStep( self._job_queue, run_context)) # Push simulation results to storage post_ensemble_results(new_ensemble_id) num_successful_realizations += arguments.get( "prev_successful_realizations", 0) self.checkHaveSufficientRealizations(num_successful_realizations) phase_string = "Post processing for iteration: %d" % iteration self.setPhaseName(phase_string, indeterminate=True) EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION, ert=ERT.ert) return num_successful_realizations, new_ensemble_id
def runSimulations__(self, arguments, run_msg): run_context = self.create_context(arguments) self.setPhase(0, "Running simulations...", indeterminate=False) self.setPhaseName("Pre processing...", indeterminate=True) self.ert().getEnkfSimulationRunner().createRunPath(run_context) # Push ensemble, parameters, observations to new storage ensemble_id = post_ensemble_data(ensemble_size=self._ensemble_size) EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION, ERT.ert) self.setPhaseName(run_msg, indeterminate=False) if FeatureToggling.is_enabled("ensemble-evaluator"): ee_config = arguments["ee_config"] num_successful_realizations = self.run_ensemble_evaluator( run_context, ee_config ) else: self._job_queue = self._queue_config.create_job_queue() num_successful_realizations = ( self.ert() .getEnkfSimulationRunner() .runEnsembleExperiment(self._job_queue, run_context) ) num_successful_realizations += arguments.get("prev_successful_realizations", 0) self.checkHaveSufficientRealizations(num_successful_realizations) self.setPhaseName("Post processing...", indeterminate=True) EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION, ERT.ert) self.setPhase(1, "Simulations completed.") # done... # Push simulation results to storage post_ensemble_results(ensemble_id) return run_context
def create_tracker( model, general_interval=5, detailed_interval=10, num_realizations=None, ee_config=None, ): """Creates a tracker tracking a @model. The provided model is updated either purely event-driven, or in two tiers: @general_interval, @detailed_interval. Whether updates are continuous or periodic depends on invocation. In the case of periodic updates, setting an interval to <=0 disables update. If @num_realizations is defined, then the intervals are scaled according to some affine transformation such that it is tractable to do tracking. This only applies to periodic updates. If @ee_host_port_tuple then the factory will produce something that can track an ensemble evaluator and emit events appropriately. """ if num_realizations is not None: general_interval, detailed_interval = scale_intervals(num_realizations) if FeatureToggling.is_enabled("ensemble-evaluator"): return EvaluatorTracker( model, ee_config.host, ee_config.port, general_interval, detailed_interval, token=ee_config.token, cert=ee_config.cert, ) return LegacyTracker( model, general_interval, detailed_interval, )
def run_gui(args): app = QApplication( []) # Early so that QT is initialized before other imports app.setWindowIcon(resourceIcon("application/window_icon_cutout")) if FeatureToggling.is_enabled("prefect"): ert_conf_path = os.path.abspath(args.config)[:-3] + "ert" with open(ert_conf_path, "w") as f: f.write("""QUEUE_SYSTEM LOCAL QUEUE_OPTION LOCAL MAX_RUNNING 50 RUNPATH out/real_%d/iter_%d NUM_REALIZATIONS 100 MIN_REALIZATIONS 1 """) res_config = ResConfig(ert_conf_path) else: res_config = ResConfig(args.config) os.chdir(res_config.config_path) ert = EnKFMain(res_config, strict=True, verbose=args.verbose) # window reference must be kept until app.exec returns window = _start_window(ert, args) return app.exec_()
def test_tracking( experiment_folder, cmd_line_arguments, num_successful, num_iters, tmpdir, source_root, ): shutil.copytree( os.path.join(source_root, "test-data", "local", f"{experiment_folder}"), os.path.join(str(tmpdir), f"{experiment_folder}"), ) config_lines = [ "INSTALL_JOB poly_eval2 POLY_EVAL\n" "SIMULATION_JOB poly_eval2\n" ] with tmpdir.as_cwd(): with open(f"{experiment_folder}/poly.ert", "a") as fh: fh.writelines(config_lines) parser = ArgumentParser(prog="test_main") parsed = ert_parser( parser, cmd_line_arguments, ) FeatureToggling.update_from_args(parsed) res_config = ResConfig(parsed.config) os.chdir(res_config.config_path) ert = EnKFMain(res_config, strict=True, verbose=parsed.verbose) notifier = ErtCliNotifier(ert, parsed.config) ERT.adapt(notifier) model, argument = create_model(parsed) ee_config = None if FeatureToggling.is_enabled("ensemble-evaluator"): ee_config = EvaluatorServerConfig() argument.update({"ee_config": ee_config}) thread = threading.Thread( name="ert_cli_simulation_thread", target=model.startSimulations, args=(argument, ), ) thread.start() tracker = create_tracker(model, general_interval=1, detailed_interval=2, ee_config=ee_config) snapshots = {} for event in tracker.track(): if isinstance(event, FullSnapshotEvent): snapshots[event.iteration] = event.snapshot if isinstance(event, SnapshotUpdateEvent): if event.partial_snapshot is not None: snapshots[event.iteration].merge( event.partial_snapshot.data()) if isinstance(event, EndEvent): break assert tracker._progress() == 1.0 assert len(snapshots) == num_iters for iter_, snapshot in snapshots.items(): successful_reals = list( filter( lambda item: item[1].status == REALIZATION_STATE_FINISHED, snapshot.get_reals().items(), )) assert len(successful_reals) == num_successful for real_id, real in successful_reals: assert (real.status == REALIZATION_STATE_FINISHED ), f"iter:{iter_} real:{real_id} was not finished" poly = real.steps["0"].jobs["0"] poly2 = real.steps["0"].jobs["1"] assert poly.name == "poly_eval" assert (poly.status == JOB_STATE_FINISHED ), f"real {real_id}/{poly['name']} was not finished" assert poly2.name == "poly_eval2" assert (poly2.status == JOB_STATE_FINISHED ), f"real {real_id}/{poly['name']} was not finished"
def runSimulations(self, arguments): prior_context = self.create_context(arguments) self.checkMinimumActiveRealizations(prior_context) self.setPhase(0, "Running simulations...", indeterminate=False) # self.setAnalysisModule(arguments["analysis_module"]) self.setPhaseName("Pre processing...", indeterminate=True) self.ert().getEnkfSimulationRunner().createRunPath(prior_context) # Push ensemble, parameters, observations to new storage ensemble_id = post_ensemble_data() EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION, ert=ERT.ert) self.setPhaseName("Running forecast...", indeterminate=False) if FeatureToggling.is_enabled("ensemble-evaluator"): ee_config = arguments["ee_config"] num_successful_realizations = self.run_ensemble_evaluator( prior_context, ee_config) else: self._job_queue = self._queue_config.create_job_queue() num_successful_realizations = self.ert().getEnkfSimulationRunner( ).runSimpleStep(self._job_queue, prior_context) self.checkHaveSufficientRealizations(num_successful_realizations) self.setPhaseName("Post processing...", indeterminate=True) EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION, ert=ERT.ert) self.setPhaseName("Analyzing...") EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_FIRST_UPDATE, ert=ERT.ert) EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_UPDATE, ert=ERT.ert) es_update = self.ert().getESUpdate() success = es_update.smootherUpdate(prior_context) if not success: raise ErtRunError("Analysis of simulation failed!") EnkfSimulationRunner.runWorkflows(HookRuntime.POST_UPDATE, ert=ERT.ert) # Push simulation results to storage post_ensemble_results(ensemble_id) self.setPhase(1, "Running simulations...") self.ert().getEnkfFsManager().switchFileSystem( prior_context.get_target_fs()) self.setPhaseName("Pre processing...") rerun_context = self.create_context(arguments, prior_context=prior_context) self.ert().getEnkfSimulationRunner().createRunPath(rerun_context) # Push ensemble, parameters, observations to new storage analysis_module_name = self.ert().analysisConfig().activeModuleName() ensemble_id = post_ensemble_data((ensemble_id, analysis_module_name)) EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION, ert=ERT.ert) self.setPhaseName("Running forecast...", indeterminate=False) if FeatureToggling.is_enabled("ensemble-evaluator"): ee_config = arguments["ee_config"] num_successful_realizations = self.run_ensemble_evaluator( rerun_context, ee_config) else: self._job_queue = self._queue_config.create_job_queue() num_successful_realizations = self.ert().getEnkfSimulationRunner( ).runSimpleStep(self._job_queue, rerun_context) self.checkHaveSufficientRealizations(num_successful_realizations) self.setPhaseName("Post processing...", indeterminate=True) EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION, ert=ERT.ert) self.setPhase(2, "Simulations completed.") # Push simulation results to storage post_ensemble_results(ensemble_id) return prior_context
def test_tracking( experiment_folder, cmd_line_arguments, num_successful, num_iters, assert_present_in_snapshot, tmpdir, source_root, ): shutil.copytree( os.path.join(source_root, "test-data", "local", f"{experiment_folder}"), os.path.join(str(tmpdir), f"{experiment_folder}"), ) config_lines = [ "INSTALL_JOB poly_eval2 POLY_EVAL\n" "SIMULATION_JOB poly_eval2\n" ] with tmpdir.as_cwd(): with open(f"{experiment_folder}/poly.ert", "a") as fh: fh.writelines(config_lines) parser = ArgumentParser(prog="test_main") parsed = ert_parser( parser, cmd_line_arguments, ) FeatureToggling.update_from_args(parsed) res_config = ResConfig(parsed.config) os.chdir(res_config.config_path) ert = EnKFMain(res_config, strict=True, verbose=parsed.verbose) notifier = ErtCliNotifier(ert, parsed.config) ERT.adapt(notifier) model, argument = create_model(parsed) ee_config = None if FeatureToggling.is_enabled("ensemble-evaluator"): ee_config = EvaluatorServerConfig() argument.update({"ee_config": ee_config}) thread = threading.Thread( name="ert_cli_simulation_thread", target=model.start_simulations_thread, args=(argument, ), ) thread.start() tracker = create_tracker(model, general_interval=1, detailed_interval=2, ee_config=ee_config) snapshots = {} for event in tracker.track(): if isinstance(event, FullSnapshotEvent): snapshots[event.iteration] = event.snapshot if isinstance(event, SnapshotUpdateEvent): if event.partial_snapshot is not None: snapshots[event.iteration].merge( event.partial_snapshot.data()) if isinstance(event, EndEvent): pass assert tracker._progress() == 1.0 assert len(snapshots) == num_iters for iter_, snapshot in snapshots.items(): successful_reals = list( filter( lambda item: item[1].status == state. REALIZATION_STATE_FINISHED, snapshot.get_reals().items(), )) assert len(successful_reals) == num_successful for ( iter_expression, snapshot_expression, expected, ) in assert_present_in_snapshot: for i, snapshot in snapshots.items(): if re.match(iter_expression, str(i)): check_expression( snapshot.to_dict(), snapshot_expression, expected, f"Snapshot {i} did not match:\n", ) thread.join()
def killAllSimulations(self) -> None: if FeatureToggling.is_enabled("ensemble-evaluator"): raise NotImplementedError( "the ensemble evaluator does not implement killAllSimulations") self._job_queue.kill_all_jobs()
def run_cli(args): res_config = ResConfig(args.config) # Create logger inside function to make sure all handlers have been added to the root-logger. logger = logging.getLogger(__name__) logger.info( "Logging forward model jobs", extra={ "workflow_jobs": str(res_config.model_config.getForwardModel().joblist()) }, ) os.chdir(res_config.config_path) ert = EnKFMain(res_config, strict=True, verbose=args.verbose) notifier = ErtCliNotifier(ert, args.config) with ERT.adapt(notifier): if args.mode == WORKFLOW_MODE: execute_workflow(args.name) return model, argument = create_model(args) # Test run does not have a current_case if "current_case" in args and args.current_case: ERT.enkf_facade.select_or_create_new_case(args.current_case) if (args.mode in [ ENSEMBLE_SMOOTHER_MODE, ITERATIVE_ENSEMBLE_SMOOTHER_MODE, ES_MDA_MODE ] and args.target_case == ERT.enkf_facade.get_current_case_name()): msg = ( "ERROR: Target file system and source file system can not be the same. " "They were both: {}.".format(args.target_case)) raise ErtCliError(msg) ee_config = None if FeatureToggling.is_enabled("ensemble-evaluator"): ee_config = EvaluatorServerConfig( custom_port_range=args.port_range) argument.update({"ee_config": ee_config}) thread = threading.Thread( name="ert_cli_simulation_thread", target=model.start_simulations_thread, args=(argument, ), ) thread.start() tracker = create_tracker(model, detailed_interval=0, ee_config=ee_config) out = open(os.devnull, "w") if args.disable_monitoring else sys.stderr monitor = Monitor(out=out, color_always=args.color_always) try: monitor.monitor(tracker) except (SystemExit, KeyboardInterrupt): print("\nKilling simulations...") tracker.request_termination() if args.disable_monitoring: out.close() thread.join() if model.hasRunFailed(): raise ErtCliError(model.getFailMessage())