def _runAndPostProcess(self, run_context, arguments, update_id=None): phase_msg = "Running iteration %d of %d simulation iterations..." % ( run_context.get_iter(), self.phaseCount() - 1, ) self.setPhase(run_context.get_iter(), phase_msg, indeterminate=False) self.setPhaseName("Pre processing...", indeterminate=True) self.ert().getEnkfSimulationRunner().createRunPath(run_context) EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION, ert=ERT.ert) # create ensemble ensemble_id = post_ensemble_data(update_id=update_id) self.setPhaseName("Running forecast...", indeterminate=False) if FeatureToggling.is_enabled("ensemble-evaluator"): ee_config = arguments["ee_config"] num_successful_realizations = self.run_ensemble_evaluator( run_context, ee_config) else: self._job_queue = self._queue_config.create_job_queue() num_successful_realizations = ( self.ert().getEnkfSimulationRunner().runSimpleStep( self._job_queue, run_context)) self.checkHaveSufficientRealizations(num_successful_realizations) self.setPhaseName("Post processing...", indeterminate=True) EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION, ert=ERT.ert) post_ensemble_results(ensemble_id) return ensemble_id
def test_post_ensemble_results(client): data = pd.DataFrame([[1, 0.1], [2, 0.2], [3, 0.3], [4, 0.4]], index=[2, 4, 6, 8]) response_name = "RES" # Add priors to ERT config builder = ErtConfigBuilder() builder.ensemble_size = 2 builder.add_general_observation("OBS", response_name, data) data = [0.0, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9] df = pd.DataFrame(data) # Create job script = dedent(f"""\ #!/usr/bin/python3 if __name__ == "__main__": output = {str(data)} with open("poly_0.out", "w") as f: f.write("\\n".join(map(str, output))) """) builder.job_script = script ert = builder.build() # Create runpath and run ERT run_context = _create_runpath(ert) _evaluate_ensemble(ert, run_context) # Post initial ensemble ensemble_id = extraction.post_ensemble_data(ert, builder.ensemble_size) # Post ensemble results extraction.post_ensemble_results(ert, ensemble_id) # Retrieve response data data = client.get( f"/ensembles/{ensemble_id}/responses/{response_name}/data") stream = io.BytesIO(data.content) response_df = pd.read_csv(stream, index_col=0, float_precision="round_trip") for realization in range(0, builder.ensemble_size): assert_array_equal(response_df.loc[realization].values, df.values.flatten())
def _simulateAndPostProcess(self, run_context, arguments, update_id: int = None): iteration = run_context.get_iter() phase_string = "Running simulation for iteration: %d" % iteration self.setPhaseName(phase_string, indeterminate=True) self.ert().getEnkfSimulationRunner().createRunPath(run_context) phase_string = "Pre processing for iteration: %d" % iteration self.setPhaseName(phase_string) EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION, ert=ERT.ert) # Push ensemble, parameters, observations to new storage new_ensemble_id = post_ensemble_data(update_id=update_id, ensemble_size=self._ensemble_size) phase_string = "Running forecast for iteration: %d" % iteration self.setPhaseName(phase_string, indeterminate=False) if FeatureToggling.is_enabled("ensemble-evaluator"): ee_config = arguments["ee_config"] num_successful_realizations = self.run_ensemble_evaluator( run_context, ee_config) else: self._job_queue = self._queue_config.create_job_queue() num_successful_realizations = ( self.ert().getEnkfSimulationRunner().runSimpleStep( self._job_queue, run_context)) # Push simulation results to storage post_ensemble_results(new_ensemble_id) num_successful_realizations += arguments.get( "prev_successful_realizations", 0) self.checkHaveSufficientRealizations(num_successful_realizations) phase_string = "Post processing for iteration: %d" % iteration self.setPhaseName(phase_string, indeterminate=True) EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION, ert=ERT.ert) return num_successful_realizations, new_ensemble_id
def runSimulations__(self, arguments, run_msg): run_context = self.create_context(arguments) self.setPhase(0, "Running simulations...", indeterminate=False) self.setPhaseName("Pre processing...", indeterminate=True) self.ert().getEnkfSimulationRunner().createRunPath(run_context) # Push ensemble, parameters, observations to new storage ensemble_id = post_ensemble_data(ensemble_size=self._ensemble_size) EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION, ERT.ert) self.setPhaseName(run_msg, indeterminate=False) if FeatureToggling.is_enabled("ensemble-evaluator"): ee_config = arguments["ee_config"] num_successful_realizations = self.run_ensemble_evaluator( run_context, ee_config ) else: self._job_queue = self._queue_config.create_job_queue() num_successful_realizations = ( self.ert() .getEnkfSimulationRunner() .runEnsembleExperiment(self._job_queue, run_context) ) num_successful_realizations += arguments.get("prev_successful_realizations", 0) self.checkHaveSufficientRealizations(num_successful_realizations) self.setPhaseName("Post processing...", indeterminate=True) EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION, ERT.ert) self.setPhase(1, "Simulations completed.") # done... # Push simulation results to storage post_ensemble_results(ensemble_id) return run_context
def _post_ensemble_results(self, ensemble_id: str) -> None: self.setPhaseName("Uploading results...") post_ensemble_results(ert=ERT.enkf_facade, ensemble_id=ensemble_id) self.setPhaseName("Uploading done")
def runSimulations(self, arguments): prior_context = self.create_context(arguments) self.checkMinimumActiveRealizations(prior_context) self.setPhase(0, "Running simulations...", indeterminate=False) # self.setAnalysisModule(arguments["analysis_module"]) self.setPhaseName("Pre processing...", indeterminate=True) self.ert().getEnkfSimulationRunner().createRunPath(prior_context) # Push ensemble, parameters, observations to new storage ensemble_id = post_ensemble_data() EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION, ert=ERT.ert) self.setPhaseName("Running forecast...", indeterminate=False) if FeatureToggling.is_enabled("ensemble-evaluator"): ee_config = arguments["ee_config"] num_successful_realizations = self.run_ensemble_evaluator( prior_context, ee_config) else: self._job_queue = self._queue_config.create_job_queue() num_successful_realizations = self.ert().getEnkfSimulationRunner( ).runSimpleStep(self._job_queue, prior_context) self.checkHaveSufficientRealizations(num_successful_realizations) self.setPhaseName("Post processing...", indeterminate=True) EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION, ert=ERT.ert) self.setPhaseName("Analyzing...") EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_FIRST_UPDATE, ert=ERT.ert) EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_UPDATE, ert=ERT.ert) es_update = self.ert().getESUpdate() success = es_update.smootherUpdate(prior_context) if not success: raise ErtRunError("Analysis of simulation failed!") EnkfSimulationRunner.runWorkflows(HookRuntime.POST_UPDATE, ert=ERT.ert) # Push simulation results to storage post_ensemble_results(ensemble_id) self.setPhase(1, "Running simulations...") self.ert().getEnkfFsManager().switchFileSystem( prior_context.get_target_fs()) self.setPhaseName("Pre processing...") rerun_context = self.create_context(arguments, prior_context=prior_context) self.ert().getEnkfSimulationRunner().createRunPath(rerun_context) # Push ensemble, parameters, observations to new storage analysis_module_name = self.ert().analysisConfig().activeModuleName() ensemble_id = post_ensemble_data((ensemble_id, analysis_module_name)) EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION, ert=ERT.ert) self.setPhaseName("Running forecast...", indeterminate=False) if FeatureToggling.is_enabled("ensemble-evaluator"): ee_config = arguments["ee_config"] num_successful_realizations = self.run_ensemble_evaluator( rerun_context, ee_config) else: self._job_queue = self._queue_config.create_job_queue() num_successful_realizations = self.ert().getEnkfSimulationRunner( ).runSimpleStep(self._job_queue, rerun_context) self.checkHaveSufficientRealizations(num_successful_realizations) self.setPhaseName("Post processing...", indeterminate=True) EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION, ert=ERT.ert) self.setPhase(2, "Simulations completed.") # Push simulation results to storage post_ensemble_results(ensemble_id) return prior_context
def runSimulations(self, arguments): phase_count = ERT.enkf_facade.get_number_of_iterations() + 1 self.setPhaseCount(phase_count) analysis_module = self.setAnalysisModule(arguments["analysis_module"]) target_case_format = arguments["target_case"] run_context = self.create_context(arguments, 0) self.ert().analysisConfig().getAnalysisIterConfig().setCaseFormat( target_case_format) self._runAndPostProcess(run_context, arguments) analysis_config = self.ert().analysisConfig() analysis_iter_config = analysis_config.getAnalysisIterConfig() num_retries_per_iteration = analysis_iter_config.getNumRetries() num_retries = 0 current_iter = 0 previous_ensemble_id = None while current_iter < ERT.enkf_facade.get_number_of_iterations( ) and num_retries < num_retries_per_iteration: pre_analysis_iter_num = analysis_module.getInt("ITER") # We run the PRE_FIRST_UPDATE hook here because the current_iter is explicitly available, versus # in the run_context inside analyzeStep if current_iter == 0: EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_FIRST_UPDATE, ert=ERT.ert) self.analyzeStep(run_context) current_iter = analysis_module.getInt("ITER") analysis_success = current_iter > pre_analysis_iter_num if analysis_success: # Push ensemble, parameters, observations to new storage analysis_module_name = self.ert().analysisConfig( ).activeModuleName() previous_ensemble_id = post_ensemble_data(( previous_ensemble_id, analysis_module_name ) if previous_ensemble_id is not None else None) post_ensemble_results(previous_ensemble_id) run_context = self.create_context(arguments, current_iter, prior_context=run_context) self._runAndPostProcess(run_context, arguments) num_retries = 0 else: run_context = self.create_context(arguments, current_iter, prior_context=run_context, rerun=True) self._runAndPostProcess(run_context, arguments) num_retries += 1 analysis_module_name = self.ert().analysisConfig().activeModuleName() previous_ensemble_id = post_ensemble_data(( previous_ensemble_id, analysis_module_name ) if previous_ensemble_id is not None else None) post_ensemble_results(previous_ensemble_id) if current_iter == (phase_count - 1): self.setPhase(phase_count, "Simulations completed.") else: raise ErtRunError( "Iterated Ensemble Smoother stopped: maximum number of iteration retries (%d retries) reached for iteration %d" % (num_retries_per_iteration, current_iter)) return run_context