def test_post_update_data(client): data = pd.DataFrame(np.random.rand(4, 2), index=[2, 4, 6, 8]) builder = ErtConfigBuilder() builder.add_general_observation("OBS", "RES", data) ert = builder.build() # Post two ensembles parent_ensemble_id = extraction.post_ensemble_data(ert, builder.ensemble_size) update_id = extraction.post_update_data(ert, parent_ensemble_id, "boruvka") child_ensemble_id = extraction.post_ensemble_data(ert, builder.ensemble_size, update_id) # Experiment should have two ensembles experiment_id = client.fetch_experiment() ensembles = client.get(f"/experiments/{experiment_id}/ensembles").json() assert len(ensembles) == 2 # Parent ensemble should have a child assert ensembles[0]["child_ensemble_ids"] == [child_ensemble_id] assert ensembles[0]["parent_ensemble_id"] is None # Child ensemble should have a parent assert ensembles[1]["child_ensemble_ids"] == [] assert ensembles[1]["parent_ensemble_id"] == parent_ensemble_id
def update(self, run_context, weight, ensemble_id): source_fs = run_context.get_sim_fs() next_iteration = run_context.get_iter() + 1 target_fs = run_context.get_target_fs() phase_string = "Analyzing iteration: %d with weight %f" % ( next_iteration, weight, ) self.setPhase(self.currentPhase() + 1, phase_string, indeterminate=True) es_update = self.ert().getESUpdate() es_update.setGlobalStdScaling(weight) success = es_update.smootherUpdate(run_context) # Push update data to new storage analysis_module_name = self.ert().analysisConfig().activeModuleName() update_id = post_update_data(parent_ensemble_id=ensemble_id, algorithm=analysis_module_name) if not success: raise UserWarning( "Analysis of simulation failed for iteration: %d!" % next_iteration) return update_id
def test_observation_transformation(client): data = pd.DataFrame([[1, 0.1], [2, 0.2], [3, 0.3], [4, 0.4]], index=[0, 1, 2, 3]) builder = ErtConfigBuilder() builder.ensemble_size = 5 builder.add_general_observation("OBS", "RES", data) builder.job_script = dedent("""\ #!/usr/bin/python3 import re from pathlib import Path # Obtain realization index by looking at the name of current directory real = int(re.match("^realization([0-9]+)$", Path.cwd().name)[1]) outlier = 1000 + real # Large number, ERT disables active_a = 1 + real active_b = 2 + real small_var = 3 # Small variation between responses, ERT disables output = [outlier, active_a, active_b, small_var] with open("poly_0.out", "w") as f: f.write("\\n".join(str(x) for x in output)) """) ert = builder.build() # Post first ensemble parent_ensemble_id = extraction.post_ensemble_data(ert, builder.ensemble_size) # Create runpath and run ERT run_context = _create_runpath(ert) _evaluate_ensemble(ert, run_context) _run_update(ert, run_context) # Post second ensemble update_id = extraction.post_update_data(ert, parent_ensemble_id, "boruvka") child_ensemble_id = extraction.post_ensemble_data(ert, builder.ensemble_size, update_id) # Ensemble should have 1 observation with transformation observations = client.get( f"/ensembles/{child_ensemble_id}/observations").json() assert len(observations) == 1 # Validate data obs = observations[0] assert obs["name"] == "OBS" assert obs["values"] == data[0].tolist() assert obs["errors"] == data[1].tolist() trans = obs["transformation"] assert trans["name"] == "OBS" assert trans["active"] == [False, True, True, False] assert trans["scale"] == [1.0] * 4 assert trans["observation_id"] == obs["id"]
def _post_update_data(self, parent_ensemble_id: str, algorithm: str) -> str: self.setPhaseName("Uploading update...") update_id = post_update_data( ert=ERT.enkf_facade, parent_ensemble_id=parent_ensemble_id, algorithm=algorithm, ) self.setPhaseName("Uploading done") return update_id
def analyzeStep(self, run_context, ensemble_id): target_fs = run_context.get_target_fs() self.setPhaseName("Analyzing...", indeterminate=True) source_fs = self.ert().getEnkfFsManager().getCurrentFileSystem() self.setPhaseName("Pre processing update...", indeterminate=True) EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_UPDATE, ert=ERT.ert) es_update = self.ert().getESUpdate() success = es_update.smootherUpdate(run_context) # Push update data to new storage analysis_module_name = self.ert().analysisConfig().activeModuleName() update_id = post_update_data(parent_ensemble_id=ensemble_id, algorithm=analysis_module_name) if not success: raise ErtRunError("Analysis of simulation failed!") self.setPhaseName("Post processing update...", indeterminate=True) EnkfSimulationRunner.runWorkflows(HookRuntime.POST_UPDATE, ert=ERT.ert) return update_id
def runSimulations(self, arguments): prior_context = self.create_context(arguments) self.checkMinimumActiveRealizations(prior_context) self.setPhase(0, "Running simulations...", indeterminate=False) # self.setAnalysisModule(arguments["analysis_module"]) self.setPhaseName("Pre processing...", indeterminate=True) self.ert().getEnkfSimulationRunner().createRunPath(prior_context) EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION, ert=ERT.ert) # Push ensemble, parameters, observations to new storage ensemble_id = post_ensemble_data(ensemble_size=self._ensemble_size) self.setPhaseName("Running forecast...", indeterminate=False) if FeatureToggling.is_enabled("ensemble-evaluator"): ee_config = arguments["ee_config"] num_successful_realizations = self.run_ensemble_evaluator( prior_context, ee_config) else: self._job_queue = self._queue_config.create_job_queue() num_successful_realizations = ( self.ert().getEnkfSimulationRunner().runSimpleStep( self._job_queue, prior_context)) # Push simulation results to storage post_ensemble_results(ensemble_id) self.checkHaveSufficientRealizations(num_successful_realizations) self.setPhaseName("Post processing...", indeterminate=True) EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION, ert=ERT.ert) self.setPhaseName("Analyzing...") EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_FIRST_UPDATE, ert=ERT.ert) EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_UPDATE, ert=ERT.ert) es_update = self.ert().getESUpdate() success = es_update.smootherUpdate(prior_context) if not success: raise ErtRunError("Analysis of simulation failed!") EnkfSimulationRunner.runWorkflows(HookRuntime.POST_UPDATE, ert=ERT.ert) # Create an update object in storage analysis_module_name = self.ert().analysisConfig().activeModuleName() update_id = post_update_data(ensemble_id, analysis_module_name) self.setPhase(1, "Running simulations...") self.ert().getEnkfFsManager().switchFileSystem( prior_context.get_target_fs()) self.setPhaseName("Pre processing...") rerun_context = self.create_context(arguments, prior_context=prior_context) self.ert().getEnkfSimulationRunner().createRunPath(rerun_context) EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION, ert=ERT.ert) # Push ensemble, parameters, observations to new storage ensemble_id = post_ensemble_data(ensemble_size=self._ensemble_size, update_id=update_id) self.setPhaseName("Running forecast...", indeterminate=False) if FeatureToggling.is_enabled("ensemble-evaluator"): ee_config = arguments["ee_config"] num_successful_realizations = self.run_ensemble_evaluator( rerun_context, ee_config) else: self._job_queue = self._queue_config.create_job_queue() num_successful_realizations = ( self.ert().getEnkfSimulationRunner().runSimpleStep( self._job_queue, rerun_context)) self.checkHaveSufficientRealizations(num_successful_realizations) self.setPhaseName("Post processing...", indeterminate=True) EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION, ert=ERT.ert) self.setPhase(2, "Simulations completed.") # Push simulation results to storage post_ensemble_results(ensemble_id) return prior_context