def test_post_update_data(client):
data = pd.DataFrame(np.random.rand(4, 2), index=[2, 4, 6, 8])
builder = ErtConfigBuilder()
builder.add_general_observation("OBS", "RES", data)
ert = builder.build()
# Post two ensembles
parent_ensemble_id = extraction.post_ensemble_data(ert,
builder.ensemble_size)
update_id = extraction.post_update_data(ert, parent_ensemble_id, "boruvka")
child_ensemble_id = extraction.post_ensemble_data(ert,
builder.ensemble_size,
update_id)
# Experiment should have two ensembles
experiment_id = client.fetch_experiment()
ensembles = client.get(f"/experiments/{experiment_id}/ensembles").json()
assert len(ensembles) == 2
# Parent ensemble should have a child
assert ensembles[0]["child_ensemble_ids"] == [child_ensemble_id]
assert ensembles[0]["parent_ensemble_id"] is None
# Child ensemble should have a parent
assert ensembles[1]["child_ensemble_ids"] == []
assert ensembles[1]["parent_ensemble_id"] == parent_ensemble_id
def update(self, run_context, weight, ensemble_id):
source_fs = run_context.get_sim_fs()
next_iteration = run_context.get_iter() + 1
target_fs = run_context.get_target_fs()
phase_string = "Analyzing iteration: %d with weight %f" % (
next_iteration,
weight,
)
self.setPhase(self.currentPhase() + 1,
phase_string,
indeterminate=True)
es_update = self.ert().getESUpdate()
es_update.setGlobalStdScaling(weight)
success = es_update.smootherUpdate(run_context)
# Push update data to new storage
analysis_module_name = self.ert().analysisConfig().activeModuleName()
update_id = post_update_data(parent_ensemble_id=ensemble_id,
algorithm=analysis_module_name)
if not success:
raise UserWarning(
"Analysis of simulation failed for iteration: %d!" %
next_iteration)
return update_id
def test_observation_transformation(client):
data = pd.DataFrame([[1, 0.1], [2, 0.2], [3, 0.3], [4, 0.4]],
index=[0, 1, 2, 3])
builder = ErtConfigBuilder()
builder.ensemble_size = 5
builder.add_general_observation("OBS", "RES", data)
builder.job_script = dedent("""\
#!/usr/bin/python3
import re
from pathlib import Path
# Obtain realization index by looking at the name of current directory
real = int(re.match("^realization([0-9]+)$", Path.cwd().name)[1])
outlier = 1000 + real # Large number, ERT disables
active_a = 1 + real
active_b = 2 + real
small_var = 3 # Small variation between responses, ERT disables
output = [outlier, active_a, active_b, small_var]
with open("poly_0.out", "w") as f:
f.write("\\n".join(str(x) for x in output))
""")
ert = builder.build()
# Post first ensemble
parent_ensemble_id = extraction.post_ensemble_data(ert,
builder.ensemble_size)
# Create runpath and run ERT
run_context = _create_runpath(ert)
_evaluate_ensemble(ert, run_context)
_run_update(ert, run_context)
# Post second ensemble
update_id = extraction.post_update_data(ert, parent_ensemble_id, "boruvka")
child_ensemble_id = extraction.post_ensemble_data(ert,
builder.ensemble_size,
update_id)
# Ensemble should have 1 observation with transformation
observations = client.get(
f"/ensembles/{child_ensemble_id}/observations").json()
assert len(observations) == 1
# Validate data
obs = observations[0]
assert obs["name"] == "OBS"
assert obs["values"] == data[0].tolist()
assert obs["errors"] == data[1].tolist()
trans = obs["transformation"]
assert trans["name"] == "OBS"
assert trans["active"] == [False, True, True, False]
assert trans["scale"] == [1.0] * 4
assert trans["observation_id"] == obs["id"]
def _post_update_data(self, parent_ensemble_id: str,
algorithm: str) -> str:
self.setPhaseName("Uploading update...")
update_id = post_update_data(
ert=ERT.enkf_facade,
parent_ensemble_id=parent_ensemble_id,
algorithm=algorithm,
)
self.setPhaseName("Uploading done")
return update_id
def analyzeStep(self, run_context, ensemble_id):
target_fs = run_context.get_target_fs()
self.setPhaseName("Analyzing...", indeterminate=True)
source_fs = self.ert().getEnkfFsManager().getCurrentFileSystem()
self.setPhaseName("Pre processing update...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_UPDATE, ert=ERT.ert)
es_update = self.ert().getESUpdate()
success = es_update.smootherUpdate(run_context)
# Push update data to new storage
analysis_module_name = self.ert().analysisConfig().activeModuleName()
update_id = post_update_data(parent_ensemble_id=ensemble_id,
algorithm=analysis_module_name)
if not success:
raise ErtRunError("Analysis of simulation failed!")
self.setPhaseName("Post processing update...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_UPDATE, ert=ERT.ert)
return update_id
def runSimulations(self, arguments):
prior_context = self.create_context(arguments)
self.checkMinimumActiveRealizations(prior_context)
self.setPhase(0, "Running simulations...", indeterminate=False)
# self.setAnalysisModule(arguments["analysis_module"])
self.setPhaseName("Pre processing...", indeterminate=True)
self.ert().getEnkfSimulationRunner().createRunPath(prior_context)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION,
ert=ERT.ert)
# Push ensemble, parameters, observations to new storage
ensemble_id = post_ensemble_data(ensemble_size=self._ensemble_size)
self.setPhaseName("Running forecast...", indeterminate=False)
if FeatureToggling.is_enabled("ensemble-evaluator"):
ee_config = arguments["ee_config"]
num_successful_realizations = self.run_ensemble_evaluator(
prior_context, ee_config)
else:
self._job_queue = self._queue_config.create_job_queue()
num_successful_realizations = (
self.ert().getEnkfSimulationRunner().runSimpleStep(
self._job_queue, prior_context))
# Push simulation results to storage
post_ensemble_results(ensemble_id)
self.checkHaveSufficientRealizations(num_successful_realizations)
self.setPhaseName("Post processing...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION,
ert=ERT.ert)
self.setPhaseName("Analyzing...")
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_FIRST_UPDATE,
ert=ERT.ert)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_UPDATE, ert=ERT.ert)
es_update = self.ert().getESUpdate()
success = es_update.smootherUpdate(prior_context)
if not success:
raise ErtRunError("Analysis of simulation failed!")
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_UPDATE, ert=ERT.ert)
# Create an update object in storage
analysis_module_name = self.ert().analysisConfig().activeModuleName()
update_id = post_update_data(ensemble_id, analysis_module_name)
self.setPhase(1, "Running simulations...")
self.ert().getEnkfFsManager().switchFileSystem(
prior_context.get_target_fs())
self.setPhaseName("Pre processing...")
rerun_context = self.create_context(arguments,
prior_context=prior_context)
self.ert().getEnkfSimulationRunner().createRunPath(rerun_context)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION,
ert=ERT.ert)
# Push ensemble, parameters, observations to new storage
ensemble_id = post_ensemble_data(ensemble_size=self._ensemble_size,
update_id=update_id)
self.setPhaseName("Running forecast...", indeterminate=False)
if FeatureToggling.is_enabled("ensemble-evaluator"):
ee_config = arguments["ee_config"]
num_successful_realizations = self.run_ensemble_evaluator(
rerun_context, ee_config)
else:
self._job_queue = self._queue_config.create_job_queue()
num_successful_realizations = (
self.ert().getEnkfSimulationRunner().runSimpleStep(
self._job_queue, rerun_context))
self.checkHaveSufficientRealizations(num_successful_realizations)
self.setPhaseName("Post processing...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION,
ert=ERT.ert)
self.setPhase(2, "Simulations completed.")
# Push simulation results to storage
post_ensemble_results(ensemble_id)
return prior_context
