def _runAndPostProcess(self, run_context, arguments, update_id=None):
phase_msg = "Running iteration %d of %d simulation iterations..." % (
run_context.get_iter(),
self.phaseCount() - 1,
)
self.setPhase(run_context.get_iter(), phase_msg, indeterminate=False)
self.setPhaseName("Pre processing...", indeterminate=True)
self.ert().getEnkfSimulationRunner().createRunPath(run_context)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION,
ert=ERT.ert)
# create ensemble
ensemble_id = post_ensemble_data(update_id=update_id)
self.setPhaseName("Running forecast...", indeterminate=False)
if FeatureToggling.is_enabled("ensemble-evaluator"):
ee_config = arguments["ee_config"]
num_successful_realizations = self.run_ensemble_evaluator(
run_context, ee_config)
else:
self._job_queue = self._queue_config.create_job_queue()
num_successful_realizations = (
self.ert().getEnkfSimulationRunner().runSimpleStep(
self._job_queue, run_context))
self.checkHaveSufficientRealizations(num_successful_realizations)
self.setPhaseName("Post processing...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION,
ert=ERT.ert)
post_ensemble_results(ensemble_id)
return ensemble_id
def test_post_ensemble_results(client):
data = pd.DataFrame([[1, 0.1], [2, 0.2], [3, 0.3], [4, 0.4]],
index=[2, 4, 6, 8])
response_name = "RES"
# Add priors to ERT config
builder = ErtConfigBuilder()
builder.ensemble_size = 2
builder.add_general_observation("OBS", response_name, data)
data = [0.0, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9]
df = pd.DataFrame(data)
# Create job
script = dedent(f"""\
#!/usr/bin/python3
if __name__ == "__main__":
output = {str(data)}
with open("poly_0.out", "w") as f:
f.write("\\n".join(map(str, output)))
""")
builder.job_script = script
ert = builder.build()
# Create runpath and run ERT
run_context = _create_runpath(ert)
_evaluate_ensemble(ert, run_context)
# Post initial ensemble
ensemble_id = extraction.post_ensemble_data(ert, builder.ensemble_size)
# Post ensemble results
extraction.post_ensemble_results(ert, ensemble_id)
# Retrieve response data
data = client.get(
f"/ensembles/{ensemble_id}/responses/{response_name}/data")
stream = io.BytesIO(data.content)
response_df = pd.read_csv(stream,
index_col=0,
float_precision="round_trip")
for realization in range(0, builder.ensemble_size):
assert_array_equal(response_df.loc[realization].values,
df.values.flatten())
def _simulateAndPostProcess(self,
run_context,
arguments,
update_id: int = None):
iteration = run_context.get_iter()
phase_string = "Running simulation for iteration: %d" % iteration
self.setPhaseName(phase_string, indeterminate=True)
self.ert().getEnkfSimulationRunner().createRunPath(run_context)
phase_string = "Pre processing for iteration: %d" % iteration
self.setPhaseName(phase_string)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION,
ert=ERT.ert)
# Push ensemble, parameters, observations to new storage
new_ensemble_id = post_ensemble_data(update_id=update_id,
ensemble_size=self._ensemble_size)
phase_string = "Running forecast for iteration: %d" % iteration
self.setPhaseName(phase_string, indeterminate=False)
if FeatureToggling.is_enabled("ensemble-evaluator"):
ee_config = arguments["ee_config"]
num_successful_realizations = self.run_ensemble_evaluator(
run_context, ee_config)
else:
self._job_queue = self._queue_config.create_job_queue()
num_successful_realizations = (
self.ert().getEnkfSimulationRunner().runSimpleStep(
self._job_queue, run_context))
# Push simulation results to storage
post_ensemble_results(new_ensemble_id)
num_successful_realizations += arguments.get(
"prev_successful_realizations", 0)
self.checkHaveSufficientRealizations(num_successful_realizations)
phase_string = "Post processing for iteration: %d" % iteration
self.setPhaseName(phase_string, indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION,
ert=ERT.ert)
return num_successful_realizations, new_ensemble_id
def runSimulations__(self, arguments, run_msg):
run_context = self.create_context(arguments)
self.setPhase(0, "Running simulations...", indeterminate=False)
self.setPhaseName("Pre processing...", indeterminate=True)
self.ert().getEnkfSimulationRunner().createRunPath(run_context)
# Push ensemble, parameters, observations to new storage
ensemble_id = post_ensemble_data(ensemble_size=self._ensemble_size)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION, ERT.ert)
self.setPhaseName(run_msg, indeterminate=False)
if FeatureToggling.is_enabled("ensemble-evaluator"):
ee_config = arguments["ee_config"]
num_successful_realizations = self.run_ensemble_evaluator(
run_context, ee_config
)
else:
self._job_queue = self._queue_config.create_job_queue()
num_successful_realizations = (
self.ert()
.getEnkfSimulationRunner()
.runEnsembleExperiment(self._job_queue, run_context)
)
num_successful_realizations += arguments.get("prev_successful_realizations", 0)
self.checkHaveSufficientRealizations(num_successful_realizations)
self.setPhaseName("Post processing...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION, ERT.ert)
self.setPhase(1, "Simulations completed.") # done...
# Push simulation results to storage
post_ensemble_results(ensemble_id)
return run_context
def _post_ensemble_results(self, ensemble_id: str) -> None:
self.setPhaseName("Uploading results...")
post_ensemble_results(ert=ERT.enkf_facade, ensemble_id=ensemble_id)
self.setPhaseName("Uploading done")
def runSimulations(self, arguments):
prior_context = self.create_context(arguments)
self.checkMinimumActiveRealizations(prior_context)
self.setPhase(0, "Running simulations...", indeterminate=False)
# self.setAnalysisModule(arguments["analysis_module"])
self.setPhaseName("Pre processing...", indeterminate=True)
self.ert().getEnkfSimulationRunner().createRunPath(prior_context)
# Push ensemble, parameters, observations to new storage
ensemble_id = post_ensemble_data()
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION,
ert=ERT.ert)
self.setPhaseName("Running forecast...", indeterminate=False)
if FeatureToggling.is_enabled("ensemble-evaluator"):
ee_config = arguments["ee_config"]
num_successful_realizations = self.run_ensemble_evaluator(
prior_context, ee_config)
else:
self._job_queue = self._queue_config.create_job_queue()
num_successful_realizations = self.ert().getEnkfSimulationRunner(
).runSimpleStep(self._job_queue, prior_context)
self.checkHaveSufficientRealizations(num_successful_realizations)
self.setPhaseName("Post processing...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION,
ert=ERT.ert)
self.setPhaseName("Analyzing...")
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_FIRST_UPDATE,
ert=ERT.ert)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_UPDATE, ert=ERT.ert)
es_update = self.ert().getESUpdate()
success = es_update.smootherUpdate(prior_context)
if not success:
raise ErtRunError("Analysis of simulation failed!")
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_UPDATE, ert=ERT.ert)
# Push simulation results to storage
post_ensemble_results(ensemble_id)
self.setPhase(1, "Running simulations...")
self.ert().getEnkfFsManager().switchFileSystem(
prior_context.get_target_fs())
self.setPhaseName("Pre processing...")
rerun_context = self.create_context(arguments,
prior_context=prior_context)
self.ert().getEnkfSimulationRunner().createRunPath(rerun_context)
# Push ensemble, parameters, observations to new storage
analysis_module_name = self.ert().analysisConfig().activeModuleName()
ensemble_id = post_ensemble_data((ensemble_id, analysis_module_name))
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION,
ert=ERT.ert)
self.setPhaseName("Running forecast...", indeterminate=False)
if FeatureToggling.is_enabled("ensemble-evaluator"):
ee_config = arguments["ee_config"]
num_successful_realizations = self.run_ensemble_evaluator(
rerun_context, ee_config)
else:
self._job_queue = self._queue_config.create_job_queue()
num_successful_realizations = self.ert().getEnkfSimulationRunner(
).runSimpleStep(self._job_queue, rerun_context)
self.checkHaveSufficientRealizations(num_successful_realizations)
self.setPhaseName("Post processing...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION,
ert=ERT.ert)
self.setPhase(2, "Simulations completed.")
# Push simulation results to storage
post_ensemble_results(ensemble_id)
return prior_context
def runSimulations(self, arguments):
phase_count = ERT.enkf_facade.get_number_of_iterations() + 1
self.setPhaseCount(phase_count)
analysis_module = self.setAnalysisModule(arguments["analysis_module"])
target_case_format = arguments["target_case"]
run_context = self.create_context(arguments, 0)
self.ert().analysisConfig().getAnalysisIterConfig().setCaseFormat(
target_case_format)
self._runAndPostProcess(run_context, arguments)
analysis_config = self.ert().analysisConfig()
analysis_iter_config = analysis_config.getAnalysisIterConfig()
num_retries_per_iteration = analysis_iter_config.getNumRetries()
num_retries = 0
current_iter = 0
previous_ensemble_id = None
while current_iter < ERT.enkf_facade.get_number_of_iterations(
) and num_retries < num_retries_per_iteration:
pre_analysis_iter_num = analysis_module.getInt("ITER")
# We run the PRE_FIRST_UPDATE hook here because the current_iter is explicitly available, versus
# in the run_context inside analyzeStep
if current_iter == 0:
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_FIRST_UPDATE,
ert=ERT.ert)
self.analyzeStep(run_context)
current_iter = analysis_module.getInt("ITER")
analysis_success = current_iter > pre_analysis_iter_num
if analysis_success:
# Push ensemble, parameters, observations to new storage
analysis_module_name = self.ert().analysisConfig(
).activeModuleName()
previous_ensemble_id = post_ensemble_data((
previous_ensemble_id, analysis_module_name
) if previous_ensemble_id is not None else None)
post_ensemble_results(previous_ensemble_id)
run_context = self.create_context(arguments,
current_iter,
prior_context=run_context)
self._runAndPostProcess(run_context, arguments)
num_retries = 0
else:
run_context = self.create_context(arguments,
current_iter,
prior_context=run_context,
rerun=True)
self._runAndPostProcess(run_context, arguments)
num_retries += 1
analysis_module_name = self.ert().analysisConfig().activeModuleName()
previous_ensemble_id = post_ensemble_data((
previous_ensemble_id, analysis_module_name
) if previous_ensemble_id is not None else None)
post_ensemble_results(previous_ensemble_id)
if current_iter == (phase_count - 1):
self.setPhase(phase_count, "Simulations completed.")
else:
raise ErtRunError(
"Iterated Ensemble Smoother stopped: maximum number of iteration retries (%d retries) reached for iteration %d"
% (num_retries_per_iteration, current_iter))
return run_context