def _runAndPostProcess(self, run_context, arguments, update_id=None):
phase_msg = "Running iteration %d of %d simulation iterations..." % (
run_context.get_iter(),
self.phaseCount() - 1,
)
self.setPhase(run_context.get_iter(), phase_msg, indeterminate=False)
self.setPhaseName("Pre processing...", indeterminate=True)
self.ert().getEnkfSimulationRunner().createRunPath(run_context)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION,
ert=ERT.ert)
# create ensemble
ensemble_id = self._post_ensemble_data(update_id=update_id)
self.setPhaseName("Running forecast...", indeterminate=False)
if FeatureToggling.is_enabled("ensemble-evaluator"):
ee_config = arguments["ee_config"]
num_successful_realizations = self.run_ensemble_evaluator(
run_context, ee_config)
else:
self._job_queue = self._queue_config.create_job_queue()
num_successful_realizations = (
self.ert().getEnkfSimulationRunner().runSimpleStep(
self._job_queue, run_context))
self.checkHaveSufficientRealizations(num_successful_realizations)
self.setPhaseName("Post processing...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION,
ert=ERT.ert)
self._post_ensemble_results(ensemble_id)
return ensemble_id
def _simulateAndPostProcess(self, run_context, arguments):
self._job_queue = self._queue_config.create_job_queue()
iteration = run_context.get_iter()
phase_string = "Running simulation for iteration: %d" % iteration
self.setPhaseName(phase_string, indeterminate=True)
self.ert().getEnkfSimulationRunner().createRunPath(run_context)
phase_string = "Pre processing for iteration: %d" % iteration
self.setPhaseName(phase_string)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION,
ert=ERT.ert)
phase_string = "Running forecast for iteration: %d" % iteration
self.setPhaseName(phase_string, indeterminate=False)
num_successful_realizations = self.ert().getEnkfSimulationRunner(
).runSimpleStep(self._job_queue, run_context)
num_successful_realizations += arguments.get(
'prev_successful_realizations', 0)
self.checkHaveSufficientRealizations(num_successful_realizations)
phase_string = "Post processing for iteration: %d" % iteration
self.setPhaseName(phase_string, indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION,
ert=ERT.ert)
return num_successful_realizations
def runSimulations__(self, arguments, run_msg):
self._job_queue = self._queue_config.create_job_queue()
run_context = self.create_context(arguments)
self.setPhase(0, "Running simulations...", indeterminate=False)
self.setPhaseName("Pre processing...", indeterminate=True)
self.ert().getEnkfSimulationRunner().createRunPath(run_context)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION, ERT.ert)
self.setPhaseName(run_msg, indeterminate=False)
num_successful_realizations = self.ert().getEnkfSimulationRunner(
).runEnsembleExperiment(self._job_queue, run_context)
num_successful_realizations += arguments.get(
'prev_successful_realizations', 0)
self.checkHaveSufficientRealizations(num_successful_realizations)
self.setPhaseName("Post processing...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION, ERT.ert)
self.setPhase(1, "Simulations completed.") # done...
return run_context
def runSimulations__(self, arguments, run_msg):
run_context = self.create_context(arguments)
self.setPhase(0, "Running simulations...", indeterminate=False)
self.setPhaseName("Pre processing...", indeterminate=True)
self.ert().getEnkfSimulationRunner().createRunPath(run_context)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION, ERT.ert)
self.setPhaseName(run_msg, indeterminate=False)
if FeatureToggling.is_enabled("ensemble-evaluator"):
num_successful_realizations = self.run_ensemble_evaluator(
run_context)
else:
self._job_queue = self._queue_config.create_job_queue()
num_successful_realizations = (
self.ert().getEnkfSimulationRunner().runEnsembleExperiment(
self._job_queue, run_context))
num_successful_realizations += arguments.get(
"prev_successful_realizations", 0)
self.checkHaveSufficientRealizations(num_successful_realizations)
self.setPhaseName("Post processing...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION, ERT.ert)
self.setPhase(1, "Simulations completed.") # done...
dump_to_new_storage()
return run_context
def _runAndPostProcess(
self,
run_context: ErtRunContext,
evaluator_server_config: EvaluatorServerConfig,
update_id: Optional[str] = None,
) -> str:
phase_msg = "Running iteration %d of %d simulation iterations..." % (
run_context.get_iter(),
self.phaseCount() - 1,
)
self.setPhase(run_context.get_iter(), phase_msg, indeterminate=False)
self.setPhaseName("Pre processing...", indeterminate=True)
self.ert().getEnkfSimulationRunner().createRunPath(run_context)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION, ert=self.ert())
# create ensemble
ensemble_id = self._post_ensemble_data(update_id=update_id)
self.setPhaseName("Running forecast...", indeterminate=False)
num_successful_realizations = self.run_ensemble_evaluator(
run_context, evaluator_server_config
)
self.checkHaveSufficientRealizations(num_successful_realizations)
self.setPhaseName("Post processing...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION, ert=self.ert())
self._post_ensemble_results(ensemble_id)
return ensemble_id
def runSimulations(self, arguments):
context = self.create_context(arguments, 0, None)
self.checkMinimumActiveRealizations(context)
weights = self.parseWeights(arguments['weights'])
iteration_count = len(weights)
self.setAnalysisModule(arguments["analysis_module"])
logger.info(
"Running MDA ES for %s iterations\t%s" %
(iteration_count, ", ".join(str(weight) for weight in weights)))
weights = self.normalizeWeights(weights)
weight_string = ", ".join(str(round(weight, 3)) for weight in weights)
logger.info("Running MDA ES on (weights normalized)\t%s" %
weight_string)
self.setPhaseCount(iteration_count + 2) # pre + post + weights
phase_string = "Running MDA ES %d iteration%s." % (iteration_count, (
's' if (iteration_count != 1) else ''))
self.setPhaseName(phase_string, indeterminate=True)
run_context = None
previous_ensemble_name = None
for iteration, weight in enumerate(weights):
run_context = self.create_context(arguments,
iteration,
prior_context=run_context)
self._simulateAndPostProcess(run_context, arguments)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_UPDATE,
ert=ERT.ert)
self.update(run_context, weights[iteration])
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_UPDATE,
ert=ERT.ert)
analysis_module_name = self.ert().analysisConfig(
).activeModuleName()
previous_ensemble_name = dump_to_new_storage(
reference=None if previous_ensemble_name is None else (
previous_ensemble_name, analysis_module_name))
self.setPhaseName("Post processing...", indeterminate=True)
run_context = self.create_context(arguments,
len(weights),
prior_context=run_context,
update=False)
self._simulateAndPostProcess(run_context, arguments)
self.setPhase(iteration_count + 2, "Simulations completed.")
analysis_module_name = self.ert().analysisConfig().activeModuleName()
previous_ensemble_name = dump_to_new_storage(
reference=None if previous_ensemble_name is None else (
previous_ensemble_name, analysis_module_name))
return run_context
def ensembleExperiment(self, line):
simulation_runner = EnkfSimulationRunner(self.ert())
now = time.time()
print("Ensemble Experiment started at: %s" %
datetime.now().isoformat(sep=" "))
iteration_count = 0
active_realization_mask = BoolVector(
default_value=True, initial_size=self.ert().getEnsembleSize())
simulation_runner.createRunPath(active_realization_mask,
iteration_count)
simulation_runner.runWorkflows(HookRuntime.PRE_SIMULATION)
print("Start simulations!")
num_successful_realizations = simulation_runner.runEnsembleExperiment()
success = self.ert().analysisConfig().haveEnoughRealisations(
num_successful_realizations,
self.ert().getEnsembleSize())
if not success:
print(
"Error: Number of successful realizations is too low.\nYou can allow more failed realizations by setting / changing the MIN_REALIZATIONS configuration element!"
)
return
print("Ensemble Experiment post processing!")
simulation_runner.runWorkflows(HookRuntime.POST_SIMULATION)
print("Ensemble Experiment completed at: %s" %
datetime.now().isoformat(sep=" "))
diff = time.time() - now
print("Running time: %d seconds" % int(diff))
def runSimulations(self, arguments):
phase_count = ERT.enkf_facade.get_number_of_iterations() + 1
self.setPhaseCount(phase_count)
analysis_module = self.setAnalysisModule(arguments["analysis_module"])
target_case_format = arguments["target_case"]
run_context = self.create_context(arguments, 0)
self.ert().analysisConfig().getAnalysisIterConfig().setCaseFormat(
target_case_format)
ensemble_id = self._runAndPostProcess(run_context, arguments)
analysis_config = self.ert().analysisConfig()
analysis_iter_config = analysis_config.getAnalysisIterConfig()
num_retries_per_iteration = analysis_iter_config.getNumRetries()
num_retries = 0
current_iter = 0
while (current_iter < ERT.enkf_facade.get_number_of_iterations()
and num_retries < num_retries_per_iteration):
pre_analysis_iter_num = analysis_module.getInt("ITER")
# We run the PRE_FIRST_UPDATE hook here because the current_iter is explicitly available, versus
# in the run_context inside analyzeStep
if current_iter == 0:
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_FIRST_UPDATE,
ert=ERT.ert)
update_id = self.analyzeStep(run_context, ensemble_id)
current_iter = analysis_module.getInt("ITER")
analysis_success = current_iter > pre_analysis_iter_num
if analysis_success:
run_context = self.create_context(arguments,
current_iter,
prior_context=run_context)
ensemble_id = self._runAndPostProcess(run_context, arguments,
update_id)
num_retries = 0
else:
run_context = self.create_context(arguments,
current_iter,
prior_context=run_context,
rerun=True)
ensemble_id = self._runAndPostProcess(run_context, arguments,
update_id)
num_retries += 1
if current_iter == (phase_count - 1):
self.setPhase(phase_count, "Simulations completed.")
else:
raise ErtRunError(
"Iterated Ensemble Smoother stopped: maximum number of iteration retries (%d retries) reached for iteration %d"
% (num_retries_per_iteration, current_iter))
return run_context
def ensembleExperiment(self, line):
simulation_runner = EnkfSimulationRunner(self.ert())
now = time.time()
print("Ensemble Experiment started at: %s" % datetime.now().isoformat(sep=" "))
iteration_count = 0
active_realization_mask = BoolVector(default_value = True, initial_size = self.ert().getEnsembleSize())
simulation_runner.createRunPath(active_realization_mask, iteration_count)
simulation_runner.runWorkflows(HookRuntime.PRE_SIMULATION)
print("Start simulations!")
num_successful_realizations = simulation_runner.runEnsembleExperiment()
success = self.ert().analysisConfig().haveEnoughRealisations(num_successful_realizations, self.ert().getEnsembleSize())
if not success:
print("Error: Number of successful realizations is too low.\nYou can allow more failed realizations by setting / changing the MIN_REALIZATIONS configuration element!")
return
print("Ensemble Experiment post processing!")
simulation_runner.runWorkflows(HookRuntime.POST_SIMULATION)
print("Ensemble Experiment completed at: %s" % datetime.now().isoformat(sep=" "))
diff = time.time() - now
print("Running time: %d seconds" % int(diff))
def analyzeStep(self, run_context):
target_fs = run_context.get_target_fs()
self.setPhaseName("Analyzing...", indeterminate=True)
source_fs = self.ert().getEnkfFsManager().getCurrentFileSystem()
self.setPhaseName("Pre processing update...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_UPDATE, ert=ERT.ert)
es_update = self.ert().getESUpdate()
success = es_update.smootherUpdate(run_context)
if not success:
raise ErtRunError("Analysis of simulation failed!")
self.setPhaseName("Post processing update...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_UPDATE, ert=ERT.ert)
def __init__(self, model_config, res_config=None, strict=True, verbose=True):
if model_config is not None and not isfile(model_config):
raise IOError('No such configuration file "%s".' % model_config)
if res_config is None:
res_config = ResConfig(model_config)
res_config.convertToCReference(self)
if res_config is None or not isinstance(res_config, ResConfig):
raise TypeError("Failed to construct EnKFMain instance due to invalid res_config.")
c_ptr = self._alloc(model_config, res_config, strict, verbose)
if c_ptr:
super(EnKFMain, self).__init__(c_ptr)
else:
raise ValueError('Failed to construct EnKFMain instance from config %s.' % model_config)
# The model_config argument can be None; the only reason to
# allow that possibility is to be able to test that the
# site-config loads correctly.
if model_config is None:
self.__simulation_runner = None
self.__fs_manager = None
self.__es_update = None
else:
self.__simulation_runner = EnkfSimulationRunner(self)
self.__fs_manager = EnkfFsManager(self)
self.__es_update = ESUpdate(self)
self.__key_manager = KeyManager(self)
def _runAndPostProcess(self, run_context):
self._job_queue = self._queue_config.create_job_queue( )
phase_msg = "Running iteration %d of %d simulation iterations..." % (run_context.get_iter(), self.phaseCount() - 1)
self.setPhase(run_context.get_iter(), phase_msg, indeterminate=False)
self.setPhaseName("Pre processing...", indeterminate=True)
self.ert().getEnkfSimulationRunner().createRunPath( run_context )
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION, ert=ERT.ert)
self.setPhaseName("Running forecast...", indeterminate=False)
num_successful_realizations = self.ert().getEnkfSimulationRunner().runSimpleStep(self._job_queue, run_context)
self.checkHaveSufficientRealizations(num_successful_realizations)
self.setPhaseName("Post processing...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION, ert=ERT.ert)
def __init__(self, config, strict=True, verbose=True):
"""
Initializes an instance of EnkfMain.
Note: @config ought to be the ResConfig instance holding the
configuration. It also accepts that config is the name of a
configuration file, this is however deprecated.
"""
res_config = self._init_res_config(config)
if res_config is None:
raise TypeError("Failed to construct EnKFMain instance due to invalid res_config.")
c_ptr = self._alloc(res_config, strict, verbose)
if c_ptr:
super(EnKFMain, self).__init__(c_ptr)
else:
raise ValueError('Failed to construct EnKFMain instance from config %s.' % res_config)
if config is None:
self.__simulation_runner = None
self.__fs_manager = None
self.__es_update = None
else:
self.__simulation_runner = EnkfSimulationRunner(self)
self.__fs_manager = EnkfFsManager(self)
self.__es_update = ESUpdate(self)
self.__key_manager = KeyManager(self)
def _simulateAndPostProcess(self,
run_context,
arguments,
update_id: int = None):
iteration = run_context.get_iter()
phase_string = "Running simulation for iteration: %d" % iteration
self.setPhaseName(phase_string, indeterminate=True)
self.ert().getEnkfSimulationRunner().createRunPath(run_context)
phase_string = "Pre processing for iteration: %d" % iteration
self.setPhaseName(phase_string)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION,
ert=ERT.ert)
# Push ensemble, parameters, observations to new storage
new_ensemble_id = post_ensemble_data(update_id=update_id,
ensemble_size=self._ensemble_size)
phase_string = "Running forecast for iteration: %d" % iteration
self.setPhaseName(phase_string, indeterminate=False)
if FeatureToggling.is_enabled("ensemble-evaluator"):
ee_config = arguments["ee_config"]
num_successful_realizations = self.run_ensemble_evaluator(
run_context, ee_config)
else:
self._job_queue = self._queue_config.create_job_queue()
num_successful_realizations = (
self.ert().getEnkfSimulationRunner().runSimpleStep(
self._job_queue, run_context))
# Push simulation results to storage
post_ensemble_results(new_ensemble_id)
num_successful_realizations += arguments.get(
"prev_successful_realizations", 0)
self.checkHaveSufficientRealizations(num_successful_realizations)
phase_string = "Post processing for iteration: %d" % iteration
self.setPhaseName(phase_string, indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION,
ert=ERT.ert)
return num_successful_realizations, new_ensemble_id
def _init_from_real_enkf_main(self, real_enkf_main):
super(EnKFMain,
self).__init__(real_enkf_main.from_param(real_enkf_main).value,
parent=real_enkf_main,
is_reference=True)
self.__simulation_runner = EnkfSimulationRunner(self)
self.__fs_manager = EnkfFsManager(self)
self.__es_update = ESUpdate(self)
def _simulateAndPostProcess(
self,
run_context: ErtRunContext,
evaluator_server_config: EvaluatorServerConfig,
update_id: str = None,
) -> Tuple[int, str]:
iteration = run_context.get_iter()
phase_string = "Running simulation for iteration: %d" % iteration
self.setPhaseName(phase_string, indeterminate=True)
self.ert().getEnkfSimulationRunner().createRunPath(run_context)
phase_string = "Pre processing for iteration: %d" % iteration
self.setPhaseName(phase_string)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION,
ert=self.ert())
# Push ensemble, parameters, observations to new storage
new_ensemble_id = self._post_ensemble_data(update_id=update_id)
phase_string = "Running forecast for iteration: %d" % iteration
self.setPhaseName(phase_string, indeterminate=False)
num_successful_realizations = self.run_ensemble_evaluator(
run_context, evaluator_server_config)
# Push simulation results to storage
self._post_ensemble_results(new_ensemble_id)
num_successful_realizations += self._simulation_arguments.get(
"prev_successful_realizations", 0)
self.checkHaveSufficientRealizations(num_successful_realizations)
phase_string = "Post processing for iteration: %d" % iteration
self.setPhaseName(phase_string, indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION,
ert=self.ert())
return num_successful_realizations, new_ensemble_id
def __init__(self, ert, sim_fs, mask, itr, case_data):
self._ert = ert
""" :type: res.enkf.EnKFMain """
max_runtime = ert.analysisConfig().get_max_runtime()
self._mask = mask
job_queue = ert.get_queue_config().create_job_queue()
job_queue.set_max_job_duration(max_runtime)
self._queue_manager = JobQueueManager(job_queue)
subst_list = self._ert.getDataKW()
path_fmt = self._ert.getModelConfig().getRunpathFormat()
jobname_fmt = self._ert.getModelConfig().getJobnameFormat()
self._run_context = ErtRunContext(
EnkfRunType.ENSEMBLE_EXPERIMENT,
sim_fs,
None,
mask,
path_fmt,
jobname_fmt,
subst_list,
itr,
)
# fill in the missing geo_id data
for sim_id, (geo_id, _) in enumerate(case_data):
if mask[sim_id]:
run_arg = self._run_context[sim_id]
run_arg.geo_id = geo_id
self._ert.getEnkfSimulationRunner().createRunPath(self._run_context)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION,
self._ert)
self._sim_thread = self._run_simulations_simple_step()
# Wait until the queue is active before we finish the creation
# to ensure sane job status while running
while self.isRunning() and not self._queue_manager.isRunning():
sleep(0.1)
def runSimulations(self, arguments):
context = self.create_context(arguments,
0,
initialize_mask_from_arguments=True)
self.checkMinimumActiveRealizations(context)
weights = self.parseWeights(arguments['weights'])
iteration_count = len(weights)
self.setAnalysisModule(arguments["analysis_module"])
logger.info(
"Running MDA ES for %s iterations\t%s" %
(iteration_count, ", ".join(str(weight) for weight in weights)))
weights = self.normalizeWeights(weights)
weight_string = ", ".join(str(round(weight, 3)) for weight in weights)
logger.info("Running MDA ES on (weights normalized)\t%s" %
weight_string)
self.setPhaseCount(iteration_count + 2) # pre + post + weights
phase_string = "Running MDA ES %d iteration%s." % (iteration_count, (
's' if (iteration_count != 1) else ''))
self.setPhaseName(phase_string, indeterminate=True)
run_context = None
current_ensemble_id = None
enumerated_weights = list(enumerate(weights))
weights_to_run = enumerated_weights[
min(arguments["start_iteration"], len(weights)):]
for iteration, weight in weights_to_run:
is_first_iteration = iteration == 0
run_context = self.create_context(
arguments,
iteration,
initialize_mask_from_arguments=is_first_iteration)
_, current_ensemble_id = self._simulateAndPostProcess(
run_context,
arguments,
previous_ensemble_id=current_ensemble_id)
if is_first_iteration:
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_FIRST_UPDATE,
ert=ERT.ert)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_UPDATE,
ert=ERT.ert)
self.update(run_context, weight)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_UPDATE,
ert=ERT.ert)
self.setPhaseName("Post processing...", indeterminate=True)
run_context = self.create_context(arguments,
len(weights),
initialize_mask_from_arguments=False,
update=False)
self._simulateAndPostProcess(run_context, arguments,
current_ensemble_id)
self.setPhase(iteration_count + 2, "Simulations completed.")
return run_context
def runSimulations(
self,
evaluator_server_config: EvaluatorServerConfig) -> ErtRunContext:
context = self.create_context(0, initialize_mask_from_arguments=True)
self._checkMinimumActiveRealizations(context)
weights = self.parseWeights(self._simulation_arguments["weights"])
iteration_count = len(weights)
self.setAnalysisModule(self._simulation_arguments["analysis_module"])
logger.info(
"Running MDA ES for %s iterations\t%s" %
(iteration_count, ", ".join(str(weight) for weight in weights)))
weights = self.normalizeWeights(weights)
weight_string = ", ".join(str(round(weight, 3)) for weight in weights)
logger.info("Running MDA ES on (weights normalized)\t%s" %
weight_string)
self.setPhaseCount(iteration_count + 1) # weights + post
phase_string = "Running MDA ES %d iteration%s." % (
iteration_count,
("s" if (iteration_count != 1) else ""),
)
self.setPhaseName(phase_string, indeterminate=True)
run_context = None
update_id = None
enumerated_weights = list(enumerate(weights))
weights_to_run = enumerated_weights[
min(self._simulation_arguments["start_iteration"], len(weights)):]
for iteration, weight in weights_to_run:
is_first_iteration = iteration == 0
run_context = self.create_context(
iteration, initialize_mask_from_arguments=is_first_iteration)
_, ensemble_id = self._simulateAndPostProcess(
run_context, evaluator_server_config, update_id=update_id)
if is_first_iteration:
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_FIRST_UPDATE,
ert=self.ert())
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_UPDATE,
ert=self.ert())
update_id = self.update(run_context=run_context,
weight=weight,
ensemble_id=ensemble_id)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_UPDATE,
ert=self.ert())
self.setPhaseName("Post processing...", indeterminate=True)
run_context = self.create_context(len(weights),
initialize_mask_from_arguments=False,
update=False)
self._simulateAndPostProcess(run_context,
evaluator_server_config,
update_id=update_id)
self.setPhase(iteration_count + 1, "Simulations completed.")
return run_context
def analyzeStep(self, run_context, ensemble_id):
target_fs = run_context.get_target_fs()
self.setPhaseName("Analyzing...", indeterminate=True)
source_fs = self.ert().getEnkfFsManager().getCurrentFileSystem()
self.setPhaseName("Pre processing update...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_UPDATE, ert=ERT.ert)
es_update = self.ert().getESUpdate()
success = es_update.smootherUpdate(run_context)
# Push update data to new storage
analysis_module_name = self.ert().analysisConfig().activeModuleName()
update_id = self._post_update_data(parent_ensemble_id=ensemble_id,
algorithm=analysis_module_name)
if not success:
raise ErtRunError("Analysis of simulation failed!")
self.setPhaseName("Post processing update...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_UPDATE, ert=ERT.ert)
return update_id
def runSimulations__(self, arguments, run_msg):
run_context = self.create_context(arguments)
self.setPhase(0, "Running simulations...", indeterminate=False)
self.setPhaseName("Pre processing...", indeterminate=True)
self.ert().getEnkfSimulationRunner().createRunPath(run_context)
# Push ensemble, parameters, observations to new storage
ensemble_id = post_ensemble_data(ensemble_size=self._ensemble_size)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION, ERT.ert)
self.setPhaseName(run_msg, indeterminate=False)
if FeatureToggling.is_enabled("ensemble-evaluator"):
ee_config = arguments["ee_config"]
num_successful_realizations = self.run_ensemble_evaluator(
run_context, ee_config
)
else:
self._job_queue = self._queue_config.create_job_queue()
num_successful_realizations = (
self.ert()
.getEnkfSimulationRunner()
.runEnsembleExperiment(self._job_queue, run_context)
)
num_successful_realizations += arguments.get("prev_successful_realizations", 0)
self.checkHaveSufficientRealizations(num_successful_realizations)
self.setPhaseName("Post processing...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION, ERT.ert)
self.setPhase(1, "Simulations completed.") # done...
# Push simulation results to storage
post_ensemble_results(ensemble_id)
return run_context
def runSimulations__(
self,
run_msg: str,
evaluator_server_config: EvaluatorServerConfig,
) -> ErtRunContext:
run_context = self.create_context()
self.setPhase(0, "Running simulations...", indeterminate=False)
self.setPhaseName("Pre processing...", indeterminate=True)
self.ert().getEnkfSimulationRunner().createRunPath(run_context)
# Push ensemble, parameters, observations to new storage
ensemble_id = self._post_ensemble_data()
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION,
self.ert())
self.setPhaseName(run_msg, indeterminate=False)
num_successful_realizations = self.run_ensemble_evaluator(
run_context, evaluator_server_config)
num_successful_realizations += self._simulation_arguments.get(
"prev_successful_realizations", 0)
self.checkHaveSufficientRealizations(num_successful_realizations)
self.setPhaseName("Post processing...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION,
self.ert())
# Push simulation results to storage
self._post_ensemble_results(ensemble_id)
self.setPhase(1, "Simulations completed.") # done...
return run_context
def analyzeStep(self, run_context: ErtRunContext, ensemble_id: str) -> str:
self.setPhaseName("Analyzing...", indeterminate=True)
self.setPhaseName("Pre processing update...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_UPDATE, ert=self.ert())
es_update = self.ert().getESUpdate()
try:
es_update.smootherUpdate(run_context)
except ErtAnalysisError as e:
raise ErtRunError(
f"Analysis of simulation failed with the following error: {e}"
) from e
# Push update data to new storage
analysis_module_name = self.ert().analysisConfig().activeModuleName()
update_id = self._post_update_data(
parent_ensemble_id=ensemble_id, algorithm=analysis_module_name
)
self.setPhaseName("Post processing update...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_UPDATE, ert=self.ert())
return update_id
def test_stop_long_running(self):
"""
This test should verify that only the jobs that are has a runtime
25% longer than the average completed are stopped when
stop_long_running_jobs is called.
"""
job_list = [
MockedJob(JobStatusType.JOB_QUEUE_WAITING) for i in range(10)
]
for i in range(5):
job_list[i].status = JobStatusType.JOB_QUEUE_DONE
job_list[i]._start_time = 0
job_list[i]._end_time = 10
for i in range(5, 8):
job_list[i].status = JobStatusType.JOB_QUEUE_RUNNING
job_list[i]._start_time = 0
job_list[i]._end_time = 20
for i in range(8, 10):
job_list[i].status = JobStatusType.JOB_QUEUE_RUNNING
job_list[i]._start_time = 0
job_list[i]._end_time = 5
queue = MockedQueue(job_list)
EnkfSimulationRunner.stop_long_running_jobs(queue, 5)
for i in range(5):
assert job_list[i].status == JobStatusType.JOB_QUEUE_DONE
for i in range(5, 8):
assert job_list[i].status == JobStatusType.JOB_QUEUE_FAILED
for i in range(8, 10):
assert job_list[i].status == JobStatusType.JOB_QUEUE_RUNNING
def runSimulations(
self,
evaluator_server_config: EvaluatorServerConfig) -> ErtRunContext:
prior_context = self.create_context()
self._checkMinimumActiveRealizations(prior_context)
self.setPhase(0, "Running simulations...", indeterminate=False)
# self.setAnalysisModule(arguments["analysis_module"])
self.setPhaseName("Pre processing...", indeterminate=True)
self.ert().getEnkfSimulationRunner().createRunPath(prior_context)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION,
ert=self.ert())
# Push ensemble, parameters, observations to new storage
ensemble_id = self._post_ensemble_data()
self.setPhaseName("Running forecast...", indeterminate=False)
num_successful_realizations = self.run_ensemble_evaluator(
prior_context, evaluator_server_config)
# Push simulation results to storage
self._post_ensemble_results(ensemble_id)
self.checkHaveSufficientRealizations(num_successful_realizations)
self.setPhaseName("Post processing...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION,
ert=self.ert())
self.setPhaseName("Analyzing...")
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_FIRST_UPDATE,
ert=self.ert())
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_UPDATE,
ert=self.ert())
es_update = self.ert().getESUpdate()
try:
es_update.smootherUpdate(prior_context)
except ErtAnalysisError as e:
raise ErtRunError(
f"Analysis of simulation failed with the following error: {e}"
) from e
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_UPDATE,
ert=self.ert())
# Create an update object in storage
analysis_module_name = self.ert().analysisConfig().activeModuleName()
update_id = self._post_update_data(ensemble_id, analysis_module_name)
self.setPhase(1, "Running simulations...")
self.ert().getEnkfFsManager().switchFileSystem(
prior_context.get_target_fs())
self.setPhaseName("Pre processing...")
rerun_context = self.create_context(prior_context=prior_context)
self.ert().getEnkfSimulationRunner().createRunPath(rerun_context)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION,
ert=self.ert())
# Push ensemble, parameters, observations to new storage
ensemble_id = self._post_ensemble_data(update_id=update_id)
self.setPhaseName("Running forecast...", indeterminate=False)
num_successful_realizations = self.run_ensemble_evaluator(
rerun_context, evaluator_server_config)
self.checkHaveSufficientRealizations(num_successful_realizations)
self.setPhaseName("Post processing...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION,
ert=self.ert())
# Push simulation results to storage
self._post_ensemble_results(ensemble_id)
self.setPhase(2, "Simulations completed.")
return prior_context
def runSimulations(self, arguments):
phase_count = ERT.enkf_facade.get_number_of_iterations() + 1
self.setPhaseCount(phase_count)
analysis_module = self.setAnalysisModule(arguments["analysis_module"])
target_case_format = arguments["target_case"]
run_context = self.create_context(arguments, 0)
self.ert().analysisConfig().getAnalysisIterConfig().setCaseFormat(
target_case_format)
self._runAndPostProcess(run_context)
analysis_config = self.ert().analysisConfig()
analysis_iter_config = analysis_config.getAnalysisIterConfig()
num_retries_per_iteration = analysis_iter_config.getNumRetries()
num_retries = 0
current_iter = 0
previous_ensemble_name = None
while current_iter < ERT.enkf_facade.get_number_of_iterations(
) and num_retries < num_retries_per_iteration:
pre_analysis_iter_num = analysis_module.getInt("ITER")
# We run the PRE_FIRST_UPDATE hook here because the current_iter is explicitly available, versus
# in the run_context inside analyzeStep
if current_iter == 0:
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_FIRST_UPDATE,
ert=ERT.ert)
self.analyzeStep(run_context)
current_iter = analysis_module.getInt("ITER")
analysis_success = current_iter > pre_analysis_iter_num
if analysis_success:
analysis_module_name = self.ert().analysisConfig(
).activeModuleName()
previous_ensemble_name = dump_to_new_storage(
reference=None if previous_ensemble_name is None else (
previous_ensemble_name, analysis_module_name))
run_context = self.create_context(arguments,
current_iter,
prior_context=run_context)
self.ert().getEnkfFsManager().switchFileSystem(
run_context.get_target_fs())
self._runAndPostProcess(run_context)
num_retries = 0
else:
run_context = self.create_context(arguments,
current_iter,
prior_context=run_context,
rerun=True)
self._runAndPostProcess(run_context)
num_retries += 1
analysis_module_name = self.ert().analysisConfig().activeModuleName()
previous_ensemble_name = dump_to_new_storage(
reference=None if previous_ensemble_name is None else (
previous_ensemble_name, analysis_module_name))
if current_iter == (phase_count - 1):
self.setPhase(phase_count, "Simulations completed.")
else:
raise ErtRunError(
"Iterated Ensemble Smoother stopped: maximum number of iteration retries (%d retries) reached for iteration %d"
% (num_retries_per_iteration, current_iter))
return run_context
def runSimulations(self, arguments):
prior_context = self.create_context(arguments)
self.checkMinimumActiveRealizations(prior_context)
self.setPhase(0, "Running simulations...", indeterminate=False)
# self.setAnalysisModule(arguments["analysis_module"])
self.setPhaseName("Pre processing...", indeterminate=True)
self.ert().getEnkfSimulationRunner().createRunPath(prior_context)
# Push ensemble, parameters, observations to new storage
ensemble_id = post_ensemble_data()
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION,
ert=ERT.ert)
self.setPhaseName("Running forecast...", indeterminate=False)
if FeatureToggling.is_enabled("ensemble-evaluator"):
ee_config = arguments["ee_config"]
num_successful_realizations = self.run_ensemble_evaluator(
prior_context, ee_config)
else:
self._job_queue = self._queue_config.create_job_queue()
num_successful_realizations = self.ert().getEnkfSimulationRunner(
).runSimpleStep(self._job_queue, prior_context)
self.checkHaveSufficientRealizations(num_successful_realizations)
self.setPhaseName("Post processing...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION,
ert=ERT.ert)
self.setPhaseName("Analyzing...")
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_FIRST_UPDATE,
ert=ERT.ert)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_UPDATE, ert=ERT.ert)
es_update = self.ert().getESUpdate()
success = es_update.smootherUpdate(prior_context)
if not success:
raise ErtRunError("Analysis of simulation failed!")
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_UPDATE, ert=ERT.ert)
# Push simulation results to storage
post_ensemble_results(ensemble_id)
self.setPhase(1, "Running simulations...")
self.ert().getEnkfFsManager().switchFileSystem(
prior_context.get_target_fs())
self.setPhaseName("Pre processing...")
rerun_context = self.create_context(arguments,
prior_context=prior_context)
self.ert().getEnkfSimulationRunner().createRunPath(rerun_context)
# Push ensemble, parameters, observations to new storage
analysis_module_name = self.ert().analysisConfig().activeModuleName()
ensemble_id = post_ensemble_data((ensemble_id, analysis_module_name))
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION,
ert=ERT.ert)
self.setPhaseName("Running forecast...", indeterminate=False)
if FeatureToggling.is_enabled("ensemble-evaluator"):
ee_config = arguments["ee_config"]
num_successful_realizations = self.run_ensemble_evaluator(
rerun_context, ee_config)
else:
self._job_queue = self._queue_config.create_job_queue()
num_successful_realizations = self.ert().getEnkfSimulationRunner(
).runSimpleStep(self._job_queue, rerun_context)
self.checkHaveSufficientRealizations(num_successful_realizations)
self.setPhaseName("Post processing...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION,
ert=ERT.ert)
self.setPhase(2, "Simulations completed.")
# Push simulation results to storage
post_ensemble_results(ensemble_id)
return prior_context
def runSimulations(self, arguments):
prior_context = self.create_context(arguments)
self.checkMinimumActiveRealizations(prior_context)
self.setPhase(0, "Running simulations...", indeterminate=False)
# self.setAnalysisModule(arguments["analysis_module"])
self.setPhaseName("Pre processing...", indeterminate=True)
self.ert().getEnkfSimulationRunner().createRunPath(prior_context)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION,
ert=ERT.ert)
self.setPhaseName("Running forecast...", indeterminate=False)
self._job_queue = self._queue_config.create_job_queue()
num_successful_realizations = self.ert().getEnkfSimulationRunner(
).runSimpleStep(self._job_queue, prior_context)
self.checkHaveSufficientRealizations(num_successful_realizations)
self.setPhaseName("Post processing...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION,
ert=ERT.ert)
self.setPhaseName("Analyzing...")
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_FIRST_UPDATE,
ert=ERT.ert)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_UPDATE, ert=ERT.ert)
es_update = self.ert().getESUpdate()
success = es_update.smootherUpdate(prior_context)
if not success:
raise ErtRunError("Analysis of simulation failed!")
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_UPDATE, ert=ERT.ert)
previous_ensemble_name = dump_to_new_storage(reference=None)
self.setPhase(1, "Running simulations...")
self.ert().getEnkfFsManager().switchFileSystem(
prior_context.get_target_fs())
self.setPhaseName("Pre processing...")
rerun_context = self.create_context(arguments,
prior_context=prior_context)
self.ert().getEnkfSimulationRunner().createRunPath(rerun_context)
EnkfSimulationRunner.runWorkflows(HookRuntime.PRE_SIMULATION,
ert=ERT.ert)
self.setPhaseName("Running forecast...", indeterminate=False)
self._job_queue = self._queue_config.create_job_queue()
num_successful_realizations = self.ert().getEnkfSimulationRunner(
).runSimpleStep(self._job_queue, rerun_context)
self.checkHaveSufficientRealizations(num_successful_realizations)
self.setPhaseName("Post processing...", indeterminate=True)
EnkfSimulationRunner.runWorkflows(HookRuntime.POST_SIMULATION,
ert=ERT.ert)
self.setPhase(2, "Simulations completed.")
analysis_module_name = self.ert().analysisConfig().activeModuleName()
dump_to_new_storage(reference=(previous_ensemble_name,
analysis_module_name))
return prior_context
def createCReference(cls, c_pointer, parent=None):
obj = super(EnKFMain, cls).createCReference(c_pointer, parent)
obj.__simulation_runner = EnkfSimulationRunner(obj)
obj.__fs_manager = EnkfFsManager(obj)
return obj
