def run_cli(args):
logging.basicConfig(level=logging.INFO, format="%(message)s")
res_config = ResConfig(args.config)
os.chdir(res_config.config_path)
ert = EnKFMain(res_config, strict=True, verbose=args.verbose)
notifier = ErtCliNotifier(ert, args.config)
ERT.adapt(notifier)
if args.mode == WORKFLOW_MODE:
execute_workflow(args.name)
return
model, argument = create_model(args)
# Test run does not have a current_case
if "current_case" in args and args.current_case:
ERT.enkf_facade.select_or_create_new_case(args.current_case)
if (
args.mode
in [ENSEMBLE_SMOOTHER_MODE, ITERATIVE_ENSEMBLE_SMOOTHER_MODE, ES_MDA_MODE]
and args.target_case == ERT.enkf_facade.get_current_case_name()
):
msg = (
"ERROR: Target file system and source file system can not be the same. "
"They were both: {}.".format(args.target_case)
)
_clear_and_exit(msg)
ee_config = None
if FeatureToggling.is_enabled("ensemble-evaluator"):
ee_config = EvaluatorServerConfig()
argument.update({"ee_config": ee_config})
thread = threading.Thread(
name="ert_cli_simulation_thread",
target=model.startSimulations,
args=(argument,),
)
thread.start()
tracker = create_tracker(model, detailed_interval=0, ee_config=ee_config)
out = open(os.devnull, "w") if args.disable_monitoring else sys.stdout
monitor = Monitor(out=out, color_always=args.color_always)
try:
monitor.monitor(tracker)
except (SystemExit, KeyboardInterrupt):
print("\nKilling simulations...")
tracker.request_termination()
if args.disable_monitoring:
out.close()
thread.join()
if model.hasRunFailed():
_clear_and_exit(1) # the monitor has already reported the error message
def test_exporter_is_not_valid():
config_file = "snake_oil.ert"
rc = ResConfig(user_config_file=config_file)
rc.convertToCReference(None)
ert = EnKFMain(rc)
notifier = ErtCliNotifier(ert, config_file)
ERT.adapt(notifier)
ex = Exporter()
assert not ex.is_valid()
def test_default_target_case_name_format_mode(self):
config_file = self.createTestPath('local/poly_example/poly.ert')
with ErtTestContext('test_default_target_case_name_format_mode', config_file) as work_area:
ert = work_area.getErt()
notifier = ErtCliNotifier(ert, config_file)
ERT.adapt(notifier)
args = Namespace(target_case=None)
res = model_factory._target_case_name(args, format_mode=True)
self.assertEqual("default_%d", res)
def test_exporter_is_valid():
with ErtPluginContext():
config_file = "snake_oil.ert"
rc = ResConfig(user_config_file=config_file)
rc.convertToCReference(None)
ert = EnKFMain(rc)
notifier = ErtCliNotifier(ert, config_file)
ERT.adapt(notifier)
ex = Exporter()
assert ex.is_valid(), "Missing CSV_EXPORT2 or EXPORT_RUNPATH jobs"
def initializeCurrentCaseFromExisting(source_case, target_case, source_report_step, parameters, members):
if caseExists(source_case) and caseIsInitialized(source_case) and caseExists(target_case):
total_member_count = getRealizationCount()
member_mask = BoolVector.createFromList(total_member_count, members)
selected_parameters = StringList(parameters)
ERT.ert.getEnkfFsManager().customInitializeCurrentFromExistingCase(source_case, source_report_step, member_mask,
selected_parameters)
ERT.emitErtChange()
def test_setup_ensemble_experiment(self):
config_file = self.createTestPath('local/poly_example/poly.ert')
with ErtTestContext('test_single_test_run', config_file) as work_area:
ert = work_area.getErt()
notifier = ErtCliNotifier(ert, config_file)
ERT.adapt(notifier)
model, argument = model_factory._setup_single_test_run()
self.assertTrue(isinstance(model, EnsembleExperiment))
self.assertEqual(1, len(argument.keys()))
self.assertTrue("active_realizations" in argument)
def test_default_target_case_name(self):
config_file = self.createTestPath("local/poly_example/poly.ert")
with ErtTestContext("test_default_target_case_name",
config_file) as work_area:
ert = work_area.getErt()
notifier = ErtCliNotifier(ert, config_file)
ERT.adapt(notifier)
args = Namespace(target_case=None)
res = model_factory._target_case_name(args)
self.assertEqual("default_smoother_update", res)
def test_custom_target_case_name(self):
config_file = self.createTestPath('local/poly_example/poly.ert')
with ErtTestContext('test_custom_target_case_name', config_file) as work_area:
ert = work_area.getErt()
notifier = ErtCliNotifier(ert, config_file)
ERT.adapt(notifier)
custom_name = "test"
args = Namespace(target_case=custom_name)
res = model_factory._target_case_name(args)
self.assertEqual(custom_name, res)
def initializeCurrentCaseFromScratch(parameters, members):
selected_parameters = StringList(parameters)
mask = BoolVector(initial_size = getRealizationCount(), default_value = False)
for member in members:
member = int(member.strip())
mask[member] = True
sim_fs = ERT.ert.getEnkfFsManager().getCurrentFileSystem()
run_context = ErtRunContext.case_init(sim_fs, mask)
ERT.ert.getEnkfFsManager().initializeFromScratch(selected_parameters, run_context)
ERT.emitErtChange()
def test_init_iteration_number(self):
config_file = self.createTestPath('local/poly_example/poly.ert')
with ErtTestContext('test_init_iteration_number', config_file) as work_area:
ert = work_area.getErt()
notifier = ErtCliNotifier(ert, config_file)
ERT.adapt(notifier)
args = Namespace(iter_num=10, realizations=None)
model, argument = model_factory._setup_ensemble_experiment(args)
run_context = model.create_context(argument)
self.assertEqual(argument['iter_num'], 10)
self.assertEqual(run_context.get_iter(), 10)
def test_setup_single_test_run(self):
config_file = self.createTestPath("local/poly_example/poly.ert")
with ErtTestContext("test_single_test_run", config_file) as work_area:
ert = work_area.getErt()
notifier = ErtCliNotifier(ert, config_file)
ERT.adapt(notifier)
model, argument = model_factory._setup_single_test_run()
self.assertTrue(isinstance(model, SingleTestRun))
self.assertEqual(1, len(argument.keys()))
self.assertTrue("active_realizations" in argument)
model.create_context(argument)
def test_custom_realizations(self):
config_file = self.createTestPath('local/poly_example/poly.ert')
with ErtTestContext('test_custom_realizations', config_file) as work_area:
ert = work_area.getErt()
notifier = ErtCliNotifier(ert, config_file)
ERT.adapt(notifier)
args = Namespace(realizations="0-4,7,8")
res = model_factory._realizations(args)
ensemble_size = ERT.ert.getEnsembleSize()
mask = BoolVector(default_value=False, initial_size=ensemble_size)
mask.updateActiveMask("0-4,7,8")
self.assertEqual(mask, res)
def runSimulation(self):
case_name = getCurrentCaseName()
message = "Are you sure you want to use case '%s' for initialization of the initial ensemble when running the simulations?" % case_name
start_simulations = QMessageBox.question(self, "Start simulations?", message, QMessageBox.Yes | QMessageBox.No )
if start_simulations == QMessageBox.Yes:
run_model = self.getCurrentSimulationModel()
arguments = self.getSimulationArguments()
dialog = RunDialog(self._config_file, run_model(), arguments)
dialog.startSimulation()
dialog.exec_()
ERT.emitErtChange() # simulations may have added new cases.
def run_cli(args):
logging.basicConfig(level=logging.INFO, format='%(message)s')
res_config = ResConfig(args.config)
os.chdir(res_config.config_path)
ert = EnKFMain(res_config, strict=True, verbose=args.verbose)
notifier = ErtCliNotifier(ert, args.config)
ERT.adapt(notifier)
if args.mode == WORKFLOW_MODE:
execute_workflow(args.name)
return
model, argument = create_model(args)
if args.disable_monitoring:
model.startSimulations(argument)
if model.hasRunFailed():
_clear_and_exit(model.getFailMessage())
else:
# Test run does not have a current_case
if "current_case" in args and args.current_case:
ERT.enkf_facade.select_or_create_new_case(args.current_case)
if (args.mode in [ENSEMBLE_SMOOTHER_MODE, ITERATIVE_ENSEMBLE_SMOOTHER_MODE, ES_MDA_MODE] and
args.target_case == ERT.enkf_facade.get_current_case_name()):
msg = (
"ERROR: Target file system and source file system can not be the same. "
"They were both: {}.".format(args.target_case)
)
_clear_and_exit(msg)
thread = threading.Thread(
name="ert_cli_simulation_thread",
target=model.startSimulations,
args=(argument,)
)
thread.start()
tracker = create_tracker(model, tick_interval=0, detailed_interval=0)
monitor = Monitor(color_always=args.color_always)
try:
monitor.monitor(tracker)
except (SystemExit, KeyboardInterrupt):
print("\nKilling simulations...")
model.killAllSimulations()
thread.join()
if model.hasRunFailed():
_clear_and_exit(1) # the monitor has already reported the error message
def test_setup_ensemble_smoother(self):
config_file = self.createTestPath('local/poly_example/poly.ert')
with ErtTestContext('test_single_test_run', config_file) as work_area:
ert = work_area.getErt()
notifier = ErtCliNotifier(ert, config_file)
ERT.adapt(notifier)
args = Namespace(realizations="0-4,7,8", target_case="test_case")
model, argument = model_factory._setup_ensemble_smoother(args)
self.assertTrue(isinstance(model, EnsembleSmoother))
self.assertEqual(3, len(argument.keys()))
self.assertTrue("active_realizations" in argument)
self.assertTrue("target_case" in argument)
self.assertTrue("analysis_module" in argument)
def test_setup_multiple_data_assimilation(self):
config_file = self.createTestPath('local/poly_example/poly.ert')
with ErtTestContext('test_single_test_run', config_file) as work_area:
ert = work_area.getErt()
notifier = ErtCliNotifier(ert, config_file)
ERT.adapt(notifier)
args = Namespace(realizations="0-4,7,8", weights="6,4,2", target_case="test_case")
model, argument = model_factory._setup_multiple_data_assimilation(args)
self.assertTrue(isinstance(model, MultipleDataAssimilation))
self.assertEqual(4, len(argument.keys()))
self.assertTrue("active_realizations" in argument)
self.assertTrue("target_case" in argument)
self.assertTrue("analysis_module" in argument)
self.assertTrue("weights" in argument)
def run_gui(args):
app = QApplication(
[]) # Early so that QT is initialized before other imports
app.setWindowIcon(resourceIcon("application/window_icon_cutout"))
res_config = ResConfig(args.config)
# Create logger inside function to make sure all handlers have been added to the root-logger.
logger = logging.getLogger(__name__)
logger.info(
"Logging forward model jobs",
extra={
"workflow_jobs":
str(res_config.model_config.getForwardModel().joblist())
},
)
os.chdir(res_config.config_path)
# Changing current working directory means we need to update the config file to
# be the base name of the original config
args.config = os.path.basename(args.config)
ert = EnKFMain(res_config, strict=True, verbose=args.verbose)
notifier = ErtNotifier(ert, args.config)
with ERT.adapt(notifier):
# window reference must be kept until app.exec returns
window = _start_window(ert, args)
return app.exec_()
def test_executing_workflow(self):
with open('test_wf', 'w') as wf_file:
wf_file.write('EXPORT_RUNPATH')
config_file = 'poly.ert'
with open(config_file, 'a') as file:
file.write("LOAD_WORKFLOW test_wf")
rc = ResConfig(user_config_file=config_file)
rc.convertToCReference(None)
ert = EnKFMain(rc)
notifier = ErtCliNotifier(ert, config_file)
ERT.adapt(notifier)
args = Namespace(name="test_wf")
execute_workflow(args.name)
assert os.path.isfile(".ert_runpath_list")
def test_run_export_pathfile(monkeypatch):
with ErtPluginContext():
run_path_file = Path("output/run_path_file/.some_new_name")
config_file = "snake_oil.ert"
with open(config_file, encoding="utf-8", mode="a") as fout:
fout.write(f"RUNPATH_FILE {run_path_file}\n")
rc = ResConfig(user_config_file=config_file)
ert = EnKFMain(rc)
notifier = ErtCliNotifier(ert, config_file)
with ERT.adapt(notifier):
run_mock = MagicMock()
run_mock.hasFailed.return_value = False
export_mock = MagicMock()
export_mock.hasFailed.return_value = False
monkeypatch.setattr(
ERT.enkf_facade,
"get_workflow_job",
MagicMock(side_effect=[export_mock, run_mock]),
)
ex = Exporter()
parameters = {
"output_file": "export.csv",
"time_index": "raw",
"column_keys": "FOPR",
}
ex.run_export(parameters)
expected_call = call(
arguments=[
f"{run_path_file.absolute()}", "export.csv", "raw", "FOPR"
],
ert=ERT.ert,
verbose=True,
)
assert export_mock.run.call_args == expected_call
def test_instantiation(self):
config_file = self.createTestPath("local/simple_config/minimum_config")
with ErtTestContext("kjell", config_file) as work_area:
ert = work_area.getErt()
notifier = ErtNotifier(ert, config_file)
with ERT.adapt(notifier):
brm = BaseRunModel(ert.get_queue_config())
assert not brm.isQueueRunning()
def test_run_export():
with ErtPluginContext():
config_file = "snake_oil.ert"
rc = ResConfig(user_config_file=config_file)
rc.convertToCReference(None)
ert = EnKFMain(rc)
notifier = ErtCliNotifier(ert, config_file)
ERT.adapt(notifier)
ex = Exporter()
parameters = {
"output_file": "export.csv",
"time_index": "raw",
"column_keys": "FOPR",
}
ex.run_export(parameters)
shutil.rmtree("storage")
with pytest.raises(UserWarning) as warn:
ex.run_export(parameters)
assert ex._export_job in str(warn)
def run_cli(args):
logging.basicConfig(level=logging.INFO, format='%(message)s')
res_config = ResConfig(args.config)
os.chdir(res_config.config_path)
ert = EnKFMain(res_config, strict=True, verbose=args.verbose)
notifier = ErtCliNotifier(ert, args.config)
ERT.adapt(notifier)
if args.mode == 'workflow':
execute_workflow(args.name)
return
model, argument = create_model(args)
if args.disable_monitoring:
model.startSimulations(argument)
if model.hasRunFailed():
sys.exit(model.getFailMessage())
else:
thread = threading.Thread(name="ert_cli_simulation_thread",
target=model.startSimulations,
args=(argument, ))
thread.start()
tracker = create_tracker(model, tick_interval=0, detailed_interval=0)
monitor = Monitor(color_always=args.color_always)
try:
monitor.monitor(tracker)
except (SystemExit, KeyboardInterrupt):
print("\nKilling simulations...")
model.killAllSimulations()
thread.join()
if model.hasRunFailed():
sys.exit(1) # the monitor has already reported the error message
def test_gui_iter_num(monkeypatch, tmpdir, qtbot, patch_enkf_main):
# won't run simulations so we mock it and test whether "iter_num" is in arguments
def _assert_iter_in_args(panel):
assert "iter_num" in panel.getSimulationArguments()
args_mock = Mock()
type(args_mock).config = PropertyMock(return_value="config.ert")
monkeypatch.setattr(
ert_gui.simulation.simulation_panel.SimulationPanel,
"runSimulation",
_assert_iter_in_args,
)
# overrides realization count to 2
monkeypatch.setattr(
ert_gui.ertwidgets.models.activerealizationsmodel,
"getRealizationCount",
Mock(return_value=2),
)
monkeypatch.setattr(
ert_gui.simulation.ensemble_experiment_panel,
"getRealizationCount",
Mock(return_value=2),
)
notifier = ErtNotifier(patch_enkf_main, args_mock.config)
with ERT.adapt(notifier):
gui = _start_window(patch_enkf_main, args_mock)
qtbot.addWidget(gui)
sim_mode = gui.findChild(qtpy.QtWidgets.QWidget,
name="Simulation_mode")
qtbot.keyClick(sim_mode, Qt.Key_Down)
sim_panel = gui.findChild(qtpy.QtWidgets.QWidget,
name="Simulation_panel")
ensemble_panel = gui.findChild(qtpy.QtWidgets.QWidget,
name="Ensemble_experiment_panel")
# simulate entering number 10 as iter_num
qtbot.keyClick(ensemble_panel._iter_field, Qt.Key_Backspace)
qtbot.keyClicks(ensemble_panel._iter_field, "10")
qtbot.keyClick(ensemble_panel._iter_field, Qt.Key_Enter)
start_simulation = gui.findChild(qtpy.QtWidgets.QWidget,
name="start_simulation")
qtbot.mouseClick(start_simulation, Qt.LeftButton)
assert sim_panel.getSimulationArguments()["iter_num"] == 10
def test_gui_load(monkeypatch, tmpdir, qtbot, patch_enkf_main):
args_mock = Mock()
type(args_mock).config = PropertyMock(return_value="config.ert")
notifier = ErtNotifier(patch_enkf_main, args_mock.config)
with ERT.adapt(notifier):
gui = _start_window(patch_enkf_main, args_mock)
qtbot.addWidget(gui)
sim_panel = gui.findChild(qtpy.QtWidgets.QWidget,
name="Simulation_panel")
single_run_panel = gui.findChild(qtpy.QtWidgets.QWidget,
name="Single_test_run_panel")
assert (sim_panel.getCurrentSimulationModel() ==
single_run_panel.getSimulationModel())
sim_mode = gui.findChild(qtpy.QtWidgets.QWidget,
name="Simulation_mode")
qtbot.keyClick(sim_mode, Qt.Key_Down)
ensemble_panel = gui.findChild(qtpy.QtWidgets.QWidget,
name="Ensemble_experiment_panel")
assert (sim_panel.getCurrentSimulationModel() ==
ensemble_panel.getSimulationModel())
def test_setup_iterative_ensemble_smoother(self):
config_file = self.createTestPath("local/poly_example/poly.ert")
with ErtTestContext("_setup_iterative_ensemble_smoother",
config_file) as work_area:
ert = work_area.getErt()
notifier = ErtCliNotifier(ert, config_file)
with ERT.adapt(notifier):
args = Namespace(
realizations="0-4,7,8",
target_case="test_case_%d",
num_iterations="10",
)
model, argument = model_factory._setup_iterative_ensemble_smoother(
args)
self.assertTrue(isinstance(model, IteratedEnsembleSmoother))
self.assertEqual(4, len(argument.keys()))
self.assertTrue("active_realizations" in argument)
self.assertTrue("target_case" in argument)
self.assertTrue("analysis_module" in argument)
self.assertTrue("num_iterations" in argument)
self.assertTrue(
ERT.enkf_facade.get_number_of_iterations() == 10)
model.create_context(argument, 0)
def test_tracking(
experiment_folder,
cmd_line_arguments,
num_successful,
num_iters,
tmpdir,
source_root,
):
shutil.copytree(
os.path.join(source_root, "test-data", "local",
f"{experiment_folder}"),
os.path.join(str(tmpdir), f"{experiment_folder}"),
)
config_lines = [
"INSTALL_JOB poly_eval2 POLY_EVAL\n"
"SIMULATION_JOB poly_eval2\n"
]
with tmpdir.as_cwd():
with open(f"{experiment_folder}/poly.ert", "a") as fh:
fh.writelines(config_lines)
parser = ArgumentParser(prog="test_main")
parsed = ert_parser(
parser,
cmd_line_arguments,
)
FeatureToggling.update_from_args(parsed)
res_config = ResConfig(parsed.config)
os.chdir(res_config.config_path)
ert = EnKFMain(res_config, strict=True, verbose=parsed.verbose)
notifier = ErtCliNotifier(ert, parsed.config)
ERT.adapt(notifier)
model, argument = create_model(parsed)
ee_config = None
if FeatureToggling.is_enabled("ensemble-evaluator"):
ee_config = EvaluatorServerConfig()
argument.update({"ee_config": ee_config})
thread = threading.Thread(
name="ert_cli_simulation_thread",
target=model.startSimulations,
args=(argument, ),
)
thread.start()
tracker = create_tracker(model,
general_interval=1,
detailed_interval=2,
ee_config=ee_config)
snapshots = {}
for event in tracker.track():
if isinstance(event, FullSnapshotEvent):
snapshots[event.iteration] = event.snapshot
if isinstance(event, SnapshotUpdateEvent):
if event.partial_snapshot is not None:
snapshots[event.iteration].merge(
event.partial_snapshot.data())
if isinstance(event, EndEvent):
break
assert tracker._progress() == 1.0
assert len(snapshots) == num_iters
for iter_, snapshot in snapshots.items():
successful_reals = list(
filter(
lambda item: item[1].status == REALIZATION_STATE_FINISHED,
snapshot.get_reals().items(),
))
assert len(successful_reals) == num_successful
for real_id, real in successful_reals:
assert (real.status == REALIZATION_STATE_FINISHED
), f"iter:{iter_} real:{real_id} was not finished"
poly = real.steps["0"].jobs["0"]
poly2 = real.steps["0"].jobs["1"]
assert poly.name == "poly_eval"
assert (poly.status == JOB_STATE_FINISHED
), f"real {real_id}/{poly['name']} was not finished"
assert poly2.name == "poly_eval2"
assert (poly2.status == JOB_STATE_FINISHED
), f"real {real_id}/{poly['name']} was not finished"
def selectOrCreateNewCase(case_name):
if getCurrentCaseName() != case_name:
fs = ERT.ert.getEnkfFsManager().getFileSystem(case_name)
ERT.ert.getEnkfFsManager().switchFileSystem(fs)
ERT.emitErtChange()
def selectOrCreateNewCase(case_name):
ERT.enkf_facade.select_or_create_new_case(case_name)
ERT.emitErtChange()
def trigger(self):
ERT.emitErtChange() # plugin may have added new cases.
def setNumberOfIterations(iteration_count):
""" @type iteration_count: int """
if iteration_count != getNumberOfIterations():
ERT.ert.analysisConfig().getAnalysisIterConfig().setNumIterations(
iteration_count)
ERT.emitErtChange()
