def test_setup_iterative_ensemble_smoother(self):
config_file = self.createTestPath("local/poly_example/poly.ert")
with ErtTestContext("_setup_iterative_ensemble_smoother",
config_file) as work_area:
ert = work_area.getErt()
facade = LibresFacade(ert)
args = Namespace(
realizations="0-4,7,8",
target_case="test_case_%d",
num_iterations="10",
)
model = model_factory._setup_iterative_ensemble_smoother(
ert,
args,
facade.get_ensemble_size(),
facade.get_current_case_name(),
)
self.assertTrue(isinstance(model, IteratedEnsembleSmoother))
self.assertEqual(4, len(model._simulation_arguments.keys()))
self.assertTrue(
"active_realizations" in model._simulation_arguments)
self.assertTrue("target_case" in model._simulation_arguments)
self.assertTrue("analysis_module" in model._simulation_arguments)
self.assertTrue("num_iterations" in model._simulation_arguments)
self.assertTrue(facade.get_number_of_iterations() == 10)
model.create_context(0)
def _load_measured_record(enkf_main):
facade = LibresFacade(enkf_main)
obs_keys = [
facade.get_observation_key(nr)
for nr, _ in enumerate(facade.get_observations())
]
return MeasuredData(facade, obs_keys)
def _create_runpath(ert: LibresFacade, iteration: int = 0) -> ErtRunContext:
"""
Instantiate an ERT runpath. This will create the parameter coefficients.
"""
enkf_main = ert._enkf_main
result_fs = ert.get_current_fs()
target_fs = ert._enkf_main.getEnkfFsManager().getFileSystem("iter")
model_config = enkf_main.getModelConfig()
runpath_fmt = model_config.getRunpathFormat()
jobname_fmt = model_config.getJobnameFormat()
subst_list = enkf_main.getDataKW()
run_context = ErtRunContext.ensemble_smoother(
result_fs,
target_fs,
BoolVector(default_value=True, initial_size=ert.get_ensemble_size()),
runpath_fmt,
jobname_fmt,
subst_list,
iteration,
)
enkf_main.getEnkfSimulationRunner().createRunPath(run_context)
return run_context
def test_setup_multiple_data_assimilation(self):
config_file = self.createTestPath("local/poly_example/poly.ert")
with ErtTestContext("test_single_test_run", config_file) as work_area:
ert = work_area.getErt()
facade = LibresFacade(ert)
args = Namespace(
realizations="0-4,7,8",
weights="6,4,2",
target_case="test_case_%d",
start_iteration="0",
)
model = model_factory._setup_multiple_data_assimilation(
ert,
args,
facade.get_ensemble_size(),
facade.get_current_case_name(),
)
self.assertTrue(isinstance(model, MultipleDataAssimilation))
self.assertEqual(5, len(model._simulation_arguments.keys()))
self.assertTrue(
"active_realizations" in model._simulation_arguments)
self.assertTrue("target_case" in model._simulation_arguments)
self.assertTrue("analysis_module" in model._simulation_arguments)
self.assertTrue("weights" in model._simulation_arguments)
self.assertTrue("start_iteration" in model._simulation_arguments)
model.create_context(0)
def __init__(self, ert: EnKFMain, notifier: ErtNotifier, config_file: str):
self.notifier = notifier
QWidget.__init__(self)
self.ert = ert
self.facade = LibresFacade(ert)
self._config_file = config_file
self.setObjectName("Simulation_panel")
layout = QVBoxLayout()
self._simulation_mode_combo = QComboBox()
self._simulation_mode_combo.setObjectName("Simulation_mode")
addHelpToWidget(self._simulation_mode_combo, "run/simulation_mode")
self._simulation_mode_combo.currentIndexChanged.connect(
self.toggleSimulationMode)
simulation_mode_layout = QHBoxLayout()
simulation_mode_layout.addSpacing(10)
simulation_mode_layout.addWidget(QLabel("Simulation mode:"), 0,
Qt.AlignVCenter)
simulation_mode_layout.addWidget(self._simulation_mode_combo, 0,
Qt.AlignVCenter)
simulation_mode_layout.addSpacing(20)
self.run_button = QToolButton()
self.run_button.setObjectName("start_simulation")
self.run_button.setText("Start simulation")
self.run_button.setIcon(resourceIcon("play_circle.svg"))
self.run_button.setIconSize(QSize(32, 32))
self.run_button.clicked.connect(self.runSimulation)
self.run_button.setToolButtonStyle(Qt.ToolButtonTextBesideIcon)
addHelpToWidget(self.run_button, "run/start_simulation")
simulation_mode_layout.addWidget(self.run_button)
simulation_mode_layout.addStretch(1)
layout.addSpacing(5)
layout.addLayout(simulation_mode_layout)
layout.addSpacing(10)
self._simulation_stack = QStackedWidget()
self._simulation_stack.setLineWidth(1)
self._simulation_stack.setFrameStyle(QFrame.StyledPanel)
layout.addWidget(self._simulation_stack)
self._simulation_widgets = OrderedDict()
""" :type: OrderedDict[BaseRunModel,SimulationConfigPanel]"""
self.addSimulationConfigPanel(SingleTestRunPanel(ert, notifier))
self.addSimulationConfigPanel(EnsembleExperimentPanel(ert, notifier))
if self.facade.have_observations:
self.addSimulationConfigPanel(EnsembleSmootherPanel(ert, notifier))
self.addSimulationConfigPanel(
MultipleDataAssimilationPanel(self.facade, notifier))
self.addSimulationConfigPanel(
IteratedEnsembleSmootherPanel(self.facade, notifier))
self.setLayout(layout)
def test_default_target_case_name_format_mode(self):
config_file = self.createTestPath("local/poly_example/poly.ert")
with ErtTestContext("test_default_target_case_name_format_mode",
config_file) as work_area:
ert = work_area.getErt()
facade = LibresFacade(ert)
args = Namespace(target_case=None)
res = model_factory._target_case_name(
ert, args, facade.get_current_case_name(), format_mode=True)
self.assertEqual("default_%d", res)
def test_custom_target_case_name(self):
config_file = self.createTestPath("local/poly_example/poly.ert")
with ErtTestContext("test_custom_target_case_name",
config_file) as work_area:
ert = work_area.getErt()
facade = LibresFacade(ert)
custom_name = "test"
args = Namespace(target_case=custom_name)
res = model_factory._target_case_name(
ert, args, facade.get_current_case_name())
self.assertEqual(custom_name, res)
def __init__(self, ert: EnKFMain, notifier: ErtNotifier):
super().__init__(EnsembleSmoother)
self.ert = ert
facade = LibresFacade(ert)
layout = QFormLayout()
self._case_selector = CaseSelector(facade, notifier)
layout.addRow("Current case:", self._case_selector)
run_path_label = QLabel(f"<b>{facade.run_path}</b>")
addHelpToWidget(run_path_label, "config/simulation/runpath")
layout.addRow("Runpath:", run_path_label)
number_of_realizations_label = QLabel(
f"<b>{facade.get_ensemble_size()}</b>")
addHelpToWidget(number_of_realizations_label,
"config/ensemble/num_realizations")
layout.addRow(QLabel("Number of realizations:"),
number_of_realizations_label)
self._target_case_model = TargetCaseModel(facade, notifier)
self._target_case_field = StringBox(self._target_case_model,
"config/simulation/target_case")
self._target_case_field.setValidator(ProperNameArgument())
layout.addRow("Target case:", self._target_case_field)
self._analysis_module_edit = AnalysisModuleEdit(
facade,
module_name=EnsembleSmootherPanel.analysis_module_name,
help_link="config/analysis/analysis_module",
)
layout.addRow("Analysis module:", self._analysis_module_edit)
active_realizations_model = ActiveRealizationsModel(facade)
self._active_realizations_field = StringBox(
active_realizations_model, "config/simulation/active_realizations")
self._active_realizations_field.setValidator(
RangeStringArgument(facade.get_ensemble_size()))
layout.addRow("Active realizations", self._active_realizations_field)
self.setLayout(layout)
self._target_case_field.getValidationSupport(
).validationChanged.connect(self.simulationConfigurationChanged)
self._active_realizations_field.getValidationSupport(
).validationChanged.connect( # noqa
self.simulationConfigurationChanged)
self._case_selector.currentIndexChanged.connect(
self._realizations_from_fs)
self._realizations_from_fs() # update with the current case
def run(self, job_config):
facade = LibresFacade(self.ert())
user_config = load_yaml(job_config)
user_config = _insert_default_group(user_config)
obs = facade.get_observations()
obs_keys = [facade.get_observation_key(nr) for nr, _ in enumerate(obs)]
obs_with_data = keys_with_data(
obs,
obs_keys,
facade.get_ensemble_size(),
facade.get_current_fs(),
)
default_values = _get_default_values(
facade.get_alpha(), facade.get_std_cutoff()
)
for config_dict in user_config:
config = ObsCorrConfig(config_dict, obs_keys, default_values)
config.validate(obs_with_data)
measured_data = _get_measured_data(
facade,
config.get_calculation_keys(),
config.get_index_lists(),
config.get_alpha(),
config.get_std_cutoff(),
)
job = ObservationScaleFactor(self.reporter, measured_data)
scale_factor = job.get_scaling_factor(config.get_threshold())
logging.info(
"Scaling factor calculated from keys: {}".format(
config.get_calculation_keys()
)
)
scale_observations(obs, scale_factor, config.get_update_keys())
def test_compare_different_jobs():
arguments = {"CALCULATE_KEYS": {"keys": [{"key": "WPR_DIFF_1"}]}}
test_data_dir = os.path.join(TEST_DATA_DIR, "local", "snake_oil")
shutil.copytree(test_data_dir, "test_data")
os.chdir(os.path.join("test_data"))
res_config = ResConfig("snake_oil.ert")
ert = EnKFMain(res_config)
facade = LibresFacade(ert)
obs = ert.getObservations()
obs_vector = obs["WPR_DIFF_1"]
assert_obs_vector(obs_vector, 1.0)
job = ert.getWorkflowList().getJob("STD_SCALE_CORRELATED_OBS")
job.run(ert, ["WPR_DIFF_1"])
# Result of old job:
assert_obs_vector(obs_vector, np.sqrt(4 / 2))
scaling_job.scaling_job(facade, arguments)
# Result of new job with no sub-indexing:
assert_obs_vector(obs_vector, np.sqrt(4 / 2))
def run(self, *args):
facade = LibresFacade(self.ert())
config_record = _fetch_config_record(args)
observations = _get_observations(facade)
config = assemble_config(config_record, observations)
config = config.snapshot
measured_record = _load_measured_record(facade, config.observations)
scaling_configs = misfit_preprocessor.run(
**{
"config": config,
"measured_data": measured_record,
"reporter": self.reporter,
})
# The execution of COS should be moved into
# misfit_preprocessor.run when COS no longer depend on self.ert
# to run.
scaling_params = _fetch_scaling_parameters(config_record, observations)
for scaling_config in scaling_configs:
scaling_config["CALCULATE_KEYS"].update(scaling_params)
try:
CorrelatedObservationsScalingJob(self.ert()).run(scaling_configs)
except EmptyDatasetException:
pass
def test_summary_obs_runtime(monkeypatch, copy_snake_oil):
"""
This is mostly a regression test, as reading SUMMARY_OBS was very slow when using
SUMMARY_OBSERVATION and not HISTORY_OBSERVATION where multiple observations
were pointing to the same response. To simulate that we load the same observations
though individual points, and also in one go. To avoid this test being flaky the
we assert on the difference in runtime. The difference in runtime we assert on is
set to 10x though it should be around 2x
"""
obs_file = pathlib.Path.cwd() / "observations" / "observations.txt"
with obs_file.open(mode="a") as fin:
fin.write(create_summary_observation())
res_config = ResConfig("snake_oil.ert")
ert = EnKFMain(res_config)
facade = LibresFacade(ert)
start_time = time.time()
foprh = MeasuredData(facade,
[f"FOPR_{restart}" for restart in range(1, 201)])
summary_obs_time = time.time() - start_time
start_time = time.time()
fopr = MeasuredData(facade, ["FOPR"])
history_obs_time = time.time() - start_time
assert (fopr.data.columns.get_level_values("data_index").values.tolist() ==
foprh.data.columns.get_level_values("data_index").values.tolist())
result = foprh.get_simulated_data().values == fopr.get_simulated_data(
).values
assert np.logical_and.reduce(result).all()
assert summary_obs_time < 10 * history_obs_time
def _setup_main_window(ert: EnKFMain, notifier: ErtNotifier,
args: argparse.Namespace):
facade = LibresFacade(ert)
config_file = args.config
window = GertMainWindow(config_file)
window.setWidget(SimulationPanel(ert, notifier, config_file))
gui_log_handler = GUILogHandler()
logging.getLogger().addHandler(gui_log_handler)
plugin_handler = PluginHandler(ert,
ert.getWorkflowList().getPluginJobs(),
window)
window.addDock("Configuration summary",
SummaryPanel(ert),
area=Qt.BottomDockWidgetArea)
window.addTool(PlotTool(ert, config_file))
window.addTool(ExportTool(ert))
window.addTool(WorkflowsTool(ert, notifier))
window.addTool(ManageCasesTool(ert, notifier))
window.addTool(PluginsTool(plugin_handler, notifier))
window.addTool(RunAnalysisTool(ert, notifier))
window.addTool(LoadResultsTool(facade))
window.addTool(EventViewerTool(gui_log_handler))
window.adjustSize()
return window
def test_main_entry_point_block_and_summary_data_calc():
arguments = {
"CALCULATE_KEYS": {
"keys": [{
"key": "FOPT"
}, {
"key": "RFT3"
}]
}
}
test_data_dir = os.path.join(TEST_DATA_DIR, "Equinor", "config",
"with_RFT")
shutil.copytree(test_data_dir, "test_data")
os.chdir(os.path.join("test_data"))
res_config = ResConfig("config")
ert = EnKFMain(res_config)
facade = LibresFacade(ert)
obs = ert.getObservations()
obs_vector = obs["RFT3"]
for index, node in enumerate(obs_vector):
assert node.getStdScaling(index) == 1.0
scaling_job.scaling_job(facade, arguments)
for index, node in enumerate(obs_vector):
assert node.getStdScaling(index) == np.sqrt(64)
def run(self, *args):
facade = LibresFacade(self.ert())
parser = spearman_job_parser()
args = parser.parse_args(args)
spearman_job(facade, args.threshold, args.dry_run)
def __init__(self, ert: EnKFMain, notifier: ErtNotifier):
self.ert = ert
self.facade = LibresFacade(ert)
super().__init__(EnsembleExperiment)
self.setObjectName("Ensemble_experiment_panel")
layout = QFormLayout()
self._case_selector = CaseSelector(self.facade, notifier)
layout.addRow("Current case:", self._case_selector)
run_path_label = QLabel(f"<b>{self.facade.run_path}</b>")
addHelpToWidget(run_path_label, "config/simulation/runpath")
layout.addRow("Runpath:", run_path_label)
number_of_realizations_label = QLabel(
f"<b>{self.facade.get_ensemble_size()}</b>")
addHelpToWidget(number_of_realizations_label,
"config/ensemble/num_realizations")
layout.addRow(QLabel("Number of realizations:"),
number_of_realizations_label)
self._active_realizations_field = StringBox(
ActiveRealizationsModel(self.facade),
"config/simulation/active_realizations",
)
self._active_realizations_field.setValidator(
RangeStringArgument(self.facade.get_ensemble_size()), )
layout.addRow("Active realizations", self._active_realizations_field)
self._iter_field = StringBox(
IterValueModel(notifier),
"config/simulation/iter_num",
)
self._iter_field.setValidator(IntegerArgument(from_value=0), )
layout.addRow("Iteration", self._iter_field)
self.setLayout(layout)
self._active_realizations_field.getValidationSupport(
).validationChanged.connect( # noqa
self.simulationConfigurationChanged)
self._case_selector.currentIndexChanged.connect(
self._realizations_from_fs)
self._realizations_from_fs() # update with the current case
def facade(self):
config_file = "snake_oil.ert"
rc = ResConfig(user_config_file=config_file)
rc.convertToCReference(None)
ert = EnKFMain(rc)
facade = LibresFacade(ert)
return facade
def _evaluate_ensemble(ert: LibresFacade, run_context: ErtRunContext):
"""
Launch ensemble experiment with the created config
"""
queue_config = ert.get_queue_config()
_job_queue = queue_config.create_job_queue()
ert._enkf_main.getEnkfSimulationRunner().runSimpleStep(_job_queue, run_context)
def test_no_storage_obs_only(monkeypatch, obs_key):
shutil.rmtree("storage")
res_config = ResConfig("snake_oil.ert")
ert = EnKFMain(res_config)
facade = LibresFacade(ert)
md = MeasuredData(facade, [obs_key], load_data=False)
assert set(md.data.columns.get_level_values(0)) == {obs_key}
class Exporter:
def __init__(self, ert):
self.ert = ert
self.facade = LibresFacade(ert)
self._export_job = "CSV_EXPORT2"
self._runpath_job = "EXPORT_RUNPATH"
def is_valid(self):
export_job = self.facade.get_workflow_job(self._export_job)
runpath_job = self.facade.get_workflow_job(self._runpath_job)
if export_job is None:
logger.error(
f"Export not available because {self._export_job} is not installed."
)
return False
if runpath_job is None:
logger.error(
f"Export not available because {self._runpath_job} is not installed."
)
return False
return True
def run_export(self, parameters):
export_job = self.facade.get_workflow_job(self._export_job)
runpath_job = self.facade.get_workflow_job(self._runpath_job)
runpath_job.run(ert=self.ert, arguments=[], verbose=True)
if runpath_job.hasFailed():
raise UserWarning(f"Failed to execute {self._runpath_job}")
export_job.run(
ert=self.ert,
arguments=[
self.ert.getRunpathList().getExportFile(),
parameters["output_file"],
parameters["time_index"],
parameters["column_keys"],
],
verbose=True,
)
if export_job.hasFailed():
raise UserWarning(f"Failed to execute {self._export_job}")
def get_responses(res: LibresFacade, ensemble_name: str):
response_names = get_response_names(res)
responses = []
active_realizations = res.get_active_realizations(ensemble_name)
for real_id in active_realizations:
for response_name in response_names:
responses.append({"name": response_name, "real_id": real_id})
return responses
def api(self):
config_file = 'snake_oil.ert'
rc = ResConfig(user_config_file=config_file)
rc.convertToCReference(None)
ert = EnKFMain(rc)
facade = LibresFacade(ert)
api = PlotApi(facade)
return api
def __init__(self, ert, config_file, parent):
QMainWindow.__init__(self, parent)
self._api = PlotApi(LibresFacade(ert))
self.setMinimumWidth(850)
self.setMinimumHeight(650)
self.setWindowTitle(f"Plotting - {config_file}")
self.activateWindow()
self._key_definitions = self._api.all_data_type_keys()
self._plot_customizer = PlotCustomizer(self, self._key_definitions)
self._plot_customizer.settingsChanged.connect(self.keySelected)
self._central_tab = QTabWidget()
central_widget = QWidget()
central_layout = QVBoxLayout()
central_layout.setContentsMargins(0, 0, 0, 0)
central_widget.setLayout(central_layout)
central_layout.addWidget(self._central_tab)
self.setCentralWidget(central_widget)
self._plot_widgets = []
""":type: list of PlotWidget"""
self.addPlotWidget(ENSEMBLE, EnsemblePlot())
self.addPlotWidget(STATISTICS, StatisticsPlot())
self.addPlotWidget(HISTOGRAM, HistogramPlot())
self.addPlotWidget(GAUSSIAN_KDE, GaussianKDEPlot())
self.addPlotWidget(DISTRIBUTION, DistributionPlot())
self.addPlotWidget(CROSS_CASE_STATISTICS, CrossCaseStatisticsPlot())
self._central_tab.currentChanged.connect(self.currentPlotChanged)
cases = self._api.get_all_cases_not_running()
case_names = [case["name"] for case in cases if not case["hidden"]]
self._data_type_keys_widget = DataTypeKeysWidget(self._key_definitions)
self._data_type_keys_widget.dataTypeKeySelected.connect(
self.keySelected)
self.addDock("Data types", self._data_type_keys_widget)
self._case_selection_widget = CaseSelectionWidget(case_names)
self._case_selection_widget.caseSelectionChanged.connect(
self.keySelected)
self.addDock("Plot case", self._case_selection_widget)
current_plot_widget = self._plot_widgets[
self._central_tab.currentIndex()]
self._data_type_keys_widget.selectDefault()
self._updateCustomizer(current_plot_widget)
def test_no_storage(monkeypatch, obs_key, expected_msg):
shutil.rmtree("storage")
res_config = ResConfig("snake_oil.ert")
ert = EnKFMain(res_config)
facade = LibresFacade(ert)
with pytest.raises(
loader.ResponseError,
match=expected_msg,
):
MeasuredData(facade, [obs_key])
def run(self, *args):
facade = LibresFacade(self.ert())
obs_keys = [
facade.get_observation_key(nr)
for nr, _ in enumerate(facade.get_observations())
]
measured_data = MeasuredData(facade, obs_keys)
parser = spearman_job_parser()
args = parser.parse_args(args)
scaling_configs = spearman_job(measured_data, args.threshold)
if not args.dry_run:
try:
CorrelatedObservationsScalingJob(
self.ert()).run(scaling_configs)
except EmptyDatasetException:
pass
def initializeCurrentCaseFromExisting(
source_case: str,
target_case: str,
source_report_step: int,
parameters: List[str],
members: List[str],
ert: EnKFMain,
):
if (
caseExists(source_case, LibresFacade(ert))
and ert.getEnkfFsManager().isCaseInitialized(source_case)
and caseExists(target_case, LibresFacade(ert))
):
member_mask = [False] * ert.getEnsembleSize()
for member in members:
member_mask[int(member)] = True
ert.getEnkfFsManager().customInitializeCurrentFromExistingCase(
source_case, source_report_step, member_mask, parameters
)
def init_facade() -> None:
global _libres_facade
global _ert
global _config
configfile = os.environ["ERT_STORAGE_RES_CONFIG"]
_config = ResConfig(configfile)
os.chdir(_config.config_path)
_ert = EnKFMain(_config, strict=True)
_libres_facade = LibresFacade(_ert)
def __init__(self, ert, notifier):
self.ert = ert
QWidget.__init__(self)
self.setWindowTitle("Run analysis")
self.activateWindow()
self.analysis_module = AnalysisModuleEdit(
LibresFacade(ert),
help_link="config/analysis/analysis_module",
)
self.target_case_text = QLineEdit()
self.source_case_selector = CaseSelector(LibresFacade(self.ert),
notifier,
update_ert=False)
layout = QFormLayout()
layout.addRow("Analysis", self.analysis_module)
layout.addRow("Target case", self.target_case_text)
layout.addRow("Source case", self.source_case_selector)
self.setLayout(layout)
def api(tmpdir, source_root):
with tmpdir.as_cwd():
test_data_root = source_root / "test-data" / "local"
test_data_dir = os.path.join(test_data_root, "snake_oil")
shutil.copytree(test_data_dir, "test_data")
os.chdir("test_data")
config_file = "snake_oil.ert"
rc = ResConfig(user_config_file=config_file)
rc.convertToCReference(None)
ert = EnKFMain(rc)
facade = LibresFacade(ert)
api = PlotApi(facade)
yield api
def test_gen_obs_runtime(monkeypatch, copy_snake_oil, snapshot):
obs_file = pathlib.Path.cwd() / "observations" / "observations.txt"
with obs_file.open(mode="a") as fin:
fin.write(create_general_observation())
res_config = ResConfig("snake_oil.ert")
ert = EnKFMain(res_config)
facade = LibresFacade(ert)
df = MeasuredData(facade,
[f"CUSTOM_DIFF_{restart}" for restart in range(500)])
snapshot.assert_match(df.data.to_csv(), "snake_oil_gendata_output.csv")