def test_setup_iterative_ensemble_smoother(self): config_file = self.createTestPath("local/poly_example/poly.ert") with ErtTestContext("_setup_iterative_ensemble_smoother", config_file) as work_area: ert = work_area.getErt() facade = LibresFacade(ert) args = Namespace( realizations="0-4,7,8", target_case="test_case_%d", num_iterations="10", ) model = model_factory._setup_iterative_ensemble_smoother( ert, args, facade.get_ensemble_size(), facade.get_current_case_name(), ) self.assertTrue(isinstance(model, IteratedEnsembleSmoother)) self.assertEqual(4, len(model._simulation_arguments.keys())) self.assertTrue( "active_realizations" in model._simulation_arguments) self.assertTrue("target_case" in model._simulation_arguments) self.assertTrue("analysis_module" in model._simulation_arguments) self.assertTrue("num_iterations" in model._simulation_arguments) self.assertTrue(facade.get_number_of_iterations() == 10) model.create_context(0)
def _load_measured_record(enkf_main): facade = LibresFacade(enkf_main) obs_keys = [ facade.get_observation_key(nr) for nr, _ in enumerate(facade.get_observations()) ] return MeasuredData(facade, obs_keys)
def _create_runpath(ert: LibresFacade, iteration: int = 0) -> ErtRunContext: """ Instantiate an ERT runpath. This will create the parameter coefficients. """ enkf_main = ert._enkf_main result_fs = ert.get_current_fs() target_fs = ert._enkf_main.getEnkfFsManager().getFileSystem("iter") model_config = enkf_main.getModelConfig() runpath_fmt = model_config.getRunpathFormat() jobname_fmt = model_config.getJobnameFormat() subst_list = enkf_main.getDataKW() run_context = ErtRunContext.ensemble_smoother( result_fs, target_fs, BoolVector(default_value=True, initial_size=ert.get_ensemble_size()), runpath_fmt, jobname_fmt, subst_list, iteration, ) enkf_main.getEnkfSimulationRunner().createRunPath(run_context) return run_context
def test_setup_multiple_data_assimilation(self): config_file = self.createTestPath("local/poly_example/poly.ert") with ErtTestContext("test_single_test_run", config_file) as work_area: ert = work_area.getErt() facade = LibresFacade(ert) args = Namespace( realizations="0-4,7,8", weights="6,4,2", target_case="test_case_%d", start_iteration="0", ) model = model_factory._setup_multiple_data_assimilation( ert, args, facade.get_ensemble_size(), facade.get_current_case_name(), ) self.assertTrue(isinstance(model, MultipleDataAssimilation)) self.assertEqual(5, len(model._simulation_arguments.keys())) self.assertTrue( "active_realizations" in model._simulation_arguments) self.assertTrue("target_case" in model._simulation_arguments) self.assertTrue("analysis_module" in model._simulation_arguments) self.assertTrue("weights" in model._simulation_arguments) self.assertTrue("start_iteration" in model._simulation_arguments) model.create_context(0)
def __init__(self, ert: EnKFMain, notifier: ErtNotifier, config_file: str): self.notifier = notifier QWidget.__init__(self) self.ert = ert self.facade = LibresFacade(ert) self._config_file = config_file self.setObjectName("Simulation_panel") layout = QVBoxLayout() self._simulation_mode_combo = QComboBox() self._simulation_mode_combo.setObjectName("Simulation_mode") addHelpToWidget(self._simulation_mode_combo, "run/simulation_mode") self._simulation_mode_combo.currentIndexChanged.connect( self.toggleSimulationMode) simulation_mode_layout = QHBoxLayout() simulation_mode_layout.addSpacing(10) simulation_mode_layout.addWidget(QLabel("Simulation mode:"), 0, Qt.AlignVCenter) simulation_mode_layout.addWidget(self._simulation_mode_combo, 0, Qt.AlignVCenter) simulation_mode_layout.addSpacing(20) self.run_button = QToolButton() self.run_button.setObjectName("start_simulation") self.run_button.setText("Start simulation") self.run_button.setIcon(resourceIcon("play_circle.svg")) self.run_button.setIconSize(QSize(32, 32)) self.run_button.clicked.connect(self.runSimulation) self.run_button.setToolButtonStyle(Qt.ToolButtonTextBesideIcon) addHelpToWidget(self.run_button, "run/start_simulation") simulation_mode_layout.addWidget(self.run_button) simulation_mode_layout.addStretch(1) layout.addSpacing(5) layout.addLayout(simulation_mode_layout) layout.addSpacing(10) self._simulation_stack = QStackedWidget() self._simulation_stack.setLineWidth(1) self._simulation_stack.setFrameStyle(QFrame.StyledPanel) layout.addWidget(self._simulation_stack) self._simulation_widgets = OrderedDict() """ :type: OrderedDict[BaseRunModel,SimulationConfigPanel]""" self.addSimulationConfigPanel(SingleTestRunPanel(ert, notifier)) self.addSimulationConfigPanel(EnsembleExperimentPanel(ert, notifier)) if self.facade.have_observations: self.addSimulationConfigPanel(EnsembleSmootherPanel(ert, notifier)) self.addSimulationConfigPanel( MultipleDataAssimilationPanel(self.facade, notifier)) self.addSimulationConfigPanel( IteratedEnsembleSmootherPanel(self.facade, notifier)) self.setLayout(layout)
def test_default_target_case_name_format_mode(self): config_file = self.createTestPath("local/poly_example/poly.ert") with ErtTestContext("test_default_target_case_name_format_mode", config_file) as work_area: ert = work_area.getErt() facade = LibresFacade(ert) args = Namespace(target_case=None) res = model_factory._target_case_name( ert, args, facade.get_current_case_name(), format_mode=True) self.assertEqual("default_%d", res)
def test_custom_target_case_name(self): config_file = self.createTestPath("local/poly_example/poly.ert") with ErtTestContext("test_custom_target_case_name", config_file) as work_area: ert = work_area.getErt() facade = LibresFacade(ert) custom_name = "test" args = Namespace(target_case=custom_name) res = model_factory._target_case_name( ert, args, facade.get_current_case_name()) self.assertEqual(custom_name, res)
def __init__(self, ert: EnKFMain, notifier: ErtNotifier): super().__init__(EnsembleSmoother) self.ert = ert facade = LibresFacade(ert) layout = QFormLayout() self._case_selector = CaseSelector(facade, notifier) layout.addRow("Current case:", self._case_selector) run_path_label = QLabel(f"<b>{facade.run_path}</b>") addHelpToWidget(run_path_label, "config/simulation/runpath") layout.addRow("Runpath:", run_path_label) number_of_realizations_label = QLabel( f"<b>{facade.get_ensemble_size()}</b>") addHelpToWidget(number_of_realizations_label, "config/ensemble/num_realizations") layout.addRow(QLabel("Number of realizations:"), number_of_realizations_label) self._target_case_model = TargetCaseModel(facade, notifier) self._target_case_field = StringBox(self._target_case_model, "config/simulation/target_case") self._target_case_field.setValidator(ProperNameArgument()) layout.addRow("Target case:", self._target_case_field) self._analysis_module_edit = AnalysisModuleEdit( facade, module_name=EnsembleSmootherPanel.analysis_module_name, help_link="config/analysis/analysis_module", ) layout.addRow("Analysis module:", self._analysis_module_edit) active_realizations_model = ActiveRealizationsModel(facade) self._active_realizations_field = StringBox( active_realizations_model, "config/simulation/active_realizations") self._active_realizations_field.setValidator( RangeStringArgument(facade.get_ensemble_size())) layout.addRow("Active realizations", self._active_realizations_field) self.setLayout(layout) self._target_case_field.getValidationSupport( ).validationChanged.connect(self.simulationConfigurationChanged) self._active_realizations_field.getValidationSupport( ).validationChanged.connect( # noqa self.simulationConfigurationChanged) self._case_selector.currentIndexChanged.connect( self._realizations_from_fs) self._realizations_from_fs() # update with the current case
def run(self, job_config): facade = LibresFacade(self.ert()) user_config = load_yaml(job_config) user_config = _insert_default_group(user_config) obs = facade.get_observations() obs_keys = [facade.get_observation_key(nr) for nr, _ in enumerate(obs)] obs_with_data = keys_with_data( obs, obs_keys, facade.get_ensemble_size(), facade.get_current_fs(), ) default_values = _get_default_values( facade.get_alpha(), facade.get_std_cutoff() ) for config_dict in user_config: config = ObsCorrConfig(config_dict, obs_keys, default_values) config.validate(obs_with_data) measured_data = _get_measured_data( facade, config.get_calculation_keys(), config.get_index_lists(), config.get_alpha(), config.get_std_cutoff(), ) job = ObservationScaleFactor(self.reporter, measured_data) scale_factor = job.get_scaling_factor(config.get_threshold()) logging.info( "Scaling factor calculated from keys: {}".format( config.get_calculation_keys() ) ) scale_observations(obs, scale_factor, config.get_update_keys())
def test_compare_different_jobs(): arguments = {"CALCULATE_KEYS": {"keys": [{"key": "WPR_DIFF_1"}]}} test_data_dir = os.path.join(TEST_DATA_DIR, "local", "snake_oil") shutil.copytree(test_data_dir, "test_data") os.chdir(os.path.join("test_data")) res_config = ResConfig("snake_oil.ert") ert = EnKFMain(res_config) facade = LibresFacade(ert) obs = ert.getObservations() obs_vector = obs["WPR_DIFF_1"] assert_obs_vector(obs_vector, 1.0) job = ert.getWorkflowList().getJob("STD_SCALE_CORRELATED_OBS") job.run(ert, ["WPR_DIFF_1"]) # Result of old job: assert_obs_vector(obs_vector, np.sqrt(4 / 2)) scaling_job.scaling_job(facade, arguments) # Result of new job with no sub-indexing: assert_obs_vector(obs_vector, np.sqrt(4 / 2))
def run(self, *args): facade = LibresFacade(self.ert()) config_record = _fetch_config_record(args) observations = _get_observations(facade) config = assemble_config(config_record, observations) config = config.snapshot measured_record = _load_measured_record(facade, config.observations) scaling_configs = misfit_preprocessor.run( **{ "config": config, "measured_data": measured_record, "reporter": self.reporter, }) # The execution of COS should be moved into # misfit_preprocessor.run when COS no longer depend on self.ert # to run. scaling_params = _fetch_scaling_parameters(config_record, observations) for scaling_config in scaling_configs: scaling_config["CALCULATE_KEYS"].update(scaling_params) try: CorrelatedObservationsScalingJob(self.ert()).run(scaling_configs) except EmptyDatasetException: pass
def test_summary_obs_runtime(monkeypatch, copy_snake_oil): """ This is mostly a regression test, as reading SUMMARY_OBS was very slow when using SUMMARY_OBSERVATION and not HISTORY_OBSERVATION where multiple observations were pointing to the same response. To simulate that we load the same observations though individual points, and also in one go. To avoid this test being flaky the we assert on the difference in runtime. The difference in runtime we assert on is set to 10x though it should be around 2x """ obs_file = pathlib.Path.cwd() / "observations" / "observations.txt" with obs_file.open(mode="a") as fin: fin.write(create_summary_observation()) res_config = ResConfig("snake_oil.ert") ert = EnKFMain(res_config) facade = LibresFacade(ert) start_time = time.time() foprh = MeasuredData(facade, [f"FOPR_{restart}" for restart in range(1, 201)]) summary_obs_time = time.time() - start_time start_time = time.time() fopr = MeasuredData(facade, ["FOPR"]) history_obs_time = time.time() - start_time assert (fopr.data.columns.get_level_values("data_index").values.tolist() == foprh.data.columns.get_level_values("data_index").values.tolist()) result = foprh.get_simulated_data().values == fopr.get_simulated_data( ).values assert np.logical_and.reduce(result).all() assert summary_obs_time < 10 * history_obs_time
def _setup_main_window(ert: EnKFMain, notifier: ErtNotifier, args: argparse.Namespace): facade = LibresFacade(ert) config_file = args.config window = GertMainWindow(config_file) window.setWidget(SimulationPanel(ert, notifier, config_file)) gui_log_handler = GUILogHandler() logging.getLogger().addHandler(gui_log_handler) plugin_handler = PluginHandler(ert, ert.getWorkflowList().getPluginJobs(), window) window.addDock("Configuration summary", SummaryPanel(ert), area=Qt.BottomDockWidgetArea) window.addTool(PlotTool(ert, config_file)) window.addTool(ExportTool(ert)) window.addTool(WorkflowsTool(ert, notifier)) window.addTool(ManageCasesTool(ert, notifier)) window.addTool(PluginsTool(plugin_handler, notifier)) window.addTool(RunAnalysisTool(ert, notifier)) window.addTool(LoadResultsTool(facade)) window.addTool(EventViewerTool(gui_log_handler)) window.adjustSize() return window
def test_main_entry_point_block_and_summary_data_calc(): arguments = { "CALCULATE_KEYS": { "keys": [{ "key": "FOPT" }, { "key": "RFT3" }] } } test_data_dir = os.path.join(TEST_DATA_DIR, "Equinor", "config", "with_RFT") shutil.copytree(test_data_dir, "test_data") os.chdir(os.path.join("test_data")) res_config = ResConfig("config") ert = EnKFMain(res_config) facade = LibresFacade(ert) obs = ert.getObservations() obs_vector = obs["RFT3"] for index, node in enumerate(obs_vector): assert node.getStdScaling(index) == 1.0 scaling_job.scaling_job(facade, arguments) for index, node in enumerate(obs_vector): assert node.getStdScaling(index) == np.sqrt(64)
def run(self, *args): facade = LibresFacade(self.ert()) parser = spearman_job_parser() args = parser.parse_args(args) spearman_job(facade, args.threshold, args.dry_run)
def __init__(self, ert: EnKFMain, notifier: ErtNotifier): self.ert = ert self.facade = LibresFacade(ert) super().__init__(EnsembleExperiment) self.setObjectName("Ensemble_experiment_panel") layout = QFormLayout() self._case_selector = CaseSelector(self.facade, notifier) layout.addRow("Current case:", self._case_selector) run_path_label = QLabel(f"<b>{self.facade.run_path}</b>") addHelpToWidget(run_path_label, "config/simulation/runpath") layout.addRow("Runpath:", run_path_label) number_of_realizations_label = QLabel( f"<b>{self.facade.get_ensemble_size()}</b>") addHelpToWidget(number_of_realizations_label, "config/ensemble/num_realizations") layout.addRow(QLabel("Number of realizations:"), number_of_realizations_label) self._active_realizations_field = StringBox( ActiveRealizationsModel(self.facade), "config/simulation/active_realizations", ) self._active_realizations_field.setValidator( RangeStringArgument(self.facade.get_ensemble_size()), ) layout.addRow("Active realizations", self._active_realizations_field) self._iter_field = StringBox( IterValueModel(notifier), "config/simulation/iter_num", ) self._iter_field.setValidator(IntegerArgument(from_value=0), ) layout.addRow("Iteration", self._iter_field) self.setLayout(layout) self._active_realizations_field.getValidationSupport( ).validationChanged.connect( # noqa self.simulationConfigurationChanged) self._case_selector.currentIndexChanged.connect( self._realizations_from_fs) self._realizations_from_fs() # update with the current case
def facade(self): config_file = "snake_oil.ert" rc = ResConfig(user_config_file=config_file) rc.convertToCReference(None) ert = EnKFMain(rc) facade = LibresFacade(ert) return facade
def _evaluate_ensemble(ert: LibresFacade, run_context: ErtRunContext): """ Launch ensemble experiment with the created config """ queue_config = ert.get_queue_config() _job_queue = queue_config.create_job_queue() ert._enkf_main.getEnkfSimulationRunner().runSimpleStep(_job_queue, run_context)
def test_no_storage_obs_only(monkeypatch, obs_key): shutil.rmtree("storage") res_config = ResConfig("snake_oil.ert") ert = EnKFMain(res_config) facade = LibresFacade(ert) md = MeasuredData(facade, [obs_key], load_data=False) assert set(md.data.columns.get_level_values(0)) == {obs_key}
class Exporter: def __init__(self, ert): self.ert = ert self.facade = LibresFacade(ert) self._export_job = "CSV_EXPORT2" self._runpath_job = "EXPORT_RUNPATH" def is_valid(self): export_job = self.facade.get_workflow_job(self._export_job) runpath_job = self.facade.get_workflow_job(self._runpath_job) if export_job is None: logger.error( f"Export not available because {self._export_job} is not installed." ) return False if runpath_job is None: logger.error( f"Export not available because {self._runpath_job} is not installed." ) return False return True def run_export(self, parameters): export_job = self.facade.get_workflow_job(self._export_job) runpath_job = self.facade.get_workflow_job(self._runpath_job) runpath_job.run(ert=self.ert, arguments=[], verbose=True) if runpath_job.hasFailed(): raise UserWarning(f"Failed to execute {self._runpath_job}") export_job.run( ert=self.ert, arguments=[ self.ert.getRunpathList().getExportFile(), parameters["output_file"], parameters["time_index"], parameters["column_keys"], ], verbose=True, ) if export_job.hasFailed(): raise UserWarning(f"Failed to execute {self._export_job}")
def get_responses(res: LibresFacade, ensemble_name: str): response_names = get_response_names(res) responses = [] active_realizations = res.get_active_realizations(ensemble_name) for real_id in active_realizations: for response_name in response_names: responses.append({"name": response_name, "real_id": real_id}) return responses
def api(self): config_file = 'snake_oil.ert' rc = ResConfig(user_config_file=config_file) rc.convertToCReference(None) ert = EnKFMain(rc) facade = LibresFacade(ert) api = PlotApi(facade) return api
def __init__(self, ert, config_file, parent): QMainWindow.__init__(self, parent) self._api = PlotApi(LibresFacade(ert)) self.setMinimumWidth(850) self.setMinimumHeight(650) self.setWindowTitle(f"Plotting - {config_file}") self.activateWindow() self._key_definitions = self._api.all_data_type_keys() self._plot_customizer = PlotCustomizer(self, self._key_definitions) self._plot_customizer.settingsChanged.connect(self.keySelected) self._central_tab = QTabWidget() central_widget = QWidget() central_layout = QVBoxLayout() central_layout.setContentsMargins(0, 0, 0, 0) central_widget.setLayout(central_layout) central_layout.addWidget(self._central_tab) self.setCentralWidget(central_widget) self._plot_widgets = [] """:type: list of PlotWidget""" self.addPlotWidget(ENSEMBLE, EnsemblePlot()) self.addPlotWidget(STATISTICS, StatisticsPlot()) self.addPlotWidget(HISTOGRAM, HistogramPlot()) self.addPlotWidget(GAUSSIAN_KDE, GaussianKDEPlot()) self.addPlotWidget(DISTRIBUTION, DistributionPlot()) self.addPlotWidget(CROSS_CASE_STATISTICS, CrossCaseStatisticsPlot()) self._central_tab.currentChanged.connect(self.currentPlotChanged) cases = self._api.get_all_cases_not_running() case_names = [case["name"] for case in cases if not case["hidden"]] self._data_type_keys_widget = DataTypeKeysWidget(self._key_definitions) self._data_type_keys_widget.dataTypeKeySelected.connect( self.keySelected) self.addDock("Data types", self._data_type_keys_widget) self._case_selection_widget = CaseSelectionWidget(case_names) self._case_selection_widget.caseSelectionChanged.connect( self.keySelected) self.addDock("Plot case", self._case_selection_widget) current_plot_widget = self._plot_widgets[ self._central_tab.currentIndex()] self._data_type_keys_widget.selectDefault() self._updateCustomizer(current_plot_widget)
def test_no_storage(monkeypatch, obs_key, expected_msg): shutil.rmtree("storage") res_config = ResConfig("snake_oil.ert") ert = EnKFMain(res_config) facade = LibresFacade(ert) with pytest.raises( loader.ResponseError, match=expected_msg, ): MeasuredData(facade, [obs_key])
def run(self, *args): facade = LibresFacade(self.ert()) obs_keys = [ facade.get_observation_key(nr) for nr, _ in enumerate(facade.get_observations()) ] measured_data = MeasuredData(facade, obs_keys) parser = spearman_job_parser() args = parser.parse_args(args) scaling_configs = spearman_job(measured_data, args.threshold) if not args.dry_run: try: CorrelatedObservationsScalingJob( self.ert()).run(scaling_configs) except EmptyDatasetException: pass
def initializeCurrentCaseFromExisting( source_case: str, target_case: str, source_report_step: int, parameters: List[str], members: List[str], ert: EnKFMain, ): if ( caseExists(source_case, LibresFacade(ert)) and ert.getEnkfFsManager().isCaseInitialized(source_case) and caseExists(target_case, LibresFacade(ert)) ): member_mask = [False] * ert.getEnsembleSize() for member in members: member_mask[int(member)] = True ert.getEnkfFsManager().customInitializeCurrentFromExistingCase( source_case, source_report_step, member_mask, parameters )
def init_facade() -> None: global _libres_facade global _ert global _config configfile = os.environ["ERT_STORAGE_RES_CONFIG"] _config = ResConfig(configfile) os.chdir(_config.config_path) _ert = EnKFMain(_config, strict=True) _libres_facade = LibresFacade(_ert)
def __init__(self, ert, notifier): self.ert = ert QWidget.__init__(self) self.setWindowTitle("Run analysis") self.activateWindow() self.analysis_module = AnalysisModuleEdit( LibresFacade(ert), help_link="config/analysis/analysis_module", ) self.target_case_text = QLineEdit() self.source_case_selector = CaseSelector(LibresFacade(self.ert), notifier, update_ert=False) layout = QFormLayout() layout.addRow("Analysis", self.analysis_module) layout.addRow("Target case", self.target_case_text) layout.addRow("Source case", self.source_case_selector) self.setLayout(layout)
def api(tmpdir, source_root): with tmpdir.as_cwd(): test_data_root = source_root / "test-data" / "local" test_data_dir = os.path.join(test_data_root, "snake_oil") shutil.copytree(test_data_dir, "test_data") os.chdir("test_data") config_file = "snake_oil.ert" rc = ResConfig(user_config_file=config_file) rc.convertToCReference(None) ert = EnKFMain(rc) facade = LibresFacade(ert) api = PlotApi(facade) yield api
def test_gen_obs_runtime(monkeypatch, copy_snake_oil, snapshot): obs_file = pathlib.Path.cwd() / "observations" / "observations.txt" with obs_file.open(mode="a") as fin: fin.write(create_general_observation()) res_config = ResConfig("snake_oil.ert") ert = EnKFMain(res_config) facade = LibresFacade(ert) df = MeasuredData(facade, [f"CUSTOM_DIFF_{restart}" for restart in range(500)]) snapshot.assert_match(df.data.to_csv(), "snake_oil_gendata_output.csv")