def test_fs_init_from_scratch_deprecated(): res_config = ResConfig("snake_oil.ert") ert = EnKFMain(res_config) sim_fs = ert.getEnkfFsManager().getFileSystem("new_case") # new case mask = BoolVector.createFromList(25, [0, 1, 2, 3, 4, 5]) run_context = ErtRunContext.case_init(sim_fs, mask) with pytest.warns(DeprecationWarning): ert.getEnkfFsManager().initializeFromScratch( StringList(["SNAKE_OIL_PARAM"]), run_context) assert len(ert.getEnkfFsManager().getStateMapForCase("new_case")) == 6
def test_fs_init_from_scratch(): res_config = ResConfig("snake_oil.ert") ert = EnKFMain(res_config) sim_fs = ert.getEnkfFsManager().getFileSystem("new_case") mask = [True] * 6 + [False] * 19 run_context = ErtRunContext.case_init(sim_fs, mask) ert.getEnkfFsManager().initializeFromScratch( StringList(["SNAKE_OIL_PARAM"]), run_context) assert len(ert.getEnkfFsManager().getStateMapForCase("new_case")) == 6
def initializeCurrentCaseFromScratch( parameters: List[str], members: List[str], ert: EnKFMain ): selected_parameters = StringList(parameters) mask = [False] * ert.getEnsembleSize() for member in members: member = int(member.strip()) mask[member] = True sim_fs = ert.getEnkfFsManager().getCurrentFileSystem() run_context = ErtRunContext.case_init(sim_fs, mask) ert.getEnkfFsManager().initializeFromScratch(selected_parameters, run_context)
def test_enkf_fs_manager_create(setup_case): # We are indirectly testing the create through the create # already in the enkf_main object. In principle we could # create a separate manager instance from the ground up, but # then the reference count will be weird. res_config = setup_case("local/snake_oil", "snake_oil.ert") ert = EnKFMain(res_config) fsm = ert.getEnkfFsManager() fsm.getCurrentFileSystem() assert fsm.isCaseMounted("default_0") assert fsm.caseExists("default_0") assert fsm.caseHasData("default_0") assert not fsm.isCaseRunning("default_0") assert fsm.getFileSystemCount() == 1 assert not fsm.isCaseMounted("newFS") assert not fsm.caseExists("newFS") assert not fsm.caseHasData("newFS") assert not fsm.isCaseRunning("newFS") fsm.getFileSystem("newFS") assert fsm.getFileSystemCount() == 2 assert fsm.isCaseMounted("newFS") assert fsm.caseExists("newFS") assert not fsm.caseHasData("newFS") assert not fsm.isCaseRunning("newFS")
def loadAllMisfitData(ert: EnKFMain, case_name) -> DataFrame: """ @type ert: EnKFMain @type case_name: str @rtype: DataFrame """ fs = ert.getEnkfFsManager().getFileSystem(case_name) realizations = MisfitCollector.createActiveList(ert, fs) misfit_keys = ert.getKeyManager().misfitKeys(sort_keys=False) misfit_sum_index = len(misfit_keys) - 1 misfit_array = numpy.empty(shape=(len(misfit_keys), len(realizations)), dtype=numpy.float64) misfit_array.fill(numpy.nan) misfit_array[misfit_sum_index] = 0.0 for column_index, obs_vector in enumerate(ert.getObservations()): for realization_index, realization_number in enumerate( realizations): misfit = obs_vector.getTotalChi2(fs, realization_number) misfit_array[column_index][realization_index] = misfit misfit_array[misfit_sum_index][realization_index] += misfit misfit_data = DataFrame(data=numpy.transpose(misfit_array), index=realizations, columns=misfit_keys) misfit_data.index.name = "Realization" return misfit_data
def loadGenData(ert: EnKFMain, case_name, key, report_step, realization_index=None): """@type ert: EnKFMain @type case_name: str @type key: str @type report_step: int @rtype: DataFrame In the returned dataframe the realisation index runs along the rows, and the gen_data element index runs vertically along the columns. """ fs = ert.getEnkfFsManager().getFileSystem(case_name) realizations = fs.realizationList(RealizationStateEnum.STATE_HAS_DATA) if realization_index: if realization_index not in realizations: raise IndexError(f"No such realization {realization_index}") realizations = IntVector.active_list(str(realization_index)) config_node = ert.ensembleConfig().getNode(key) config_node.getModelConfig() ensemble_data = EnsemblePlotGenData(config_node, fs, report_step) data_array = ensemble_data.getRealizations(realizations) realizations = numpy.array(realizations) return DataFrame(data=data_array, columns=realizations)
def test_with_enkf_fs(self): config_file = self.createTestPath("Statoil/config/with_data/config") with TestAreaContext("enkf/summary_key_set/enkf_fs", store_area=True) as context: context.copy_parent_content(config_file) fs = EnkfFs("storage/default") summary_key_set = fs.getSummaryKeySet() summary_key_set.addSummaryKey("FOPT") summary_key_set.addSummaryKey("WWCT") summary_key_set.addSummaryKey("WOPR") fs.umount() res_config = ResConfig("config") ert = EnKFMain(res_config) fs = ert.getEnkfFsManager().getCurrentFileSystem() summary_key_set = fs.getSummaryKeySet() self.assertTrue("FOPT" in summary_key_set) self.assertTrue("WWCT" in summary_key_set) self.assertTrue("WOPR" in summary_key_set) ensemble_config = ert.ensembleConfig() self.assertTrue("FOPT" in ensemble_config) self.assertTrue("WWCT" in ensemble_config) self.assertTrue("WOPR" in ensemble_config) self.assertFalse("TCPU" in ensemble_config)
def loadAllGenKwData(ert: EnKFMain, case_name, keys=None, realization_index=None): """ @type ert: EnKFMain @type case_name: str @type keys: list of str @rtype: DataFrame """ fs = ert.getEnkfFsManager().getFileSystem(case_name) realizations = GenKwCollector.createActiveList(ert, fs) if realization_index is not None: if realization_index not in realizations: raise IndexError(f"No such realization ({realization_index})") realizations = [realization_index] gen_kw_keys = ert.getKeyManager().genKwKeys() if keys is not None: gen_kw_keys = [ key for key in keys if key in gen_kw_keys ] # ignore keys that doesn't exist gen_kw_array = _lib.enkf_fs_keyword_data.keyword_data_get_realizations( ert.ensembleConfig(), fs, gen_kw_keys, realizations ) gen_kw_data = DataFrame( data=gen_kw_array, index=realizations, columns=gen_kw_keys ) gen_kw_data.index.name = "Realization" return gen_kw_data
def test_assert_symlink_deleted(setup_case): res_config = setup_case("local/snake_oil_field", "snake_oil.ert") ert = EnKFMain(res_config) runpath_list = ert.getRunpathList() runner = ert.getEnkfSimulationRunner() # create directory structure model_config = ert.getModelConfig() run_context = ErtRunContext.ensemble_experiment( ert.getEnkfFsManager().getCurrentFileSystem(), [True], model_config.getRunpathFormat(), model_config.getJobnameFormat(), ert.getDataKW(), 0, ) runner.createRunPath(run_context) # replace field file with symlink linkpath = "%s/permx.grdcel" % str(runpath_list[0].runpath) targetpath = "%s/permx.grdcel.target" % str(runpath_list[0].runpath) open(targetpath, "a").close() os.remove(linkpath) os.symlink(targetpath, linkpath) # recreate directory structure runner.createRunPath(run_context) # ensure field symlink is replaced by file assert not os.path.islink(linkpath)
def test_load_results_manually2(setup_case, caplog, monkeypatch, lazy_load): """ This little test does not depend on Equinor-data and only verifies the lazy_load flag in forward_load_context plus memory-logging """ if lazy_load: monkeypatch.setenv("ERT_LAZY_LOAD_SUMMARYDATA", str(lazy_load)) res_config = setup_case("local/snake_oil", "snake_oil.ert") ert = EnKFMain(res_config, strict=True) load_from = ert.getEnkfFsManager().getFileSystem("default_0") ert.getEnkfFsManager().switchFileSystem(load_from) realisations = [False] * 25 realisations[0] = True # only need one to test what we want with caplog.at_level(logging.INFO): loaded = ert.loadFromForwardModel(realisations, 0, load_from) assert 0 == loaded # they will in fact all fail, but that's ok assert f"lazy={lazy_load}".lower() in caplog.text
def test_localization(setup_case, expected_target_gen_kw): """ Note that this is now a snapshot test, so there is no guarantee that the snapshots are correct, they are just documenting the current behavior. """ res_config = setup_case("local/snake_oil", "snake_oil.ert") ert = EnKFMain(res_config) es_update = ESUpdate(ert) fsm = ert.getEnkfFsManager() sim_fs = fsm.getFileSystem("default_0") target_fs = fsm.getFileSystem("target") # perform localization localized_idxs = (1, 2) local_config = ert.getLocalConfig() local_config.clear() obs = local_config.createObsdata("OBSSET_LOCA") obs.addNode("WOPR_OP1_72") ministep = local_config.createMinistep("MINISTEP_LOCA") ministep.addActiveData("SNAKE_OIL_PARAM") # replace dataset.addNode() active_list = ministep.getActiveList("SNAKE_OIL_PARAM") for i in localized_idxs: active_list.addActiveIndex(i) ministep.attachObsset(obs) updatestep = local_config.getUpdatestep() updatestep.attachMinistep(ministep) # Run ensemble smoother mask = [True] * ert.getEnsembleSize() model_config = ert.getModelConfig() path_fmt = model_config.getRunpathFormat() jobname_fmt = model_config.getJobnameFormat() subst_list = None run_context = ErtRunContext.ensemble_smoother( sim_fs, target_fs, mask, path_fmt, jobname_fmt, subst_list, 0 ) es_update.smootherUpdate(run_context) conf = ert.ensembleConfig()["SNAKE_OIL_PARAM"] sim_node = EnkfNode(conf) target_node = EnkfNode(conf) node_id = NodeId(0, 0) sim_node.load(sim_fs, node_id) target_node.load(target_fs, node_id) sim_gen_kw = list(sim_node.asGenKw()) target_gen_kw = list(target_node.asGenKw()) # Test that the localized values has been updated assert sim_gen_kw[1:3] != target_gen_kw[1:3] # test that all the other values are left unchanged assert sim_gen_kw[3:] == target_gen_kw[3:] assert sim_gen_kw[0] == target_gen_kw[0] assert target_gen_kw == pytest.approx(expected_target_gen_kw)
def initializeCurrentCaseFromExisting( source_case: str, target_case: str, source_report_step: int, parameters: List[str], members: List[str], ert: EnKFMain, ): if ( caseExists(source_case, LibresFacade(ert)) and ert.getEnkfFsManager().isCaseInitialized(source_case) and caseExists(target_case, LibresFacade(ert)) ): member_mask = [False] * ert.getEnsembleSize() for member in members: member_mask[int(member)] = True ert.getEnkfFsManager().customInitializeCurrentFromExistingCase( source_case, source_report_step, member_mask, parameters )
def test_custom_init_runs(state_mask, expected_length): res_config = ResConfig("snake_oil.ert") ert = EnKFMain(res_config) new_fs = ert.getEnkfFsManager().getFileSystem("new_case") ert.getEnkfFsManager().switchFileSystem(new_fs) ert.getEnkfFsManager().customInitializeCurrentFromExistingCase( "default_0", 0, state_mask, StringList(["SNAKE_OIL_PARAM"])) assert len(ert.getEnkfFsManager().getStateMapForCase( "new_case")) == expected_length
def test_update_report(setup_case, snapshot): """ Note that this is now a snapshot test, so there is no guarantee that the snapshots are correct, they are just documenting the current behavior. """ res_config = setup_case("local/snake_oil", "snake_oil.ert") ert = EnKFMain(res_config) es_update = ESUpdate(ert) fsm = ert.getEnkfFsManager() sim_fs = fsm.getFileSystem("default_0") target_fs = fsm.getFileSystem("target") run_context = ErtRunContext.ensemble_smoother_update(sim_fs, target_fs) es_update.smootherUpdate(run_context) log_file = Path(ert.analysisConfig().get_log_path()) / "deprecated" snapshot.assert_match(log_file.read_text("utf-8"), "update_log")
def test_large_case(self): with open("config", "w") as fp: fp.write( """NUM_REALIZATIONS 10 GRID CASE.EGRID FIELD PORO PARAMETER poro.grdecl INIT_FILES:fields/poro%d.grdecl SUMMARY WBHP OBS_CONFIG observations.txt TIME_MAP timemap.txt """ ) for f in ["timemap.txt", "observations.txt"]: src_file = self.createTestPath(os.path.join("local/row_scaling", f)) shutil.copy(src_file, "./") # The grid size must be greater than 250000 (the default matrix size in # enkf_main_update()) grid = EclGridGenerator.create_rectangular((70, 70, 70), (1, 1, 1)) grid.save_EGRID("CASE.EGRID") res_config = ResConfig(user_config_file="config") main = EnKFMain(res_config) init_fs = init_data(main) # Configure the local updates local_config = main.getLocalConfig() local_config.clear() local_data = local_config.createDataset("LOCAL") local_data.addNode("PORO") obs = local_config.createObsdata("OBSSET_LOCAL") obs.addNode("WBHP0") ministep = local_config.createMinistep("MINISTEP_LOCAL") ministep.attachDataset(local_data) ministep.attachObsset(obs) updatestep = local_config.getUpdatestep() updatestep.attachMinistep(ministep) # Apply the row scaling row_scaling = local_data.row_scaling("PORO") ens_config = main.ensembleConfig() poro_config = ens_config["PORO"] field_config = poro_config.getFieldModelConfig() grid = main.eclConfig().getGrid() row_scaling.assign(field_config.get_data_size(), ScalingTest(grid)) es_update = ESUpdate(main) update_fs = main.getEnkfFsManager().getFileSystem("target2") run_context = ErtRunContext.ensemble_smoother_update(init_fs, update_fs) es_update.smootherUpdate(run_context)
def test_snapshot_alpha(setup_case, alpha, expected): """ Note that this is now a snapshot test, so there is no guarantee that the snapshots are correct, they are just documenting the current behavior. """ res_config = setup_case("local/snake_oil", "snake_oil.ert") obs_file = Path("observations") / "observations.txt" with obs_file.open(mode="w") as fin: fin.write( """ SUMMARY_OBSERVATION LOW_STD { VALUE = 10; ERROR = 0.1; DATE = 2015-06-23; KEY = FOPR; }; SUMMARY_OBSERVATION HIGH_STD { VALUE = 10; ERROR = 1.0; DATE = 2015-06-23; KEY = FOPR; }; SUMMARY_OBSERVATION EXTREMELY_HIGH_STD { VALUE = 10; ERROR = 10.0; DATE = 2015-06-23; KEY = FOPR; }; """ ) ert = EnKFMain(res_config) es_update = ESUpdate(ert) ert.analysisConfig().selectModule("IES_ENKF") fsm = ert.getEnkfFsManager() sim_fs = fsm.getFileSystem("default_0") target_fs = fsm.getFileSystem("target") run_context = ErtRunContext.ensemble_smoother_update(sim_fs, target_fs) ert.analysisConfig().setEnkfAlpha(alpha) es_update.smootherUpdate(run_context) result_snapshot = ert.update_snapshots[run_context.get_id()] assert result_snapshot.alpha == alpha assert result_snapshot.ministep_snapshots["ALL_ACTIVE"].obs_status == expected
def test_custom_init_runs(state_mask, expected_length): res_config = ResConfig("snake_oil.ert") ert = EnKFMain(res_config) new_fs = ert.getEnkfFsManager().getFileSystem("new_case") # new case ert.getEnkfFsManager().switchFileSystem(new_fs) index_list = [i for i, flag in enumerate(state_mask) if flag] bool_vector = BoolVector.createFromList(len(state_mask), index_list) ert.getEnkfFsManager().customInitializeCurrentFromExistingCase( "default_0", 0, bool_vector, StringList(["SNAKE_OIL_PARAM"])) assert len(ert.getEnkfFsManager().getStateMapForCase( "new_case")) == expected_length
def test_ecl_config_creation(self): with TestAreaContext("enkf_library_test") as work_area: work_area.copy_directory(self.case_directory) res_config = ResConfig("simple_config/minimum_config") main = EnKFMain(res_config) self.assertIsInstance(main.analysisConfig(), AnalysisConfig) self.assertIsInstance(main.eclConfig(), EclConfig) with self.assertRaises(AssertionError): # Null pointer! self.assertIsInstance(main.eclConfig().getRefcase(), EclSum) file_system = main.getEnkfFsManager().getCurrentFileSystem() self.assertEqual(file_system.getCaseName(), "default") time_map = file_system.getTimeMap() self.assertIsInstance(time_map, TimeMap)
def test_update(setup_case, module, expected_gen_kw): """ Note that this is now a snapshot test, so there is no guarantee that the snapshots are correct, they are just documenting the current behavior. """ res_config = setup_case("local/snake_oil", "snake_oil.ert") ert = EnKFMain(res_config) es_update = ESUpdate(ert) ert.analysisConfig().selectModule(module) fsm = ert.getEnkfFsManager() sim_fs = fsm.getFileSystem("default_0") target_fs = fsm.getFileSystem("target") run_context = ErtRunContext.ensemble_smoother_update(sim_fs, target_fs) es_update.smootherUpdate(run_context) conf = ert.ensembleConfig()["SNAKE_OIL_PARAM"] sim_node = EnkfNode(conf) target_node = EnkfNode(conf) node_id = NodeId(0, 0) sim_node.load(sim_fs, node_id) target_node.load(target_fs, node_id) sim_gen_kw = list(sim_node.asGenKw()) target_gen_kw = list(target_node.asGenKw()) assert sim_gen_kw != target_gen_kw assert sim_gen_kw == pytest.approx( [ -1.3035319087841115, 0.8222709205428339, -1.1400029486153482, 0.7477534046493867, -0.10400064074767973, -1.7223242794585338, 0.0761604027734105, 0.4039137216428462, 0.10001691562080614, 0.09549338450036506, ] ) assert target_gen_kw == pytest.approx(expected_gen_kw)
def test_validate_no_realizations(test_data_root): """ Ensamble has not run """ test_data_dir = os.path.join(test_data_root, "poly_normal") shutil.copytree(test_data_dir, "test_data") os.chdir(os.path.join("test_data")) res_config = ResConfig("poly.ert") ert = EnKFMain(res_config) observations = ert.getObservations() result = keys_with_data( observations, ["POLY_OBS"], ert.getEnsembleSize(), ert.getEnkfFsManager().getCurrentFileSystem(), ) assert result == []
def test_validate_failed_realizations(test_data_root): """ Config has several failed realisations """ test_data_dir = os.path.join(test_data_root, "failed_runs_in_storage") shutil.copytree(test_data_dir, "test_data") os.chdir(os.path.join("test_data")) res_config = ResConfig("mini_fail_config") ert = EnKFMain(res_config) observations = ert.getObservations() result = keys_with_data( observations, ["GEN_PERLIN_1"], ert.getEnsembleSize(), ert.getEnkfFsManager().getCurrentFileSystem(), ) assert result == ["GEN_PERLIN_1"]
def loadAllSummaryData(ert: EnKFMain, case_name, keys=None, realization_index=None): """ @type ert: EnKFMain @type case_name: str @type keys: list of str @rtype: DataFrame """ fs = ert.getEnkfFsManager().getFileSystem(case_name) time_map = fs.getTimeMap() dates = [ time_map[index].datetime() for index in range(1, len(time_map)) ] realizations = SummaryCollector.createActiveList(ert, fs) if realization_index is not None: if realization_index not in realizations: raise IndexError(f"No such realization {realization_index}") realizations = [realization_index] summary_keys = ert.getKeyManager().summaryKeys() if keys is not None: summary_keys = [key for key in keys if key in summary_keys ] # ignore keys that doesn't exist summary_data = _lib.enkf_fs_summary_data.get_summary_data( ert.ensembleConfig(), fs, summary_keys, realizations, len(dates)) multi_index = MultiIndex.from_product([realizations, dates], names=["Realization", "Date"]) df = DataFrame(data=summary_data, index=multi_index, columns=summary_keys) return df
def test_transfer_var(use_tmpdir): # Write a minimal config file with env with open("config_file.ert", "w") as fout: fout.write( dedent(""" NUM_REALIZATIONS 1 JOBNAME a_name_%d SETENV FIRST TheFirstValue SETENV SECOND TheSecondValue UPDATE_PATH THIRD TheThirdValue UPDATE_PATH FOURTH TheFourthValue """)) res_config = ResConfig("config_file.ert") ert = EnKFMain(res_config) fs_manager = ert.getEnkfFsManager() model_config = ert.getModelConfig() run_context = ErtRunContext.ensemble_experiment( fs_manager.getCurrentFileSystem(), [True], model_config.getRunpathFormat(), model_config.getJobnameFormat(), ert.getDataKW(), 0, ) ert.getEnkfSimulationRunner().createRunPath(run_context) os.chdir("simulations/realization0") with open("jobs.json", "r") as f: data = json.load(f) env_data = data["global_environment"] assert env_data["FIRST"] == "TheFirstValue" assert env_data["SECOND"] == "TheSecondValue" path_data = data["global_update_path"] assert "TheThirdValue" == path_data["THIRD"] assert "TheFourthValue" == path_data["FOURTH"]
def test_assert_export(use_tmpdir): # Write a minimal config file with env with open("config_file.ert", "w") as fout: fout.write( dedent( """ NUM_REALIZATIONS 1 JOBNAME a_name_%d RUNPATH_FILE directory/test_runpath_list.txt """ ) ) res_config = ResConfig("config_file.ert") ert = EnKFMain(res_config) runpath_list = ert.getRunpathList() assert not os.path.isfile(runpath_list.getExportFile()) fs_manager = ert.getEnkfFsManager() model_config = ert.getModelConfig() run_context = ErtRunContext.ensemble_experiment( fs_manager.getCurrentFileSystem(), [True], model_config.getRunpathFormat(), model_config.getJobnameFormat(), ert.getDataKW(), 0, ) ert.getEnkfSimulationRunner().createRunPath(run_context) assert os.path.isfile(runpath_list.getExportFile()) assert "test_runpath_list.txt" == os.path.basename(runpath_list.getExportFile()) assert ( Path(runpath_list.getExportFile()).read_text("utf-8") == f"000 {os.getcwd()}/simulations/realization0 a_name_0 000\n" )
class BatchSimulator(object): def __init__(self, res_config, controls, results, callback=None): """Will create simulator which can be used to run multiple simulations. The @res_config argument should be a ResConfig object, representing the fully configured state of libres. The @controls argument configures which parameters the simulator should get when actually simulating. The @controls argument should be a dictionary like this : controls = { "cmode": ["Well", "Group"], "order": "W" : ["01", "02", "03"] } In this example, the first group of controls "cmode" includes two controls, called "Well" and "Group". The second group of controls "order" has one control "W" with three suffixes. Note that: - Either no variable in a group has suffixes or all the variables in the group have suffixes. - Suffixes must be specified as non-empty collections of strings. - No duplicate groups/controls/suffixes are allowed When actually simulating, these values will be written to json files looking like this: cmode.json = {"Well": 1.0, "Group": 2.0} order.json = { "W": "01": 0.3, "02": 1.0, "03": 4.2 } When later invoking the start() method the simulator expects to get values for all parameters configured with the @controls argument, otherwise an exception will be raised. Internally in libres code the controls will be implemented as 'ext_param' instances. The @results argument is a list of keys of results which the simulator expects to be generated by the forward model. If argument @results looks like: results = ["CMODE", "order"] The simulator will look for the files 'CMODE_0' and 'order_0' in the simulation folder. If those files are not produced by the simulator an exception will be raised. The optional argument callback can be used to provide a callable which will be called as: callback(run_context) When the simulator has started. For the callable passed as callback you are encouraged to use the future proof signature: def callback(*args, **kwargs): .... """ if not isinstance(res_config, ResConfig): raise ValueError( "The first argument must be valid ResConfig instance") self.res_config = res_config self.ert = EnKFMain(self.res_config) self.control_keys = set(controls.keys()) self.result_keys = set(results) self.callback = callback ens_config = self.res_config.ensemble_config for control_name, variables in controls.items(): ens_config.addNode( EnkfConfigNode.create_ext_param(control_name, variables)) for key in results: ens_config.addNode( EnkfConfigNode.create_gen_data(key, "{}_%d".format(key))) def _setup_sim(self, sim_id, controls, file_system): def _set_ext_param(ext_param, key, assignment): if isinstance(assignment, dict): # handle suffixes suffixes = ext_param.config[key] if len(assignment) != len(suffixes): raise KeyError( "Key {} is missing values for these suffixes: {}". format( key, set(suffixes).difference(set(assignment.keys())))) for suffix, value in assignment.items(): ext_node[key, suffix] = value else: # assume assignment is a single numerical value ext_node[key] = assignment node_id = NodeId(0, sim_id) if set(controls.keys()) != self.control_keys: err_msg = "Mismatch between initialized and provided control names." raise KeyError(err_msg) for control_name, control in controls.items(): ens_config = self.res_config.ensemble_config node = EnkfNode(ens_config[control_name]) ext_node = node.as_ext_param() if len(ext_node) != len(control.keys()): raise KeyError(("Expected {} variables for control {}, " "received {}.").format(len(ext_node), control_name, len(control.keys()))) for var_name, var_setting in control.items(): _set_ext_param(ext_node, var_name, var_setting) node.save(file_system, node_id) def start(self, case_name, case_data): """Start batch simulation, return a simulation context The start method will submit simulations to the queue system and then return a BatchContext handle which can be used to query for simulation status and results. The @case_name argument should just be string which will be used as name for the storage of these simulations in the system. The @controls argument is the set of control values, and the corresponding ID of the external realisation used for the simulations. The @control argument must match the control argument used when the simulator was instantiated. Assuming the following @control argument was passed to simulator construction: controls = { "cmode": ["Well", "Group"], "order": "W" : ["01", "02", "03"] } Then the following @case_data argument can be used in the start method to simulate four simulations: [ (1, { "cmode": {"Well": 2, "Group": 2}, "order": { "W": "01": 2, "02": 2, "03": 5}, }), (1, { "cmode": {"Well": 1, "Group": 3}, "order": {"W": ...}, }), (1, { "cmode": {"Well": 1, "Group": 7}, "order": {"W": ...}, }), (2, { "cmode": {"Well": 1, "Group": -1}, "order": {"W": ...}, }), ] The first integer argument in the tuple is the realisation id, so this simulation batch will consist of a total of four simulations, where the first three are based on realisation 1, and the last is based on realisation 2. Observe that only one BatchSimulator should actually be running at a time, so when you have called the 'start' method you need to let that batch complete before you start a new batch. """ self.ert.addDataKW("<CASE_NAME>", _slug(case_name)) file_system = self.ert.getEnkfFsManager().getFileSystem(case_name) for sim_id, (geo_id, controls) in enumerate(case_data): assert isinstance(geo_id, int) self._setup_sim(sim_id, controls, file_system) # The input should be validated before we instantiate the BatchContext # object, at that stage a job_queue object with multiple threads is # started, and things will typically be in a quite sorry state if an # exception occurs. itr = 0 mask = BoolVector(default_value=True, initial_size=len(case_data)) sim_context = BatchContext(self.result_keys, self.ert, file_system, mask, itr, case_data) if self.callback: self.callback(sim_context) return sim_context
#!/usr/bin/env python import sys import time from res.enkf import EnKFMain, RunArg, NodeId, ResConfig from res.enkf.data import EnkfNode from ert.job_queue import JobQueueManager res_config = ResConfig( sys.argv[1] ) ert = EnKFMain( res_config ) fs_manager = ert.getEnkfFsManager( ) fs = fs_manager.getCurrentFileSystem( ) # Initialize the realisations. for iens in range( ert.getEnsembleSize()): realisation = ert.getRealisation( iens ) realisation.initialize( fs ) # Fetch out the job_queue from the SiteConfig object. In addition we # create a JobQueueManager objects which wraps the queue. The purpose # of this manager object is to let the queue run nonblocking in the # background. site_config = ert.siteConfig( ) queue_manager = JobQueueManager( site_config.getJobQueue( ) ) queue_manager.startQueue( ert.getEnsembleSize( ) , verbose = False ) # Create list of RunArg instances which hold metadata for one running # realisation, create the directory where the simulation should run # and submit the simulation.
#!/usr/bin/env python import sys import time from res.enkf import EnKFMain, RunArg, NodeId, ResConfig from res.enkf.data import EnkfNode from ert.job_queue import JobQueueManager res_config = ResConfig(sys.argv[1]) ert = EnKFMain(res_config) fs_manager = ert.getEnkfFsManager() fs = fs_manager.getCurrentFileSystem() # Initialize the realisations. for iens in range(ert.getEnsembleSize()): realisation = ert.getRealisation(iens) realisation.initialize(fs) # Fetch out the job_queue from the SiteConfig object. In addition we # create a JobQueueManager objects which wraps the queue. The purpose # of this manager object is to let the queue run nonblocking in the # background. site_config = ert.siteConfig() queue_manager = JobQueueManager(site_config.getJobQueue()) queue_manager.startQueue(ert.getEnsembleSize(), verbose=False) # Create list of RunArg instances which hold metadata for one running # realisation, create the directory where the simulation should run # and submit the simulation. path_fmt = "/tmp/run%d" arg_list = [ RunArg.createEnsembleExperimentRunArg(fs, iens, path_fmt % iens)
class BatchSimulator(object): def __init__(self, res_config, controls, results): """Will create simulator which can be used to run multiple simulations. The @res_config argument should be a ResConfig object, representing the fully configured state of libres. The @controls argument configures which parameters the simulator should get when actually simulating. The @controls argument should be a dictionary like this: controls = {"cmode": ["Well","Group"], "order" : ["W1", "W2", "W3"]} In this example the simulator will expect two arrays 'cmode' and 'order', consisting of two and three elements respectively. When actually simualating these values will be written to json files looking like: cmode.json = {"Well" : 1.0, "Group" : 2.0} order.json = {"W1" : 1, "W2" : 1.0, "W3": 1.0} When later invoking the start() method the simulator expects to get values for all parameters configured with the @controls argument, otherwise an exception will be raised. Internally in libres code the controls will be implemented as 'ext_param' instances. The @results argument is a list of keys of results which the simulator expects to be generated by the forward model. If argument @results looks like: results = ["CMODE", "order"] The simulator will look for the files 'CMODE_0' and 'order_0' in the simulation folder. If those files are not produced by the simulator an exception will be raised. """ if not isinstance(res_config, ResConfig): raise ValueError( "The first argument must be valid ResConfig instance") self.res_config = res_config self.ert = EnKFMain(self.res_config) self.control_keys = [] self.result_keys = [] ens_config = self.res_config.ensemble_config for key in controls.keys(): ens_config.addNode( EnkfConfigNode.create_ext_param(key, controls[key])) self.control_keys.append(key) for key in results: ens_config.addNode( EnkfConfigNode.create_gen_data(key, "{}_%d".format(key))) self.result_keys.append(key) def start(self, case_name, controls): """Will start batch simulation, returning a handle to query status and results. The start method will submit simulations to the queue system and then return a RobustContext handle which can be used to query for simulation status and results. The @case_name argument should just be string which will be used as name for the storage of these simulations in the system. The @controls argument is the set of control values, and the corresponding ID of the external realisation used for the simulations. The @control argument must match the control argument used when the simulator was instantiated. Assuming the following @control argument was passed to simulator construction: controls = {"cmode": ["Well","Group"], "order" : ["W1", "W2", "W3"]} Then the following @controls argument can be used in the start method to simulate four simulations: [ (1, {"cmode" : [1 ,2], "order" : [2,2,5]}), (1, {"cmode" : [1, 3], "order" : [2,2,7]}), (1, {"cmode" : [1, 7], "order" : [2,0,5]}), (2, {"cmode" : [1,-1], "order" : [2,2,1]})] The first integer argument in the tuple is the realisation id, so this simulation batch will consist of a total of four simulations, where the three first are based on realisation 1, and the last is based on realisation 2. Observe that only one BatchSimulator should actually be running at a time, so when you have called the 'start' method you need to let that batch complete before you start a new batch. """ ens_config = self.res_config.ensemble_config fsm = self.ert.getEnkfFsManager() fs = fsm.getFileSystem(case_name) for sim_id, (geo_id, control_dict) in enumerate(controls): assert isinstance(geo_id, int) node_id = NodeId(0, sim_id) if len(control_dict) != len(self.control_keys): raise ValueError("Not all keys supplied in controls") for key in control_dict.keys(): config_node = ens_config[key] ext_config = config_node.getModelConfig() values = control_dict[key] if not len(values) == len(ext_config): raise ValueError("Wrong number of values for:%s" % key) node = EnkfNode(config_node) ext_node = node.as_ext_param() ext_node.set_vector(values) node.save(fs, node_id) # The input should be validated before we instantiate the BatchContext # object, at that stage a job_queue object with multiple threads is # started, and things will typically be in a quite sorry state if an # exception occurs. itr = 0 mask = BoolVector(default_value=True, initial_size=len(controls)) sim_context = BatchContext(self.result_keys, self.ert, fs, mask, itr) for sim_id, (geo_id, control_dict) in enumerate(controls): sim_context.addSimulation(sim_id, geo_id) return sim_context