def createActiveList(self, fs): state_map = fs.getStateMap() ens_mask = BoolVector(False, self.ert().getEnsembleSize()) state_map.selectMatching(ens_mask, RealizationStateEnum.STATE_HAS_DATA) active_list = BoolVector.createActiveList(ens_mask) return active_list
def test_it(self): state_size = 10 with ErtTestContext("update", self.config_file) as tc: analysis = self.createAnalysisModule() ert = tc.getErt() obs = ert.getObservations() local_obsdata = obs.getAllActiveLocalObsdata() fs = ert.getEnkfFsManager().getCurrentFileSystem() mask = BoolVector(initial_size=ert.getEnsembleSize(), default_value=True) meas_data = MeasData(mask) obs_data = ObsData() obs.getObservationAndMeasureData(fs, local_obsdata, mask.createActiveList(), meas_data, obs_data) update(self.rng, mask, analysis, ert, meas_data, obs_data, state_size) mask[0] = False mask[4] = False meas_data = MeasData(mask) obs_data = ObsData() obs.getObservationAndMeasureData(fs, local_obsdata, mask.createActiveList(), meas_data, obs_data) update(self.rng, mask, analysis, ert, meas_data, obs_data, state_size)
def test_bool_vector(self): b = BoolVector() b.setDefault(True) b[4] = False self.assertEqual(list(b), [True, True, True, True, False])
def test_analysis_module(self): rng = RandomNumberGenerator( ) module = self.createAnalysisModule() ens_size = 12 obs_size = 1 state_size = 2 true_params = [1.25 , 0.75] true_state = forward_model( true_params ) obs = [(measure( true_state ) , 0.75)] A = Matrix( state_size , ens_size ) ens = [] for iens in range(ens_size): param = [ random.gauss( 1.00 , 1.00 ) , random.gauss(1.00 , 1.00)] ens.append( param ) mask = BoolVector(default_value = True , initial_size = ens_size) mask[2] = False (A , S , E , D , R , dObs) = init_matrices( ens , mask , obs , rng ) module.initUpdate( mask , S , R , dObs , E , D ) module.updateA( A , S , R , dObs , E , D ) mask[10] = False mask[5] = False (A , S , E , D , R , dObs) = init_matrices( ens , mask , obs , rng ) self.assertEqual( S.dims() , (obs_size , mask.countEqual( True ))) self.assertEqual( E.dims() , (obs_size , mask.countEqual( True ))) self.assertEqual( D.dims() , (obs_size , mask.countEqual( True ))) module.initUpdate( mask , S , R , dObs , E , D ) module.updateA( A , S , R , dObs , E , D )
def load(self, fs, report_step, input_mask=None): """ @type fs: EnkfFs @type report_step: int @type input_mask: BoolVector @rtype: PlotBlockData """ state_map = fs.getStateMap() ensemble_size = len(state_map) if not input_mask is None: mask = BoolVector.copy(input_mask) else: mask = BoolVector(False, ensemble_size) state_map.selectMatching(mask, RealizationStateEnum.STATE_HAS_DATA) depth = self.getDepthValues(report_step) self.__permutation_vector = depth.permutationSort() depth.permute(self.__permutation_vector) plot_block_data = PlotBlockData(depth) thread_pool = ThreadPool() for index in range(ensemble_size): if mask[index]: thread_pool.addTask(self.loadVector, plot_block_data, fs, report_step, index) thread_pool.nonBlockingStart() thread_pool.join() return plot_block_data
def createActiveList(ert, fs): state_map = fs.getStateMap() ens_mask = BoolVector(False, ert.getEnsembleSize()) state_map.selectMatching( ens_mask, RealizationStateEnum.STATE_INITIALIZED | RealizationStateEnum.STATE_HAS_DATA) active_list = BoolVector.createActiveList(ens_mask) return [iens for iens in active_list]
def realizationList(self, state_value): """ Will create a list of all realisations with state equal to state_value. @type state_value: RealizationStateEnum @rtype: ecl.util.IntVector """ mask = BoolVector(False, len(self)) self.selectMatching(mask, state_value) return BoolVector.createActiveList(mask)
def test_update_active_mask(self): vec = BoolVector(False, 10) self.assertTrue(vec.updateActiveMask("1-2,5")) self.assertTrue(vec[1]) self.assertTrue(vec[2]) self.assertTrue(vec[5]) self.assertFalse(vec[4]) vec = BoolVector(False, 10) self.assertTrue(vec.updateActiveMask("1-5,2,3")) self.assertTrue(vec[1]) self.assertTrue(vec[2]) self.assertTrue(vec[3]) self.assertTrue(vec[4]) self.assertTrue(vec[5]) self.assertFalse(vec[0]) self.assertFalse(vec[6]) vec = BoolVector(False, 10) self.assertTrue(vec.updateActiveMask("5,6,7,15")) self.assertTrue(vec[5]) self.assertTrue(vec[6]) self.assertTrue(vec[7]) self.assertFalse(vec[4]) self.assertFalse(vec[8]) self.assertEqual(len(vec), 16)
def getActiveRealizationsMask(self): count = getRealizationCount() mask = BoolVector(default_value=False, initial_size = count) if not mask.updateActiveMask(self.getValue()): raise ValueError('Error while parsing range string "%s"!' % self.getValue()) if len(mask) != count: raise ValueError("Mask size changed %d != %d!" % (count, len(mask))) return mask
def getActiveRealizationsMask(self): count = getRealizationCount() mask = BoolVector(default_value=False, initial_size=count) if not mask.updateActiveMask(self.getValue()): raise ValueError('Error while parsing range string "%s"!' % self.getValue()) if len(mask) != count: raise ValueError("Mask size changed %d != %d!" % (count, len(mask))) return mask
def test_activeMask(self): active_list = BoolVector.createActiveMask("1 , 4 - 7 , 10") self.assertTrue(len(active_list) == 11) self.assertTrue(active_list[1]) self.assertTrue(active_list[4]) self.assertTrue(active_list[10]) self.assertFalse(active_list[9]) self.assertFalse(active_list[8]) self.assertEqual(6, active_list.count(True)) active_list = BoolVector.createActiveMask("1,4-7,10X") self.assertFalse(active_list)
def test_update(self): key = "OBS" obs_size = 4 ens_size = 10 ens_mask = BoolVector( default_value = True , initial_size = ens_size ) block = MeasBlock( key , obs_size , ens_mask) with self.assertRaises(TypeError): block["String"] = 10 with self.assertRaises(TypeError): block[10] = 10 with self.assertRaises(IndexError): block[obs_size,0] = 10 with self.assertRaises(IndexError): block[0,ens_size] = 10 #----------------------------------------------------------------- with self.assertRaises(TypeError): a = block["String"] with self.assertRaises(TypeError): a = block[10] with self.assertRaises(IndexError): val = block[obs_size,0] with self.assertRaises(IndexError): val = block[0,ens_size] block[1,2] = 3 self.assertEqual( 3 , block[1,2] )
def test_load_results_manually(self): with ErtTestContext("manual_load_test", self.config_file) as test_context: ert = test_context.getErt() load_into_case = "A1" load_from_case = "default" load_into = ert.getEnkfFsManager().getFileSystem(load_into_case) load_from = ert.getEnkfFsManager().getFileSystem(load_from_case) ert.getEnkfFsManager().switchFileSystem(load_from) realisations = BoolVector(default_value=True, initial_size=25) realisations[7] = False iteration = 0 loaded = ert.loadFromForwardModel(realisations, iteration, load_into) load_into_case_state_map = load_into.getStateMap() load_into_states = [state for state in load_into_case_state_map] expected = [RealizationStateEnum.STATE_HAS_DATA] * 25 expected[7] = RealizationStateEnum.STATE_UNDEFINED self.assertListEqual(load_into_states, expected) self.assertEqual(24, loaded) self.assertEqual(25, len(expected)) self.assertEqual(25, len(realisations))
def testObs(self): with ErtTestContext("obs_test", self.config_file) as test_context: ert = test_context.getErt() obs = ert.getObservations() self.assertEqual(32, len(obs)) for v in obs: self.assertTrue(isinstance(v, ObsVector)) self.assertEqual(obs[-1].getKey(), 'RFT_TEST') self.assertEqual(obs[-1].getDataKey(), '4289383') self.assertEqual(obs[-1].getObsKey(), 'RFT_TEST') with self.assertRaises(IndexError): v = obs[-40] with self.assertRaises(IndexError): v = obs[40] with self.assertRaises(KeyError): v = obs["No-this-does-not-exist"] v1 = obs["WWCT:OP_3"] v2 = obs["GOPT:OP"] mask = BoolVector(True, ert.getEnsembleSize()) current_fs = ert.getEnkfFsManager().getCurrentFileSystem() self.assertTrue(v1.hasData(mask, current_fs)) self.assertFalse(v2.hasData(mask, current_fs)) local_node = v1.createLocalObs() for t in v1.getStepList(): self.assertTrue(local_node.tstepActive(t))
def test_without_gen_kw(self): case_directory = self.createTestPath('local/snake_oil_no_data/') with TestAreaContext('test_enkf_runpath', store_area=True) as work_area: work_area.copy_directory(case_directory) res_config = ResConfig('snake_oil_no_data/snake_oil_no_gen_kw.ert') main = EnKFMain(res_config) iactive = BoolVector(initial_size=main.getEnsembleSize(), default_value=False) iactive[0] = True fs = main.getEnkfFsManager().getCurrentFileSystem() run_context = main.getRunContextENSEMPLE_EXPERIMENT(fs, iactive) main.createRunpath(run_context) self.assertDirectoryExists( 'storage/snake_oil_no_gen_kw/runpath/realisation-0/iter-0') self.assertFileDoesNotExist( 'storage/snake_oil_no_gen_kw/runpath/realisation-0/iter-0/parameters.txt' ) self.assertEqual( len(os.listdir('storage/snake_oil_no_gen_kw/runpath')), 1) self.assertEqual( len( os.listdir( 'storage/snake_oil_no_gen_kw/runpath/realisation-0')), 1)
def createActiveList(self, fs): state_map = fs.getStateMap() ens_mask = BoolVector(False, self.ert().getEnsembleSize()) state_map.selectMatching(ens_mask, RealizationStateEnum.STATE_HAS_DATA) active_list = BoolVector.createActiveList(ens_mask) return active_list
def test_run_context(self): with TestAreaContext("enkf_test") as work_area: work_area.copy_directory(self.case_directory) res_config = ResConfig("simple_config/minimum_config") main = EnKFMain(res_config) fs_manager = main.getEnkfFsManager() fs = fs_manager.getCurrentFileSystem( ) iactive = BoolVector(initial_size = 10 , default_value = True) iactive[0] = False iactive[1] = False run_context = main.getRunContextENSEMPLE_EXPERIMENT( fs , iactive ) self.assertEqual( len(run_context) , 8 ) with self.assertRaises(IndexError): run_context[8] with self.assertRaises(TypeError): run_context["String"] run_arg = run_context[0] self.assertTrue( isinstance( run_arg , RunArg )) with self.assertRaises(ValueError): run_context.iensGet(0) with self.assertRaises(ValueError): run_context.iensGet(1) arg0 = run_context[0] arg2 = run_context.iensGet( 2 )
def test_simulated_custom_kw(self): config = self.createTestPath("local/custom_kw/mini_config") with ErtTestContext("python/enkf/data/custom_kw_simulated", config) as context: ert = context.getErt() ensemble_config = ert.ensembleConfig() self.assertTrue("AGGREGATED" in ensemble_config) config = ensemble_config.getNode( "AGGREGATED").getCustomKeywordModelConfig() self.assertEqual(len(config.getKeys()), 0) simulation_runner = EnkfSimulationRunner(ert) iteration_count = 0 active = BoolVector(default_value=True, initial_size=ert.getEnsembleSize()) simulation_runner.createRunPath(active, iteration_count) simulation_runner.runEnsembleExperiment() config = ensemble_config.getNode( "AGGREGATED").getCustomKeywordModelConfig() self.assertEqual(len(config.getKeys()), 4) self.assertItemsEqual( config.getKeys(), ["PERLIN_1", "PERLIN_2", "PERLIN_3", "STATE"])
def test_assert_symlink_deleted(self): with ErtTestContext("create_runpath2" , self.createTestPath("local/snake_oil_field/snake_oil.ert")) as tc: ert = tc.getErt( ) runpath_list = ert.getRunpathList( ) ens_size = ert.getEnsembleSize( ) runner = ert.getEnkfSimulationRunner( ) mask = BoolVector( initial_size = ens_size , default_value = True ) fs_manager = ert.getEnkfFsManager( ) init_fs = fs_manager.getFileSystem("init_fs") # create directory structure runpath_fmt = ert.getModelConfig().getRunpathFormat( ) subst_list = SubstitutionList( ) itr = 0 run_context = ErtRunContext( EnkfRunType.INIT_ONLY , init_fs, None , mask , runpath_fmt, subst_list , itr ) runner.createRunPath( run_context ) # replace field file with symlink linkpath = '%s/permx.grdcel' % str(runpath_list[0].runpath) targetpath = '%s/permx.grdcel.target' % str(runpath_list[0].runpath) open(targetpath, 'a').close() remove(linkpath) symlink(targetpath, linkpath) # recreate directory structure runner.createRunPath( run_context ) # ensure field symlink is replaced by file self.assertFalse( path.islink(linkpath) )
def update(self, line): arguments = splitArguments(line) if len(arguments) == 1: case_name = arguments[0] ert = self.ert() fs_manager = ert.getEnkfFsManager() ert.getEnkfSimulationRunner().runWorkflows(HookRuntime.PRE_UPDATE) es_update = ESUpdate(ert) target_fs = fs_manager.getFileSystem(case_name) source_fs = fs_manager.getCurrentFileSystem() model_config = ert.getModelConfig() runpath_fmt = model_config.getRunpathFormat() subst_list = ert.getDataKW() mask = BoolVector(default_value=True, initial_size=ert.getEnsembleSize()) run_context = ErtRunContext.ensemble_smoother( source_fs, target_fs, mask, runpath_fmt, subst_list, 0) success = es_update.smootherUpdate(run_context) if not success: self.lastCommandFailed("Unable to perform update") ert.getEnkfSimulationRunner().runWorkflows(HookRuntime.POST_UPDATE) else: self.lastCommandFailed( "Expected one argument: <target_fs> received: '%s'" % line)
def test_assert_export(self): with ErtTestContext( "create_runpath1", self.createTestPath( "local/snake_oil_no_data/snake_oil.ert")) as tc: ert = tc.getErt() runpath_list = ert.getRunpathList() self.assertFalse(path.isfile(runpath_list.getExportFile())) ens_size = ert.getEnsembleSize() runner = ert.getEnkfSimulationRunner() fs_manager = ert.getEnkfFsManager() init_fs = fs_manager.getFileSystem("init_fs") mask = BoolVector(initial_size=100, default_value=True) runpath_fmt = ert.getModelConfig().getRunpathFormat() subst_list = SubstitutionList() itr = 0 run_context1 = ErtRunContext(EnkfRunType.INIT_ONLY, init_fs, None, None, mask, runpath_fmt, subst_list, itr) runner.createRunPath(run_context1) self.assertTrue(path.isfile(runpath_list.getExportFile())) self.assertEqual("test_runpath_list.txt", path.basename(runpath_list.getExportFile()))
def test_create(self): with TestAreaContext("run_context"): arg = None sim_fs = EnkfFs.createFileSystem("sim_fs", EnKFFSType.BLOCK_FS_DRIVER_ID, arg) target_fs = None mask = BoolVector(initial_size=100, default_value=True) runpath_fmt = PathFormat("path/to/sim%d") subst_list = SubstitutionList() itr = 0 run_context1 = ErtRunContext(EnkfRunType.ENSEMBLE_EXPERIMENT, sim_fs, target_fs, mask, runpath_fmt, subst_list, itr) run_id1 = run_context1.get_id() run_arg0 = run_context1[0] self.assertEqual(run_id1, run_arg0.get_run_id()) run_context2 = ErtRunContext(EnkfRunType.ENSEMBLE_EXPERIMENT, sim_fs, target_fs, mask, runpath_fmt, subst_list, itr) run_id2 = run_context2.get_id() self.assertFalse(run_id1 == run_id2)
def load(self, args): arguments = splitArguments(args) if len(arguments) < 1: self.lastCommandFailed("Loading requires a realization mask.") return False realization_count = self.ert().getEnsembleSize() mask = BoolVector(False, realization_count) mask_success = mask.updateActiveMask(arguments[0]) if not mask_success: self.lastCommandFailed("The realization mask: '%s' is not valid." % arguments[0]) return False fs = self.ert().getEnkfFsManager().getCurrentFileSystem() self.ert().loadFromForwardModel(mask, 0, fs)
def runEnsembleExperiment(self, job_queue, active_realization_mask=None): """ @rtype: int """ if active_realization_mask is None: count = self.ert.getEnsembleSize() active_realization_mask = BoolVector(default_value=True, initial_size=count) iter_nr = 0 return self.runSimpleStep(job_queue, active_realization_mask, EnkfInitModeEnum.INIT_CONDITIONAL, iter_nr)
def filterObsKeys(self, obs_keys, fs): active_mask = BoolVector(True, self.ert().getEnsembleSize()) ert_obs = self.ert().getObservations() result = [] for obs_key in obs_keys: obsVector = ert_obs[obs_key] if obsVector.hasData(active_mask, fs): result.append(obs_key) return result
def initializeCurrentCaseFromExisting(source_case, target_case, source_report_step, parameters, members): if caseExists(source_case) and caseIsInitialized(source_case) and caseExists(target_case): total_member_count = getRealizationCount() member_mask = BoolVector.createFromList(total_member_count, members) selected_parameters = StringList(parameters) ERT.ert.getEnkfFsManager().customInitializeCurrentFromExistingCase(source_case, source_report_step, member_mask, selected_parameters) ERT.emitErtChange()
def test_inactive(self): key = "OBS" obs_size = 2 ens_size = 10 ens_mask = BoolVector( default_value = True , initial_size = ens_size ) ens_mask[5] = False block = MeasBlock( key , obs_size , ens_mask) self.assertFalse( block.iensActive( 5 )) with self.assertRaises(ValueError): block[0,5] = 10
def test_repr(self): primes = [2, 3, 5, 7, 11, 13, 17, 19] b = BoolVector() for i in primes: b[i] = True pfx = 'BoolVector(size = 20, content = "00110101000101000101")' print(repr(b)) self.assertEqual(pfx, repr(b)[:len(pfx)]) b[30] = True pfx = 'BoolVector(size = 31, content = "001101010...00000001")' print(repr(b)) self.assertEqual(pfx, repr(b)[:len(pfx)])
def test_assert_export(self): with ErtTestContext("create_runpath1" , self.createTestPath("local/snake_oil_no_data/snake_oil.ert")) as tc: ert = tc.getErt( ) runpath_list = ert.getRunpathList( ) self.assertFalse( path.isfile( runpath_list.getExportFile( ) )) ens_size = ert.getEnsembleSize( ) runner = ert.getEnkfSimulationRunner( ) mask = BoolVector( initial_size = ens_size , default_value = True ) runner.createRunPath( mask , 0 ) self.assertTrue( path.isfile( runpath_list.getExportFile( ) )) self.assertEqual( "test_runpath_list.txt" , path.basename( runpath_list.getExportFile( ) ))
def test_setitem_getitem(self): primes = [2, 3, 5, 7, 11, 13, 17, 19] primep = [i in primes for i in range(20)] b = BoolVector(initial_size=20) b[2] = True b[3:8:2] = True b[11::2] = True self.assertTrue(b[15]) self.assertTrue(b[-5]) self.assertTrue(b[17]) self.assertTrue(b[19]) b[-5] = False self.assertEqual(list(b), primep)
def calculatePrincipalComponent(self, fs, local_obsdata, truncation_or_ncomp=3): pc = Matrix(1, 1) pc_obs = Matrix(1, 1) singular_values = DoubleVector() state_map = fs.getStateMap() ens_mask = BoolVector(False, self.ert().getEnsembleSize()) state_map.selectMatching(ens_mask, RealizationStateEnum.STATE_HAS_DATA) active_list = ens_mask.createActiveList( ) if len(ens_mask) > 0: meas_data = MeasData(ens_mask) obs_data = ObsData() self.ert().getObservations().getObservationAndMeasureData(fs, local_obsdata, active_list, meas_data, obs_data) meas_data.deactivateZeroStdSamples(obs_data) active_size = len(obs_data) if active_size > 0: S = meas_data.createS() D_obs = obs_data.createDObs() truncation, ncomp = self.truncationOrNumberOfComponents(truncation_or_ncomp) obs_data.scale(S, D_obs=D_obs) EnkfLinalg.calculatePrincipalComponents(S, D_obs, truncation, ncomp, pc, pc_obs, singular_values) if self.__prior_singular_values is None: self.__prior_singular_values = singular_values else: for row in range(pc.rows()): factor = singular_values[row]/self.__prior_singular_values[row] pc.scaleRow( row , factor ) pc_obs.scaleRow( row , factor ) return PcaPlotData(local_obsdata.getName(), pc , pc_obs , singular_values) return None
def test_add_all(self): with ErtTestContext("add_all_runpath_dump", model_config=self.config_path) as ctx: res = ctx.getErt() fs_manager = res.getEnkfFsManager() sim_fs = fs_manager.getFileSystem("sim_fs") num_realizations = 25 mask = BoolVector(initial_size=num_realizations, default_value=True) mask[13] = False runpath_fmt = "simulations/<GEO_ID>/realisation-%d/iter-%d" jobname_fmt = "SNAKE_OIL_%d" itr = 0 subst_list = res.resConfig().subst_config.subst_list run_context = ErtRunContext.ensemble_experiment( sim_fs, mask, PathFormat(runpath_fmt), jobname_fmt, subst_list, itr) res.initRun(run_context) for i, run_arg in enumerate(run_context): if mask[i]: run_arg.geo_id = 10 * i res.createRunpath(run_context) for i, run_arg in enumerate(run_context): if not mask[i]: continue self.assertTrue( os.path.isdir("simulations/%d" % run_arg.geo_id)) runpath_list_path = ".ert_runpath_list" self.assertTrue(os.path.isfile(runpath_list_path)) exp_runpaths = [ runpath_fmt.replace("<GEO_ID>", str(run_arg.geo_id)) % (iens, itr) for iens, run_arg in enumerate(run_context) if mask[iens] ] exp_runpaths = map(os.path.realpath, exp_runpaths) with open(runpath_list_path, 'r') as f: dumped_runpaths = zip( *[line.split() for line in f.readlines()])[1] self.assertEqual(list(exp_runpaths), list(dumped_runpaths))