def createActiveList(self, fs):
state_map = fs.getStateMap()
ens_mask = BoolVector(False, self.ert().getEnsembleSize())
state_map.selectMatching(ens_mask, RealizationStateEnum.STATE_HAS_DATA)
active_list = BoolVector.createActiveList(ens_mask)
return active_list
def test_it(self):
state_size = 10
with ErtTestContext("update", self.config_file) as tc:
analysis = self.createAnalysisModule()
ert = tc.getErt()
obs = ert.getObservations()
local_obsdata = obs.getAllActiveLocalObsdata()
fs = ert.getEnkfFsManager().getCurrentFileSystem()
mask = BoolVector(initial_size=ert.getEnsembleSize(),
default_value=True)
meas_data = MeasData(mask)
obs_data = ObsData()
obs.getObservationAndMeasureData(fs, local_obsdata,
mask.createActiveList(),
meas_data, obs_data)
update(self.rng, mask, analysis, ert, meas_data, obs_data,
state_size)
mask[0] = False
mask[4] = False
meas_data = MeasData(mask)
obs_data = ObsData()
obs.getObservationAndMeasureData(fs, local_obsdata,
mask.createActiveList(),
meas_data, obs_data)
update(self.rng, mask, analysis, ert, meas_data, obs_data,
state_size)
def test_bool_vector(self):
b = BoolVector()
b.setDefault(True)
b[4] = False
self.assertEqual(list(b), [True, True, True, True, False])
def test_analysis_module(self):
rng = RandomNumberGenerator( )
module = self.createAnalysisModule()
ens_size = 12
obs_size = 1
state_size = 2
true_params = [1.25 , 0.75]
true_state = forward_model( true_params )
obs = [(measure( true_state ) , 0.75)]
A = Matrix( state_size , ens_size )
ens = []
for iens in range(ens_size):
param = [ random.gauss( 1.00 , 1.00 ) , random.gauss(1.00 , 1.00)]
ens.append( param )
mask = BoolVector(default_value = True , initial_size = ens_size)
mask[2] = False
(A , S , E , D , R , dObs) = init_matrices( ens , mask , obs , rng )
module.initUpdate( mask , S , R , dObs , E , D )
module.updateA( A , S , R , dObs , E , D )
mask[10] = False
mask[5] = False
(A , S , E , D , R , dObs) = init_matrices( ens , mask , obs , rng )
self.assertEqual( S.dims() , (obs_size , mask.countEqual( True )))
self.assertEqual( E.dims() , (obs_size , mask.countEqual( True )))
self.assertEqual( D.dims() , (obs_size , mask.countEqual( True )))
module.initUpdate( mask , S , R , dObs , E , D )
module.updateA( A , S , R , dObs , E , D )
def load(self, fs, report_step, input_mask=None):
"""
@type fs: EnkfFs
@type report_step: int
@type input_mask: BoolVector
@rtype: PlotBlockData
"""
state_map = fs.getStateMap()
ensemble_size = len(state_map)
if not input_mask is None:
mask = BoolVector.copy(input_mask)
else:
mask = BoolVector(False, ensemble_size)
state_map.selectMatching(mask, RealizationStateEnum.STATE_HAS_DATA)
depth = self.getDepthValues(report_step)
self.__permutation_vector = depth.permutationSort()
depth.permute(self.__permutation_vector)
plot_block_data = PlotBlockData(depth)
thread_pool = ThreadPool()
for index in range(ensemble_size):
if mask[index]:
thread_pool.addTask(self.loadVector, plot_block_data, fs,
report_step, index)
thread_pool.nonBlockingStart()
thread_pool.join()
return plot_block_data
def createActiveList(ert, fs):
state_map = fs.getStateMap()
ens_mask = BoolVector(False, ert.getEnsembleSize())
state_map.selectMatching(
ens_mask, RealizationStateEnum.STATE_INITIALIZED
| RealizationStateEnum.STATE_HAS_DATA)
active_list = BoolVector.createActiveList(ens_mask)
return [iens for iens in active_list]
def realizationList(self, state_value):
"""
Will create a list of all realisations with state equal to state_value.
@type state_value: RealizationStateEnum
@rtype: ecl.util.IntVector
"""
mask = BoolVector(False, len(self))
self.selectMatching(mask, state_value)
return BoolVector.createActiveList(mask)
def test_update_active_mask(self):
vec = BoolVector(False, 10)
self.assertTrue(vec.updateActiveMask("1-2,5"))
self.assertTrue(vec[1])
self.assertTrue(vec[2])
self.assertTrue(vec[5])
self.assertFalse(vec[4])
vec = BoolVector(False, 10)
self.assertTrue(vec.updateActiveMask("1-5,2,3"))
self.assertTrue(vec[1])
self.assertTrue(vec[2])
self.assertTrue(vec[3])
self.assertTrue(vec[4])
self.assertTrue(vec[5])
self.assertFalse(vec[0])
self.assertFalse(vec[6])
vec = BoolVector(False, 10)
self.assertTrue(vec.updateActiveMask("5,6,7,15"))
self.assertTrue(vec[5])
self.assertTrue(vec[6])
self.assertTrue(vec[7])
self.assertFalse(vec[4])
self.assertFalse(vec[8])
self.assertEqual(len(vec), 16)
def getActiveRealizationsMask(self):
count = getRealizationCount()
mask = BoolVector(default_value=False, initial_size = count)
if not mask.updateActiveMask(self.getValue()):
raise ValueError('Error while parsing range string "%s"!' % self.getValue())
if len(mask) != count:
raise ValueError("Mask size changed %d != %d!" % (count, len(mask)))
return mask
def getActiveRealizationsMask(self):
count = getRealizationCount()
mask = BoolVector(default_value=False, initial_size=count)
if not mask.updateActiveMask(self.getValue()):
raise ValueError('Error while parsing range string "%s"!' %
self.getValue())
if len(mask) != count:
raise ValueError("Mask size changed %d != %d!" %
(count, len(mask)))
return mask
def test_activeMask(self):
active_list = BoolVector.createActiveMask("1 , 4 - 7 , 10")
self.assertTrue(len(active_list) == 11)
self.assertTrue(active_list[1])
self.assertTrue(active_list[4])
self.assertTrue(active_list[10])
self.assertFalse(active_list[9])
self.assertFalse(active_list[8])
self.assertEqual(6, active_list.count(True))
active_list = BoolVector.createActiveMask("1,4-7,10X")
self.assertFalse(active_list)
def test_update(self):
key = "OBS"
obs_size = 4
ens_size = 10
ens_mask = BoolVector( default_value = True , initial_size = ens_size )
block = MeasBlock( key , obs_size , ens_mask)
with self.assertRaises(TypeError):
block["String"] = 10
with self.assertRaises(TypeError):
block[10] = 10
with self.assertRaises(IndexError):
block[obs_size,0] = 10
with self.assertRaises(IndexError):
block[0,ens_size] = 10
#-----------------------------------------------------------------
with self.assertRaises(TypeError):
a = block["String"]
with self.assertRaises(TypeError):
a = block[10]
with self.assertRaises(IndexError):
val = block[obs_size,0]
with self.assertRaises(IndexError):
val = block[0,ens_size]
block[1,2] = 3
self.assertEqual( 3 , block[1,2] )
def test_load_results_manually(self):
with ErtTestContext("manual_load_test",
self.config_file) as test_context:
ert = test_context.getErt()
load_into_case = "A1"
load_from_case = "default"
load_into = ert.getEnkfFsManager().getFileSystem(load_into_case)
load_from = ert.getEnkfFsManager().getFileSystem(load_from_case)
ert.getEnkfFsManager().switchFileSystem(load_from)
realisations = BoolVector(default_value=True, initial_size=25)
realisations[7] = False
iteration = 0
loaded = ert.loadFromForwardModel(realisations, iteration,
load_into)
load_into_case_state_map = load_into.getStateMap()
load_into_states = [state for state in load_into_case_state_map]
expected = [RealizationStateEnum.STATE_HAS_DATA] * 25
expected[7] = RealizationStateEnum.STATE_UNDEFINED
self.assertListEqual(load_into_states, expected)
self.assertEqual(24, loaded)
self.assertEqual(25, len(expected))
self.assertEqual(25, len(realisations))
def testObs(self):
with ErtTestContext("obs_test", self.config_file) as test_context:
ert = test_context.getErt()
obs = ert.getObservations()
self.assertEqual(32, len(obs))
for v in obs:
self.assertTrue(isinstance(v, ObsVector))
self.assertEqual(obs[-1].getKey(), 'RFT_TEST')
self.assertEqual(obs[-1].getDataKey(), '4289383')
self.assertEqual(obs[-1].getObsKey(), 'RFT_TEST')
with self.assertRaises(IndexError):
v = obs[-40]
with self.assertRaises(IndexError):
v = obs[40]
with self.assertRaises(KeyError):
v = obs["No-this-does-not-exist"]
v1 = obs["WWCT:OP_3"]
v2 = obs["GOPT:OP"]
mask = BoolVector(True, ert.getEnsembleSize())
current_fs = ert.getEnkfFsManager().getCurrentFileSystem()
self.assertTrue(v1.hasData(mask, current_fs))
self.assertFalse(v2.hasData(mask, current_fs))
local_node = v1.createLocalObs()
for t in v1.getStepList():
self.assertTrue(local_node.tstepActive(t))
def test_without_gen_kw(self):
case_directory = self.createTestPath('local/snake_oil_no_data/')
with TestAreaContext('test_enkf_runpath',
store_area=True) as work_area:
work_area.copy_directory(case_directory)
res_config = ResConfig('snake_oil_no_data/snake_oil_no_gen_kw.ert')
main = EnKFMain(res_config)
iactive = BoolVector(initial_size=main.getEnsembleSize(),
default_value=False)
iactive[0] = True
fs = main.getEnkfFsManager().getCurrentFileSystem()
run_context = main.getRunContextENSEMPLE_EXPERIMENT(fs, iactive)
main.createRunpath(run_context)
self.assertDirectoryExists(
'storage/snake_oil_no_gen_kw/runpath/realisation-0/iter-0')
self.assertFileDoesNotExist(
'storage/snake_oil_no_gen_kw/runpath/realisation-0/iter-0/parameters.txt'
)
self.assertEqual(
len(os.listdir('storage/snake_oil_no_gen_kw/runpath')), 1)
self.assertEqual(
len(
os.listdir(
'storage/snake_oil_no_gen_kw/runpath/realisation-0')),
1)
def createActiveList(self, fs):
state_map = fs.getStateMap()
ens_mask = BoolVector(False, self.ert().getEnsembleSize())
state_map.selectMatching(ens_mask, RealizationStateEnum.STATE_HAS_DATA)
active_list = BoolVector.createActiveList(ens_mask)
return active_list
def test_run_context(self):
with TestAreaContext("enkf_test") as work_area:
work_area.copy_directory(self.case_directory)
res_config = ResConfig("simple_config/minimum_config")
main = EnKFMain(res_config)
fs_manager = main.getEnkfFsManager()
fs = fs_manager.getCurrentFileSystem( )
iactive = BoolVector(initial_size = 10 , default_value = True)
iactive[0] = False
iactive[1] = False
run_context = main.getRunContextENSEMPLE_EXPERIMENT( fs , iactive )
self.assertEqual( len(run_context) , 8 )
with self.assertRaises(IndexError):
run_context[8]
with self.assertRaises(TypeError):
run_context["String"]
run_arg = run_context[0]
self.assertTrue( isinstance( run_arg , RunArg ))
with self.assertRaises(ValueError):
run_context.iensGet(0)
with self.assertRaises(ValueError):
run_context.iensGet(1)
arg0 = run_context[0]
arg2 = run_context.iensGet( 2 )
def test_simulated_custom_kw(self):
config = self.createTestPath("local/custom_kw/mini_config")
with ErtTestContext("python/enkf/data/custom_kw_simulated",
config) as context:
ert = context.getErt()
ensemble_config = ert.ensembleConfig()
self.assertTrue("AGGREGATED" in ensemble_config)
config = ensemble_config.getNode(
"AGGREGATED").getCustomKeywordModelConfig()
self.assertEqual(len(config.getKeys()), 0)
simulation_runner = EnkfSimulationRunner(ert)
iteration_count = 0
active = BoolVector(default_value=True,
initial_size=ert.getEnsembleSize())
simulation_runner.createRunPath(active, iteration_count)
simulation_runner.runEnsembleExperiment()
config = ensemble_config.getNode(
"AGGREGATED").getCustomKeywordModelConfig()
self.assertEqual(len(config.getKeys()), 4)
self.assertItemsEqual(
config.getKeys(),
["PERLIN_1", "PERLIN_2", "PERLIN_3", "STATE"])
def test_assert_symlink_deleted(self):
with ErtTestContext("create_runpath2" , self.createTestPath("local/snake_oil_field/snake_oil.ert")) as tc:
ert = tc.getErt( )
runpath_list = ert.getRunpathList( )
ens_size = ert.getEnsembleSize( )
runner = ert.getEnkfSimulationRunner( )
mask = BoolVector( initial_size = ens_size , default_value = True )
fs_manager = ert.getEnkfFsManager( )
init_fs = fs_manager.getFileSystem("init_fs")
# create directory structure
runpath_fmt = ert.getModelConfig().getRunpathFormat( )
subst_list = SubstitutionList( )
itr = 0
run_context = ErtRunContext( EnkfRunType.INIT_ONLY , init_fs, None , mask , runpath_fmt, subst_list , itr )
runner.createRunPath( run_context )
# replace field file with symlink
linkpath = '%s/permx.grdcel' % str(runpath_list[0].runpath)
targetpath = '%s/permx.grdcel.target' % str(runpath_list[0].runpath)
open(targetpath, 'a').close()
remove(linkpath)
symlink(targetpath, linkpath)
# recreate directory structure
runner.createRunPath( run_context )
# ensure field symlink is replaced by file
self.assertFalse( path.islink(linkpath) )
def update(self, line):
arguments = splitArguments(line)
if len(arguments) == 1:
case_name = arguments[0]
ert = self.ert()
fs_manager = ert.getEnkfFsManager()
ert.getEnkfSimulationRunner().runWorkflows(HookRuntime.PRE_UPDATE)
es_update = ESUpdate(ert)
target_fs = fs_manager.getFileSystem(case_name)
source_fs = fs_manager.getCurrentFileSystem()
model_config = ert.getModelConfig()
runpath_fmt = model_config.getRunpathFormat()
subst_list = ert.getDataKW()
mask = BoolVector(default_value=True,
initial_size=ert.getEnsembleSize())
run_context = ErtRunContext.ensemble_smoother(
source_fs, target_fs, mask, runpath_fmt, subst_list, 0)
success = es_update.smootherUpdate(run_context)
if not success:
self.lastCommandFailed("Unable to perform update")
ert.getEnkfSimulationRunner().runWorkflows(HookRuntime.POST_UPDATE)
else:
self.lastCommandFailed(
"Expected one argument: <target_fs> received: '%s'" % line)
def test_assert_export(self):
with ErtTestContext(
"create_runpath1",
self.createTestPath(
"local/snake_oil_no_data/snake_oil.ert")) as tc:
ert = tc.getErt()
runpath_list = ert.getRunpathList()
self.assertFalse(path.isfile(runpath_list.getExportFile()))
ens_size = ert.getEnsembleSize()
runner = ert.getEnkfSimulationRunner()
fs_manager = ert.getEnkfFsManager()
init_fs = fs_manager.getFileSystem("init_fs")
mask = BoolVector(initial_size=100, default_value=True)
runpath_fmt = ert.getModelConfig().getRunpathFormat()
subst_list = SubstitutionList()
itr = 0
run_context1 = ErtRunContext(EnkfRunType.INIT_ONLY, init_fs, None,
None, mask, runpath_fmt, subst_list,
itr)
runner.createRunPath(run_context1)
self.assertTrue(path.isfile(runpath_list.getExportFile()))
self.assertEqual("test_runpath_list.txt",
path.basename(runpath_list.getExportFile()))
def test_create(self):
with TestAreaContext("run_context"):
arg = None
sim_fs = EnkfFs.createFileSystem("sim_fs",
EnKFFSType.BLOCK_FS_DRIVER_ID,
arg)
target_fs = None
mask = BoolVector(initial_size=100, default_value=True)
runpath_fmt = PathFormat("path/to/sim%d")
subst_list = SubstitutionList()
itr = 0
run_context1 = ErtRunContext(EnkfRunType.ENSEMBLE_EXPERIMENT,
sim_fs, target_fs, mask, runpath_fmt,
subst_list, itr)
run_id1 = run_context1.get_id()
run_arg0 = run_context1[0]
self.assertEqual(run_id1, run_arg0.get_run_id())
run_context2 = ErtRunContext(EnkfRunType.ENSEMBLE_EXPERIMENT,
sim_fs, target_fs, mask, runpath_fmt,
subst_list, itr)
run_id2 = run_context2.get_id()
self.assertFalse(run_id1 == run_id2)
def load(self, args):
arguments = splitArguments(args)
if len(arguments) < 1:
self.lastCommandFailed("Loading requires a realization mask.")
return False
realization_count = self.ert().getEnsembleSize()
mask = BoolVector(False, realization_count)
mask_success = mask.updateActiveMask(arguments[0])
if not mask_success:
self.lastCommandFailed("The realization mask: '%s' is not valid." % arguments[0])
return False
fs = self.ert().getEnkfFsManager().getCurrentFileSystem()
self.ert().loadFromForwardModel(mask, 0, fs)
def runEnsembleExperiment(self, job_queue, active_realization_mask=None):
""" @rtype: int """
if active_realization_mask is None:
count = self.ert.getEnsembleSize()
active_realization_mask = BoolVector(default_value=True,
initial_size=count)
iter_nr = 0
return self.runSimpleStep(job_queue, active_realization_mask,
EnkfInitModeEnum.INIT_CONDITIONAL, iter_nr)
def filterObsKeys(self, obs_keys, fs):
active_mask = BoolVector(True, self.ert().getEnsembleSize())
ert_obs = self.ert().getObservations()
result = []
for obs_key in obs_keys:
obsVector = ert_obs[obs_key]
if obsVector.hasData(active_mask, fs):
result.append(obs_key)
return result
def initializeCurrentCaseFromExisting(source_case, target_case, source_report_step, parameters, members):
if caseExists(source_case) and caseIsInitialized(source_case) and caseExists(target_case):
total_member_count = getRealizationCount()
member_mask = BoolVector.createFromList(total_member_count, members)
selected_parameters = StringList(parameters)
ERT.ert.getEnkfFsManager().customInitializeCurrentFromExistingCase(source_case, source_report_step, member_mask,
selected_parameters)
ERT.emitErtChange()
def test_inactive(self):
key = "OBS"
obs_size = 2
ens_size = 10
ens_mask = BoolVector( default_value = True , initial_size = ens_size )
ens_mask[5] = False
block = MeasBlock( key , obs_size , ens_mask)
self.assertFalse( block.iensActive( 5 ))
with self.assertRaises(ValueError):
block[0,5] = 10
def test_repr(self):
primes = [2, 3, 5, 7, 11, 13, 17, 19]
b = BoolVector()
for i in primes:
b[i] = True
pfx = 'BoolVector(size = 20, content = "00110101000101000101")'
print(repr(b))
self.assertEqual(pfx, repr(b)[:len(pfx)])
b[30] = True
pfx = 'BoolVector(size = 31, content = "001101010...00000001")'
print(repr(b))
self.assertEqual(pfx, repr(b)[:len(pfx)])
def test_assert_export(self):
with ErtTestContext("create_runpath1" , self.createTestPath("local/snake_oil_no_data/snake_oil.ert")) as tc:
ert = tc.getErt( )
runpath_list = ert.getRunpathList( )
self.assertFalse( path.isfile( runpath_list.getExportFile( ) ))
ens_size = ert.getEnsembleSize( )
runner = ert.getEnkfSimulationRunner( )
mask = BoolVector( initial_size = ens_size , default_value = True )
runner.createRunPath( mask , 0 )
self.assertTrue( path.isfile( runpath_list.getExportFile( ) ))
self.assertEqual( "test_runpath_list.txt" , path.basename( runpath_list.getExportFile( ) ))
def test_setitem_getitem(self):
primes = [2, 3, 5, 7, 11, 13, 17, 19]
primep = [i in primes for i in range(20)]
b = BoolVector(initial_size=20)
b[2] = True
b[3:8:2] = True
b[11::2] = True
self.assertTrue(b[15])
self.assertTrue(b[-5])
self.assertTrue(b[17])
self.assertTrue(b[19])
b[-5] = False
self.assertEqual(list(b), primep)
def calculatePrincipalComponent(self, fs, local_obsdata, truncation_or_ncomp=3):
pc = Matrix(1, 1)
pc_obs = Matrix(1, 1)
singular_values = DoubleVector()
state_map = fs.getStateMap()
ens_mask = BoolVector(False, self.ert().getEnsembleSize())
state_map.selectMatching(ens_mask, RealizationStateEnum.STATE_HAS_DATA)
active_list = ens_mask.createActiveList( )
if len(ens_mask) > 0:
meas_data = MeasData(ens_mask)
obs_data = ObsData()
self.ert().getObservations().getObservationAndMeasureData(fs, local_obsdata, active_list, meas_data, obs_data)
meas_data.deactivateZeroStdSamples(obs_data)
active_size = len(obs_data)
if active_size > 0:
S = meas_data.createS()
D_obs = obs_data.createDObs()
truncation, ncomp = self.truncationOrNumberOfComponents(truncation_or_ncomp)
obs_data.scale(S, D_obs=D_obs)
EnkfLinalg.calculatePrincipalComponents(S, D_obs, truncation, ncomp, pc, pc_obs, singular_values)
if self.__prior_singular_values is None:
self.__prior_singular_values = singular_values
else:
for row in range(pc.rows()):
factor = singular_values[row]/self.__prior_singular_values[row]
pc.scaleRow( row , factor )
pc_obs.scaleRow( row , factor )
return PcaPlotData(local_obsdata.getName(), pc , pc_obs , singular_values)
return None
def test_add_all(self):
with ErtTestContext("add_all_runpath_dump",
model_config=self.config_path) as ctx:
res = ctx.getErt()
fs_manager = res.getEnkfFsManager()
sim_fs = fs_manager.getFileSystem("sim_fs")
num_realizations = 25
mask = BoolVector(initial_size=num_realizations,
default_value=True)
mask[13] = False
runpath_fmt = "simulations/<GEO_ID>/realisation-%d/iter-%d"
jobname_fmt = "SNAKE_OIL_%d"
itr = 0
subst_list = res.resConfig().subst_config.subst_list
run_context = ErtRunContext.ensemble_experiment(
sim_fs, mask, PathFormat(runpath_fmt), jobname_fmt, subst_list,
itr)
res.initRun(run_context)
for i, run_arg in enumerate(run_context):
if mask[i]:
run_arg.geo_id = 10 * i
res.createRunpath(run_context)
for i, run_arg in enumerate(run_context):
if not mask[i]:
continue
self.assertTrue(
os.path.isdir("simulations/%d" % run_arg.geo_id))
runpath_list_path = ".ert_runpath_list"
self.assertTrue(os.path.isfile(runpath_list_path))
exp_runpaths = [
runpath_fmt.replace("<GEO_ID>", str(run_arg.geo_id)) %
(iens, itr) for iens, run_arg in enumerate(run_context)
if mask[iens]
]
exp_runpaths = map(os.path.realpath, exp_runpaths)
with open(runpath_list_path, 'r') as f:
dumped_runpaths = zip(
*[line.split() for line in f.readlines()])[1]
self.assertEqual(list(exp_runpaths), list(dumped_runpaths))