def addNodeAndRange(self, key, step_1, step_2):
assert isinstance(key, str)
assert isinstance(step_1, int)
assert isinstance(step_2, int)
node = LocalObsdataNode(key)
self.addNode( node )
node.addRange(step_1, step_2)
def addNodeAndRange(self, key, step_1, step_2):
""" @rtype: bool """
assert isinstance(key, str)
assert isinstance(step_1, int)
assert isinstance(step_2, int)
node = LocalObsdataNode(key)
node.addRange(step_1, step_2)
return self.addNode( node )
def addNodeAndRange(self, key, step_1, step_2):
""" @rtype: bool """
assert isinstance(key, str)
assert isinstance(step_1, int)
assert isinstance(step_2, int)
node = LocalObsdataNode(key)
node.addRange(step_1, step_2)
node.convertToCReference(self)
already_exists_node_for_key = LocalObsdata.cNamespace().add_node(self, node)
return already_exists_node_for_key
def test_tstep(self):
node = LocalObsdataNode("KEY")
self.assertTrue(node.allTimeStepActive())
self.assertTrue(node.tstepActive(10))
self.assertTrue(node.tstepActive(0))
node.addTimeStep(10)
self.assertFalse(node.allTimeStepActive())
self.assertTrue(node.tstepActive(10))
self.assertFalse(node.tstepActive(0))
def addNode(self, key, add_all_timesteps = True):
""" @rtype: LocalObsdataNode """
assert isinstance(key, str)
if key in self.obs:
node = LocalObsdataNode(key , add_all_timesteps)
if node not in self:
node.convertToCReference(self)
self._add_node(node)
return node
else:
raise KeyError("Tried to add existing observation key:%s " % key)
else:
raise KeyError("The observation node: %s is not recognized observation key" % key)
def addNode(self, key, add_all_timesteps = True):
""" @rtype: LocalObsdataNode """
assert isinstance(key, str)
if key in self.obs:
node = LocalObsdataNode(key , add_all_timesteps)
if node not in self:
node.convertToCReference(self)
LocalObsdata.cNamespace().add_node(self, node)
return node
else:
raise KeyError("Tried to add existing observation key:%s " % key)
else:
raise KeyError("The observation node: %s is not recognized observation key" % key)
def test_tstep(self):
node = LocalObsdataNode("KEY")
self.assertTrue( node.allTimeStepActive() )
self.assertTrue( node.tstepActive( 10 ))
self.assertTrue( node.tstepActive( 0 ))
node.addTimeStep(10)
self.assertFalse( node.allTimeStepActive() )
self.assertTrue( node.tstepActive( 10 ))
self.assertFalse( node.tstepActive( 0 ))
def test_tstep(self):
node = LocalObsdataNode("KEY")
self.assertTrue(node.allTimeStepActive())
with self.assertRaises(ValueError):
tstep_list = node.getStepList()
node.addTimeStep(10)
self.assertFalse(node.allTimeStepActive())
tstep_list = node.getStepList()
self.assertEqual(len(tstep_list), 1)
self.assertEqual(tstep_list[0], 10)
def test_scale_obs(self):
with ErtTestContext("obs_test", self.config_file) as test_context:
ert = test_context.getErt()
obs = ert.getObservations()
obs1 = obs["WWCT:OP_1"].getNode( 50 )
obs2 = obs["WWCT:OP_1_50"].getNode( 50 )
self.assertEqual( obs1.getStandardDeviation( ) , obs2.getStandardDeviation( ))
std0 = obs1.getStandardDeviation( )
local_obsdata = LocalObsdata("obs" , obs)
node1 = LocalObsdataNode( "WWCT:OP_1" )
node1.addRange( 50 , 50 )
node2 = LocalObsdataNode( "WWCT:OP_1_50" )
node2.addRange( 50 , 50 )
local_obsdata.addNode( node1 )
local_obsdata.addNode( node2 )
mask = BoolVector( default_value = True )
mask[2] = True
meas_data = MeasData(mask)
obs_data = ObsData( )
fs = ert.getEnkfFsManager().getCurrentFileSystem()
active_list = IntVector()
active_list.initRange(0,2,1)
obs.getObservationAndMeasureData( fs , local_obsdata , EnkfStateType.FORECAST , active_list , meas_data , obs_data )
self.assertEqual( 2 , len(obs_data) )
v1 = obs_data[0]
v2 = obs_data[1]
self.assertEqual( v1[1] , std0 )
self.assertEqual( v2[1] , std0 )
meas_data = MeasData(mask)
obs_data = ObsData( 10 )
obs.getObservationAndMeasureData( fs , local_obsdata , EnkfStateType.FORECAST , active_list , meas_data , obs_data )
self.assertEqual( 2 , len(obs_data) )
v1 = obs_data[0]
v2 = obs_data[1]
self.assertEqual( v1[1] , std0*10)
self.assertEqual( v2[1] , std0*10 )
actl = ActiveList()
obs1.updateStdScaling( 10 , actl)
obs2.updateStdScaling( 20 , actl)
meas_data = MeasData(mask)
obs_data = ObsData( )
obs.getObservationAndMeasureData( fs , local_obsdata , EnkfStateType.FORECAST , active_list , meas_data , obs_data )
self.assertEqual( 2 , len(obs_data) )
v1 = obs_data[0]
v2 = obs_data[1]
self.assertEqual( v1[1] , std0*10)
self.assertEqual( v2[1] , std0*20)
def test_tstep(self):
node = LocalObsdataNode("KEY")
self.assertTrue( node.allTimeStepActive() )
with self.assertRaises(ValueError):
tstep_list = node.getStepList()
node.addTimeStep(10)
self.assertFalse( node.allTimeStepActive() )
tstep_list = node.getStepList()
self.assertEqual( len(tstep_list) , 1 )
self.assertEqual( tstep_list[0] , 10)
class PcaDataFetcher(DataFetcher):
def __init__(self, ert):
super(PcaDataFetcher, self).__init__(ert)
self.__prior_singular_values = None
def fetchSupportedKeys(self):
summary_keys = EnsembleDataFetcher(self.ert()).getSupportedKeys()
keys = []
for key in summary_keys:
obs_keys = self.ert().ensembleConfig().getNode(key).getObservationKeys()
if len(obs_keys) > 0:
keys.append(key)
keys += BlockObservationDataFetcher(self.ert()).getSupportedKeys()
keys += ObservationGenDataFetcher(self.ert()).getSupportedKeys()
return keys
def truncationOrNumberOfComponents(self, truncation_or_ncomp):
""" @rtype: (float, int) """
truncation = -1
ncomp = -1
if truncation_or_ncomp < 1:
truncation = truncation_or_ncomp
else:
ncomp = int(truncation_or_ncomp)
return truncation, ncomp
def calculatePrincipalComponent(self, fs, local_obsdata, truncation_or_ncomp=3):
pc = Matrix(1, 1)
pc_obs = Matrix(1, 1)
singular_values = DoubleVector()
state_map = fs.getStateMap()
ens_mask = BoolVector(False, self.ert().getEnsembleSize())
state_map.selectMatching(ens_mask, RealizationStateEnum.STATE_HAS_DATA)
active_list = BoolVector.createActiveList(ens_mask)
if len(active_list) > 0:
state = EnkfStateType.FORECAST
meas_data = MeasData(active_list)
obs_data = ObsData()
self.ert().getObservations().getObservationAndMeasureData(fs, local_obsdata, state, active_list, meas_data, obs_data)
meas_data.deactivateZeroStdSamples(obs_data)
active_size = len(obs_data)
if active_size > 0:
S = meas_data.createS(active_size)
D_obs = obs_data.createDobs(active_size)
truncation, ncomp = self.truncationOrNumberOfComponents(truncation_or_ncomp)
obs_data.scale(S, D_obs=D_obs)
EnkfLinalg.calculatePrincipalComponents(S, D_obs, truncation, ncomp, pc, pc_obs, singular_values)
if self.__prior_singular_values is None:
self.__prior_singular_values = singular_values
else:
for row in range(pc.rows()):
factor = singular_values[row]/self.__prior_singular_values[row]
pc.scaleRow( row , factor )
pc_obs.scaleRow( row , factor )
return PcaPlotData(local_obsdata.getName(), pc , pc_obs , singular_values)
return None
def getAllObsKeys(self):
observations = self.ert().getObservations()
summary_obs_keys = observations.getTypedKeylist(EnkfObservationImplementationType.SUMMARY_OBS)
gen_data_obs_keys = observations.getTypedKeylist(EnkfObservationImplementationType.GEN_OBS)
block_obs_keys = observations.getTypedKeylist(EnkfObservationImplementationType.BLOCK_OBS)
summary_obs_keys = [key for key in summary_obs_keys]
gen_data_obs_keys = [key for key in gen_data_obs_keys]
block_obs_keys = [key for key in block_obs_keys]
return summary_obs_keys + gen_data_obs_keys + block_obs_keys
def getObsKeys(self, data_key):
ensemble_data_fetcher = EnsembleDataFetcher(self.ert())
block_observation_data_fetcher = BlockObservationDataFetcher(self.ert())
gen_data_observation_data_fetcher = ObservationGenDataFetcher(self.ert())
if ensemble_data_fetcher.supportsKey(data_key):
return self.ert().ensembleConfig().getNode(data_key).getObservationKeys()
elif block_observation_data_fetcher.supportsKey(data_key):
return [data_key]
elif gen_data_observation_data_fetcher.supportsKey(data_key):
return gen_data_observation_data_fetcher.getAllObsKeysForKey(data_key)
def filterObsKeys(self, obs_keys, fs):
active_mask = BoolVector(True, self.ert().getEnsembleSize())
ert_obs = self.ert().getObservations()
result = []
for obs_key in obs_keys:
obsVector = ert_obs[obs_key]
if obsVector.hasData(active_mask, fs):
result.append(obs_key)
return result
def fetchData(self, obs_keys, case=None):
data = {"x": None,
"y": None,
"obs_y": None,
"min_y": None,
"max_y": None,
"min_x": None,
"max_x": None}
fs = self.ert().getEnkfFsManager().getFileSystem(case)
obs_keys = self.filterObsKeys(obs_keys, fs)
step_1 = 0
step_2 = self.ert().getHistoryLength()
local_obsdata = LocalObsdata("PCA Observations %s" % case)
for obs_key in obs_keys:
if not obs_key in local_obsdata:
obs_node = LocalObsdataNode(obs_key)
obs_node.addRange(step_1, step_2)
local_obsdata.addNode(obs_node)
if len(local_obsdata) > 0:
pca_data = self.calculatePrincipalComponent(fs, local_obsdata)
if pca_data is not None:
data["x"] = []
data["y"] = []
data["obs_y"] = []
data["min_x"] = 1
data["max_x"] = len(pca_data)
component_number = 0
for pca_vector in pca_data:
component_number += 1
data["x"].append(component_number)
obs_y = pca_vector.getObservation()
if data["min_y"] is None or data["min_y"] > obs_y:
data["min_y"] = obs_y
if data["max_y"] is None or data["max_y"] < obs_y:
data["max_y"] = obs_y
data["obs_y"].append(obs_y)
for index, value in enumerate(pca_vector):
if len(data["y"]) == index:
data["y"].append([])
y = data["y"][index]
y.append(value)
if data["min_y"] is None or data["min_y"] > value:
data["min_y"] = value
if data["max_y"] is None or data["max_y"] < value:
data["max_y"] = value
return data
def fetchData(self, obs_keys, case=None):
data = {"x": None,
"y": None,
"obs_y": None,
"min_y": None,
"max_y": None,
"min_x": None,
"max_x": None}
fs = self.ert().getEnkfFsManager().getFileSystem(case)
obs_keys = self.filterObsKeys(obs_keys, fs)
step_1 = 0
step_2 = self.ert().getHistoryLength()
local_obsdata = LocalObsdata("PCA Observations %s" % case)
for obs_key in obs_keys:
if not obs_key in local_obsdata:
obs_node = LocalObsdataNode(obs_key)
obs_node.addRange(step_1, step_2)
local_obsdata.addNode(obs_node)
if len(local_obsdata) > 0:
pca_data = self.calculatePrincipalComponent(fs, local_obsdata)
if pca_data is not None:
data["x"] = []
data["y"] = []
data["obs_y"] = []
data["min_x"] = 1
data["max_x"] = len(pca_data)
component_number = 0
for pca_vector in pca_data:
component_number += 1
data["x"].append(component_number)
obs_y = pca_vector.getObservation()
if data["min_y"] is None or data["min_y"] > obs_y:
data["min_y"] = obs_y
if data["max_y"] is None or data["max_y"] < obs_y:
data["max_y"] = obs_y
data["obs_y"].append(obs_y)
for index, value in enumerate(pca_vector):
if len(data["y"]) == index:
data["y"].append([])
y = data["y"][index]
y.append(value)
if data["min_y"] is None or data["min_y"] > value:
data["min_y"] = value
if data["max_y"] is None or data["max_y"] < value:
data["max_y"] = value
return data