Beispiel #1
0
 def addNodeAndRange(self, key, step_1, step_2):
     assert isinstance(key, str)
     assert isinstance(step_1, int)
     assert isinstance(step_2, int)        
     node = LocalObsdataNode(key)                
     self.addNode( node )
     node.addRange(step_1, step_2)
Beispiel #2
0
 def addNodeAndRange(self, key, step_1, step_2):
     """ @rtype: bool """
     assert isinstance(key, str)
     assert isinstance(step_1, int)
     assert isinstance(step_2, int)
     
     node = LocalObsdataNode(key)  
     node.addRange(step_1, step_2)      
     return self.addNode( node )
Beispiel #3
0
 def addNodeAndRange(self, key, step_1, step_2):
     """ @rtype: bool """
     assert isinstance(key, str)
     assert isinstance(step_1, int)
     assert isinstance(step_2, int)
     
     node = LocalObsdataNode(key)  
     node.addRange(step_1, step_2)      
     node.convertToCReference(self)
     already_exists_node_for_key = LocalObsdata.cNamespace().add_node(self, node)
     return already_exists_node_for_key
    def test_tstep(self):
        node = LocalObsdataNode("KEY")
        self.assertTrue(node.allTimeStepActive())
        self.assertTrue(node.tstepActive(10))
        self.assertTrue(node.tstepActive(0))

        node.addTimeStep(10)
        self.assertFalse(node.allTimeStepActive())

        self.assertTrue(node.tstepActive(10))
        self.assertFalse(node.tstepActive(0))
Beispiel #5
0
 def addNode(self, key, add_all_timesteps = True):
     """ @rtype: LocalObsdataNode """
     assert isinstance(key, str)
     if key in self.obs:
         node = LocalObsdataNode(key , add_all_timesteps)
         if node not in self:
             node.convertToCReference(self)
             self._add_node(node)
             return node
         else:
             raise KeyError("Tried to add existing observation key:%s " % key)
     else:
         raise KeyError("The observation node: %s is not recognized observation key" % key)
Beispiel #6
0
 def addNode(self, key, add_all_timesteps = True):
     """ @rtype: LocalObsdataNode """           
     assert isinstance(key, str)
     if key in self.obs:
         node = LocalObsdataNode(key , add_all_timesteps) 
         if node not in self:
             node.convertToCReference(self)
             LocalObsdata.cNamespace().add_node(self, node)
             return node
         else:
             raise KeyError("Tried to add existing observation key:%s " % key)
     else:
         raise KeyError("The observation node: %s is not recognized observation key" % key)
    def test_tstep(self):
        node = LocalObsdataNode("KEY")
        self.assertTrue( node.allTimeStepActive() )
        self.assertTrue( node.tstepActive( 10 ))
        self.assertTrue( node.tstepActive( 0 ))
        
        node.addTimeStep(10)
        self.assertFalse( node.allTimeStepActive() )

        self.assertTrue( node.tstepActive( 10 ))
        self.assertFalse( node.tstepActive( 0 ))
    def test_tstep(self):
        node = LocalObsdataNode("KEY")
        self.assertTrue(node.allTimeStepActive())

        with self.assertRaises(ValueError):
            tstep_list = node.getStepList()

        node.addTimeStep(10)
        self.assertFalse(node.allTimeStepActive())

        tstep_list = node.getStepList()
        self.assertEqual(len(tstep_list), 1)
        self.assertEqual(tstep_list[0], 10)
Beispiel #9
0
    def test_scale_obs(self):
        with ErtTestContext("obs_test", self.config_file) as test_context:
            ert = test_context.getErt()
            obs = ert.getObservations()

            obs1 = obs["WWCT:OP_1"].getNode( 50 )
            obs2 = obs["WWCT:OP_1_50"].getNode( 50 )
            
            self.assertEqual( obs1.getStandardDeviation( ) , obs2.getStandardDeviation( ))
            std0 = obs1.getStandardDeviation( )

            local_obsdata = LocalObsdata("obs" , obs)
            node1 = LocalObsdataNode( "WWCT:OP_1" )
            node1.addRange( 50 , 50 )
            node2 = LocalObsdataNode( "WWCT:OP_1_50" )
            node2.addRange( 50 , 50 )
            local_obsdata.addNode( node1 )
            local_obsdata.addNode( node2 )

            mask = BoolVector( default_value = True )
            mask[2] = True
            meas_data = MeasData(mask)
            obs_data = ObsData( )
            fs = ert.getEnkfFsManager().getCurrentFileSystem()
            active_list = IntVector()
            active_list.initRange(0,2,1)
            obs.getObservationAndMeasureData( fs , local_obsdata , EnkfStateType.FORECAST , active_list , meas_data , obs_data )
            self.assertEqual( 2 , len(obs_data) )

            v1 = obs_data[0]
            v2 = obs_data[1]

            self.assertEqual( v1[1] , std0 )
            self.assertEqual( v2[1] , std0 )

            meas_data = MeasData(mask)
            obs_data = ObsData( 10 )
            obs.getObservationAndMeasureData( fs , local_obsdata , EnkfStateType.FORECAST , active_list , meas_data , obs_data )
            self.assertEqual( 2 , len(obs_data) )
            
            v1 = obs_data[0]
            v2 = obs_data[1]
            
            self.assertEqual( v1[1] , std0*10)
            self.assertEqual( v2[1] , std0*10 )

            actl = ActiveList()
            obs1.updateStdScaling( 10 , actl)
            obs2.updateStdScaling( 20 , actl)
            meas_data = MeasData(mask)
            obs_data = ObsData( )
            obs.getObservationAndMeasureData( fs , local_obsdata , EnkfStateType.FORECAST , active_list , meas_data , obs_data )
            self.assertEqual( 2 , len(obs_data) )
            
            v1 = obs_data[0]
            v2 = obs_data[1]
            
            self.assertEqual( v1[1] , std0*10)
            self.assertEqual( v2[1] , std0*20)
    def test_tstep(self):
        node = LocalObsdataNode("KEY")
        self.assertTrue( node.allTimeStepActive() )
        
        with self.assertRaises(ValueError):
            tstep_list = node.getStepList()

        node.addTimeStep(10)
        self.assertFalse( node.allTimeStepActive() )

        tstep_list = node.getStepList()
        self.assertEqual( len(tstep_list) , 1 )
        self.assertEqual( tstep_list[0] , 10)
Beispiel #11
0
class PcaDataFetcher(DataFetcher):
    def __init__(self, ert):
        super(PcaDataFetcher, self).__init__(ert)
        self.__prior_singular_values = None

    def fetchSupportedKeys(self):
        summary_keys = EnsembleDataFetcher(self.ert()).getSupportedKeys()

        keys = []
        for key in summary_keys:
            obs_keys = self.ert().ensembleConfig().getNode(key).getObservationKeys()
            if len(obs_keys) > 0:
                keys.append(key)

        keys += BlockObservationDataFetcher(self.ert()).getSupportedKeys()
        keys += ObservationGenDataFetcher(self.ert()).getSupportedKeys()

        return keys


    def truncationOrNumberOfComponents(self, truncation_or_ncomp):
        """ @rtype: (float, int) """
        truncation = -1
        ncomp = -1

        if truncation_or_ncomp < 1:
            truncation = truncation_or_ncomp
        else:
            ncomp = int(truncation_or_ncomp)

        return truncation, ncomp


    def calculatePrincipalComponent(self, fs, local_obsdata, truncation_or_ncomp=3):
        pc = Matrix(1, 1)
        pc_obs = Matrix(1, 1)
        singular_values = DoubleVector()

        state_map = fs.getStateMap()
        ens_mask = BoolVector(False, self.ert().getEnsembleSize())
        state_map.selectMatching(ens_mask, RealizationStateEnum.STATE_HAS_DATA)
        active_list = BoolVector.createActiveList(ens_mask)

        if len(active_list) > 0:
            state = EnkfStateType.FORECAST
            meas_data = MeasData(active_list)
            obs_data = ObsData()

            self.ert().getObservations().getObservationAndMeasureData(fs, local_obsdata, state, active_list, meas_data, obs_data)

            meas_data.deactivateZeroStdSamples(obs_data)

            active_size = len(obs_data)

            if active_size > 0:
                S = meas_data.createS(active_size)
                D_obs = obs_data.createDobs(active_size)

                truncation, ncomp = self.truncationOrNumberOfComponents(truncation_or_ncomp)

                obs_data.scale(S, D_obs=D_obs)
                EnkfLinalg.calculatePrincipalComponents(S, D_obs, truncation, ncomp, pc, pc_obs, singular_values)
                if self.__prior_singular_values is None:
                    self.__prior_singular_values = singular_values
                else:
                    for row in range(pc.rows()):
                        factor = singular_values[row]/self.__prior_singular_values[row]
                        pc.scaleRow( row , factor )
                        pc_obs.scaleRow( row , factor )


                return PcaPlotData(local_obsdata.getName(), pc , pc_obs , singular_values)
        return None



    def getAllObsKeys(self):
        observations = self.ert().getObservations()
        summary_obs_keys = observations.getTypedKeylist(EnkfObservationImplementationType.SUMMARY_OBS)
        gen_data_obs_keys =  observations.getTypedKeylist(EnkfObservationImplementationType.GEN_OBS)
        block_obs_keys =  observations.getTypedKeylist(EnkfObservationImplementationType.BLOCK_OBS)

        summary_obs_keys = [key for key in summary_obs_keys]
        gen_data_obs_keys = [key for key in gen_data_obs_keys]
        block_obs_keys = [key for key in block_obs_keys]

        return summary_obs_keys + gen_data_obs_keys + block_obs_keys


    def getObsKeys(self, data_key):
        ensemble_data_fetcher = EnsembleDataFetcher(self.ert())
        block_observation_data_fetcher = BlockObservationDataFetcher(self.ert())
        gen_data_observation_data_fetcher = ObservationGenDataFetcher(self.ert())

        if ensemble_data_fetcher.supportsKey(data_key):
            return self.ert().ensembleConfig().getNode(data_key).getObservationKeys()
        elif block_observation_data_fetcher.supportsKey(data_key):
            return [data_key]
        elif gen_data_observation_data_fetcher.supportsKey(data_key):
            return gen_data_observation_data_fetcher.getAllObsKeysForKey(data_key)


    def filterObsKeys(self, obs_keys, fs):
        active_mask = BoolVector(True, self.ert().getEnsembleSize())
        ert_obs = self.ert().getObservations()

        result = []
        for obs_key in obs_keys:
            obsVector = ert_obs[obs_key]
            if obsVector.hasData(active_mask, fs):
                result.append(obs_key)
        return result


    def fetchData(self, obs_keys, case=None):
        data = {"x": None,
                "y": None,
                "obs_y": None,
                "min_y": None,
                "max_y": None,
                "min_x": None,
                "max_x": None}

        fs = self.ert().getEnkfFsManager().getFileSystem(case)
        obs_keys = self.filterObsKeys(obs_keys, fs)

        step_1 = 0
        step_2 = self.ert().getHistoryLength()

        local_obsdata = LocalObsdata("PCA Observations %s" % case)

        for obs_key in obs_keys:
            if not obs_key in local_obsdata:
                obs_node = LocalObsdataNode(obs_key)
                obs_node.addRange(step_1, step_2)
                local_obsdata.addNode(obs_node)

        if len(local_obsdata) > 0:
            pca_data = self.calculatePrincipalComponent(fs, local_obsdata)

            if pca_data is not None:
                data["x"] = []
                data["y"] = []
                data["obs_y"] = []

                data["min_x"] = 1
                data["max_x"] = len(pca_data)

                component_number = 0
                for pca_vector in pca_data:
                    component_number += 1
                    data["x"].append(component_number)

                    obs_y = pca_vector.getObservation()

                    if data["min_y"] is None or data["min_y"] > obs_y:
                        data["min_y"] = obs_y

                    if data["max_y"] is None or data["max_y"] < obs_y:
                        data["max_y"] = obs_y

                    data["obs_y"].append(obs_y)
                    for index, value in enumerate(pca_vector):
                        if len(data["y"]) == index:
                            data["y"].append([])

                        y = data["y"][index]
                        y.append(value)

                        if data["min_y"] is None or data["min_y"] > value:
                            data["min_y"] = value

                        if data["max_y"] is None or data["max_y"] < value:
                            data["max_y"] = value

        return data
Beispiel #12
0
    def fetchData(self, obs_keys, case=None):
        data = {"x": None,
                "y": None,
                "obs_y": None,
                "min_y": None,
                "max_y": None,
                "min_x": None,
                "max_x": None}

        fs = self.ert().getEnkfFsManager().getFileSystem(case)
        obs_keys = self.filterObsKeys(obs_keys, fs)

        step_1 = 0
        step_2 = self.ert().getHistoryLength()

        local_obsdata = LocalObsdata("PCA Observations %s" % case)

        for obs_key in obs_keys:
            if not obs_key in local_obsdata:
                obs_node = LocalObsdataNode(obs_key)
                obs_node.addRange(step_1, step_2)
                local_obsdata.addNode(obs_node)

        if len(local_obsdata) > 0:
            pca_data = self.calculatePrincipalComponent(fs, local_obsdata)

            if pca_data is not None:
                data["x"] = []
                data["y"] = []
                data["obs_y"] = []

                data["min_x"] = 1
                data["max_x"] = len(pca_data)

                component_number = 0
                for pca_vector in pca_data:
                    component_number += 1
                    data["x"].append(component_number)

                    obs_y = pca_vector.getObservation()

                    if data["min_y"] is None or data["min_y"] > obs_y:
                        data["min_y"] = obs_y

                    if data["max_y"] is None or data["max_y"] < obs_y:
                        data["max_y"] = obs_y

                    data["obs_y"].append(obs_y)
                    for index, value in enumerate(pca_vector):
                        if len(data["y"]) == index:
                            data["y"].append([])

                        y = data["y"][index]
                        y.append(value)

                        if data["min_y"] is None or data["min_y"] > value:
                            data["min_y"] = value

                        if data["max_y"] is None or data["max_y"] < value:
                            data["max_y"] = value

        return data