Ejemplo n.º 1
0
    def compare_vector(self, mean=True, revers=True):
        """
        comparison of vectors
        
        mean: compute vector mean?
        revers: compare observations with revers id from and to?
        """
        
        self.revers =revers
        
        # find vectors in cluster list
        self.obsToComp1 = ObsClusterList() # cluster list with vectors
        self.obsToComp2 = ObsClusterList()

        for cl in self.data1.obsClusterList:
            if isinstance(cl, ObsClusterVector):
                self.obsToComp1.append_cluster(cl)
        
        for cl in self.data2.obsClusterList:
            if isinstance(cl, ObsClusterVector):
                self.obsToComp2.append_cluster(cl)
        
        if sum([len(c) for c in self.obsToComp1]) is 0:
            return
            #raise CompareObsError, "No vectors found in data #1"
        
        if sum([len(c) for c in self.obsToComp2]) is 0:
            return
            #raise CompareObsError, "No vectors found in data #2"
    
        if mean:
            self.obsToComp1, self.meanList1, self.obsMeanList1, self.resList1 =\
                self._mean_vector(self.obsToComp1)
    
            self.obsToComp2, self.meanList2, self.obsMeanList2, self.resList2 =\
                self._mean_vector(self.obsToComp2)

        # cluster for compared vectors
        # covariance matrix - initial zero dimension 
        self._cluster = ObsClusterVector(covmat=CovMat(dim=0, band=2))

        self.compObsClList.append_cluster(self._cluster)

        # find vectors with same fromid and toid and compare
        for c1 in self.obsToComp1:
            for vec1 in c1:
                found, vec2, reversed = self._find_obs(vec1, self.obsToComp2)
                if found:
                        self._cmp_vector(vec1, vec2, reversed=reversed)

        # find uncompared vectors
        self._uncompared_obs()
Ejemplo n.º 2
0
    def _mean_vector(self, data):
        "returns mean of vector with same fromid and toid"
        
        # add vectors to list
        lvec = [] # helper list of vectors
        for c in data:
            for v in c:
                lvec.append(v)

        datamean = ObsClusterList() 
        dmcluster = ObsClusterVector(covmat=CovMat(dim=0, band=2))
        datamean.append_cluster(dmcluster) 
        # list of vector means and single vectors (without mean)

        cll_mean = ObsClusterList()
        cll_obs = ObsClusterList()
        cll_res = ObsClusterList()
        
        if len(lvec) <= 1:
            return # nothink to compare
        while len(lvec) > 1:
            # find vectors with same fromid and toid
            v1 = lvec.pop()
            fromid = v1.fromid
            toid = v1.toid
            lvecm = [v1] # list of vectors for mean
            lsign = [1.0] # list of signs: for revers vectors -1.0
            for v2 in lvec:
                if v2.fromid == fromid and v2.toid == toid:
                    lvecm.append(v2)
                    lvec.remove(v2)
                    lsign.append(1.0)
                    continue
                if self.revers and\
                v2.fromid == toid and v2.toid == fromid:
                    lvecm.append(v2)
                    lvec.remove(v2)
                    lsign.append(-1.0)

            # compute mean
            if float(len(lvecm)) == 1:
                # nothink to compute
                vec = lvecm[0]
                covmat = vec.covmat
                dmcluster.append_obs(vec)
                dmcluster.covmat.extend_dim(3)
                vec.covmat = covmat
                continue

            
            from numpy import mat, mean
            
            ll = [mat([i.dx, i.dy, i.dz])*sign for i,sign in zip(lvecm,lsign)] 
            # row vectors of observ.

            #lm = mean(ll, axis=0)
            #print "Mean  0:", lm
            #print "covmat0:", lvecm[0].covmat.mat

            # compute covariance matrix 
            # covariances between vectors not handled
            
            sil = [i.covmat.mat.I for i in lvecm] 
            # covariance matrice inverses inv(Sigma)
            
            sill = [si*l.T for si,l in zip(sil,ll)]
            # product inv(Sigma)*l
            
            cmm = sum(sil).I # covariance matrix of mean
            mn = cmm * sum(sill) # mean value
            #print "Mean  1:", mn
            #print "covmat1:", cmm
        
            # append clusters 
            cl_mean = ObsClusterVector(covmat=CovMat(dim=3, band=2))
            cl_obs  = ObsClusterVector(covmat=CovMat(dim=0, band=2))
            cl_res  = ObsClusterVector(covmat=None)
            cll_mean.append_cluster(cl_mean)
            cll_obs.append_cluster(cl_obs)
            cll_res.append_cluster(cl_res)
            
            # append vector mean
            vec1 = ObsVector(fromid=lvecm[0].fromid, toid=lvecm[0].toid,
                            dx=float(mn[0]), dy=float(mn[1]), dz=float(mn[2]))
            vec2 = ObsVector(fromid=lvecm[0].fromid, toid=lvecm[0].toid,
                            dx=float(mn[0]), dy=float(mn[1]), dz=float(mn[2]))
            # vec1 == vec2
            cl_mean.append_obs(vec1)
            dmcluster.append_obs(vec2)

            # set covmat of mean
            covmat = CovMat(dim=3, band=2)
            covmat.mat = cmm
            vec1.covmat = covmat
            dmcluster.covmat.extend_dim(3)
            vec2.covmat = covmat
            
            # append observations with covmat
            for v in lvecm:
                vcovmat = v.covmat
                cl_obs.append_obs(v)
                cl_obs.covmat.extend_dim(3)
                v.covmat = vcovmat

            # append residuals
            for l,v,s in zip(ll,lvecm,lsign):
                res = (mn - l.T) * s
                vec = ObsVector(fromid=v.fromid, toid=v.toid,
                                dx=float(res[0]), 
                                dy=float(res[1]),
                                dz=float(res[2]))
                cl_res.append_obs(vec)
        
        return datamean, cll_mean, cll_obs, cll_res
Ejemplo n.º 3
0
class CompareObs(object):
    """
    compares measured data
    """

    def __init__(self, data1, data2, revers=True):
        """
        data1, data2: observation data - GamaLocalDataObs instance
        revers: handle fromid_1 == toid_2 and toid_1 == fromid_2 
        """

        self.data1 = data1
        self.data2 = data2
        
        self.revers = revers

        self.compObsClList = ObsClusterList() # list of differences from comparison
        
        # data 1
        self.meanList1 = None # list of means 
        self.obsMeanList1 = None # list of observations for mean
        self.resList1  = None # list of residuals
        self.obsToComp1 = None # observations in cluster list to be compared
        
        # data 2
        self.meanList2 = None 
        self.obsMeanList2 = None
        self.resList2  = None
        self.obsToComp2 = None

        # make list of uncompared observations
        self.uncompObsClList1 = ObsClusterList() # cluster list with observations uncompared
        self.uncompObsClList2 = ObsClusterList()

        self._cluster = None # actual cluster


    def _mean_vector(self, data):
        "returns mean of vector with same fromid and toid"
        
        # add vectors to list
        lvec = [] # helper list of vectors
        for c in data:
            for v in c:
                lvec.append(v)

        datamean = ObsClusterList() 
        dmcluster = ObsClusterVector(covmat=CovMat(dim=0, band=2))
        datamean.append_cluster(dmcluster) 
        # list of vector means and single vectors (without mean)

        cll_mean = ObsClusterList()
        cll_obs = ObsClusterList()
        cll_res = ObsClusterList()
        
        if len(lvec) <= 1:
            return # nothink to compare
        while len(lvec) > 1:
            # find vectors with same fromid and toid
            v1 = lvec.pop()
            fromid = v1.fromid
            toid = v1.toid
            lvecm = [v1] # list of vectors for mean
            lsign = [1.0] # list of signs: for revers vectors -1.0
            for v2 in lvec:
                if v2.fromid == fromid and v2.toid == toid:
                    lvecm.append(v2)
                    lvec.remove(v2)
                    lsign.append(1.0)
                    continue
                if self.revers and\
                v2.fromid == toid and v2.toid == fromid:
                    lvecm.append(v2)
                    lvec.remove(v2)
                    lsign.append(-1.0)

            # compute mean
            if float(len(lvecm)) == 1:
                # nothink to compute
                vec = lvecm[0]
                covmat = vec.covmat
                dmcluster.append_obs(vec)
                dmcluster.covmat.extend_dim(3)
                vec.covmat = covmat
                continue

            
            from numpy import mat, mean
            
            ll = [mat([i.dx, i.dy, i.dz])*sign for i,sign in zip(lvecm,lsign)] 
            # row vectors of observ.

            #lm = mean(ll, axis=0)
            #print "Mean  0:", lm
            #print "covmat0:", lvecm[0].covmat.mat

            # compute covariance matrix 
            # covariances between vectors not handled
            
            sil = [i.covmat.mat.I for i in lvecm] 
            # covariance matrice inverses inv(Sigma)
            
            sill = [si*l.T for si,l in zip(sil,ll)]
            # product inv(Sigma)*l
            
            cmm = sum(sil).I # covariance matrix of mean
            mn = cmm * sum(sill) # mean value
            #print "Mean  1:", mn
            #print "covmat1:", cmm
        
            # append clusters 
            cl_mean = ObsClusterVector(covmat=CovMat(dim=3, band=2))
            cl_obs  = ObsClusterVector(covmat=CovMat(dim=0, band=2))
            cl_res  = ObsClusterVector(covmat=None)
            cll_mean.append_cluster(cl_mean)
            cll_obs.append_cluster(cl_obs)
            cll_res.append_cluster(cl_res)
            
            # append vector mean
            vec1 = ObsVector(fromid=lvecm[0].fromid, toid=lvecm[0].toid,
                            dx=float(mn[0]), dy=float(mn[1]), dz=float(mn[2]))
            vec2 = ObsVector(fromid=lvecm[0].fromid, toid=lvecm[0].toid,
                            dx=float(mn[0]), dy=float(mn[1]), dz=float(mn[2]))
            # vec1 == vec2
            cl_mean.append_obs(vec1)
            dmcluster.append_obs(vec2)

            # set covmat of mean
            covmat = CovMat(dim=3, band=2)
            covmat.mat = cmm
            vec1.covmat = covmat
            dmcluster.covmat.extend_dim(3)
            vec2.covmat = covmat
            
            # append observations with covmat
            for v in lvecm:
                vcovmat = v.covmat
                cl_obs.append_obs(v)
                cl_obs.covmat.extend_dim(3)
                v.covmat = vcovmat

            # append residuals
            for l,v,s in zip(ll,lvecm,lsign):
                res = (mn - l.T) * s
                vec = ObsVector(fromid=v.fromid, toid=v.toid,
                                dx=float(res[0]), 
                                dy=float(res[1]),
                                dz=float(res[2]))
                cl_res.append_obs(vec)
        
        return datamean, cll_mean, cll_obs, cll_res

    
    def _find_obs(self, obs, cl):
        """
        finds observation in cluster list
        returns found, observation, reversed
        """
        fromid = obs.fromid
        toid = obs.toid
        for c in cl:
            for o in c:
                if o.fromid == fromid and o.toid == toid:
                    return True, o, False
                if self.revers and \
                   o.fromid == toid and o.toid == fromid:
                    return True, o, True

        return False, None, False


    def compare_vector(self, mean=True, revers=True):
        """
        comparison of vectors
        
        mean: compute vector mean?
        revers: compare observations with revers id from and to?
        """
        
        self.revers =revers
        
        # find vectors in cluster list
        self.obsToComp1 = ObsClusterList() # cluster list with vectors
        self.obsToComp2 = ObsClusterList()

        for cl in self.data1.obsClusterList:
            if isinstance(cl, ObsClusterVector):
                self.obsToComp1.append_cluster(cl)
        
        for cl in self.data2.obsClusterList:
            if isinstance(cl, ObsClusterVector):
                self.obsToComp2.append_cluster(cl)
        
        if sum([len(c) for c in self.obsToComp1]) is 0:
            return
            #raise CompareObsError, "No vectors found in data #1"
        
        if sum([len(c) for c in self.obsToComp2]) is 0:
            return
            #raise CompareObsError, "No vectors found in data #2"
    
        if mean:
            self.obsToComp1, self.meanList1, self.obsMeanList1, self.resList1 =\
                self._mean_vector(self.obsToComp1)
    
            self.obsToComp2, self.meanList2, self.obsMeanList2, self.resList2 =\
                self._mean_vector(self.obsToComp2)

        # cluster for compared vectors
        # covariance matrix - initial zero dimension 
        self._cluster = ObsClusterVector(covmat=CovMat(dim=0, band=2))

        self.compObsClList.append_cluster(self._cluster)

        # find vectors with same fromid and toid and compare
        for c1 in self.obsToComp1:
            for vec1 in c1:
                found, vec2, reversed = self._find_obs(vec1, self.obsToComp2)
                if found:
                        self._cmp_vector(vec1, vec2, reversed=reversed)

        # find uncompared vectors
        self._uncompared_obs()
    
    
    def _uncompared_obs(self):

        for cl in self.obsToComp1:
            # add cluster to cluster list
            uc1 = cl.__class__()
            added = False # added some observation

            for obs in cl:
                found, x, x = self._find_obs(obs, self.compObsClList)
                if not found:
                    uc1.append_obs(obs)
                    added = True
            
            if added:
                self.uncompObsClList1.append_cluster(uc1)
        
        for cl in self.obsToComp2:
            # add cluster to cluster list
            uc2 = cl.__class__()
            added = False # added some observation
            
            for obs in cl:
                found, x ,x = self._find_obs(obs, self.compObsClList)
                if not found:
                    uc2.append_obs(obs)
                    added = True
            
            if added:
                self.uncompObsClList2.append_cluster(uc2)


    def _cmp_vector(self, vec1, vec2, reversed=False):
        "comparison of two vectors - dx, dy, dz, var_dx, var_dy, var_dz"
        if reversed:
            ddx = vec1.dx + vec2.dx
            ddy = vec1.dy + vec2.dy
            ddz = vec1.dz + vec2.dz
        else:
            ddx = vec1.dx - vec2.dx
            ddy = vec1.dy - vec2.dy
            ddz = vec1.dz - vec2.dz
        
        # variances
        var1 = vec1.var
        var2 = vec2.var
        var_ddx = var1[0] + var2[0]
        var_ddy = var1[1] + var2[1]
        var_ddz = var1[2] + var2[2]

        # covariances
        cov1 = vec1.cov
        cov2 = vec2.cov
        cov_xy = cov1[0] + cov2[0]
        cov_xz = cov1[1] + cov2[1]
        cov_yz = cov1[2] + cov2[2]

        #if excludeZero:
        #    if (ddx, ddy, ddz) == (0, 0, 0):
        #        return

        # append vector 
        vec = ObsVector(fromid=vec1.fromid, toid=vec1.toid,
                        dx=ddx, dy=ddy, dz=ddz)
        self._cluster.append_obs(vec)
        
        # extend covariance matrix and add values
        self._cluster.covmat.extend_dim(3)
        vec.var = (var_ddx, var_ddy, var_ddz)
        vec.cov = (cov_xy, cov_xz, cov_yz)

    
    def str_mean(self):
        str = ["\nObservations 1: mean and residuals",
               "=================================="]

        str.extend(["\nmean:%s\n\nobservations:%s\n\nresiduals:%s"\
                    % (mean, obs, res) \
                    for mean, obs, res in zip(self.meanList1,
                                              self.obsMeanList1,
                                              self.resList1)])

        str.extend(["\nObservations 2: mean and residuals",
                    "=================================="])

        str.extend(["\nmean:%s\n\nobservations:%s\n\nresiduals:%s"\
                    % (mean, obs, res) \
                    for mean, obs, res in zip(self.meanList2,
                                              self.obsMeanList2,
                                              self.resList2)])

        return "\n".join(str)

    def str_summary(self):
        tt = TextTable([("", "%10s"),
                        ("ep 0", "%4i"),
                        ("ep 1", "%4i")])
        str = [tt.make_table_head()]
        
        sum1 = sum([len(cl) for cl in self.obsToComp1])
        sum2 = sum([len(cl) for cl in self.obsToComp2])
        sumComp = sum([len(cl) for cl in self.compObsClList])
        sumu1 = sum([len(cl) for cl in self.uncompObsClList1])
        sumu2 = sum([len(cl) for cl in self.uncompObsClList2])
    

        str.append(tt.make_table_row("sum obs", sum1, sum2))
        str.append(tt.make_table_row("obs comp", sumComp, sumComp))
        str.append(tt.make_table_row("obs uncomp", sumu1, sumu2))

        str.append(tt.make_table_foot())

        return "".join(str)


    def str_uncompared(self):

        str = ["\nUncompared observations 1:",
               "=========================="]
        str.append(self.uncompObsClList1.__str__())
        str.extend(["\nUncompared observations 2:",
                    "=========================="])
        str.append(self.uncompObsClList2.__str__())

        return "\n".join(str)


    def str_compared(self):
        str = ["\nObservations compared:",
                 "======================"]
        
        str.append(self.compObsClList.__str__())
        return "\n".join(str)

    def __str__(self):
        return self.str_compared()