def compare_vector(self, mean=True, revers=True): """ comparison of vectors mean: compute vector mean? revers: compare observations with revers id from and to? """ self.revers =revers # find vectors in cluster list self.obsToComp1 = ObsClusterList() # cluster list with vectors self.obsToComp2 = ObsClusterList() for cl in self.data1.obsClusterList: if isinstance(cl, ObsClusterVector): self.obsToComp1.append_cluster(cl) for cl in self.data2.obsClusterList: if isinstance(cl, ObsClusterVector): self.obsToComp2.append_cluster(cl) if sum([len(c) for c in self.obsToComp1]) is 0: return #raise CompareObsError, "No vectors found in data #1" if sum([len(c) for c in self.obsToComp2]) is 0: return #raise CompareObsError, "No vectors found in data #2" if mean: self.obsToComp1, self.meanList1, self.obsMeanList1, self.resList1 =\ self._mean_vector(self.obsToComp1) self.obsToComp2, self.meanList2, self.obsMeanList2, self.resList2 =\ self._mean_vector(self.obsToComp2) # cluster for compared vectors # covariance matrix - initial zero dimension self._cluster = ObsClusterVector(covmat=CovMat(dim=0, band=2)) self.compObsClList.append_cluster(self._cluster) # find vectors with same fromid and toid and compare for c1 in self.obsToComp1: for vec1 in c1: found, vec2, reversed = self._find_obs(vec1, self.obsToComp2) if found: self._cmp_vector(vec1, vec2, reversed=reversed) # find uncompared vectors self._uncompared_obs()
def _mean_vector(self, data): "returns mean of vector with same fromid and toid" # add vectors to list lvec = [] # helper list of vectors for c in data: for v in c: lvec.append(v) datamean = ObsClusterList() dmcluster = ObsClusterVector(covmat=CovMat(dim=0, band=2)) datamean.append_cluster(dmcluster) # list of vector means and single vectors (without mean) cll_mean = ObsClusterList() cll_obs = ObsClusterList() cll_res = ObsClusterList() if len(lvec) <= 1: return # nothink to compare while len(lvec) > 1: # find vectors with same fromid and toid v1 = lvec.pop() fromid = v1.fromid toid = v1.toid lvecm = [v1] # list of vectors for mean lsign = [1.0] # list of signs: for revers vectors -1.0 for v2 in lvec: if v2.fromid == fromid and v2.toid == toid: lvecm.append(v2) lvec.remove(v2) lsign.append(1.0) continue if self.revers and\ v2.fromid == toid and v2.toid == fromid: lvecm.append(v2) lvec.remove(v2) lsign.append(-1.0) # compute mean if float(len(lvecm)) == 1: # nothink to compute vec = lvecm[0] covmat = vec.covmat dmcluster.append_obs(vec) dmcluster.covmat.extend_dim(3) vec.covmat = covmat continue from numpy import mat, mean ll = [mat([i.dx, i.dy, i.dz])*sign for i,sign in zip(lvecm,lsign)] # row vectors of observ. #lm = mean(ll, axis=0) #print "Mean 0:", lm #print "covmat0:", lvecm[0].covmat.mat # compute covariance matrix # covariances between vectors not handled sil = [i.covmat.mat.I for i in lvecm] # covariance matrice inverses inv(Sigma) sill = [si*l.T for si,l in zip(sil,ll)] # product inv(Sigma)*l cmm = sum(sil).I # covariance matrix of mean mn = cmm * sum(sill) # mean value #print "Mean 1:", mn #print "covmat1:", cmm # append clusters cl_mean = ObsClusterVector(covmat=CovMat(dim=3, band=2)) cl_obs = ObsClusterVector(covmat=CovMat(dim=0, band=2)) cl_res = ObsClusterVector(covmat=None) cll_mean.append_cluster(cl_mean) cll_obs.append_cluster(cl_obs) cll_res.append_cluster(cl_res) # append vector mean vec1 = ObsVector(fromid=lvecm[0].fromid, toid=lvecm[0].toid, dx=float(mn[0]), dy=float(mn[1]), dz=float(mn[2])) vec2 = ObsVector(fromid=lvecm[0].fromid, toid=lvecm[0].toid, dx=float(mn[0]), dy=float(mn[1]), dz=float(mn[2])) # vec1 == vec2 cl_mean.append_obs(vec1) dmcluster.append_obs(vec2) # set covmat of mean covmat = CovMat(dim=3, band=2) covmat.mat = cmm vec1.covmat = covmat dmcluster.covmat.extend_dim(3) vec2.covmat = covmat # append observations with covmat for v in lvecm: vcovmat = v.covmat cl_obs.append_obs(v) cl_obs.covmat.extend_dim(3) v.covmat = vcovmat # append residuals for l,v,s in zip(ll,lvecm,lsign): res = (mn - l.T) * s vec = ObsVector(fromid=v.fromid, toid=v.toid, dx=float(res[0]), dy=float(res[1]), dz=float(res[2])) cl_res.append_obs(vec) return datamean, cll_mean, cll_obs, cll_res
class CompareObs(object): """ compares measured data """ def __init__(self, data1, data2, revers=True): """ data1, data2: observation data - GamaLocalDataObs instance revers: handle fromid_1 == toid_2 and toid_1 == fromid_2 """ self.data1 = data1 self.data2 = data2 self.revers = revers self.compObsClList = ObsClusterList() # list of differences from comparison # data 1 self.meanList1 = None # list of means self.obsMeanList1 = None # list of observations for mean self.resList1 = None # list of residuals self.obsToComp1 = None # observations in cluster list to be compared # data 2 self.meanList2 = None self.obsMeanList2 = None self.resList2 = None self.obsToComp2 = None # make list of uncompared observations self.uncompObsClList1 = ObsClusterList() # cluster list with observations uncompared self.uncompObsClList2 = ObsClusterList() self._cluster = None # actual cluster def _mean_vector(self, data): "returns mean of vector with same fromid and toid" # add vectors to list lvec = [] # helper list of vectors for c in data: for v in c: lvec.append(v) datamean = ObsClusterList() dmcluster = ObsClusterVector(covmat=CovMat(dim=0, band=2)) datamean.append_cluster(dmcluster) # list of vector means and single vectors (without mean) cll_mean = ObsClusterList() cll_obs = ObsClusterList() cll_res = ObsClusterList() if len(lvec) <= 1: return # nothink to compare while len(lvec) > 1: # find vectors with same fromid and toid v1 = lvec.pop() fromid = v1.fromid toid = v1.toid lvecm = [v1] # list of vectors for mean lsign = [1.0] # list of signs: for revers vectors -1.0 for v2 in lvec: if v2.fromid == fromid and v2.toid == toid: lvecm.append(v2) lvec.remove(v2) lsign.append(1.0) continue if self.revers and\ v2.fromid == toid and v2.toid == fromid: lvecm.append(v2) lvec.remove(v2) lsign.append(-1.0) # compute mean if float(len(lvecm)) == 1: # nothink to compute vec = lvecm[0] covmat = vec.covmat dmcluster.append_obs(vec) dmcluster.covmat.extend_dim(3) vec.covmat = covmat continue from numpy import mat, mean ll = [mat([i.dx, i.dy, i.dz])*sign for i,sign in zip(lvecm,lsign)] # row vectors of observ. #lm = mean(ll, axis=0) #print "Mean 0:", lm #print "covmat0:", lvecm[0].covmat.mat # compute covariance matrix # covariances between vectors not handled sil = [i.covmat.mat.I for i in lvecm] # covariance matrice inverses inv(Sigma) sill = [si*l.T for si,l in zip(sil,ll)] # product inv(Sigma)*l cmm = sum(sil).I # covariance matrix of mean mn = cmm * sum(sill) # mean value #print "Mean 1:", mn #print "covmat1:", cmm # append clusters cl_mean = ObsClusterVector(covmat=CovMat(dim=3, band=2)) cl_obs = ObsClusterVector(covmat=CovMat(dim=0, band=2)) cl_res = ObsClusterVector(covmat=None) cll_mean.append_cluster(cl_mean) cll_obs.append_cluster(cl_obs) cll_res.append_cluster(cl_res) # append vector mean vec1 = ObsVector(fromid=lvecm[0].fromid, toid=lvecm[0].toid, dx=float(mn[0]), dy=float(mn[1]), dz=float(mn[2])) vec2 = ObsVector(fromid=lvecm[0].fromid, toid=lvecm[0].toid, dx=float(mn[0]), dy=float(mn[1]), dz=float(mn[2])) # vec1 == vec2 cl_mean.append_obs(vec1) dmcluster.append_obs(vec2) # set covmat of mean covmat = CovMat(dim=3, band=2) covmat.mat = cmm vec1.covmat = covmat dmcluster.covmat.extend_dim(3) vec2.covmat = covmat # append observations with covmat for v in lvecm: vcovmat = v.covmat cl_obs.append_obs(v) cl_obs.covmat.extend_dim(3) v.covmat = vcovmat # append residuals for l,v,s in zip(ll,lvecm,lsign): res = (mn - l.T) * s vec = ObsVector(fromid=v.fromid, toid=v.toid, dx=float(res[0]), dy=float(res[1]), dz=float(res[2])) cl_res.append_obs(vec) return datamean, cll_mean, cll_obs, cll_res def _find_obs(self, obs, cl): """ finds observation in cluster list returns found, observation, reversed """ fromid = obs.fromid toid = obs.toid for c in cl: for o in c: if o.fromid == fromid and o.toid == toid: return True, o, False if self.revers and \ o.fromid == toid and o.toid == fromid: return True, o, True return False, None, False def compare_vector(self, mean=True, revers=True): """ comparison of vectors mean: compute vector mean? revers: compare observations with revers id from and to? """ self.revers =revers # find vectors in cluster list self.obsToComp1 = ObsClusterList() # cluster list with vectors self.obsToComp2 = ObsClusterList() for cl in self.data1.obsClusterList: if isinstance(cl, ObsClusterVector): self.obsToComp1.append_cluster(cl) for cl in self.data2.obsClusterList: if isinstance(cl, ObsClusterVector): self.obsToComp2.append_cluster(cl) if sum([len(c) for c in self.obsToComp1]) is 0: return #raise CompareObsError, "No vectors found in data #1" if sum([len(c) for c in self.obsToComp2]) is 0: return #raise CompareObsError, "No vectors found in data #2" if mean: self.obsToComp1, self.meanList1, self.obsMeanList1, self.resList1 =\ self._mean_vector(self.obsToComp1) self.obsToComp2, self.meanList2, self.obsMeanList2, self.resList2 =\ self._mean_vector(self.obsToComp2) # cluster for compared vectors # covariance matrix - initial zero dimension self._cluster = ObsClusterVector(covmat=CovMat(dim=0, band=2)) self.compObsClList.append_cluster(self._cluster) # find vectors with same fromid and toid and compare for c1 in self.obsToComp1: for vec1 in c1: found, vec2, reversed = self._find_obs(vec1, self.obsToComp2) if found: self._cmp_vector(vec1, vec2, reversed=reversed) # find uncompared vectors self._uncompared_obs() def _uncompared_obs(self): for cl in self.obsToComp1: # add cluster to cluster list uc1 = cl.__class__() added = False # added some observation for obs in cl: found, x, x = self._find_obs(obs, self.compObsClList) if not found: uc1.append_obs(obs) added = True if added: self.uncompObsClList1.append_cluster(uc1) for cl in self.obsToComp2: # add cluster to cluster list uc2 = cl.__class__() added = False # added some observation for obs in cl: found, x ,x = self._find_obs(obs, self.compObsClList) if not found: uc2.append_obs(obs) added = True if added: self.uncompObsClList2.append_cluster(uc2) def _cmp_vector(self, vec1, vec2, reversed=False): "comparison of two vectors - dx, dy, dz, var_dx, var_dy, var_dz" if reversed: ddx = vec1.dx + vec2.dx ddy = vec1.dy + vec2.dy ddz = vec1.dz + vec2.dz else: ddx = vec1.dx - vec2.dx ddy = vec1.dy - vec2.dy ddz = vec1.dz - vec2.dz # variances var1 = vec1.var var2 = vec2.var var_ddx = var1[0] + var2[0] var_ddy = var1[1] + var2[1] var_ddz = var1[2] + var2[2] # covariances cov1 = vec1.cov cov2 = vec2.cov cov_xy = cov1[0] + cov2[0] cov_xz = cov1[1] + cov2[1] cov_yz = cov1[2] + cov2[2] #if excludeZero: # if (ddx, ddy, ddz) == (0, 0, 0): # return # append vector vec = ObsVector(fromid=vec1.fromid, toid=vec1.toid, dx=ddx, dy=ddy, dz=ddz) self._cluster.append_obs(vec) # extend covariance matrix and add values self._cluster.covmat.extend_dim(3) vec.var = (var_ddx, var_ddy, var_ddz) vec.cov = (cov_xy, cov_xz, cov_yz) def str_mean(self): str = ["\nObservations 1: mean and residuals", "=================================="] str.extend(["\nmean:%s\n\nobservations:%s\n\nresiduals:%s"\ % (mean, obs, res) \ for mean, obs, res in zip(self.meanList1, self.obsMeanList1, self.resList1)]) str.extend(["\nObservations 2: mean and residuals", "=================================="]) str.extend(["\nmean:%s\n\nobservations:%s\n\nresiduals:%s"\ % (mean, obs, res) \ for mean, obs, res in zip(self.meanList2, self.obsMeanList2, self.resList2)]) return "\n".join(str) def str_summary(self): tt = TextTable([("", "%10s"), ("ep 0", "%4i"), ("ep 1", "%4i")]) str = [tt.make_table_head()] sum1 = sum([len(cl) for cl in self.obsToComp1]) sum2 = sum([len(cl) for cl in self.obsToComp2]) sumComp = sum([len(cl) for cl in self.compObsClList]) sumu1 = sum([len(cl) for cl in self.uncompObsClList1]) sumu2 = sum([len(cl) for cl in self.uncompObsClList2]) str.append(tt.make_table_row("sum obs", sum1, sum2)) str.append(tt.make_table_row("obs comp", sumComp, sumComp)) str.append(tt.make_table_row("obs uncomp", sumu1, sumu2)) str.append(tt.make_table_foot()) return "".join(str) def str_uncompared(self): str = ["\nUncompared observations 1:", "=========================="] str.append(self.uncompObsClList1.__str__()) str.extend(["\nUncompared observations 2:", "=========================="]) str.append(self.uncompObsClList2.__str__()) return "\n".join(str) def str_compared(self): str = ["\nObservations compared:", "======================"] str.append(self.compObsClList.__str__()) return "\n".join(str) def __str__(self): return self.str_compared()