def load(self, dataPath, labelsPath, test=False, verbose=True): # load data as previously super(CNNDataset,self).load(dataPath,labelsPath,test=test) # don't compress the test data if test: return # create the compressed search data remIdx = [0] it = 0 self.tree = spatial.cKDTree(self.data[remIdx]) changed = True "Iteratively updating the search tree" while len(remIdx) > 0: # print iteration number print "Iteration %s" % it it +=1 # add data to the search tree changed = False for i,d in enumerate(self.data): if i in remIdx: continue temp = self.classify(d) print i, len(remIdx) if temp != self.labels[i]: remIdx.append(i) if len(remIdx) % 10 == 0: self.tree = spatial.cKDTree(self.data[remIdx]) changed = True
def interlinks(cluster1,cluster2,neighbours=None): ''' This function searches a certain set of neighbours between two clusters. Parameters ---------- cluster1,cluster2 : list of 1d ndarray The coordinates of the clusters. neighbours : dict, optional The neighbour-length map of the links to be searched. Returns ------- list of Link The searched links. ''' result=[] if len(cluster1)>0 and len(cluster2)>0: if neighbours is None: neighbours={0:0.0} smatrix=cKDTree(cluster1).sparse_distance_matrix(cKDTree(cluster2),np.max(list(neighbours.values()))+RZERO) for (i,j),dist in smatrix.items(): for neighbour,length in neighbours.items(): if abs(length-dist)<RZERO: result.append(Link(neighbour,sindex=i,eindex=j,disp=0)) break return result
def test_ckdtree_box(): # check ckdtree periodic boundary n = 2000 m = 2 k = 3 np.random.seed(1234) data = np.random.uniform(size=(n, m)) kdtree = cKDTree(data, leafsize=1, boxsize=1.0) # use the standard python KDTree for the simulated periodic box kdtree2 = cKDTree(data, leafsize=1) dd, ii = kdtree.query(data, k) dd1, ii1 = kdtree.query(data + 1.0, k) assert_almost_equal(dd, dd1) assert_equal(ii, ii1) dd1, ii1 = kdtree.query(data - 1.0, k) assert_almost_equal(dd, dd1) assert_equal(ii, ii1) dd2, ii2 = simulate_periodic_box(kdtree2, data, k, boxsize=1.0) assert_almost_equal(dd, dd2) assert_equal(ii, ii2)
def mutual_information2(x, y, k=3): """ Return mutual information estimate based on Ross B C (PlosOne 2014). x contains discrete variables, y contains continuous variables. """ x = np.array(x) y = np.array(y) discrete_vars = set(x) cats = dict((d, y[x == d][:,np.newaxis]) for d in discrete_vars) Nx = dict((d, cats[d].shape[0]) for d in discrete_vars) # one KDTree for each discrete variable kdtrees = dict((d, cKDTree(cats[d])) for d in discrete_vars) # one KDTree for all data points full_tree = cKDTree(y[:,np.newaxis]) N = len(x) # vectorized KD tree. Calculation of the m_i is the bottle neck # no idea how to speed it up further I = 0 for d, data in cats.iteritems(): ds, __ = kdtrees[d].query(data, k=k+1, p=np.inf) ds = ds[:,-1] mis = np.array([len(full_tree.query_ball_point(dat, r, p=np.inf)) - 1.0 for dat, r in izip(data, ds)]) I -= psi(mis).sum() + len(data)*psi(Nx[d]) return I/N + psi(N) + psi(k)
def bench_query_ball_point(self): print print ' Query ball point kd-tree' print '===============================================================' print ' dim | # points | # queries | probe radius | KDTree | cKDTree | flat cKDTree' for (m, n, r, repeat) in [(3,10000,1000,3)]:#, # (8,10000,1000,3), # (16,10000,1000,3)]: for probe_radius in (0.2, 0.5): print '%4s | %8s | %9s | %11.1f ' % (m, n, r, probe_radius), sys.stdout.flush() data = np.concatenate((np.random.randn(n//2,m), np.random.randn(n-n//2,m)+np.ones(m))) queries = np.concatenate((np.random.randn(r//2,m), np.random.randn(r-r//2,m)+np.ones(m))) T1 = KDTree(data) T2 = cKDTree(data) T3 = cKDTree(data,leafsize=n) print '| %6.3fs ' % (measure('T1.query_ball_point(queries, probe_radius)', 1) / 1), sys.stdout.flush() print '| %6.3fs' % (measure('T2.query_ball_point(queries, probe_radius)', repeat) / repeat), sys.stdout.flush() print '| %6.3fs' % (measure('T3.query_ball_point(queries, probe_radius)', repeat) / repeat), sys.stdout.flush() print ''
def bench_sparse_distance_matrix(self): print print ' Sparse distance matrix kd-tree' print '====================================================================' print ' dim | # points T1 | # points T2 | probe radius | KDTree | cKDTree' for (m, n1, n2, repeat) in [(3,1000,1000,30), (8,1000,1000,30), (16,1000,1000,30)]: data1 = np.concatenate((np.random.randn(n1//2,m), np.random.randn(n1-n1//2,m)+np.ones(m))) data2 = np.concatenate((np.random.randn(n2//2,m), np.random.randn(n2-n2//2,m)+np.ones(m))) T1 = KDTree(data1) T2 = KDTree(data2) cT1 = cKDTree(data1) cT2 = cKDTree(data2) for probe_radius in (0.2, 0.5): print '%4s | %11s | %11s | %11.1f ' % (m, n1, n2, probe_radius), sys.stdout.flush() print '| %6.3fs ' % (measure('T1.sparse_distance_matrix(T2, probe_radius)', 1) / 1), sys.stdout.flush() print '| %6.3fs ' % (measure('cT1.sparse_distance_matrix(cT2, probe_radius)', repeat) / repeat), sys.stdout.flush() print ''
def kraskov_mi(x,y,k=5): ''' Estimate the mutual information I(X;Y) of X and Y from samples {x_i, y_i}_{i=1}^N Using KSG mutual information estimator Input: x: 2D list of size N*d_x y: 2D list of size N*d_y k: k-nearest neighbor parameter Output: one number of I(X;Y) ''' assert len(x)==len(y), "Lists should have same length" assert k <= len(x)-1, "Set k smaller than num. samples - 1" N = len(x) dx = len(x[0]) dy = len(y[0]) data = np.concatenate((x,y),axis=1) tree_xy = ss.cKDTree(data) tree_x = ss.cKDTree(x) tree_y = ss.cKDTree(y) knn_dis = [tree_xy.query(point,k+1,p=float('inf'))[0][k] for point in data] ans_xy = -digamma(k) + digamma(N) + (dx+dy)*log(2)#2*log(N-1) - digamma(N) #+ vd(dx) + vd(dy) - vd(dx+dy) ans_x = digamma(N) + dx*log(2) ans_y = digamma(N) + dy*log(2) for i in range(N): ans_xy += (dx+dy)*log(knn_dis[i])/N ans_x += -digamma(len(tree_x.query_ball_point(x[i],knn_dis[i]-1e-15,p=float('inf'))))/N+dx*log(knn_dis[i])/N ans_y += -digamma(len(tree_y.query_ball_point(y[i],knn_dis[i]-1e-15,p=float('inf'))))/N+dy*log(knn_dis[i])/N return ans_x+ans_y-ans_xy
def getkernmat(dirsout, dirsin, kappa, eps, chiral_flip): "Assume dirsout and dirsin and unit norm vectors" arghash = hash((dirsout[::10].tostring(), dirsin[::10].tostring(), kappa, eps, chiral_flip)) if arghash in precomputed_kernmats: return precomputed_kernmats[arghash] else: kdout = ss.cKDTree( dirsout * (1.0 + eps) ) # this is to make sure the distance between identicals is nonzero (bad hack but it works) kdin = ss.cKDTree(dirsin) if chiral_flip: kdin2 = ss.cKDTree(-dirsin) distthres = n.sqrt(-2.0 / kappa * n.log(eps)) distmat = kdout.sparse_distance_matrix(kdin, distthres).tocsr() if chiral_flip: distmat += kdout.sparse_distance_matrix(kdin2, distthres).tocsr() distmat = distmat.tocoo() kerndata = n.exp(-kappa / 2.0 * distmat.data ** 2) kernmat = sp.coo_matrix((kerndata, (distmat.row, distmat.col)), distmat.shape) precomputed_kernmats[arghash] = kernmat return kernmat
def axisBasedInit(allowed,arraySize,strongDist,weakDist,strongAxis,weakAxis,connectionPoint): #This initializer attempts to find the Strong (or most damaging) axis of the wind profile and place turbines off it. #The separation distance is estimated using a formula that finds the optimal distance assuming two turbines are connected using roads and cables #This is obviously not true, but should be a good starting point. zippedAllowed = numpy.array(zip(allowed[0],allowed[1])) allowedTree = cKDTree(zippedAllowed) #We start byfinding the closest allowed position to the connection point. distance, ind = allowedTree.query(connectionPoint, k =1) positions = numpy.zeros((arraySize,2)) positions[0] = allowedTree.data[ind] positionsTree = cKDTree(positions) axes = numpy.zeros((4,2)) axes[0] = weakAxis axes[1] = -weakAxis axes[2] = strongAxis axes[3] = -strongAxis index = 0 distances = numpy.array([weakDist,weakDist,strongDist,strongDist]) while len(positions) < arraySize: departurePt = positions[index] positions = searchOnAxes(distances,departurePt,axes,allowedTree,positions) numpy.concatenate((positions,newPositions)) index = index + 1 if len(positions) > arraySize: diff = len(positions) - arraySize for i in range(diff): r = random.randint(0,len(positions)) positions = numpy.delete(positions,r,0) return positions, allowedTree
def __init__(self,ncfile='woa13_decav_04v2.nc'): nc = netCDF4.Dataset(ncfile) sv = nc.variables latslice=slice(173,262) # 45.-67. lonslice=slice(320,455) # -20.-13.625 self.lon = sv['lon'][lonslice] self.lat = sv['lat'][latslice] self.d = -sv['depth'][:] self.time = sv['time'][:] self.timeunits = sv['time'].units self.s = sv['s_mn'][:,:,latslice,lonslice] self.t = sv['t_mn'][:,:,latslice,lonslice] self.lon2,self.lat2 = meshgrid(self.lon,self.lat) if True: self.s_tree={} self.t_tree={} self.s_var={} self.t_var={} for tidx,time in enumerate(self.time): print(' build trees for month=%d'%(tidx+1)) self.d3,self.lat3,self.lon3 = meshgrid(self.d,self.lat*100.,self.lon*100.,indexing='ij') vlon3 = self.lon3[where(self.s.mask[tidx]==False)] vlat3 = self.lat3[where(self.s.mask[tidx]==False)] vd3 = self.d3[where(self.s.mask[tidx]==False)] self.s_tree[tidx]=cKDTree(zip(vlon3,vlat3,vd3)) vlon3 = self.lon3[where(self.t.mask[tidx]==False)] vlat3 = self.lat3[where(self.t.mask[tidx]==False)] vd3 = self.d3[where(self.t.mask[tidx]==False)] self.t_tree[tidx]=cKDTree(zip(vlon3,vlat3,vd3)) self.s_var[tidx] = self.s[tidx][where(self.s.mask[tidx]==False)].flatten() self.t_var[tidx] = self.t[tidx][where(self.t.mask[tidx]==False)].flatten()
def setUp(self): n = 50 m = 4 np.random.seed(0) self.T1 = cKDTree(np.random.randn(n,m),leafsize=2) self.T2 = cKDTree(np.random.randn(n,m),leafsize=2) self.r = 0.3
def max_neighbor(self, in_lon, in_lat, radius=0.05): """ Finds the largest value within a given radius of a point on the interpolated grid. Args: in_lon: 2D array of longitude values in_lat: 2D array of latitude values radius: radius of influence for largest neighbor search in degrees Returns: Array of interpolated data """ out_data = np.zeros((self.data.shape[0], in_lon.shape[0], in_lon.shape[1])) in_tree = cKDTree(np.vstack((in_lat.ravel(), in_lon.ravel())).T) out_indices = np.indices(out_data.shape[1:]) out_rows = out_indices[0].ravel() out_cols = out_indices[1].ravel() for d in range(self.data.shape[0]): nz_points = np.where(self.data[d] > 0) if len(nz_points[0]) > 0: nz_vals = self.data[d][nz_points] nz_rank = np.argsort(nz_vals) original_points = cKDTree(np.vstack((self.lat[nz_points[0][nz_rank]], self.lon[nz_points[1][nz_rank]])).T) all_neighbors = original_points.query_ball_tree(in_tree, radius, p=2, eps=0) for n, neighbors in enumerate(all_neighbors): if len(neighbors) > 0: out_data[d, out_rows[neighbors], out_cols[neighbors]] = nz_vals[nz_rank][n] return out_data
def bench_count_neighbors(self): print() print(' Count neighbors kd-tree') print('====================================================================') print(' dim | # points T1 | # points T2 | probe radius | KDTree | cKDTree') for (m, n1, n2, repeat) in [(3,1000,1000,30), (8,1000,1000,30), (16,1000,1000,30)]: data1 = np.concatenate((np.random.randn(n1//2,m), np.random.randn(n1-n1//2,m)+np.ones(m))) data2 = np.concatenate((np.random.randn(n2//2,m), np.random.randn(n2-n2//2,m)+np.ones(m))) T1 = KDTree(data1) T2 = KDTree(data2) cT1 = cKDTree(data1) cT2 = cKDTree(data2) for probe_radius in (0.2, 0.5): print('%4s | %11s | %11s | %11.1f ' % (m, n1, n2, probe_radius), end=' ') sys.stdout.flush() print('| %6.3fs ' % (measure('T1.count_neighbors(T2, probe_radius)', 1) / 1), end=' ') sys.stdout.flush() print('| %6.3fs ' % (measure('cT1.count_neighbors(cT2, probe_radius)', repeat) / repeat), end=' ') sys.stdout.flush() print('')
def test_ckdtree_box_upper_bounds(): data = np.linspace(0, 2, 10).reshape(-1, 2) data[:, 1] += 10 assert_raises(ValueError, cKDTree, data, leafsize=1, boxsize=1.0) assert_raises(ValueError, cKDTree, data, leafsize=1, boxsize=(0.0, 2.0)) # skip a dimension. cKDTree(data, leafsize=1, boxsize=(2.0, 0.0))
def bench_query(self): print() print(' Querying kd-tree') print('===============================================================') print(' dim | # points | # queries | KDTree | cKDTree | flat cKDTree') for (m, n, r, repeat) in [(3,10000,1000,3), (8,10000,1000,3), (16,10000,1000,3)]: print('%4s | %8s | %8s ' % (m, n, r), end=' ') sys.stdout.flush() data = np.concatenate((np.random.randn(n//2,m), np.random.randn(n-n//2,m)+np.ones(m))) queries = np.concatenate((np.random.randn(r//2,m), np.random.randn(r-r//2,m)+np.ones(m))) T1 = KDTree(data) T2 = cKDTree(data) T3 = cKDTree(data,leafsize=n) print('| %6.3fs ' % (measure('T1.query(queries)', 1) / 1), end=' ') sys.stdout.flush() print('| %6.3fs' % (measure('T2.query(queries)', repeat) / repeat), end=' ') sys.stdout.flush() print('| %6.3fs' % (measure('T3.query(queries)', repeat) / repeat), end=' ') sys.stdout.flush() print('')
def bench_query_pairs(self): print() print(' Query pairs kd-tree') print('==================================================================') print(' dim | # points | probe radius | KDTree | cKDTree | flat cKDTree') for (m, n, repeat) in [(3,1000,30), (8,1000,30), (16,1000,30)]: for probe_radius in (0.2, 0.5): print('%4s | %8s | %11.1f ' % (m, n, probe_radius), end=' ') sys.stdout.flush() data = np.concatenate((np.random.randn(n//2,m), np.random.randn(n-n//2,m)+np.ones(m))) T1 = KDTree(data) T2 = cKDTree(data) T3 = cKDTree(data,leafsize=n) print('| %6.3fs ' % (measure('T1.query_pairs(probe_radius)', 1) / 1), end=' ') sys.stdout.flush() print('| %6.3fs' % (measure('T2.query_pairs(probe_radius)', repeat) / repeat), end=' ') sys.stdout.flush() print('| %6.3fs' % (measure('T3.query_pairs(probe_radius)', repeat) / repeat), end=' ') sys.stdout.flush() print('')
def search_tree(self, centers, radius): """ Searches all the pairs within `radius` between `centers` and ``coords`` ``coords`` are the already initialized coordinates in the tree during :meth:`set_coords`. ``centers`` are wrapped around the primary unit cell if PBC is desired. Minimum image convention (PBC) is activated if the `box` argument is provided during class initialization Parameters ---------- centers: array_like (N,3) coordinate array to search for neighbors radius: float maximum distance to search for neighbors. Returns ------- pairs : array all the pairs between ``coords`` and ``centers`` Note ---- This method constructs another tree from the ``centers`` and queries the previously built tree (built in :meth:`set_coords`) """ if not self._built: raise RuntimeError('Unbuilt tree. Run tree.set_coords(...)') centers = np.asarray(centers) if centers.shape == (self.dim, ): centers = centers.reshape((1, self.dim)) # Sanity check if self.pbc: if self.cutoff < radius: raise RuntimeError('Set cutoff greater or equal to the radius.') # Bring all query points to the central cell wrapped_centers = apply_PBC(centers, self.box) other_tree = cKDTree(wrapped_centers, leafsize=self.leafsize) pairs = other_tree.query_ball_tree(self.ckdt, radius) pairs = np.array([[i, j] for i, lst in enumerate(pairs) for j in lst], dtype=np.int64) if pairs.size > 0: pairs[:, 1] = undo_augment(pairs[:, 1], self.mapping, len(self.coords)) else: other_tree = cKDTree(centers, leafsize=self.leafsize) pairs = other_tree.query_ball_tree(self.ckdt, radius) pairs = np.array([[i, j] for i, lst in enumerate(pairs) for j in lst], dtype=np.int64) if pairs.size > 0: pairs = unique_rows(pairs) return pairs
def period_neighborhood_probability(self, radius, smoothing, threshold, stride,start_time,end_time): """ Calculate the neighborhood probability over the full period of the forecast Args: radius: circular radius from each point in km smoothing: width of Gaussian smoother in km threshold: intensity of exceedance stride: number of grid points to skip for reduced neighborhood grid Returns: (neighborhood probabilities) """ neighbor_x = self.x[::stride, ::stride] neighbor_y = self.y[::stride, ::stride] neighbor_kd_tree = cKDTree(np.vstack((neighbor_x.ravel(), neighbor_y.ravel())).T) neighbor_prob = np.zeros((self.data.shape[0], neighbor_x.shape[0], neighbor_x.shape[1])) print('Forecast Hours: {0}-{1}'.format(start_time, end_time)) for m in range(len(self.members)): period_max = self.data[m,start_time:end_time,:,:].max(axis=0) valid_i, valid_j = np.where(period_max >= threshold) print(self.members[m], len(valid_i)) if len(valid_i) > 0: var_kd_tree = cKDTree(np.vstack((self.x[valid_i, valid_j], self.y[valid_i, valid_j])).T) exceed_points = np.unique(np.concatenate(var_kd_tree.query_ball_tree(neighbor_kd_tree, radius))).astype(int) exceed_i, exceed_j = np.unravel_index(exceed_points, neighbor_x.shape) neighbor_prob[m][exceed_i, exceed_j] = 1 if smoothing > 0: neighbor_prob[m] = gaussian_filter(neighbor_prob[m], smoothing,mode='constant') return neighbor_prob
def period_neighborhood_probability(self, radius, smoothing, threshold, stride, x, y, dx): """ Calculate the neighborhood probability over the full period of the forecast Args: radius: circular radius from each point in km smoothing: width of Gaussian smoother in km threshold: intensity of exceedance stride: number of grid points to skip for reduced neighborhood grid x: x-coordinate array in km y: y-coordinate array in km dx: distance between grid points in km Returns: neighborhood probablities """ neighbor_x = x[::stride, ::stride] neighbor_y = y[::stride, ::stride] neighbor_kd_tree = cKDTree(np.vstack((neighbor_x.ravel(), neighbor_y.ravel())).T) neighbor_prob = np.zeros((neighbor_x.shape[0], neighbor_x.shape[1])) period_max = self.data.max(axis=0) valid_i, valid_j = np.where(period_max >= threshold) if len(valid_i) > 0: var_kd_tree = cKDTree(np.vstack((x[valid_i, valid_j], y[valid_i, valid_j])).T) exceed_points = np.unique(np.concatenate(var_kd_tree.query_ball_tree(neighbor_kd_tree, radius))).astype(int) exceed_i, exceed_j = np.unravel_index(exceed_points, neighbor_x.shape) neighbor_prob[exceed_i, exceed_j] = 1 if smoothing > 0: neighbor_prob = gaussian_filter(neighbor_prob, int(smoothing / dx / stride)) return neighbor_prob
def set_coords(self, coords): """Unclear what this does. James??""" # Normalize coordinates 0-1 if np.any(coords.max(0) > 1) or np.any(coords.min(0) < 0): coords -= coords.min(0) coords /= coords.max(0) # Renormalize coordinates to shape of svg self.coords = coords * self.svgshape # Update of scipy (0.16+) means that cKDTree hangs / takes absurdly long to compute with new default # balanced_tree=True. Seems only to be true on Mac OS, for whatever reason. Possibly a broken # C library, unclear. Setting balanced_tree=False seems to resolve the issue, thus going with that for now # See http://stackoverflow.com/questions/31819778/scipy-spatial-ckdtree-running-slowly try: # not compatible with scipy version < 0.16 self.kdt = cKDTree(self.coords, balanced_tree=False) except: # Older call signature self.kdt = cKDTree(self.coords) for layer in self: for name in layer.labels.elements: for element in layer.labels.elements[name]: x, y = float(element.get("x")), float(element.get("y")) dist, idx = self.kdt.query((x, self.svgshape[1]-y)) if idx >= len(self.kdt.data): idx = 0 element.attrib['data-ptidx'] = str(idx)
def init_element_tree(self,latlon=True): """ build element tree for xy or latlon coordinates (default: latlon=True) """ from scipy.spatial import cKDTree #print(' schism.py: build element tree') if self.element_depth == None: self.element_depth={} calc_depths = True else: calc_depths = False if latlon: self.element_lon={} self.element_lat={} for el in self.nvdict: self.element_lon[el] = sum([self.londict[idx] for idx in self.nvdict[el]])/len(self.nvdict[el]) self.element_lat[el] = sum([self.latdict[idx] for idx in self.nvdict[el]])/len(self.nvdict[el]) if calc_depths: self.element_depth[el] = sum([self.depthsdict[idx] for idx in self.nvdict[el]])/len(self.nvdict[el]) self.element_tree_latlon = cKDTree(list(zip(self.element_lon.values(),self.element_lat.values()))) self.element_tree_ids = self.element_lon.keys() else: self.element_x={} self.element_y={} for el in self.nvdict: self.element_x[el] = sum([self.xdict[idx] for idx in self.nvdict[el]])/len(self.nvdict[el]) self.element_y[el] = sum([self.ydict[idx] for idx in self.nvdict[el]])/len(self.nvdict[el]) if calc_depths: self.element_depth[el] = sum([self.depthsdict[idx] for idx in self.nvdict[el]])/len(self.nvdict[el]) self.element_tree_xy = cKDTree(list(zip(self.element_x.values(),self.element_y.values()))) self.element_tree_ids = self.element_x.keys()
def kldiv(x,xp,k=3,base=2): """ KL Divergence between p and q for x~p(x),xp~q(x) x,xp should be a list of vectors, e.g. x = [[1.3],[3.7],[5.1],[2.4]] if x is a one-dimensional scalar and we have four samples """ assert k <= len(x) - 1, "Set k smaller than num. samples - 1" assert k <= len(xp) - 1, "Set k smaller than num. samples - 1" assert len(x[0]) == len(xp[0]), "Two distributions must have same dim." d = len(x[0]) n = len(x) m = len(xp) const = log(m) - log(n-1) tree = ss.cKDTree(x) treep = ss.cKDTree(xp) nn_old = [tree.query(point,k+1,p=float('inf'))[0][k] for point in x] nnp_old = [treep.query(point,k,p=float('inf'))[0][k-1] for point in x] # This isn't right! nn = [] nnp = [] for val in nn_old: if val > 0: nn.append(val) for val in nnp_old: if val > 0: nnp.append(val) return (const + d*np.mean(map(log,nnp))-d*np.mean(map(log,nn)))/log(base)
def init_node_tree(self,latlon=True): #print(' build node tree') from scipy.spatial import cKDTree if latlon: self.node_tree_latlon = cKDTree(list(zip(self.lon,self.lat))) else: self.node_tree_xy = cKDTree(list(zip(self.x,self.y)))
def get_ctmmap(subject, **kwargs): """ Parameters ---------- subject : str Subject name Returns ------- lnew : rnew : """ from scipy.spatial import cKDTree from . import brainctm jsfile = get_ctmpack(subject, **kwargs) ctmfile = os.path.splitext(jsfile)[0]+".ctm" try: left, right = db.get_surf(subject, "pia") except IOError: left, right = db.get_surf(subject, "fiducial") lmap, rmap = cKDTree(left[0]), cKDTree(right[0]) left, right = brainctm.read_pack(ctmfile) lnew = lmap.query(left[0])[1] rnew = rmap.query(right[0])[1] return lnew, rnew
def fitDataRigidScaleNoCorr_TwoSurfaces( X1,X2,data1,data2, xtol=1e-5, maxfev=0, t0=None ): """ fit list of points X to list of points data by minimising least squares distance between each point in X and closest neighbour in data """ if t0==None: t0 = scipy.array([0.0,0.0,0.0,0.0,0.0,0.0,1.0]) data1Tree = cKDTree( data1 ) data2Tree = cKDTree( data2 ) X1 = scipy.array(X1) X2 = scipy.array(X2) def obj( t ): x1R = transformRigid3D( X1, t[:6] ) x1RS = transformScale3D( x1R, scipy.ones(3)*t[6] ) d1 = data1Tree.query( list(x1RS) )[0] x2R = transformRigid3D( X2, t[:6] ) x2RS = transformScale3D( x2R, scipy.ones(3)*t[6] ) d2 = data2Tree.query( list(x2RS) )[0] d=concatenate((d1,d2),0) return d*d tOpt = leastsq( obj, t0, xtol=xtol, maxfev=maxfev )[0] X1Opt = transformRigid3D( X1, tOpt[:6] ) X2Opt = transformRigid3D( X2, tOpt[:6] ) X1Opt = transformScale3D( X1Opt, tOpt[6:] ) X2Opt = transformScale3D( X2Opt, tOpt[6:] ) return tOpt, X1Opt,X2Opt
def revised_mi(x,y,k=5,q=float('inf')): ''' Estimate the mutual information I(X;Y) of X and Y from samples {x_i, y_i}_{i=1}^N Using *REVISED* KSG mutual information estimator (see arxiv.org/abs/1604.03006) Input: x: 2D list of size N*d_x y: 2D list of size N*d_y k: k-nearest neighbor parameter q: l_q norm used to decide k-nearest distance Output: one number of I(X;Y) ''' assert len(x)==len(y), "Lists should have same length" assert k <= len(x)-1, "Set k smaller than num. samples - 1" N = len(x) dx = len(x[0]) dy = len(y[0]) data = np.concatenate((x,y),axis=1) tree_xy = ss.cKDTree(data) tree_x = ss.cKDTree(x) tree_y = ss.cKDTree(y) knn_dis = [tree_xy.query(point,k+1,p=q)[0][k] for point in data] ans_xy = -digamma(k) + log(N) + vd(dx+dy,q) ans_x = log(N) + vd(dx,q) ans_y = log(N) + vd(dy,q) for i in range(N): ans_xy += (dx+dy)*log(knn_dis[i])/N ans_x += -log(len(tree_x.query_ball_point(x[i],knn_dis[i]+1e-15,p=q))-1)/N+dx*log(knn_dis[i])/N ans_y += -log(len(tree_y.query_ball_point(y[i],knn_dis[i]+1e-15,p=q))-1)/N+dy*log(knn_dis[i])/N return ans_x+ans_y-ans_xy
def map_cells(input_cells, output_cells, max_distance=5, min_overlap=0.001): input_position_array = np.array([cell.center.coords[0] for cell in input_cells]) output_position_array = np.array([cell.center.coords[0] for cell in output_cells]) input_tree = cKDTree(input_position_array) output_tree = cKDTree(output_position_array) # retrieve lists of (output) neighbors around input cells print '>> Fetching neighbors for the', len(input_cells), 'input cells' neighbors = input_tree.query_ball_tree(output_tree, max_distance) overlaps = [] print '>> Computing overlaps' for i,cells in enumerate(neighbors): input_cell = input_cells[i] if len(cells)==0: raise RuntimeError('Cannot match input cell to any output cell') intersection_candidates = [output_cells[j] for j in cells] areas = [] for intersection_candidate in intersection_candidates: intersection = input_cell.vertices.intersection(intersection_candidate.vertices) areas.append(intersection.area) areas = np.array(areas) areas /= np.sum(areas) cells = np.array(cells) intersection_indices = areas>min_overlap intersection_cells = cells[intersection_indices] intersection_sum = np.sum(areas[intersection_indices]) overlaps.append(zip(intersection_cells, areas[intersection_indices]/intersection_sum)) overlaps_dict = {} for cell,overlap in zip(input_cells,overlaps): overlaps_dict[cell.id] = [] for output_cell, area in overlap: overlaps_dict[cell.id].append((output_cells[output_cell].id, area)) return overlaps, overlaps_dict
def match_radec(r1, d1, r2, d2, rad=1./60./60., nneighbor=0, notself=False): """Match r1, d1, to r2, d2, within radius rad.""" if notself and nneighbor > 0: nneighbor += 1 uv1 = lb2uv(r1, d1) uv2 = lb2uv(r2, d2) from scipy.spatial import cKDTree tree = cKDTree(uv2) dub = 2*numpy.sin(numpy.radians(rad)/2) if nneighbor > 0: d12, m2 = tree.query(uv1, nneighbor, distance_upper_bound=dub) if nneighbor > 1: m2 = m2.reshape(-1) d12 = d12.reshape(-1) m1 = numpy.arange(len(r1)*nneighbor, dtype='i4') // nneighbor d12 = 2*numpy.arcsin(numpy.clip(d12 / 2, 0, 1))*180/numpy.pi m = m2 < len(r2) else: tree1 = cKDTree(uv1) res = tree.query_ball_tree(tree1, dub) lens = [len(r) for r in res] m2 = numpy.repeat(numpy.arange(len(r2), dtype='i4'), lens) m1 = numpy.concatenate([r for r in res if len(r) > 0]) d12 = gc_dist(r1[m1], d1[m1], r2[m2], d2[m2]) m = numpy.ones(len(m1), dtype='bool') if notself: m = m & (m1 != m2) return m1[m], m2[m], d12[m]
def test_ckdtree_box_lower_bounds(): data = np.linspace(-1, 1, 10) try: cKDTree(data, leafsize=1, boxsize=1.0) except ValueError: return raise AssertionError("ValueError is not raised")
def set_coords(self, coords, cutoff=None): """Constructs KDTree from the coordinates Wrapping of coordinates to the primary unit cell is enforced before any distance evaluations. If periodic boundary conditions are enabled, then duplicate particles are generated in the vicinity of the box. An additional array `mapping` is also generated which can be later used to trace the origin of duplicate particle coordinates. For non-periodic calculations, cutoff should not be provided the parameter is only required for periodic calculations. Parameters ---------- coords: array_like Coordinate array of shape ``(N, 3)`` for N atoms. cutoff: float Specified cutoff distance to create duplicate images Typically equivalent to the desired search radius or the maximum of the desired cutoff radius. Relevant images corresponding to every atom which lies within ``cutoff`` distance from either of the box boundary will be generated. See Also -------- MDAnalysis.lib.distances.augment_coordinates """ # If no cutoff distance is provided but PBC aware if self.pbc and (cutoff is None): raise RuntimeError('Provide a cutoff distance' ' with tree.set_coords(...)') # set coords dtype to float32 # augment coordinates will work only with float32 coords = np.asarray(coords, dtype=np.float32) if self.pbc: self.cutoff = cutoff # Bring the coordinates in the central cell self.coords = apply_PBC(coords, self.box) # generate duplicate images self.aug, self.mapping = augment_coordinates(self.coords, self.box, self.cutoff) # Images + coords self.all_coords = np.concatenate([self.coords, self.aug]) self.ckdt = cKDTree(self.all_coords, leafsize=self.leafsize) else: # if cutoff distance is provided for non PBC calculations if cutoff is not None: raise RuntimeError('Donot provide cutoff distance for' ' non PBC aware calculations') self.coords = coords self.ckdt = cKDTree(self.coords, self.leafsize) self._built = True
else: mesh = meshInput print("Calculating active cells from topo") # Compute active cells activeCells = Utils.surface2ind_topo(mesh, topo, gridLoc='CC') Mesh.TreeMesh.writeUBC(mesh, outDir + 'OctreeMeshGlobal.msh', models={outDir + 'ActiveSurface.act': activeCells}) if "adjust_clearance" in list(input_dict.keys()): print("Forming cKDTree for clearance calculations") tree = cKDTree(mesh.gridCC[activeCells, :]) # Get the layer of cells directly below topo nC = int(activeCells.sum()) # Number of active cells # Create active map to go from reduce set to full activeCellsMap = Maps.InjectActiveCells(mesh, activeCells, ndv) # Create identity map if input_dict["inversion_type"].lower() == 'mvi': wrGlobal = np.zeros(3 * nC) else: idenMap = Maps.IdentityMap(nP=nC) wrGlobal = np.zeros(nC) # Loop over different tile size and break problem until
from transfer.frontends.swiftvr import SWIFTSnapshotData from scipy.spatial import cKDTree import numpy final = SWIFTSnapshotData( '/cosma6/data/dp004/dc-borr1/snap7/new_randomness_runs/adiabatic/Run_0/eagle_0007.hdf5', '/cosma6/data/dp004/dc-borr1/snap7/new_randomness_runs/adiabatic/Run_0/halo_0007.properties', ) boxsize = final.boxsize boxsize.convert_to_units('Mpc') dm_coords = final.dark_matter.coordinates halo_coords = final.halo_coordinates dm_coords.convert_to_units('Mpc') halo_coords.convert_to_units('Mpc') tree = cKDTree(halo_coords, boxsize=boxsize.value) d, i = tree.query(dm_coords, k=1, n_jobs=-1) numpy.savetxt('/cosma5/data/durham/dc-murr1/adi_dm_nearest_halo_distance.txt', d) numpy.savetxt('/cosma5/data/durham/dc-murr1/adi_dm_nearest_halos.txt', i)
def pars(kx, ky, k, sens, tx=None, ty=None, kernel_size=25, kernel_radius=None, coil_axis=-1): '''Parallel MRI with adaptive radius in k‐space. Parameters ---------- kx, ky : array_like Sample points in kspace corresponding to measurements k. kx, kx are 1D arrays. k : array_like Complex kspace coil measurements corresponding to points (kx, ky). sens : array_like Coil sensitivity maps with shape of desired reconstruction. tx, ty : array_like Sample points in kspace defining the grid of ifft2(sens). If None, then tx, ty will be generated from a meshgrid with endpoints [min(kx), max(kx), min(ky), max(ky)]. kernel_size : int, optional Number of nearest neighbors to use when interpolating kspace. kernel_radius : float, optional Raidus in kspace (units same as (kx, ky)) to select neighbors when training kernels. coil_axis : int, optional Dimension holding coil data. Returns ------- res : array_like Reconstructed image space on a Cartesian grid with the same shape as sens. Notes ----- Implements the algorithm described in [1]_. Using kernel_radius seems to perform better than kernel_size. References ---------- .. [1] Yeh, Ernest N., et al. "3Parallel magnetic resonance imaging with adaptive radius in k‐space (PARS): Constrained image reconstruction using k‐space locality in radiofrequency coil encoded data." Magnetic Resonance in Medicine: An Official Journal of the International Society for Magnetic Resonance in Medicine 53.6 (2005): 1383-1392. ''' # Move coil axis to the back k = np.moveaxis(k, coil_axis, -1) sens = np.moveaxis(sens, coil_axis, -1) kxy = np.concatenate((kx[:, None], ky[:, None]), axis=-1) # Oversample the sensitivity maps by a factor of 2 t0 = time() sensr = zoom(sens.real, (2, 2, 1), order=1) sensi = zoom(sens.imag, (2, 2, 1), order=1) sens = sensr + 1j * sensi print('Took %g seconds to upsample sens' % (time() - t0)) # We want to resample onto a Cartesian grid sx, sy, nc = sens.shape[:] if tx is None or ty is None: tx, ty = np.meshgrid(np.linspace(np.min(kx), np.max(kx), sx), np.linspace(np.min(ky), np.max(ky), sy)) tx, ty = tx.flatten(), ty.flatten() txy = np.concatenate((tx[:, None], ty[:, None]), axis=-1) # Make a kd-tree and find all point around targets t0 = time() kdtree = cKDTree(kxy) if kernel_radius is None: _, idx = kdtree.query(txy, k=kernel_size) else: idx = kdtree.query_ball_point(txy, r=kernel_radius) print('Took %g seconds to find nearest neighbors' % (time() - t0)) # Scale kspace coordinates to be within [-.5, .5] kxy0 = np.concatenate( (kx[:, None] / np.max(kx), ky[:, None] / np.max(ky)), axis=-1) / 2 txy0 = np.concatenate( (tx[:, None] / np.max(tx), ty[:, None] / np.max(ty)), axis=-1) / 2 # Encoding matrix is much too large to invert, so we'll go # kernel by kernel to grid/reconstruct kspace sens = np.reshape(sens, (-1, nc)) res = np.zeros(sens.shape, dtype=sens.dtype) t0 = time() for ii, idx0 in tqdm(enumerate(idx), total=idx.shape[0], leave=False, desc='PARS'): dk = kxy0[idx0, :] r = txy0[ii, :] # Create local encoding matrix and train weights E = np.exp(1j * (-dk @ r)) E = E[:, None] * sens[ii, :] W = sens[ii, :] @ E.conj().T @ np.linalg.pinv(E @ E.conj().T) # Grid the sample: res[ii, :] = W @ k[idx0, :] print('Took %g seconds to regrid' % (time() - t0)) # Return image at correct resolution with coil axis in right place ax = (0, 1) sx4 = int(sx / 4) return np.moveaxis( np.fft.fftshift(np.fft.ifft2(np.fft.ifftshift(np.reshape( res, (sx, sy, nc), 'F'), axes=ax), axes=ax), axes=ax)[sx4:-sx4, sx4:-sx4, :], -1, coil_axis)
# Transform centers reduced_centers = pca_model.transform(centers) # Step size of mesh for plotting h = .02 # Plot the decision boundary. For that, we will assign a color to each x_min, x_max = reduced_data[:, 0].min() - 1, reduced_data[:, 0].max() + 1 y_min, y_max = reduced_data[:, 1].min() - 1, reduced_data[:, 1].max() + 1 xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h)) # Get k-means classifications for the grid points xx_pt = list(xx.ravel()) yy_pt = list(yy.ravel()) xy_pts = np.array([[x, y] for x, y in zip(xx_pt, yy_pt)]) mytree = cKDTree(reduced_centers) dist, indexes = mytree.query(xy_pts) # Put the result into a color plot indexes = indexes.reshape(xx.shape) plt.figure(1) plt.clf() plt.imshow(indexes, interpolation='nearest', extent=(xx.min(), xx.max(), yy.min(), yy.max()), cmap=plt.cm.Paired, aspect='auto', origin='lower') # Plot each of the true iris data groups symbols = ['o', '^', 'D']
def process_frame(u, residues, prot, notH): '''Calculate distance descriptors for all requested residues of the current frame of a molecular dynamics simulation. Provide selections of the protein atoms and all notH atoms''' bb_atoms = ["N", "O"] res = prot.atoms.residues # Load this frame into a spatial cKDTree for fast nearest-neighbor search tree = spatial.cKDTree(np.array([x.pos for x in notH.atoms])) ret = dict() #indexed by resid # Simultaneously iterate through each residue and corresponding open file for r in residues: i = r.id - 1 assert res[i] == r ###################################################### ## Process the N dome ###################################################### # Original atom selection # orig_cloud = u.select_atoms("around 5.0 atom SYSTEM " + str(res[i].id) + " H") cloud = [] processed_cloud = [] atom_pattern_N = '' atom_pattern_dist = [] current_res_H_pos = r.H.pos x = r.N.pos - current_res_H_pos res_i_O_pos = res[i].O.pos res_i_m1_N_pos = res[i - 1].N.pos res_i_p1_N_pos = res[i + 1].N.pos res_i_m2_O_pos = res[i - 2].O.pos # First round of processing from enhanced atom selection for j in tree.query_ball_point(current_res_H_pos, 5.0): atom = notH.atoms[j] atom_pos = atom.pos ref_name = atom_to_pattern(atom) # Check to ensure backbone atoms within (+/-) 2 residues are not included if atom.name in bb_atoms and abs(r.id - atom.resid) < 3: continue else: if ref_name: # Variables for calculating the vector angles of each atom v = atom_pos - current_res_H_pos # Eliminate any atoms that have vector angles less than 90.0 if abs(vangle(x, v)) < 90.0: continue else: ###################################################### ## Compute distance between atoms based on types ## Aromatic atoms: within 5.0 A ## Carbon atoms: within 4.0 A ## Other atoms: wtihin 2.5 A ###################################################### dist = distance(atom_pos, current_res_H_pos) if ref_name == "R": cloud.append([atom, dist]) elif ref_name == "C": if dist <= 4.0: cloud.append([atom, dist]) elif dist <= 2.5: cloud.append([atom, dist]) # Second round of processing if cloud is None: atom_pattern_N = "Z:" atom_pattern_dist += [0.0, 0.0, 0.0, 0.0, 0.0] else: processed_cloud = find_closest_atom(cloud, res[i].H) if processed_cloud == None: atom_pattern_N = "Z:" atom_pattern_dist += [0.0, 0.0, 0.0, 0.0, 0.0] else: # Rank atoms based on the atom type preferences noted above processed_cloud.sort(key=lambda atom: (atom_ranking[ atom_to_pattern(atom[0])], atom[1])) # Generate the atom pattern with their corresponding distances for atom in processed_cloud: atom_pattern_N += atom_to_pattern(atom[0]) + ':' apos = atom[0].pos atom_pattern_dist += [ distance(current_res_H_pos, apos), distance(res_i_m1_N_pos, apos), distance(res_i_m2_O_pos, apos), distance(res_i_O_pos, apos), distance(res_i_p1_N_pos, apos) ] # Write the output to the open file nvals = [ atom_pattern_N, u.trajectory.frame, distance(res_i_m1_N_pos, current_res_H_pos), distance(res_i_m2_O_pos, current_res_H_pos), distance(res[i - 1].O.pos, current_res_H_pos), distance(res_i_O_pos, current_res_H_pos), distance(res[i + 1].N.pos, current_res_H_pos) ] + atom_pattern_dist # Intentionally offset the oxygen output o = i - 1 ###################################################### ## Process the O dome ###################################################### # Original atom selection # orig_cloud = u.select_atoms("around 3.9 atom SYSTEM " + str(res[o].id) + " O") cloud = [] processed_cloud = [] atom_pattern_O = '' atom_pattern_dist = [] current_res_O_pos = res[o].O.pos x = res[o + 1].N.pos - current_res_O_pos res_o_N_pos = res[o].N.pos res_o_m1_O_pos = res[o - 1].O.pos res_o_p1_O_pos = res[o + 1].O.pos res_o_p2_N_pos = res[o + 2].N.pos # First round of processing from enhanced atom selection for j in tree.query_ball_point(current_res_O_pos, 3.9): atom = notH.atoms[j] atom_pos = atom.pos ref_name = atom_to_pattern(atom) # Check to ensure backbone atoms within (+/-) 2 residues are not included if atom.name in bb_atoms and abs(res[o].id - atom.resid) < 3: continue else: if ref_name: # Eliminate all Carbon atoms if "C" == ref_name: continue else: # Variables for calculating the vector angles of each atom v = atom_pos - current_res_O_pos # Eliminate any atoms that have vector angles less than 90.0 if abs(vangle(x, v)) < 90.0: continue elif ref_name: cloud.append( [atom, distance(atom_pos, current_res_O_pos)]) # Second round of processing if cloud is None: atom_pattern_O = "Z:" atom_pattern_dist += [0.0, 0.0, 0.0, 0.0, 0.0] else: processed_cloud = find_closest_atom(cloud, res[o].O) if processed_cloud == None or len(processed_cloud) < 1: atom_pattern_O = "Z:" atom_pattern_dist += [0.0, 0.0, 0.0, 0.0, 0.0] else: # Rank atoms based on the atom type preferences noted above processed_cloud.sort(key=lambda atom: (atom_ranking[ atom_to_pattern(atom[0])], atom[1])) # Generate the atom pattern with their corresponding distances for atom in processed_cloud: atom_pattern_O += atom_to_pattern(atom[0]) + ":" apos = atom[0].pos atom_pattern_dist += [ distance(current_res_O_pos, apos), distance(res_o_N_pos, apos), distance(res_o_m1_O_pos, apos), distance(res_o_p1_O_pos, apos), distance(res_o_p2_N_pos, apos) ] # Write the output to the open file ovals = [ atom_pattern_O, u.trajectory.frame, distance(res_o_N_pos, current_res_O_pos), distance(res_o_m1_O_pos, current_res_O_pos), distance(res[o + 1].N.pos, current_res_O_pos), distance(res_o_p1_O_pos, current_res_O_pos), distance(res_o_p2_N_pos, current_res_O_pos) ] + atom_pattern_dist ret[i] = (nvals, ovals) return ret
for record in disFile: inXYT.append([ float(record.split(" ")[1]), float(record.split(" ")[2]), float(record.split(" ")[3]) ]) disFile.close() #number of points numPts = len(inXYT) #sort dataset temporally inXYT_s = np.array(sorted(inXYT, key=lambda a: a[2])) #index dataset sTree = spatial.cKDTree(inXYT_s[:, :2]) stNeighIndex = [] #maximum number of neighbors mNN = np.inf #current number of neighbors NN = numPts # output folder outDir = 'outputs/ST_MC_1/sim_' + sim if not os.path.exists(outDir): os.makedirs(outDir) outFile = open(outDir + os.sep + 'bandwidths_' + str(neighThres) + '.txt', 'w')
def set_hbonds_in_selection_and_water_around(self, around_radius, not_water_water=False): sfilter = _np.array([(ids.split('-')[3].startswith('S')) for ids in self._all_ids_atomwise]) result = {} frame_count = 0 frames = self.nb_frames for ts in self._universe.trajectory[self._trajectory_slice]: water_coordinates = self._water.positions selection_coordinates = self._da_selection.positions selection_tree = _sp.cKDTree(selection_coordinates) if self._nb_acceptors > 0 and self._nb_donors > 0: d_tree = _sp.cKDTree(self._donors.positions) a_tree = _sp.cKDTree(self._acceptors.positions) hydrogen_coordinates = self._hydrogen.positions water_tree = _sp.cKDTree(water_coordinates, leafsize=32) local_water_index = [] [ local_water_index.extend(l) for l in water_tree.query_ball_point(selection_coordinates, float(around_radius)) ] local_water_index = _np.unique(local_water_index) local_water_coordinates = water_coordinates[local_water_index] local_water_tree = _sp.cKDTree(local_water_coordinates) local_pairs = [(i, local_water_index[j] + self._first_water_id) for i, bla in enumerate( selection_tree.query_ball_tree( local_water_tree, self.distance)) for j in bla] water_pairs = [(local_water_index[p[0]] + self._first_water_id, local_water_index[p[1]] + self._first_water_id) for p in local_water_tree.query_pairs(self.distance) ] if self._nb_acceptors > 0 and self._nb_donors > 0: da_pairs = _np.array([[i, j] for i, donors in enumerate( a_tree.query_ball_tree(d_tree, self.distance)) for j in donors]) da_pairs[:, 0] += self._nb_donors else: da_pairs = [] if da_pairs != []: da_pairs = da_pairs[_np.logical_not( _np.all(sfilter[da_pairs], axis=1))] if self.check_angle: all_coordinates = _np.vstack( (selection_coordinates, water_coordinates)) if not_water_water: hbonds = _hf.check_angle( list(da_pairs) + local_pairs, self.heavy2hydrogen, all_coordinates, hydrogen_coordinates, self.cut_angle) else: hbonds = _hf.check_angle( list(da_pairs) + water_pairs + local_pairs, self.heavy2hydrogen, all_coordinates, hydrogen_coordinates, self.cut_angle) else: if not_water_water: hbonds = list(da_pairs) + local_pairs else: hbonds = list(da_pairs) + water_pairs + local_pairs hbonds = _np.array(hbonds) sorted_bonds = _np.sort(hbonds) check = self._resids[sorted_bonds] check = check[:, 0] < check[:, 1] if self.residuewise: frame_res = [ self._all_ids[i] + ':' + self._all_ids[j] if check[ii] else self._all_ids[j] + ':' + self._all_ids[i] for ii, (i, j) in enumerate(sorted_bonds) ] else: frame_res = [ self._all_ids_atomwise[i] + ':' + self._all_ids_atomwise[j] if check[ii] else self._all_ids_atomwise[j] + ':' + self._all_ids_atomwise[i] for ii, (i, j) in enumerate(sorted_bonds) ] for bond in frame_res: a, b = bond.split(':') if a == b: continue try: result[bond][frame_count] = True except: result[bond] = _np.zeros(frames, dtype=bool) result[bond][frame_count] = True frame_count += 1 if self.progress_callback is not None: self.progress_callback.emit( 'Computing H bonds in frame {}/{}'.format( frame_count, self.nb_frames)) self._set_results(result)
def rebuildKDT(self): """ Force the internal KDTree to be rebuilt. """ self.kdt = cKDTree(self.data[:, :-1], leafsize=self.leafsize) self.rebuild_tree = False
def point2plane(self, steps=1, neigh=10, inside=True, subset=None, scale=None, smooth=1, fixBrim=False, error='norm'): r""" Point to Plane method for registration between the two meshes Parameters ---------- steps: int, default 1 Number of iterations int, default 10 Number of nearest neighbours to interrogate for each baseline point inside: bool, default True If True, a barycentric centre check is made to ensure the registered point lines within the target triangle subset: array_like, default None Indicies of the baseline nodes to include in the registration, default is none so all are used scale: float, default None If not None scale the baseline mesh to match the target mesh in the z-direction, the value of scale will be used as a plane from which the nodes are scaled. Nodes with a higher z value will not be scaled. smooth: int, default 1 Indicate number of Laplacian smooth steps in between the steps fixBrim: bool, default False If True, the nodes on the brim line will not be included in the smooth error: bool, default False If True, the polarity will be included when calculating the distance between the target and baseline mesh """ # Calc FaceCentroids fC = self.t.vert[self.t.faces].mean(axis=1) # Construct knn tree tTree = spatial.cKDTree(fC) bData = dict( zip(['vert', 'faces', 'values'], [self.b.vert, self.b.faces, self.b.values])) regData = copy.deepcopy(bData) self.reg = AmpObject(regData, stype='reg') self.disp = AmpObject({ 'vert': np.zeros(self.reg.vert.shape), 'faces': self.reg.faces, 'values': self.reg.values }) if scale is not None: tmin = self.t.vert.min(axis=0)[2] rmin = self.reg.vert.min(axis=0)[2] SF = ((tmin - scale) / (rmin - scale)) - 1 logic = self.reg.vert[:, 2] < scale d = (self.reg.vert[logic, 2] - scale) * SF self.disp.vert[logic, 2] += d self.reg.vert = self.b.vert + self.disp.vert normals = np.cross( self.t.vert[self.t.faces[:, 1]] - self.t.vert[self.t.faces[:, 0]], self.t.vert[self.t.faces[:, 2]] - self.t.vert[self.t.faces[:, 0]]) mag = (normals**2).sum(axis=1) for step in np.arange(steps, 0, -1, dtype=float): # Index of 10 centroids nearest to each baseline vertex ind = tTree.query(self.reg.vert, neigh)[1] # Define normals for faces of nearest faces norms = normals[ind] # Get a point on each face fPoints = self.t.vert[self.t.faces[ind, 0]] # Calculate dot product between point on face and normals d = np.einsum('ijk, ijk->ij', norms, fPoints) t = (d - np.einsum('ijk, ik->ij', norms, self.reg.vert)) / mag[ind] # Calculate the vector from old point to new point G = self.reg.vert[:, None, :] + np.einsum('ijk, ij->ijk', norms, t) # Ensure new points lie inside points otherwise set to 99999 # Find smallest distance from old to new point if inside is False: G = G - self.reg.vert[:, None, :] GMag = np.sqrt(np.einsum('ijk, ijk->ij', G, G)) GInd = GMag.argmin(axis=1) else: G, GInd = self.__calcBarycentric(self.reg.vert, G, ind) # Define vector from baseline point to intersect point D = G[np.arange(len(G)), GInd, :] # rVert += D/step self.disp.vert += D / step if smooth > 0 and step > 1: self.disp.lp_smooth(smooth, brim=fixBrim) self.reg.vert = self.b.vert + self.disp.vert else: self.reg.vert = self.b.vert + self.disp.vert self.reg.calcNorm() self.reg.calcStruct(vNorm=True) self.reg.values[:] = self.calcError(error)
setup = schism_setup() xmin = min(setup.x) xmax = max(setup.x) ymin = min(setup.y) ymax = max(setup.y) # make new grid dx=2000. dy=2000. xn = arange(xmin-dx,xmax+2*dx,dx) yn = arange(ymin-dy,ymax+2*dy,dy) XN,YN = meshgrid(xn,yn) # build tree print(' build tree') tree = cKDTree(zip(x,y)) # find weights and distances print(' query weights') dist,inds = tree.query(zip(XN.flat[:],YN.flat[:]), k=100) weights = 1.0 / dist**2 # do interpolation print(' interpolate') depths = np.min(d[inds],axis=1) #depths = np.sum( weights*d[inds],axis=1) / np.sum(weights,axis=1) # create size field maxlength=4000. minlength=500. maxdepth=50.
def config(): """ Starts the program up with all necessary things. Reads the inputs, creates the Singleton objects properly, sets up the heightmap for later, makes sure all Vertices in the axiom have the correct neighbor. Could need a rework in which the Singletons are unified and not broken as they are now. Returns ------- variables : Variables object Singleton with all numeric values which are not to be changed at runtime singleton.global_lists : singleton.global_lists object Singleton with the Global Lists which will be altered at runtime """ import json from collections import namedtuple import os import procedural_city_generation from procedural_city_generation.additional_stuff.Singleton import Singleton path = os.path.dirname(procedural_city_generation.__file__) singleton = Singleton("roadmap") #Creates Singleton-Variables object from namedtuple from procedural_city_generation.roadmap.Vertex import Vertex, set_plotbool #Creates Vertex objects from coordinates singleton.axiom = [ Vertex(np.array([float(x[0]), float(x[1])])) for x in singleton.axiom ] singleton.border = np.array([singleton.border_x, singleton.border_y]) set_plotbool(singleton.plot) #Finds the longest possible length of a connection between to vertices singleton.maxLength = max(singleton.radiallMax, singleton.gridlMax, singleton.organiclMax, singleton.minor_roadlMax, singleton.seedlMax) import os from procedural_city_generation.roadmap.config_functions.input_image_setup import input_image_setup singleton.img, singleton.img2 = input_image_setup( singleton.rule_image_name, singleton.density_image_name) with open(path + "/temp/" + singleton.output_name + "_densitymap.txt", 'w') as f: f.write(singleton.density_image_name.split(".")[0] + "diffused.png") from procedural_city_generation.roadmap.config_functions.find_radial_centers import find_radial_centers singleton.center = find_radial_centers(singleton) singleton.center = [ np.array([ singleton.border[0] * ((x[1] / singleton.img.shape[1]) - 0.5) * 2, singleton.border[1] * (((singleton.img.shape[0] - x[0]) / singleton.img.shape[0]) - 0.5) * 2 ]) for x in singleton.center ] from procedural_city_generation.roadmap.config_functions.setup_heightmap import setup_heightmap setup_heightmap(singleton, path) singleton.global_lists = Global_Lists() singleton.global_lists.vertex_list.extend(singleton.axiom) singleton.global_lists.coordsliste = [ x.coords for x in singleton.global_lists.vertex_list ] def setNeighbours(vertex): """ Correctly Sets up the neighbors for a vertex from the axiom. Parameters ---------- vertex : vertex Object """ d = np.inf neighbour = None for v in singleton.axiom: if v is not vertex: dneu = np.linalg.norm(v.coords - vertex.coords) if dneu < d: d = dneu neighbour = v vertex.neighbours = [neighbour] for k in singleton.axiom: setNeighbours(k) from scipy.spatial import cKDTree singleton.global_lists.tree = cKDTree(singleton.global_lists.coordsliste, leafsize=160) return singleton
rest_pose[0, 0] = np.pi / 2 rest_pose[1, 2] = .15 rest_pose[2, 2] = -.15 smpl_path = os.path.dirname(os.path.abspath(__file__)) + '/../Model/smpl/' SMPL = { 0: SMPLModel(smpl_path + 'model_f.pkl', rest_pose), 1: SMPLModel(smpl_path + 'model_m.pkl', rest_pose) } # get sample list samples = os.listdir(SRC) N = len(samples) # for each sample for i, sample in enumerate(samples): print(str(i + 1) + " / " + str(N)) src = SRC + sample + '/' src_pre = SRC_PREPROCESS + sample + '/' dst = src_pre + 'weights.npy' # load info info = loadInfo(src + 'info.mat') shape = info['shape'] gender = info['gender'] # load template outfit T = readPC2(src_pre + 'rest.pc16', True)['V'][0] # compute SMPL in rest pose _, B = SMPL[gender].set_params(pose=rest_pose, beta=shape, with_body=True) # nearest neighbour tree = cKDTree(B) _, idx = tree.query(T) W_prior = SMPL[gender].weights[idx] np.save(dst, W_prior)
def __init__(self, data): cart_data = self.spherical2cartesian(data) self.data = data self.kd_tree = cKDTree(data=cart_data, leafsize=10)
def __init__(self, data, leafsize = 10): self.real_tree = cKDTree(data.T, leafsize)
try: if i > 2: values[i] = float(v) if int(v) == 99999: values[i] = None except: continue lat, lon = values[6], values[7] if lat and lon: # Some stations have no lat-long; this isn't useful stations.append(IntegratedSurfaceDatabaseStation(*values)) _latlongs.append((lat, lon)) _latlongs = np.array(_latlongs) station_count = len(stations) kd_tree = cKDTree( _latlongs) # _latlongs must be unchanged as data is not copied def get_closest_station(latitude, longitude, minumum_recent_data=20140000, match_max=100): """Query function to find the nearest weather station to a particular set of coordinates. Optionally allows for a recent date by which the station is required to be still active at. Parameters ---------- latitude : float Latitude to search for nearby weather stations at, [degrees] longitude : float
def _calculate_novelty(self, bc, _archive): kd = spatial.cKDTree(_archive) distances, idxs = kd.query(bc, k=self.k) distances = distances[distances < float('inf')] novelty = np.sum(distances) / np.linalg.norm(_archive) return novelty
def do_kdtree(combined_x_y_arrays, points): mytree = cKDTree(combined_x_y_arrays) dist, indexes = mytree.query(points) return dist, indexes
def adi(pts_est, pts_gt): nn_index = spatial.cKDTree(pts_est) nn_dists, _ = nn_index.query(pts_gt, k=1) e = nn_dists.mean() return e
def set_hbonds_in_selection(self): sfilter = _np.array([(ids.split('-')[3].startswith('S')) for ids in self._all_ids_atomwise]) frame_count = 0 frames = self.nb_frames result = {} #t0 = time.time() for ts in self._universe.trajectory[self._trajectory_slice]: selection_coordinates = self._da_selection.positions d_tree = _sp.cKDTree(self._donors.positions) a_tree = _sp.cKDTree(self._acceptors.positions) hydrogen_coordinates = self._hydrogen.positions da_pairs = _np.array([[i, j] for i, donors in enumerate( a_tree.query_ball_tree(d_tree, self.distance)) for j in donors]) da_pairs[:, 0] += self._nb_donors da_pairs = da_pairs[_np.logical_not( _np.all(sfilter[da_pairs], axis=1))] if self.check_angle: all_coordinates = selection_coordinates local_hbonds = _hf.check_angle(da_pairs, self.heavy2hydrogen, all_coordinates, hydrogen_coordinates, self.cut_angle) else: local_hbonds = da_pairs sorted_bonds = _np.sort(local_hbonds) check = self._resids[sorted_bonds] check = check[:, 0] < check[:, 1] if self.residuewise: frame_res = [ self._all_ids[i] + ':' + self._all_ids[j] if check[ii] else self._all_ids[j] + ':' + self._all_ids[i] for ii, (i, j) in enumerate(sorted_bonds) ] else: frame_res = [ self._all_ids_atomwise[i] + ':' + self._all_ids_atomwise[j] if check[ii] else self._all_ids_atomwise[j] + ':' + self._all_ids_atomwise[i] for ii, (i, j) in enumerate(sorted_bonds) ] for bond in frame_res: a, b = bond.split(':') if self.residuewise and (a.split('-')[:3] == b.split('-')[:3]): continue try: result[bond][frame_count] = True except: result[bond] = _np.zeros(frames, dtype=bool) result[bond][frame_count] = True frame_count += 1 if self.progress_callback is not None: self.progress_callback.emit( 'Computing H bonds in frame {}/{}'.format( frame_count, self.nb_frames)) #print('Time to compute {} H-bonds: {}s'.format(len(result), _np.round(time.time()-t0,5))) self._set_results(result)
def run(in_jsons: List[params.preprocessing_content_type], out_dir: str, resolution: float = 0.5, min_elevation_offset: float = None, max_elevation_offset: float = None, epsg: int = None, sigma: float = None, dsm_radius: int = 1, dsm_no_data: int = -32768, msk_no_data: int = 65535, color_no_data: int = 0, corr_config: Dict = None, output_stats: bool = False, mode: str = "local_dask", nb_workers: int = 4, walltime: str = "00:59:00", roi: Tuple[List[int], int] = None, use_geoid_alt: bool = False, use_sec_disp: bool = False, snap_to_img1: bool = False, align: bool = False, cloud_small_components_filter: bool = True, cloud_statistical_outliers_filter: bool = True, epi_tile_size: int = None): """ Main function for the compute_dsm pipeline subcommand It computes independent tiles of the final DSM, with the following steps: 1. Epipolar resampling (including mask) 2. Disparity map estimation 3. Triangulation of disparity map 4. Rasterization to DSM :param in_jsons: Input pair dictionaries (as produced by cars prepare step) :param out_dir: Computed raster and color image output directory :param resolution: DSM resolution to produce :param min_elevation_offset: Override minimum disparity from prepare step with this offset in meters :param max_elevation_offset: Override maximum disparity from prepare step with this offset in meters :param epsg: Output DSM Coordinate Reference System EPSG code :param sigma: Rasterization width of gaussian weight :param dsm_radius: Rasterization radius around a cell for gathering points :param dsm_no_data: No data value to use in the final DSM file :param color_no_data: No data value to use in the final colored image :param msk_no_data: No data value to use in the final mask image :param corr_config: Correlator configuration :param output_stats: Ouput DSM associated quality statistics flag boolean :param mode: Parallelization mode :param nb_workers: Number of dask workers to use for the sift matching step :param walltime: Walltime of the dask workers :param roi: DSM Region Of Interest in final projection with EPSG reference ([xmin, ymin, xmax, ymax], roi_epsg)) (roi_epsg can be set to None if the ROI is in final projection) :param use_geoid_alt: Geoid height reference for DSM altitude flag. :param use_sec_disp: Secondary disparity map activation flag. :param snap_to_img1: Force Img2 / Img1 Lines of Sight crossing flag. :param align: If this is True, use the correction estimated during prepare to align to lowres DEM (if available) :param cloud_small_components_filter: Activating the points cloud small components filtering. The filter's parameters are set in static configuration json. :param cloud_statistical_outliers_filter: Activating the points cloud statistical outliers filtering. The filter's parameters are set in static configuration json. :param epi_tile_size: Force the size of epipolar tiles (None by default) """ out_dir = os.path.abspath(out_dir) # Ensure that outdir exists try: os.makedirs(out_dir) except OSError as exc: if exc.errno == errno.EEXIST and os.path.isdir(out_dir): pass else: raise tmp_dir = os.path.join(out_dir, 'tmp') utils.add_log_file(out_dir, 'compute_dsm') logging.info("Received {} stereo pairs configurations".format( len(in_jsons))) # Retrieve static parameters (rasterization and cloud filtering) static_params = static_cfg.get_cfg() # Initiate ouptut json dictionary out_json = { params.stereo_inputs_section_tag: [], params.stereo_section_tag: { params.stereo_version_tag: utils.get_version(), params.stereo_parameters_section_tag: { params.resolution_tag: resolution, params.sigma_tag: sigma, params.dsm_radius_tag: dsm_radius }, params.static_params_tag: static_params[static_cfg.compute_dsm_tag], params.stereo_output_section_tag: {} } } if use_geoid_alt: geoid_data = utils.read_geoid_file() out_json[params.stereo_section_tag][params.stereo_output_section_tag][ params.alt_reference_tag] = 'geoid' else: geoid_data = None out_json[params.stereo_section_tag][params.stereo_output_section_tag][ params.alt_reference_tag] = 'ellipsoid' if epsg is not None: out_json[params.stereo_section_tag][ params.stereo_parameters_section_tag][params.epsg_tag] = epsg roi_epsg = None if roi is not None: (roi_xmin, roi_ymin, roi_xmax, roi_ymax), roi_epsg = roi roi_poly = Polygon([(roi_xmin, roi_ymin), (roi_xmax, roi_ymin), (roi_xmax, roi_ymax), (roi_xmin, roi_ymax), (roi_xmin, roi_ymin)]) # set the timeout for each job in multiprocessing mode (in seconds) per_job_timeout = 600 configurations_data = {} config_idx = 1 ref_left_img = None write_msk = False for in_json in in_jsons: # Build config id config_id = "config_{}".format(config_idx) # Check configuration with respect to schema configuration = utils.check_json(in_json, params.preprocessing_content_schema) # retrieve masks classes usages mask1_classes = configuration[params.input_section_tag].get( params.mask1_classes_tag, None) mask2_classes = configuration[params.input_section_tag].get( params.mask2_classes_tag, None) classes_usage = dict() if mask1_classes is not None: mask1_classes_dict = mask_classes.read_mask_classes(mask1_classes) classes_usage[params.mask1_ignored_by_corr_tag] =\ mask1_classes_dict.get(mask_classes.ignored_by_corr_tag, None) classes_usage[params.mask1_set_to_ref_alt_tag] = \ mask1_classes_dict.get(mask_classes.set_to_ref_alt_tag, None) if mask2_classes is not None: mask2_classes_dict = mask_classes.read_mask_classes(mask2_classes) classes_usage[params.mask2_ignored_by_corr_tag] = \ mask2_classes_dict.get(mask_classes.ignored_by_corr_tag, None) classes_usage[params.mask2_set_to_ref_alt_tag] = \ mask2_classes_dict.get(mask_classes.set_to_ref_alt_tag, None) # Append input configuration to output json out_json_config = { params.stereo_input_tag: configuration, } if mask1_classes is not None or mask2_classes is not None: out_json_config[ params.stereo_mask_classes_usage_tag] = classes_usage out_json[params.stereo_inputs_section_tag].append(out_json_config) configurations_data[config_id] = {} configurations_data[config_id]['configuration'] = configuration # Get local conf left image for this in_json iteration conf_left_img = configuration[params.input_section_tag][ params.img1_tag] # Check left image and raise a warning # if different left images are used along with snap_to_img1 mode if ref_left_img is None: ref_left_img = conf_left_img else: if snap_to_img1 and ref_left_img != conf_left_img: logging.warning( "--snap_to_left_image mode is used but input " "configurations have different images as their " "left image in pair. This may result in " "increasing registration discrepencies between pairs") # If mask1 and/or mask2 are set in the prepare input configuration json # then the DSM rasterized mask will be written alongside the DSM # TODO : Mask 2 ? mask1 = \ configuration[params.input_section_tag].get(params.mask1_tag, None) if mask1 is not None: write_msk = True # Get Preprocessing output config preprocessing_output_config = configuration[ params.preprocessing_section_tag][ params.preprocessing_output_section_tag] # Get largest epipolar regions from configuration file largest_epipolar_region =\ [0, 0, preprocessing_output_config[params.epipolar_size_x_tag], preprocessing_output_config[params.epipolar_size_y_tag]] configurations_data[config_id]['largest_epipolar_region'] =\ largest_epipolar_region disp_min = preprocessing_output_config[params.minimum_disparity_tag] disp_max = preprocessing_output_config[params.maximum_disparity_tag] disp_to_alt_ratio = preprocessing_output_config[ params.disp_to_alt_ratio_tag] # Check if we need to override disp_min if min_elevation_offset is not None: user_disp_min = min_elevation_offset / disp_to_alt_ratio if user_disp_min > disp_min: logging.warning( ('Overriden disparity minimum = {:.3f} pix. (= {:.3f} m.) ' 'is greater than disparity minimum estimated ' 'in prepare step = {:.3f} pix. (or {:.3f} m.) ' 'for configuration {}').format( user_disp_min, min_elevation_offset, disp_min, disp_min * disp_to_alt_ratio, config_id)) disp_min = user_disp_min # Check if we need to override disp_max if max_elevation_offset is not None: user_disp_max = max_elevation_offset / disp_to_alt_ratio if user_disp_max < disp_max: logging.warning(( 'Overriden disparity maximum = {:.3f} pix. (or {:.3f} m.) ' 'is lower than disparity maximum estimated ' 'in prepare step = {:.3f} pix. (or {:.3f} m.) ' 'for configuration {}').format( user_disp_max, max_elevation_offset, disp_max, disp_max * disp_to_alt_ratio, config_id)) disp_max = user_disp_max logging.info( 'Disparity range for config {}: [{:.3f} pix., {:.3f} pix.] ' '(or [{:.3f} m., {:.3f} m.])'.format(config_id, disp_min, disp_max, disp_min * disp_to_alt_ratio, disp_max * disp_to_alt_ratio)) configurations_data[config_id]['disp_min'] = disp_min configurations_data[config_id]['disp_max'] = disp_max origin = [ preprocessing_output_config[params.epipolar_origin_x_tag], preprocessing_output_config[params.epipolar_origin_y_tag] ] spacing = [ preprocessing_output_config[params.epipolar_spacing_x_tag], preprocessing_output_config[params.epipolar_spacing_y_tag] ] configurations_data[config_id]['origin'] = origin configurations_data[config_id]['spacing'] = spacing logging.info( "Size of epipolar image: {}".format(largest_epipolar_region)) logging.debug("Origin of epipolar grid: {}".format(origin)) logging.debug("Spacing of epipolar grid: {}".format(spacing)) # Warning if align is set but correction is missing param_lowres_tag = params.lowres_dem_splines_fit_tag if align and param_lowres_tag not in preprocessing_output_config: logging.warning( ('Align with low resolution DSM option is set but splines ' 'correction file is not available for configuration {}. ' 'Correction will not be applied for this configuration' ).format(config_id)) # Numpy array with corners of largest epipolar region. # Order does not matter here, # since it will be passed to stereo.compute_epipolar_grid_min_max corners = np.array( [[[largest_epipolar_region[0], largest_epipolar_region[1]], [largest_epipolar_region[0], largest_epipolar_region[3]]], [[largest_epipolar_region[2], largest_epipolar_region[3]], [largest_epipolar_region[2], largest_epipolar_region[1]]]], dtype=np.float64) # get UTM zone with the middle point of terrain_min if epsg is None if epsg is None: # Compute epipolar image terrain position corners # for min and max disparity terrain_dispmin, terrain_dispmax =\ stereo.compute_epipolar_grid_min_max( corners, 4326, configuration, disp_min, disp_max) epsg = rasterization.get_utm_zone_as_epsg_code( *np.mean(terrain_dispmin, axis=0)) logging.info("EPSG code: {}".format(epsg)) # Compute terrain min and max again, this time using estimated epsg code terrain_dispmin, terrain_dispmax =\ stereo.compute_epipolar_grid_min_max( corners, epsg, configuration, disp_min, disp_max) if roi_epsg is not None: if roi_epsg != epsg: roi_poly =\ projection.polygon_projection(roi_poly, roi_epsg, epsg) # Compute bounds from epipolar image corners and dispmin/dispmax terrain_bounds = np.stack((terrain_dispmin, terrain_dispmax), axis=0) terrain_min = np.amin(terrain_bounds, axis=(0, 1)) terrain_max = np.amax(terrain_bounds, axis=(0, 1)) terrain_area =\ (terrain_max[0]-terrain_min[0])*(terrain_max[1]-terrain_min[1]) configurations_data[config_id]['terrain_area'] = terrain_area logging.info( "Terrain area covered: {} square meters (or square degrees)". format(terrain_area)) # Retrieve bounding box of the ground intersection of the envelopes inter_poly, inter_epsg = utils.read_vector( preprocessing_output_config[params.envelopes_intersection_tag]) if epsg != inter_epsg: inter_poly =\ projection.polygon_projection(inter_poly, inter_epsg, epsg) (inter_xmin, inter_ymin, inter_xmax, inter_ymax) = inter_poly.bounds # Align bounding box to integer resolution steps xmin, ymin, xmax, ymax = tiling.snap_to_grid(inter_xmin, inter_ymin, inter_xmax, inter_ymax, resolution) logging.info("Terrain bounding box : [{}, {}] x [{}, {}]".format( xmin, xmax, ymin, ymax)) configurations_data[config_id]['terrain_bounding_box'] = [ xmin, ymin, xmax, ymax ] if roi is not None: if not roi_poly.intersects(inter_poly): logging.warning("The pair composed of {} and {} " "does not intersect the requested ROI".format( configuration[params.input_section_tag][ params.img1_tag], configuration[params.input_section_tag][ params.img2_tag])) # Get optimal tile size if epi_tile_size is not None: opt_epipolar_tile_size = epi_tile_size else: opt_epipolar_tile_size =\ stereo.optimal_tile_size_pandora_plugin_libsgm( disp_min, disp_max, margin=static_cfg.get_epi_tile_margin_percent()) logging.info( "Optimal tile size for epipolar regions: " "{size}x{size} pixels".format(size=opt_epipolar_tile_size)) configurations_data[config_id]['opt_epipolar_tile_size'] =\ opt_epipolar_tile_size # Split epipolar image in pieces epipolar_regions = tiling.split( 0, 0, preprocessing_output_config[params.epipolar_size_x_tag], preprocessing_output_config[params.epipolar_size_y_tag], opt_epipolar_tile_size, opt_epipolar_tile_size) epipolar_regions_grid = tiling.grid( 0, 0, preprocessing_output_config[params.epipolar_size_x_tag], preprocessing_output_config[params.epipolar_size_y_tag], opt_epipolar_tile_size, opt_epipolar_tile_size) configurations_data[config_id]['epipolar_regions'] = epipolar_regions configurations_data[config_id]['epipolar_regions_grid'] =\ epipolar_regions_grid logging.info("Epipolar image will be processed in {} splits".format( len(epipolar_regions))) # Increment config index config_idx += 1 xmin, ymin, xmax, ymax = tiling.union([ conf['terrain_bounding_box'] for config_id, conf in configurations_data.items() ]) if roi is not None: # terrain bounding box polygon terrain_poly = Polygon([(xmin, ymin), (xmax, ymin), (xmax, ymax), (xmin, ymax), (xmin, ymin)]) if not roi_poly.intersects(terrain_poly): raise Exception( 'None of the input pairs intersect the requested ROI') # Show ROI if valid (no exception raised) : logging.info('Setting terrain bounding box to the requested ROI') xmin, ymin, xmax, ymax = roi_poly.bounds xmin, ymin, xmax, ymax = tiling.snap_to_grid(xmin, ymin, xmax, ymax, resolution) logging.info("Total terrain bounding box : [{}, {}] x [{}, {}]".format( xmin, xmax, ymin, ymax)) # Compute optimal terrain tile size optimal_terrain_tile_widths = [] for config_id, conf in configurations_data.items(): # Compute terrain area covered by a single epipolar tile terrain_area_covered_by_epipolar_tile = conf["terrain_area"] / len( epipolar_regions) # Compute tile width in pixels optimal_terrain_tile_widths.append( math.sqrt(terrain_area_covered_by_epipolar_tile)) # In case of multiple json configuration, take the average optimal size, # and align to multiple of resolution optimal_terrain_tile_width = int( math.ceil( np.mean(optimal_terrain_tile_widths) / resolution)) * resolution logging.info("Optimal terrain tile size: {}x{} pixels".format( int(optimal_terrain_tile_width / resolution), int(optimal_terrain_tile_width / resolution))) # Split terrain bounding box in pieces terrain_grid = tiling.grid(xmin, ymin, xmax, ymax, optimal_terrain_tile_width, optimal_terrain_tile_width) number_of_terrain_splits = (terrain_grid.shape[0] - 1) * (terrain_grid.shape[1] - 1) logging.info("Terrain bounding box will be processed in {} splits".format( number_of_terrain_splits)) # Start dask cluster cluster = None client = None # Use dask use_dask = {"local_dask": True, "pbs_dask": True, "mp": False} if mode not in use_dask.keys(): raise NotImplementedError('{} mode is not implemented'.format(mode)) if use_dask[mode]: if mode == "local_dask": cluster, client = start_local_cluster(nb_workers) else: cluster, client = start_cluster(nb_workers, walltime, out_dir) # Add plugin to monitor memory of workers plugin = ComputeDSMMemoryLogger(out_dir) client.register_worker_plugin(plugin) geoid_data_futures = None if geoid_data is not None: # Broadcast geoid data to all dask workers geoid_data_futures = client.scatter(geoid_data, broadcast=True) # Retrieve the epsg code which will be used # for the triangulation's output points clouds # (ecef if filters are activated) if cloud_small_components_filter or cloud_statistical_outliers_filter: stereo_out_epsg = 4978 else: stereo_out_epsg = epsg # Submit all epipolar regions to be processed as delayed tasks, and # project terrain grid to epipolar for config_id, conf in configurations_data.items(): # This list will hold the different epipolar tiles to be # processed as points cloud delayed_point_clouds = [] if use_dask[mode]: # Use Dask delayed for region in conf['epipolar_regions']: delayed_point_clouds.append( dask.delayed(stereo.images_pair_to_3d_points)( conf['configuration'], region, corr_config, disp_min=conf['disp_min'], disp_max=conf['disp_max'], geoid_data=geoid_data_futures, out_epsg=stereo_out_epsg, use_sec_disp=use_sec_disp, snap_to_img1=snap_to_img1, align=align, add_msk_info=write_msk)) logging.info("Submitted {} epipolar delayed tasks to dask " "for stereo configuration {}".format( len(delayed_point_clouds), config_id)) else: # Use multiprocessing module # create progress bar with an update callback pbar = tqdm(total=len(conf['epipolar_regions'])) def update(args): #pylint: disable=unused-argument pbar.update() # create a thread pool pool = multiprocessing.Pool(nb_workers) # launch several 'write_3d_points()' to process each epipolar region for region in conf['epipolar_regions']: delayed_point_clouds.append( pool.apply_async(write_3d_points, args=(conf['configuration'], region, corr_config, tmp_dir, config_id), kwds={ 'disp_min': conf['disp_min'], 'disp_max': conf['disp_max'], 'geoid_data': geoid_data, 'out_epsg': stereo_out_epsg, 'use_sec_disp': use_sec_disp, "add_msk_info": write_msk }, callback=update)) # Wait computation results (timeout in seconds) and replace the # async objects by the actual output of write_3d_points(), meaning # the paths to cloud files delayed_point_clouds =\ [delayed_pc.get(timeout=per_job_timeout) for delayed_pc in delayed_point_clouds] # closing thread pool when computation is done pool.close() pool.join() configurations_data[config_id]['delayed_point_clouds'] =\ delayed_point_clouds # build list of epipolar region hashes configurations_data[config_id]['epipolar_regions_hash'] = [ region_hash_string(k) for k in conf['epipolar_regions'] ] # Compute disp_min and disp_max location for epipolar grid epipolar_grid_min, epipolar_grid_max = \ stereo.compute_epipolar_grid_min_max( conf['epipolar_regions_grid'], epsg, conf["configuration"], conf['disp_min'], conf['disp_max']) epipolar_regions_grid_flat = conf['epipolar_regions_grid'].reshape( -1, conf['epipolar_regions_grid'].shape[-1]) # in the following code a factor is used to increase the precision spatial_ref = osr.SpatialReference() spatial_ref.ImportFromEPSG(epsg) if spatial_ref.IsGeographic(): precision_factor = 1000.0 else: precision_factor = 1.0 # Build delaunay_triangulation delaunay_min = Delaunay(epipolar_grid_min * precision_factor) delaunay_max = Delaunay(epipolar_grid_max * precision_factor) # Build kdtrees tree_min = cKDTree(epipolar_grid_min * precision_factor) tree_max = cKDTree(epipolar_grid_max * precision_factor) # Look-up terrain_grid with Delaunay s_min = tsearch(delaunay_min, terrain_grid * precision_factor) s_max = tsearch(delaunay_max, terrain_grid * precision_factor) points_disp_min =\ epipolar_regions_grid_flat[delaunay_min.simplices[s_min]] points_disp_max =\ epipolar_regions_grid_flat[delaunay_max.simplices[s_max]] nn_disp_min =\ epipolar_regions_grid_flat[ tree_min.query(terrain_grid*precision_factor)[1]] nn_disp_max =\ epipolar_regions_grid_flat[ tree_max.query(terrain_grid*precision_factor)[1]] points_disp_min_min = np.min(points_disp_min, axis=2) points_disp_min_max = np.max(points_disp_min, axis=2) points_disp_max_min = np.min(points_disp_max, axis=2) points_disp_max_max = np.max(points_disp_max, axis=2) # Use either Delaunay search or NN search # if delaunay search fails (point outside triangles) points_disp_min_min = np.where( np.stack((s_min, s_min), axis=-1) != -1, points_disp_min_min, nn_disp_min) points_disp_min_max = np.where( np.stack((s_min, s_min), axis=-1) != -1, points_disp_min_max, nn_disp_min) points_disp_max_min = np.where( np.stack((s_max, s_max), axis=-1) != -1, points_disp_max_min, nn_disp_max) points_disp_max_max = np.where( np.stack((s_max, s_max), axis=-1) != -1, points_disp_max_max, nn_disp_max) points = np.stack((points_disp_min_min, points_disp_min_max, points_disp_max_min, points_disp_max_max), axis=0) points_min = np.min(points, axis=0) points_max = np.max(points, axis=0) configurations_data[config_id]['epipolar_points_min'] = points_min configurations_data[config_id]['epipolar_points_max'] = points_max # Retrieve number of bands if params.color1_tag in configuration[params.input_section_tag]: nb_bands = utils.rasterio_get_nb_bands( configuration[params.input_section_tag][params.color1_tag]) else: logging.info( 'No color image has been given in input, ' '{} will be used as the color image'.format( configuration[params.input_section_tag][params.img1_tag])) nb_bands = utils.rasterio_get_nb_bands( configuration[params.input_section_tag][params.img1_tag]) logging.info("Number of bands in color image: {}".format(nb_bands)) rank = [] # This list will contained the different raster tiles to be written by cars delayed_dsm_tiles = [] number_of_epipolar_tiles_per_terrain_tiles = [] if not use_dask[mode]: # create progress bar with update callback pbar = tqdm( total=number_of_terrain_splits, desc="Finding correspondences between terrain and epipolar tiles") def update(args): #pylint: disable=unused-argument pbar.update() # initialize a thread pool for multiprocessing mode pool = multiprocessing.Pool(nb_workers) # Loop on terrain regions and derive dependency to epipolar regions for terrain_region_dix in tqdm(range(number_of_terrain_splits), total=number_of_terrain_splits, desc="Delaunay look-up"): j = int(terrain_region_dix / (terrain_grid.shape[1] - 1)) i = terrain_region_dix % (terrain_grid.shape[1] - 1) logging.debug("Processing tile located at {},{} in tile grid".format( i, j)) terrain_region = [ terrain_grid[j, i, 0], terrain_grid[j, i, 1], terrain_grid[j + 1, i + 1, 0], terrain_grid[j + 1, i + 1, 1] ] logging.debug( "Corresponding terrain region: {}".format(terrain_region)) # This list will hold the required points clouds for this terrain tile required_point_clouds = [] # For each stereo configuration for config_id, conf in configurations_data.items(): epipolar_points_min = conf['epipolar_points_min'] epipolar_points_max = conf['epipolar_points_max'] tile_min = np.minimum( np.minimum( np.minimum(epipolar_points_min[j, i], epipolar_points_min[j + 1, i]), np.minimum(epipolar_points_min[j + 1, i + 1], epipolar_points_min[j, i + 1])), np.minimum( np.minimum(epipolar_points_max[j, i], epipolar_points_max[j + 1, i]), np.minimum(epipolar_points_max[j + 1, i + 1], epipolar_points_max[j, i + 1]))) tile_max = np.maximum( np.maximum( np.maximum(epipolar_points_min[j, i], epipolar_points_min[j + 1, i]), np.maximum(epipolar_points_min[j + 1, i + 1], epipolar_points_min[j, i + 1])), np.maximum( np.maximum(epipolar_points_max[j, i], epipolar_points_max[j + 1, i]), np.maximum(epipolar_points_max[j + 1, i + 1], epipolar_points_max[j, i + 1]))) # Bouding region of corresponding cell epipolar_region_minx = tile_min[0] epipolar_region_miny = tile_min[1] epipolar_region_maxx = tile_max[0] epipolar_region_maxy = tile_max[1] # This mimics the previous code that was using # transform_terrain_region_to_epipolar epipolar_region = [ epipolar_region_minx, epipolar_region_miny, epipolar_region_maxx, epipolar_region_maxy ] # Crop epipolar region to largest region epipolar_region = tiling.crop(epipolar_region, conf['largest_epipolar_region']) logging.debug( "Corresponding epipolar region: {}".format(epipolar_region)) # Check if the epipolar region contains any pixels to process if tiling.empty(epipolar_region): logging.debug("Skipping terrain region " "because corresponding epipolar region is empty") else: # Loop on all epipolar tiles covered by epipolar region for epipolar_tile in tiling.list_tiles( epipolar_region, conf['largest_epipolar_region'], conf['opt_epipolar_tile_size']): cur_hash = region_hash_string(epipolar_tile) # Look for corresponding hash in delayed point clouds # dictionnary if cur_hash in conf['epipolar_regions_hash']: # If hash can be found, append it to the required # clouds to compute for this terrain tile pos = conf['epipolar_regions_hash'].index(cur_hash) required_point_clouds.append( conf['delayed_point_clouds'][pos]) # start and size parameters for the rasterization function xstart, ystart, xsize, ysize = tiling.roi_to_start_and_size( terrain_region, resolution) # cloud filtering params if cloud_small_components_filter: small_cpn_filter_params =\ static_cfg.get_small_components_filter_params() else: small_cpn_filter_params = None if cloud_statistical_outliers_filter: statistical_filter_params =\ static_cfg.get_statistical_outliers_filter_params() else: statistical_filter_params = None # rasterization grid division factor rasterization_params = static_cfg.get_rasterization_params() grid_points_division_factor =\ getattr( rasterization_params, static_cfg.grid_points_division_factor_tag) if len(required_point_clouds) > 0: logging.debug( "Number of clouds to process for this terrain tile: {}".format( len(required_point_clouds))) if use_dask[mode]: # Delayed call to rasterization operations using all required # point clouds rasterized = dask.delayed(rasterization_wrapper)( required_point_clouds, resolution, epsg, xstart=xstart, ystart=ystart, xsize=xsize, ysize=ysize, radius=dsm_radius, sigma=sigma, dsm_no_data=dsm_no_data, color_no_data=color_no_data, msk_no_data=msk_no_data, small_cpn_filter_params=small_cpn_filter_params, statistical_filter_params=statistical_filter_params, grid_points_division_factor=grid_points_division_factor) # Keep track of delayed raster tiles delayed_dsm_tiles.append(rasterized) rank.append(i * i + j * j) else: # Launch asynchrone job for write_dsm_by_tile() delayed_dsm_tiles.append( pool.apply_async( write_dsm_by_tile, args=(required_point_clouds, resolution, epsg, tmp_dir, nb_bands, static_cfg.get_color_image_encoding(), output_stats, write_msk), kwds={ 'xstart': xstart, 'ystart': ystart, 'xsize': xsize, 'ysize': ysize, 'radius': dsm_radius, 'sigma': sigma, 'dsm_no_data': dsm_no_data, 'color_no_data': color_no_data, 'small_cpn_filter_params': small_cpn_filter_params, 'statistical_filter_params': statistical_filter_params, 'grid_points_division_factor': grid_points_division_factor, 'msk_no_data': msk_no_data }, callback=update)) number_of_epipolar_tiles_per_terrain_tiles.append( len(required_point_clouds)) logging.info( "Average number of epipolar tiles " "for each terrain tile: {}".format( int(np.round( np.mean(number_of_epipolar_tiles_per_terrain_tiles))))) logging.info("Max number of epipolar tiles " "for each terrain tile: {}".format( np.max(number_of_epipolar_tiles_per_terrain_tiles))) bounds = (xmin, ymin, xmax, ymax) # Derive output image files parameters to pass to rasterio xsize, ysize = tiling.roi_to_start_and_size([xmin, ymin, xmax, ymax], resolution)[2:] out_dsm = os.path.join(out_dir, "dsm.tif") out_clr = os.path.join(out_dir, "clr.tif") out_msk = None if write_msk: out_msk = os.path.join(out_dir, "msk.tif") out_dsm_mean = os.path.join(out_dir, "dsm_mean.tif") out_dsm_std = os.path.join(out_dir, "dsm_std.tif") out_dsm_n_pts = os.path.join(out_dir, "dsm_n_pts.tif") out_dsm_points_in_cell = os.path.join(out_dir, "dsm_pts_in_cell.tif") if use_dask[mode]: # Sort tiles according to rank delayed_dsm_tiles = [ delayed for _, delayed in sorted(zip(rank, delayed_dsm_tiles), key=lambda pair: pair[0]) ] logging.info("Submitting {} tasks to dask".format( len(delayed_dsm_tiles))) # Transform all delayed raster tiles to futures (computation starts # immediatly on workers, assynchronously) future_dsm_tiles = client.compute(delayed_dsm_tiles) logging.info("DSM output image size: {}x{} pixels".format( xsize, ysize)) readwrite.write_geotiff_dsm( future_dsm_tiles, out_dir, xsize, ysize, bounds, resolution, epsg, nb_bands, dsm_no_data, color_no_data, color_dtype=static_cfg.get_color_image_encoding(), write_color=True, write_stats=output_stats, write_msk=write_msk, msk_no_data=msk_no_data) # stop cluster stop_cluster(cluster, client) else: logging.info("Computing DSM tiles ...") # Wait for asynchrone jobs (timeout in seconds) and replace them by # write_dsm_by_tile() output delayed_dsm_tiles = [ delayed_tile.get(timeout=per_job_timeout) for delayed_tile in delayed_dsm_tiles ] # closing the tread pool after computation pool.close() pool.join() # vrt to tif logging.info("Building VRT") vrt_options = gdal.BuildVRTOptions(resampleAlg='nearest') def vrt_mosaic(tiles_glob, vrt_name, vrt_options, output): vrt_file = os.path.join(out_dir, vrt_name) tiles_list = glob(os.path.join(out_dir, 'tmp', tiles_glob)) gdal.BuildVRT(vrt_file, tiles_list, options=vrt_options) vrt_file_descriptor = gdal.Open(vrt_file) vrt_file_descriptor = gdal.Translate(output, vrt_file_descriptor) vrt_file_descriptor = None vrt_mosaic('*_dsm.tif', 'dsm.vrt', vrt_options, out_dsm) vrt_mosaic('*_clr.tif', 'clr.vrt', vrt_options, out_clr) if write_msk: vrt_mosaic('*_msk.tif', 'msk.vrt', vrt_options, out_msk) if output_stats: vrt_mosaic('*_dsm_mean.tif', 'dsm_mean.vrt', vrt_options, out_dsm_mean) vrt_mosaic('*_dsm_std.tif', 'dsm_std.vrt', vrt_options, out_dsm_std) vrt_mosaic('*_dsm_n_pts.tif', 'dsm_n_pts.vrt', vrt_options, out_dsm_n_pts) vrt_mosaic('*_pts_in_cell.tif', 'dsm_pts_in_cell.vrt', vrt_options, out_dsm_points_in_cell) # Fill output json file out_json[params.stereo_section_tag][params.stereo_output_section_tag][ params.epsg_tag] = epsg out_json[params.stereo_section_tag][params.stereo_output_section_tag][ params.dsm_tag] = out_dsm out_json[params.stereo_section_tag][params.stereo_output_section_tag][ params.dsm_no_data_tag] = float(dsm_no_data) out_json[params.stereo_section_tag][params.stereo_output_section_tag][ params.color_no_data_tag] = float(color_no_data) out_json[params.stereo_section_tag][params.stereo_output_section_tag][ params.color_tag] = out_clr if write_msk: out_json[params.stereo_section_tag][params.stereo_output_section_tag][ params.msk_tag] = out_msk if output_stats: out_json[params.stereo_section_tag][params.stereo_output_section_tag][ params.dsm_mean_tag] = out_dsm_mean out_json[params.stereo_section_tag][params.stereo_output_section_tag][ params.dsm_std_tag] = out_dsm_std out_json[params.stereo_section_tag][params.stereo_output_section_tag][ params.dsm_n_pts_tag] = out_dsm_n_pts out_json[params.stereo_section_tag][params.stereo_output_section_tag][ params.dsm_points_in_cell_tag] = out_dsm_points_in_cell # Write the output json out_json_path = os.path.join(out_dir, "content.json") try: utils.check_json(out_json, params.stereo_content_schema) except CheckerError as check_error: logging.warning( "content.json does not comply with schema: {}".format(check_error)) params.write_stereo_content_file(out_json, out_json_path)
def get_subsampling_index2(data_process, standard_scale = True, cutoff_sig = 0.02, rate = 0.3, \ method = "pykdtree", verbose = 1): """ Using Nearest-Neighbor search based algorithm, find the list of indices of the subsampled dataset Parameters ------------- data_process: List. the list of datapoints, with selected features standard_scale [True]: Boolean. Whether to apply standard scaler to the dataset prior to subsampling cutoff_sig [0.02]: Float. cutoff significance. the cutoff distance equals to the Euclidean norm of the standard deviations in all dimensions of the data points rate [0.3]: Float. possibility of deletion method ["pykdtree"]: String. which backend nearest neighbour model to use. possible choices: ["pykdtree", "nmslib", "sklearn", "scipy", "annoy", "flann"] verbose [1]: integer. level of verbosity Return ------------- overall_keep_list: The list of indices of the final subsampled entries """ if verbose >=1: print("Started NN-subsampling, original length: {}".format(len(data_process))) method = method.lower() start = time.time() if method == "flann": if verbose >=1: print("use flann backend") elif method == "pykdtree": if verbose >=1: print("use pykdtree backend") elif method == "sklearn": if verbose >=1: print("use slearn nearest neighbors backend") elif method == "scipy": if verbose >=1: print("use scipy cKDTree backend") elif method == "annoy": if verbose >=1: print("use annoy backend") elif method == "nmslib": if verbose >=1: print("use nmslib backend") else: print("method {} not impletemented".format(method)) raise NotImplemented # apply standard scaling if standard_scale: if verbose >= 2: print("Subample with standard scaled data") data_process = StandardScaler().fit_transform(np.asarray(data_process).copy()) else: if verbose >= 2: print("Subample with original data") data_process = np.asarray(data_process).copy() #set cutoff distance list_of_descs = zip(*data_process) sum_std2 = 0. for descs in list_of_descs: temp_std = np.std(descs) sum_std2 += temp_std**2 cutoff = cutoff_sig * np.sqrt(sum_std2) #initialize the index overall_keep_list = np.arange(len(data_process)).tolist() keep_going = True iter_count = 1 while keep_going: if verbose >= 2: print('start iteration {}, total length: {}'.format(iter_count, len(overall_keep_list))) start_cycle = time.time() temp_data_process = get_array_based_on_index(data_process.copy(), overall_keep_list) #build and query nearest neighbour model if method == "flann": flann = FLANN() indices, distances = flann.nn(temp_data_process, temp_data_process, 2, algorithm="kmeans") elif method == "scipy": kd_tree = cKDTree(temp_data_process) distances, indices = kd_tree.query(temp_data_process, k=2) elif method == "pykdtree": kd_tree = KDTree(temp_data_process,leafsize=6) distances, indices = kd_tree.query(temp_data_process, k=2) elif method == "sklearn": nbrs = NearestNeighbors(n_neighbors=2, algorithm='kd_tree',n_jobs=-1).fit(temp_data_process) distances, indices = nbrs.kneighbors(temp_data_process) elif method == "annoy": annoy = AnnoyIndex(len(temp_data_process[0]),metric='euclidean') for i in range(len(temp_data_process)): annoy.add_item(i,temp_data_process[i]) annoy.build(1) distances = [] indices = [] for i in range(len(temp_data_process)): temp_index, temp_dist = annoy.get_nns_by_vector(temp_data_process[i], 2, include_distances=True) indices.append([i, temp_index[1]]) distances.append([0.0, temp_dist[1]]) elif method == "nmslib": index = nmslib.init(method='hnsw', space='l2') index.addDataPointBatch(temp_data_process) index.createIndex( print_progress=False) neighbours = index.knnQueryBatch(temp_data_process, k=2) distances = [] indices = [] for item in neighbours: indices.append(item[0]) distances.append(item[1]) else: raise NotImplemented # if distance between each point and its nearest neighbor is below cutoff distance, # add the nearest neighbout to the candidate removal list remove_index_li = [] index_li = [] for index, distance in zip(indices, distances): index_li.append(index[0]) if distance[1] <= cutoff: remove_index_li.append(index[1]) # randomly select datapoints in the candidate removal list (based on rate) # and form the final removal list of this iteration # stop the cycle if the final removal list is empty temp_num = int(ceil(float(len(remove_index_li))*rate)) if temp_num == 0: keep_going = False remove_index_li = random_subsampling(remove_index_li,temp_num) temp_keep_list = remove_list_from_list(index_li, remove_index_li) overall_keep_list = [overall_keep_list[i] for i in temp_keep_list ] if verbose >= 2: print('end iteration {}. length: {}\t time:{}'.format(iter_count, len(overall_keep_list), time.time()-start_cycle)) iter_count += 1 if verbose >= 1: print('end NN-subsampling. length: {}\t time:{}'.format(len(overall_keep_list), time.time()-start)) return overall_keep_list
def execute(self, style, xpoints, ypoints, mask=None, backend='vectorized', n_closest_points=None): """Calculates a kriged grid and the associated variance. This is now the method that performs the main kriging calculation. Note that currently measurements (i.e., z values) are considered 'exact'. This means that, when a specified coordinate for interpolation is exactly the same as one of the data points, the variogram evaluated at the point is forced to be zero. Also, the diagonal of the kriging matrix is also always forced to be zero. In forcing the variogram evaluated at data points to be zero, we are effectively saying that there is no variance at that point (no uncertainty, so the value is 'exact'). In the future, the code may include an extra 'exact_values' boolean flag that can be adjusted to specify whether to treat the measurements as 'exact'. Setting the flag to false would indicate that the variogram should not be forced to be zero at zero distance (i.e., when evaluated at data points). Instead, the uncertainty in the point will be equal to the nugget. This would mean that the diagonal of the kriging matrix would be set to the nugget instead of to zero. Parameters ---------- style : str Specifies how to treat input kriging points. Specifying 'grid' treats xpoints and ypoints as two arrays of x and y coordinates that define a rectangular grid. Specifying 'points' treats xpoints and ypoints as two arrays that provide coordinate pairs at which to solve the kriging system. Specifying 'masked' treats xpoints and ypoints as two arrays of x and y coordinates that define a rectangular grid and uses mask to only evaluate specific points in the grid. xpoints : array_like, shape (N,) or (N, 1) If style is specific as 'grid' or 'masked', x-coordinates of MxN grid. If style is specified as 'points', x-coordinates of specific points at which to solve kriging system. ypoints : array_like, shape (M,) or (M, 1) If style is specified as 'grid' or 'masked', y-coordinates of MxN grid. If style is specified as 'points', y-coordinates of specific points at which to solve kriging system. Note that in this case, xpoints and ypoints must have the same dimensions (i.e., M = N). mask : bool, array_like, shape (M, N), optional Specifies the points in the rectangular grid defined by xpoints and ypoints that are to be excluded in the kriging calculations. Must be provided if style is specified as 'masked'. False indicates that the point should not be masked, so the kriging system will be solved at the point. True indicates that the point should be masked, so the kriging system should will not be solved at the point. backend : str, optional Specifies which approach to use in kriging. Specifying 'vectorized' will solve the entire kriging problem at once in a vectorized operation. This approach is faster but also can consume a significant amount of memory for large grids and/or large datasets. Specifying 'loop' will loop through each point at which the kriging system is to be solved. This approach is slower but also less memory-intensive. Specifying 'C' will utilize a loop in Cython. Default is 'vectorized'. n_closest_points : int, optional For kriging with a moving window, specifies the number of nearby points to use in the calculation. This can speed up the calculation for large datasets, but should be used with caution. As Kitanidis notes, kriging with a moving window can produce unexpected oddities if the variogram model is not carefully chosen. Returns ------- zvalues : ndarray, shape (M, N) or (N, 1) Z-values of specified grid or at the specified set of points. If style was specified as 'masked', zvalues will be a numpy masked array. sigmasq : ndarray, shape (M, N) or (N, 1) Variance at specified grid points or at the specified set of points. If style was specified as 'masked', sigmasq will be a numpy masked array. """ if self.verbose: print("Executing Ordinary Kriging...\n") if style != 'grid' and style != 'masked' and style != 'points': raise ValueError("style argument must be 'grid', " "'points', or 'masked'") if n_closest_points is not None and n_closest_points <= 1: # If this is not checked, nondescriptive errors emerge # later in the code. raise ValueError("n_closest_points has to be at least two!") xpts = np.atleast_1d(np.squeeze(np.array(xpoints, copy=True))) ypts = np.atleast_1d(np.squeeze(np.array(ypoints, copy=True))) n = self.X_ADJUSTED.shape[0] nx = xpts.size ny = ypts.size a = self._get_kriging_matrix(n) if style in ['grid', 'masked']: if style == 'masked': if mask is None: raise IOError("Must specify boolean masking array " "when style is 'masked'.") if mask.shape[0] != ny or mask.shape[1] != nx: if mask.shape[0] == nx and mask.shape[1] == ny: mask = mask.T else: raise ValueError("Mask dimensions do not match " "specified grid dimensions.") mask = mask.flatten() npt = ny * nx grid_x, grid_y = np.meshgrid(xpts, ypts) xpts = grid_x.flatten() ypts = grid_y.flatten() elif style == 'points': if xpts.size != ypts.size: raise ValueError("xpoints and ypoints must have " "same dimensions when treated as " "listing discrete points.") npt = nx else: raise ValueError("style argument must be 'grid', " "'points', or 'masked'") if self.coordinates_type == 'euclidean': xpts, ypts = _adjust_for_anisotropy( np.vstack((xpts, ypts)).T, [self.XCENTER, self.YCENTER], [self.anisotropy_scaling], [self.anisotropy_angle]).T # Prepare for cdist: xy_data = np.concatenate( (self.X_ADJUSTED[:, np.newaxis], self.Y_ADJUSTED[:, np.newaxis]), axis=1) xy_points = np.concatenate( (xpts[:, np.newaxis], ypts[:, np.newaxis]), axis=1) elif self.coordinates_type == 'geographic': # In spherical coordinates, we do not correct for anisotropy. # Also, we don't use scipy.spatial.cdist, so we do not have to # format the input data accordingly. pass if style != 'masked': mask = np.zeros(npt, dtype='bool') c_pars = None if backend == 'C': try: from .lib.c*k import _c_exec_loop, _c_exec_loop_moving_window except ImportError: print('Warning: failed to load Cython extensions.\n' ' See https://github.com/bsmurphy/PyKrige/issues/8 \n' ' Falling back to a pure python backend...') backend = 'loop' except: raise RuntimeError("Unknown error in trying to " "load Cython extension.") c_pars = { key: getattr(self, key) for key in [ 'Z', 'eps', 'variogram_model_parameters', 'variogram_function' ] } if n_closest_points is not None: if self.coordinates_type == 'geographic': # To make use of the KDTree, we have to convert the # spherical coordinates into three dimensional Euclidean # coordinates, since the standard KDTree cannot handle # the periodicity. # Do the conversion just for the step involving the KDTree: lon_d = self.X_ADJUSTED[:, np.newaxis] * np.pi / 180.0 lat_d = self.Y_ADJUSTED[:, np.newaxis] * np.pi / 180.0 xy_data = np.concatenate( (np.cos(lon_d) * np.cos(lat_d), np.sin(lon_d) * np.cos(lat_d), np.sin(lat_d)), axis=1) lon_p = xpts[:, np.newaxis] * np.pi / 180.0 lat_p = ypts[:, np.newaxis] * np.pi / 180.0 xy_points = np.concatenate( (np.cos(lon_p) * np.cos(lat_p), np.sin(lon_p) * np.cos(lat_p), np.sin(lat_p)), axis=1) from scipy.spatial import cKDTree tree = cKDTree(xy_data) bd, bd_idx = tree.query(xy_points, k=n_closest_points, eps=0.0) if self.coordinates_type == 'geographic': # Between the nearest neighbours from Euclidean search, # calculate the great circle distance using the standard method: x_points = np.tile(xpts[:, np.newaxis], (1, n_closest_points)) y_points = np.tile(ypts[:, np.newaxis], (1, n_closest_points)) bd = core.great_circle_distance(x_points, y_points, self.X_ADJUSTED[bd_idx], self.Y_ADJUSTED[bd_idx]) if backend == 'loop': zvalues, sigmasq = \ self._exec_loop_moving_window(a, bd, mask, bd_idx) elif backend == 'C': zvalues, sigmasq = \ _c_exec_loop_moving_window(a, bd, mask.astype('int8'), bd_idx, self.X_ADJUSTED.shape[0], c_pars) else: raise ValueError('Specified backend {} for a moving window ' 'is not supported.'.format(backend)) else: if self.coordinates_type == 'euclidean': bd = cdist(xy_points, xy_data, 'euclidean') elif self.coordinates_type == 'geographic': bd = core.great_circle_distance(xpts[:, np.newaxis], ypts[:, np.newaxis], self.X_ADJUSTED, self.Y_ADJUSTED) if backend == 'vectorized': zvalues, sigmasq = self._exec_vector(a, bd, mask) elif backend == 'loop': zvalues, sigmasq = self._exec_loop(a, bd, mask) elif backend == 'C': zvalues, sigmasq = _c_exec_loop(a, bd, mask.astype('int8'), self.X_ADJUSTED.shape[0], c_pars) else: raise ValueError('Specified backend {} is not supported for ' '2D ordinary kriging.'.format(backend)) if style == 'masked': zvalues = np.ma.array(zvalues, mask=mask) sigmasq = np.ma.array(sigmasq, mask=mask) if style in ['masked', 'grid']: zvalues = zvalues.reshape((ny, nx)) sigmasq = sigmasq.reshape((ny, nx)) return zvalues, sigmasq
def count_i(land_name): data = land_name.split('/')[-1].split('.')[0] #file1 = open('txt_file_m18_m/'+data+'.txt','w') maxNum = 0 maxName = '' for name in glob(feature_db + '*'): #print (name) name2 = name.split('/') number = name2[-1].split('.')[0] # Read features. locations_1, _, descriptors_1, _, _ = feature_io.ReadFromFile( land_name) #cmd_args.features_1_path) num_features_1 = locations_1.shape[0] #tf.logging.info("Loaded image 1's %d features" % num_features_1) locations_2, _, descriptors_2, _, _ = feature_io.ReadFromFile(name) #cmd_args.features_2_path) num_features_2 = locations_2.shape[0] #tf.logging.info("Loaded image 2's %d features" % num_features_2) # Find nearest-neighbor matches using a KD tree. d1_tree = cKDTree(descriptors_1) _, indices = d1_tree.query(descriptors_2, distance_upper_bound=_DISTANCE_THRESHOLD) # Select feature locations for putative matches. locations_2_to_use = np.array([ locations_2[i, ] for i in range(num_features_2) if indices[i] != num_features_1 ]) locations_1_to_use = np.array([ locations_1[indices[i], ] for i in range(num_features_2) if indices[i] != num_features_1 ]) # Perform geometric verification using RANSAC. try: _, inliers = ransac((locations_1_to_use, locations_2_to_use), AffineTransform, min_samples=3, residual_threshold=20, max_trials=1000) #print(sum(inliers)) #print(maxNum) #""" #tf.logging.info('Found %d inliers' % sum(inliers)) #print(sum(inliers)) #print(maxNum) score = int(sum(inliers)) #file1.write(name+'\t'+str(sum(inliers))+'\n') #maxName = num2name[number] num = name.split('/')[-1].split('.')[0] if score > 35 and score > maxNum: maxNum = score maxName = num2name[number] print(maxName) print(maxNum) #break #""" except: #print('fail') a = 0 #break #print (maxName) #result_dic[data] = maxName return maxName
def make_pos_neg_dataset(tensor_first_view, tensor_second_view, ss, C, view1_id, view2_id, default_sigma=0.8,size=5, save_tiff_files=False, find_negative=True): tensor_first_view = tensor_first_view.astype(np.float32) tensor_second_view = tensor_second_view.astype(np.float32) H, W, D = ss.info['Height'], ss.info['Width'], ss.info['Depth'] patchlen = (1 + 2 * size) ** 3 margin = ss.plist['Margin'] / 2 c_array = np.array([[c.x, c.y, c.z] for c in C]) trunc=1.5 fixed_radiiassigned={c:default_sigma for c in C if inside_margin(c,ss) > 0 } cccc=fixed_radiiassigned.copy() kdt = cKDTree(c_array) dist,ind = kdt.query(c_array,k=c_array.shape[0],distance_upper_bound=(2*default_sigma*trunc)+2.) for c,id_c in zip(C,xrange(c_array.shape[0])): if inside_margin(c, ss) > 0: for j in xrange(2,c_array.shape[0]): if not np.isinf(dist[id_c][j]): if inside_margin(C[ind[id_c][j]], ss) > 0: if trunc*(fixed_radiiassigned[c]+fixed_radiiassigned[C[ind[id_c][j]]]) > dist[id_c][j] - 2.: new_sigma=((dist[id_c][j]-1.)/trunc)*fixed_radiiassigned[c]/(fixed_radiiassigned[c]+fixed_radiiassigned[C[ind[id_c][j]]]) fixed_radiiassigned[C[ind[id_c][j]]]=(new_sigma*fixed_radiiassigned[C[ind[id_c][j]]])/fixed_radiiassigned[c] fixed_radiiassigned[c]=new_sigma target_tensor_3d = np.zeros(tensor_first_view.shape,dtype=np.uint8) for c in C: if inside_margin(c, ss) > 0: target_tensor_3d_tmp = np.zeros(tensor_first_view.shape) target_tensor_3d_tmp[c.z, c.y, c.x] = 1 sigma=fixed_radiiassigned[c] target_tensor_3d_tmp = gfilter.gaussian_filter(target_tensor_3d_tmp, sigma, mode='constant', cval=0.0, truncate=trunc) #sigma=3.5,mode='constant',cval=0.0,truncate=1.5 target_tensor_3d_tmp = (target_tensor_3d_tmp.astype(np.float32) / np.max(target_tensor_3d_tmp)) target_tensor_3d = np.maximum(np.array(target_tensor_3d_tmp * 255.0, dtype=np.uint8), target_tensor_3d) if save_tiff_files: debug_path='/tmp/debug/sigma_'+str(default_sigma) mkdir_p(debug_path) minz = 0 imtensor.save_tensor_as_tif(target_tensor_3d, debug_path+'/'+ss.substack_id+'_'+view1_id+'_'+view2_id, minz) print('num markers =',len(C)) target_tensor_3d = (target_tensor_3d.astype(np.float32) / np.max(target_tensor_3d)) tensor_first_view = tensor_first_view.astype(np.float32) / 255.0 tensor_second_view = tensor_second_view.astype(np.float32) / 255.0 nrej_intensity = 0 num_negative_targets = 0 num_positive_targets = 0 step_x = 1 step_y = 1 step_z = 1 num_iterations = (W - 2*margin +1) * (H- 2*margin) * (D - 2*margin) / (step_x * step_y * step_z) pbar = ProgressBar(widgets=['Making positive and negative examples for %d points: ' % num_iterations, Percentage()], maxval=num_iterations).start() X_positive = np.zeros((num_iterations, patchlen*2), dtype=np.float32) y_positive = np.zeros((num_iterations, patchlen), dtype=np.float32) X_negative = [] y_negative = [] if find_negative == True: X_negative = np.zeros((num_iterations, patchlen*2), dtype=np.float32) y_negative = np.zeros((num_iterations, patchlen), dtype=np.float32) pbi = 0 for x0 in range(margin, W - margin, step_x): for y0 in range(margin, H - margin, step_y): for z0 in range(margin, D - margin, step_z): if inside_patch(c_array, x0, y0, z0, size, offset=2.0): patch_left = tensor_first_view[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1] patch_right = tensor_second_view[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1] X_positive[num_positive_targets, 0:patchlen] = np.reshape(patch_left, (patchlen,)) X_positive[num_positive_targets, patchlen:2*patchlen] = np.reshape(patch_right, (patchlen,)) ypatch = target_tensor_3d[z0 - size:z0 + size + 1, y0 - size:y0 + size + 1, x0 - size:x0 + size + 1] y_positive[num_positive_targets, :] = np.reshape(ypatch, (patchlen,)) num_positive_targets += 1 elif find_negative == True: patch_left = tensor_first_view[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1] patch_right = tensor_second_view[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1] if np.mean(patch_left * 255) > 5 or np.mean(patch_right * 255) > 5: X_negative[num_negative_targets, 0:patchlen] = np.reshape(patch_left, (patchlen,)) X_negative[num_negative_targets, patchlen:2*patchlen] = np.reshape(patch_right, (patchlen,)) ypatch = target_tensor_3d[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1] y_negative[num_negative_targets, :] = np.reshape(ypatch, (patchlen,)) num_negative_targets += 1 else: nrej_intensity += 1 pbar.update(pbi+ 1) pbi += 1 pbar.finish() X_positive = X_positive[0: num_positive_targets] y_positive = y_positive[0: num_positive_targets] print('Total positive examples for substack (', ss.substack_id, '):', num_positive_targets) if find_negative == True: print('Total negative examples for substack (', ss.substack_id, '):', num_negative_targets) print('Rejected by intensity ', nrej_intensity) X_negative = X_negative[0: num_negative_targets] y_negative = y_negative[0: num_negative_targets] if (num_negative_targets > num_positive_targets): print('Shuffling negative examples...') perm = np.random.permutation(len(X_negative)).astype(int) X_negative = X_negative[perm] y_negative = y_negative[perm] print('Shuffling done') X_negative = X_negative[0: num_positive_targets] y_negative = y_negative[0: num_positive_targets] return X_positive, y_positive, X_negative, y_negative
def processPlumes(options): eps = 1e-5 #=============================== Local functions def checkConduitContinuity(nnodes): maxVrShallow = -1e20 maxVrDeep = -1e20 selectedNodeShallow = -1 selectedNodeDeep = -1 for n in nnodes: if ((cdata[n, 5] > maxVrShallow) and (cdata[n, 9] == 0)): maxVrShallow = cdata[n, 5] selectedNodeShallow = n #end if if ((cdata[n, 11] > maxVrDeep) and (cdata[n, 15] == 0)): maxVrDeep = cdata[n, 11] selectedNodeDeep = n #end if #end for # Check for continuity of identified plume-region at depth if (cdata[selectedNodeShallow, 9] == cdata[selectedNodeDeep, 15]): return selectedNodeShallow, selectedNodeDeep, True, else: return selectedNodeShallow, selectedNodeDeep, False #end if #end func def isPlumeEruption(plumeIds, theta, phi): mindist = 1e20 for p in plumeIds: dist = haversine(cdata[p, 2], cdata[p, 1], phi, theta) if (dist < mindist): mindist = dist #end for if (mindist > MIN_PLUME_DIST): return True else: return False #end func def shouldAdd(plist, pid): nxyz = rtp2xyz(cdata[pid,0]*RADIUS, \ (90-cdata[pid,1])*math.pi/180, \ cdata[pid,2]*math.pi/180) for epid in plist: exyz = rtp2xyz(cdata[epid,0]*RADIUS, \ (90-cdata[epid,1])*math.pi/180, \ cdata[epid,2]*math.pi/180) dist = (nxyz[0] - exyz[0])**2 + \ (nxyz[1] - exyz[1])**2 + \ (nxyz[2] - exyz[2])**2 dist = math.sqrt(dist) if (dist < MIN_PLUME_DIST): return 0 #end for return 1 #end func def getNeighbours(nbFileName): nf = open(nbFileName) neighbours = defaultdict(list) for line in nf: vals = line.split(' ') for i in range(len(vals)): if (i > 0): neighbours[int(vals[0])].append(int(vals[i])) #end for #end for nf.close() return neighbours #end func def getNeighbouringNodes(tree, xyz, radius): l = tree.query_ball_point(xyz, radius) return l #end func #=============================== Begin processing def key_func(x): return int(x.split('.')[-2]) fileNames = glob.glob('%s*.txt' % (options.baseName)) fileNames = sorted(fileNames, key=key_func) minNablaV = 1e30 maxNablaV = -1e30 cdataList = [] for fileName in fileNames: ''' Data-format========================================= r, theta, phi, time, magGradVr, v_r, T, Eta, data cid magGradVrVD, v_rVD, TVD, EtaVD, dataVD cidVD tracComp 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 */ =====================================================''' # Load clusteredData cdata = np.loadtxt(fileName) cdataList.append(cdata) if (np.min(cdata[:, 4]) < minNablaV): minNablaV = np.min(cdata[:, 4]) if (np.max(cdata[:, 4]) > maxNablaV): maxNablaV = np.max(cdata[:, 4]) #end for nodes = [] tree = None plumeIdsList = [] # List of identified plumes for each time-slice outFile = open(options.outFileName, 'w') for counter, fileName in enumerate(fileNames): print 'Processing: %s..' % (fileName) currTime = int(fileName.split('.')[-2]) dirName = os.path.dirname(options.outFileName) if (len(dirName) == 0): dirName = './' baseName = os.path.basename(fileName) suppDirName = '%s/supp/' % (dirName) # create supplimentary directory if it doesn't exist if (not os.path.exists(suppDirName)): os.makedirs(suppDirName) #end if cdata = cdataList[counter] if (counter == 0): for i in range(np.shape(cdata)[0]): xyz = rtp2xyz(RADIUS, math.pi / 2 - cdata[i, 1] * math.pi / 180, cdata[i, 2] * math.pi / 180) nodes.append(xyz) #end for tree = spatial.cKDTree(nodes, 16) #end if dataOut = np.zeros((361, 181)) xOut = np.zeros((361, 181)) yOut = np.zeros((361, 181)) for i in range(361): for j in range(181): theta = float(j) phi = float(i) xOut[i, j] = phi * math.pi / 180 yOut[i, j] = theta * math.pi / 180 xyz = rtp2xyz(RADIUS, yOut[i, j], xOut[i, j]) (d, l) = tree.query(xyz) dataOut[i, j] = cdata[l, 4] * THERM_DIFF / RADIUS #magGradVrDepth #dataOut[i,j] = cdata[l, 9] #cidDepth #dataOut[i,j] = cdata[l, 15] #cidValidationDepth #end for #end for #=======================================Compute fluxes numNodes = np.shape(cdata)[0] plumeIds = [] meanTemp = np.mean(cdata[:, 6], axis=0) meanBgTemp = 0. hitCount = 0. for i in range(numNodes): if (cdata[i, 6] > meanTemp): meanBgTemp += cdata[i, 6] hitCount += 1. #end if #end for meanBgTemp /= hitCount print 'mean: %f, meanBg: %f' % (meanTemp, meanBgTemp) for i in range(numNodes): if ((cdata[i, 9] == 0)): nn = np.array( getNeighbouringNodes(tree, nodes[i], MAX_PLUME_OFFSET)) selectedNodeShallow, selectedNodeDeep, conduitIsContinuous = checkConduitContinuity( nn) if (conduitIsContinuous == False): continue # Check if plume-conduit meets the minimum radial velocity criteria if (cdata[selectedNodeShallow, 5] * VSCALE < options.radialVelocityMin): continue if (shouldAdd(plumeIds, selectedNodeShallow)): plumeIds.append(selectedNodeShallow) #end if #end if #end for # Save plume-ids list plumeIdsList.append(plumeIds) flux = np.zeros(len(plumeIds)) area = np.zeros(len(plumeIds)) avgConduitTemp = np.zeros(len(plumeIds)) avgConduitConcentration = np.zeros(len(plumeIds)) for i, pid in enumerate(plumeIds): #print "plume %d: T(%d) = %f"%(i, pid, cdata[pid,6]) suppFigName = '%s/%s.plume%d.png' % (suppDirName, baseName, i) nn = np.array( getNeighbouringNodes(tree, nodes[pid], PLUME_PLOT_RADIUS)) p = Plume( options, np.array((cdata[nn, 2], 90 - cdata[nn, 1], cdata[nn, 6], cdata[nn, 5], cdata[nn, 16])).T, meanBgTemp, suppFigName) result = p.ComputeBuoyancyFlux() flux[i] = result['flux'] area[i] = result['area'] avgConduitTemp[i] = result['avgConduitTemp'] avgConduitConcentration[i] = result['avgConduitConcentration'] # Report plume======= mlatp = 90 - p.mlat mlonp = p.mlon if (mlonp > 180): mlonp = mlonp - 360 print '\tplume(%d) loc: (%f %f)' % (i, mlatp, mlonp) #end for #================================== Done computing fluxes #================================== Prepare data and axes dataOutFlipped = np.zeros(np.shape(dataOut)) dataOutFlipped[0:181:, :] = dataOut[180:361, :] dataOutFlipped[181:361:, :] = dataOut[0:180, :] fig, gax = plt.subplots(nrows=1, ncols=1) fig.set_size_inches(11.69, 11.69) m = Basemap(ax=gax, projection='moll', lon_0=0, resolution='c') #=======================================Plot flux pxOut, pyOut = m(xOut * 180 / math.pi - 180, 90 - yOut * 180 / math.pi) cmap = cm.get_cmap(name='jet', lut=None) cbInfo = m.pcolormesh(pxOut, pyOut, dataOutFlipped, cmap=cmap) m.colorbar(cbInfo, "bottom", size="5%", pad="15%") legList = [] labels = [] for i, pid in enumerate(plumeIds): #print "plume %d: T(%d) = %f"%(i, pid, cdata[pid,6]) #coords = circle(m, cdata[pid,2], cdata[pid,1], 50*flux[i]) #casa = SPolygon(coords) #cpatch = PolygonPatch(casa, fc='magenta', ec='#333333', lw=0, alpha=.25, zorder=2) #gax.add_patch(cpatch) if (flux[i] > 0): plumeEruption = 0 if (counter): plumeEruption = int( isPlumeEruption(plumeIdsList[counter - 1], cdata[pid, 1], cdata[pid, 2])) #end if #(time, theta, phi, T, Vr, Flux, Area, Eruption, meanBgTemp, avgConduitTemp, concentration) outFile.write('%f %f %f %f %f %f %f %f %f %f %f\n' % (currTime, cdata[pid, 1], cdata[pid, 2], cdata[pid, 6], cdata[pid, 5], flux[i], area[i], plumeEruption, meanBgTemp, avgConduitTemp[i], avgConduitConcentration[i])) px, py = m(cdata[pid, 2], cdata[pid, 1]) #m.scatter(px,py,flux[i]*SYMBOL_SCALE, marker='o',color='green', edgecolors='none', alpha=0.5) m.scatter(px, py, flux[i] * SYMBOL_SCALE, marker='o', color='none', edgecolors='white', lw=0.5, alpha=1) leg = m.scatter([], [], s=0, edgecolors='black', color='none') legList.append(leg) labels.append('(%d) : %0.1f' % (i, flux[i])) phil = cdata[pid, 2] thetal = cdata[pid, 1] pxl, pyl = m(phil, thetal) gax.text(pxl, pyl, '%d' % (i), color='white', alpha=0.5) #gax.text(px,py,'%.1f Mg/s'%(flux[i]),color='white') #gax.text(px,py,'%d: %.0f km2'%(i, area[i]),color='white') #gax.text(px,py,'%.3f'%(cdata[pid, 6]),color='white') # temperature else: print 'Negative flux: %f detected for plume %d..' % (flux[i], i) #end if #end for #=======================================Done plotting flux #==================================================Plot reconstruction #m.drawparallels(np.arange(-90.,210.,30.)) #m.drawmeridians(np.arange(-180.,240.,60.)) age = options.startAge - int(fileName.split('.')[-2]) if (age < 0): age = 0 if 1: sfName = '%s/reconstructed_%d.00Ma.shp' % (options.reconsPath, age) sf = fiona.open(sfName) features = [] for feature in sf: #print feature['geometry']['type'] features.append(shape(feature['geometry'])) #end for patches = [] for feature in features: if (isinstance(feature, Polygon)): polygon = feature x, y = polygon.exterior.coords.xy px, py = m(x, y) ppolygon = Polygon(zip(px, py)) patches.append( PolygonPatch(ppolygon, fill=False, edgecolor=(0.5, 0.5, 0.5), alpha=0.5)) else: multiPolugon = feature for polygon in multiPolugon: x, y = polygon.exterior.coords.xy px, py = m(x, y) ppolygon = Polygon(zip(px, py)) patches.append( PolygonPatch(ppolygon, fill=False, edgecolor=(0.5, 0.5, 0.5), alpha=0.5)) #end for #end for gax.add_collection(PatchCollection(patches, match_original=True)) #end if m.drawmapboundary() #=======================================Done plotting reconstruction leg = plt.legend(legList, labels, ncol=6, frameon=False, fontsize=12, handlelength=1, loc='lower center', borderaxespad=-13, handletextpad=0.2, title='Flux / (Mg/s)', scatterpoints=1, labelspacing=0.2) gax.set_title('Plumes Identified at %d Ma' % (age)) gax.set_xlabel('$\\Vert\\nabla V_r \\Vert [s^{-1}]$') plt.savefig('%s/%s.png' % (dirName, os.path.basename(fileName)), dpi=300) plt.close() #end for outFile.close()
def idw_serie_temporal(): """ Método de interpolación inverse distance weighted con distance elevada a una potencia, normalmente 2 Interpola la variable en una serie de puntos dados en 3D. Si no tienes valor de Z pon en todos los puntos un valor cte, por ej 0 Los datos son una serie temporal; los puntos con datos varían en el tiempo """ from datetime import date, timedelta from os.path import join from time import time import interpolation_param as par time_steps = ('diaria', 'day', 'mensual', 'month', 'anual', 'year') cstrAemet = r'DRIVER={Microsoft Access Driver (*.mdb, *.accdb)}; ' +\ f'DBQ={par.dbMeteoro};' PRINT_INTERVAL = 5. if par.time_step in time_steps[:2]: tstep = 1 time_step = timedelta(days=1) datei = date(par.year1, par.month1, par.day1) datefin = date(par.year2, par.month2, par.day2) else: raise ValueError(f'valor de time_step {par.time_step} ' +\ 'no implementado') start_time = time() # datos donde hay que interpolar rows = points2interpolate_get(join(par.PATHIN, par.FPOINTS), par.skip_lines) fidi = rows[:, 0] # array con los id de los puntos if rows.shape[1] == 3: # 2D xi = rows[:, [1, 2]].astype(np.float32) # array con las coordenadas elif rows.shape[1] == 4: #3D xi = rows[:, [1, 2, 3]].astype(np.float32) else: raise ValueError('El núm de columnas en el fichero de puntos debe ' +\ '3 o 4: id del punto, x, y, (z)') rows = None z = np.empty((len(xi), par.kidw), np.float32) zi = np.empty((len(xi)), np.float32) # array con los valores interpolados # datos para hacer las interpolaciones con = pyodbc.connect(cstrAemet) cur = con.cursor() # fichero de salida dst = file_name_out(par.PATHOUT, par.FPOINTS, par.SHORT_NAME_4_VARIABLE, 'idw') f = open(dst, 'w') f.write('fid\tfecha\tvalor\n') t0 = PRINT_INTERVAL while datei <= datefin: if tstep == 1: dateStr = datei.strftime('%d/%m/%Y') if time() - t0 > PRINT_INTERVAL: t0 = time() print(dateStr) cur.execute(par.SELECT1, (datei,)) data = [row for row in cur] if xi.shape[1] == 2: data = np.array([[row.X, row.Y, row.v] for row in data]) tree = spatial.cKDTree(data[:, [0, 1]]) else: data = np.array([[row.X, row.Y, row.Z, row.v] for row in data]) tree = spatial.cKDTree(data[:, [0, 1, 2]]) dist, ii = tree.query(xi, k=par.kidw) dist = np.where(dist<=par.epsidw, par.mindistidw, dist) idwcore(data, xi.shape[1], dist, ii, par.poweridw, z, zi) for i in range(len(fidi)): f.write(f'{fidi[i]}\t{dateStr}\t{zi[i]:0.1f}\n') datei = datei + time_step elapsed_time = time() - start_time print(f'La interpolación tardó {elapsed_time:0.1f} s') # fichero de metadatos dst = file_name_out(par.PATHOUT, par.FPOINTS, par.SHORT_NAME_4_VARIABLE, 'idw_matadata') with open(dst, 'w') as f: f.write(f'idw\n') f.write(f'potencia a la que se eleva la dist, {par.poweridw:f}\n') f.write(f'núm datos por interpolación, {par.kidw:d}\n') f.write('distancia máx. a la que se considera que las ' +\ 'localizaciones de una estación y un punto coinciden, ' +\ f'{par.epsidw:f}\n') f.write('corrección de la distancia de un punto para evitar ' +\ f'infinitos en 1/dist, {par.mindistidw:f}\n') f.write(f'db de los datos, {par.dbMeteoro}\n') f.write(f'número de puntos interpolados {fidi.size:d}\n') f.write(f'tiempo transcurrido, {elapsed_time:0.1f} s\n') _ = input(MSG_FIN_PROCESO)
def adi(self): kd_tree = spatial.cKDTree(self.transformed_pts_est) kd_tree_distances, ids = kd_tree.query(self.transformed_pts_gt, k=1) adi_error = kd_tree_distances.mean() return adi_error
def convexHull(vertices, tol=1e-6): """Compute the 3D convex hull of a set of vertices and merge coplanar faces. Args: vertices (list): List of (x, y, z) coordinates tol (float): Floating point tolerance for merging coplanar faces Returns an array of vertices and a list of faces (vertex indices) for the convex hull of the given set of vertice. .. note:: This method uses scipy's quickhull wrapper and therefore requires scipy. """ from scipy.spatial import cKDTree, ConvexHull from scipy.sparse.csgraph import connected_components hull = ConvexHull(vertices) # Triangles in the same face will be defined by the same linear equalities dist = cKDTree(hull.equations) trianglePairs = dist.query_pairs(tol) connectivity = np.zeros((len(hull.simplices), len(hull.simplices)), dtype=np.int32) for (i, j) in trianglePairs: connectivity[i, j] = connectivity[j, i] = 1 # connected_components returns (number of faces, cluster index for each input) (_, joinTarget) = connected_components(connectivity, directed=False) faces = defaultdict(list) norms = defaultdict(list) for (idx, target) in enumerate(joinTarget): faces[target].append(idx) norms[target] = hull.equations[idx][:3] # a list of sets of all vertex indices in each face faceVerts = [ set(hull.simplices[faces[faceIndex]].flat) for faceIndex in sorted(faces) ] # normal vector for each face faceNorms = [norms[faceIndex] for faceIndex in sorted(faces)] # polygonal faces polyFaces = [] for (norm, faceIndices) in zip(faceNorms, faceVerts): face = np.array(list(faceIndices), dtype=np.uint32) N = len(faceIndices) r = hull.points[face] rcom = np.mean(r, axis=0) # plane_{a, b}: basis vectors in the plane plane_a = r[0] - rcom plane_a /= np.sqrt(np.sum(plane_a**2)) plane_b = np.cross(norm, plane_a) dr = r - rcom[np.newaxis, :] thetas = np.arctan2(dr.dot(plane_b), dr.dot(plane_a)) sortidx = np.argsort(thetas) face = face[sortidx] polyFaces.append(face) return (hull.points, polyFaces)
#print(VSXdata.iloc[1,1]) listdatara = datara.tolist() listdatadec = datadec.tolist() radectemp = [] for i in range(0, len(listdatara)): RA = np.float32(listdatara[i]) DEC = np.float32(listdatadec[i]) radectemp.append(RA) radectemp.append(DEC) VSXradec = np.float32(radectemp).reshape(-1, 2) kdt = cKDTree(radecname[:, 0:2]) dist, indices = kdt.query(VSXradec) temp = [] for i in range(len(indices)): index = indices[i] temp.append(radecname[index]) nptemp = np.array(temp) VSXradec = np.column_stack((dist, VSXradec)) allradec = np.column_stack((VSXradec, nptemp)) lista = ['distance', 'VSXRA', 'VSXDEC', 'cg20RA', 'cg20DEC', 'cg20name'] dfallradec = pd.DataFrame(allradec, columns=lista) df4 = [VSXdata, dfallradec]
lon_rad = np.radians(lon) lat_rad = np.radians(lat) # convert to cartesian coordinates r_n = A / (np.sqrt(1 - E2 * (np.sin(lat_rad)**2))) x = r_n * np.cos(lat_rad) * np.cos(lon_rad) y = r_n * np.cos(lat_rad) * np.sin(lon_rad) z = r_n * (1 - E2) * np.sin(lat_rad) return x, y, z xs, ys, zs = lon_lat_to_cartesian(lon_source.values.flatten(), lat_source.values.flatten()) xt, yt, zt = lon_lat_to_cartesian(lon_target2d.flatten(), lat_target2d.flatten()) tree = cKDTree(np.column_stack((xs, ys, zs))) d, inds = tree.query(np.column_stack((xt, yt, zt)), k=10) def interpolate(source, target, d, inds): nt = source["time"].shape[0] tmp = [] for t in range(0, nt): w = 1.0 / d**2 air_idw = np.sum(w * source.tmax[t].values.flatten()[inds], axis=1) / np.sum(w, axis=1) air_idw.shape = target.shape tmp.append(air_idw) return tmp