def load(self, dataPath, labelsPath, test=False, verbose=True): # load data as previously super(CNNDataset,self).load(dataPath,labelsPath,test=test) # don't compress the test data if test: return # create the compressed search data remIdx = [0] it = 0 self.tree = spatial.cKDTree(self.data[remIdx]) changed = True "Iteratively updating the search tree" while len(remIdx) > 0: # print iteration number print "Iteration %s" % it it +=1 # add data to the search tree changed = False for i,d in enumerate(self.data): if i in remIdx: continue temp = self.classify(d) print i, len(remIdx) if temp != self.labels[i]: remIdx.append(i) if len(remIdx) % 10 == 0: self.tree = spatial.cKDTree(self.data[remIdx]) changed = True
def interlinks(cluster1,cluster2,neighbours=None):
'''
This function searches a certain set of neighbours between two clusters.
Parameters
----------
cluster1,cluster2 : list of 1d ndarray
The coordinates of the clusters.
neighbours : dict, optional
The neighbour-length map of the links to be searched.
Returns
-------
list of Link
The searched links.
'''
result=[]
if len(cluster1)>0 and len(cluster2)>0:
if neighbours is None: neighbours={0:0.0}
smatrix=cKDTree(cluster1).sparse_distance_matrix(cKDTree(cluster2),np.max(list(neighbours.values()))+RZERO)
for (i,j),dist in smatrix.items():
for neighbour,length in neighbours.items():
if abs(length-dist)<RZERO:
result.append(Link(neighbour,sindex=i,eindex=j,disp=0))
break
return result
def test_ckdtree_box():
# check ckdtree periodic boundary
n = 2000
m = 2
k = 3
np.random.seed(1234)
data = np.random.uniform(size=(n, m))
kdtree = cKDTree(data, leafsize=1, boxsize=1.0)
# use the standard python KDTree for the simulated periodic box
kdtree2 = cKDTree(data, leafsize=1)
dd, ii = kdtree.query(data, k)
dd1, ii1 = kdtree.query(data + 1.0, k)
assert_almost_equal(dd, dd1)
assert_equal(ii, ii1)
dd1, ii1 = kdtree.query(data - 1.0, k)
assert_almost_equal(dd, dd1)
assert_equal(ii, ii1)
dd2, ii2 = simulate_periodic_box(kdtree2, data, k, boxsize=1.0)
assert_almost_equal(dd, dd2)
assert_equal(ii, ii2)
def mutual_information2(x, y, k=3):
""" Return mutual information estimate based on Ross B C (PlosOne 2014).
x contains discrete variables, y contains continuous variables.
"""
x = np.array(x)
y = np.array(y)
discrete_vars = set(x)
cats = dict((d, y[x == d][:,np.newaxis]) for d in discrete_vars)
Nx = dict((d, cats[d].shape[0]) for d in discrete_vars)
# one KDTree for each discrete variable
kdtrees = dict((d, cKDTree(cats[d])) for d in discrete_vars)
# one KDTree for all data points
full_tree = cKDTree(y[:,np.newaxis])
N = len(x)
# vectorized KD tree. Calculation of the m_i is the bottle neck
# no idea how to speed it up further
I = 0
for d, data in cats.iteritems():
ds, __ = kdtrees[d].query(data, k=k+1, p=np.inf)
ds = ds[:,-1]
mis = np.array([len(full_tree.query_ball_point(dat, r, p=np.inf))
- 1.0
for dat, r in izip(data, ds)])
I -= psi(mis).sum() + len(data)*psi(Nx[d])
return I/N + psi(N) + psi(k)
def bench_query_ball_point(self):
print
print ' Query ball point kd-tree'
print '==============================================================='
print ' dim | # points | # queries | probe radius | KDTree | cKDTree | flat cKDTree'
for (m, n, r, repeat) in [(3,10000,1000,3)]:#,
# (8,10000,1000,3),
# (16,10000,1000,3)]:
for probe_radius in (0.2, 0.5):
print '%4s | %8s | %9s | %11.1f ' % (m, n, r, probe_radius),
sys.stdout.flush()
data = np.concatenate((np.random.randn(n//2,m),
np.random.randn(n-n//2,m)+np.ones(m)))
queries = np.concatenate((np.random.randn(r//2,m),
np.random.randn(r-r//2,m)+np.ones(m)))
T1 = KDTree(data)
T2 = cKDTree(data)
T3 = cKDTree(data,leafsize=n)
print '| %6.3fs ' % (measure('T1.query_ball_point(queries, probe_radius)', 1) / 1),
sys.stdout.flush()
print '| %6.3fs' % (measure('T2.query_ball_point(queries, probe_radius)', repeat) / repeat),
sys.stdout.flush()
print '| %6.3fs' % (measure('T3.query_ball_point(queries, probe_radius)', repeat) / repeat),
sys.stdout.flush()
print ''
def bench_sparse_distance_matrix(self):
print
print ' Sparse distance matrix kd-tree'
print '===================================================================='
print ' dim | # points T1 | # points T2 | probe radius | KDTree | cKDTree'
for (m, n1, n2, repeat) in [(3,1000,1000,30),
(8,1000,1000,30),
(16,1000,1000,30)]:
data1 = np.concatenate((np.random.randn(n1//2,m),
np.random.randn(n1-n1//2,m)+np.ones(m)))
data2 = np.concatenate((np.random.randn(n2//2,m),
np.random.randn(n2-n2//2,m)+np.ones(m)))
T1 = KDTree(data1)
T2 = KDTree(data2)
cT1 = cKDTree(data1)
cT2 = cKDTree(data2)
for probe_radius in (0.2, 0.5):
print '%4s | %11s | %11s | %11.1f ' % (m, n1, n2, probe_radius),
sys.stdout.flush()
print '| %6.3fs ' % (measure('T1.sparse_distance_matrix(T2, probe_radius)', 1) / 1),
sys.stdout.flush()
print '| %6.3fs ' % (measure('cT1.sparse_distance_matrix(cT2, probe_radius)', repeat) / repeat),
sys.stdout.flush()
print ''
def kraskov_mi(x,y,k=5):
'''
Estimate the mutual information I(X;Y) of X and Y from samples {x_i, y_i}_{i=1}^N
Using KSG mutual information estimator
Input: x: 2D list of size N*d_x
y: 2D list of size N*d_y
k: k-nearest neighbor parameter
Output: one number of I(X;Y)
'''
assert len(x)==len(y), "Lists should have same length"
assert k <= len(x)-1, "Set k smaller than num. samples - 1"
N = len(x)
dx = len(x[0])
dy = len(y[0])
data = np.concatenate((x,y),axis=1)
tree_xy = ss.cKDTree(data)
tree_x = ss.cKDTree(x)
tree_y = ss.cKDTree(y)
knn_dis = [tree_xy.query(point,k+1,p=float('inf'))[0][k] for point in data]
ans_xy = -digamma(k) + digamma(N) + (dx+dy)*log(2)#2*log(N-1) - digamma(N) #+ vd(dx) + vd(dy) - vd(dx+dy)
ans_x = digamma(N) + dx*log(2)
ans_y = digamma(N) + dy*log(2)
for i in range(N):
ans_xy += (dx+dy)*log(knn_dis[i])/N
ans_x += -digamma(len(tree_x.query_ball_point(x[i],knn_dis[i]-1e-15,p=float('inf'))))/N+dx*log(knn_dis[i])/N
ans_y += -digamma(len(tree_y.query_ball_point(y[i],knn_dis[i]-1e-15,p=float('inf'))))/N+dy*log(knn_dis[i])/N
return ans_x+ans_y-ans_xy
def getkernmat(dirsout, dirsin, kappa, eps, chiral_flip):
"Assume dirsout and dirsin and unit norm vectors"
arghash = hash((dirsout[::10].tostring(), dirsin[::10].tostring(), kappa, eps, chiral_flip))
if arghash in precomputed_kernmats:
return precomputed_kernmats[arghash]
else:
kdout = ss.cKDTree(
dirsout * (1.0 + eps)
) # this is to make sure the distance between identicals is nonzero (bad hack but it works)
kdin = ss.cKDTree(dirsin)
if chiral_flip:
kdin2 = ss.cKDTree(-dirsin)
distthres = n.sqrt(-2.0 / kappa * n.log(eps))
distmat = kdout.sparse_distance_matrix(kdin, distthres).tocsr()
if chiral_flip:
distmat += kdout.sparse_distance_matrix(kdin2, distthres).tocsr()
distmat = distmat.tocoo()
kerndata = n.exp(-kappa / 2.0 * distmat.data ** 2)
kernmat = sp.coo_matrix((kerndata, (distmat.row, distmat.col)), distmat.shape)
precomputed_kernmats[arghash] = kernmat
return kernmat
def axisBasedInit(allowed,arraySize,strongDist,weakDist,strongAxis,weakAxis,connectionPoint): #This initializer attempts to find the Strong (or most damaging) axis of the wind profile and place turbines off it. #The separation distance is estimated using a formula that finds the optimal distance assuming two turbines are connected using roads and cables #This is obviously not true, but should be a good starting point. zippedAllowed = numpy.array(zip(allowed[0],allowed[1])) allowedTree = cKDTree(zippedAllowed) #We start byfinding the closest allowed position to the connection point. distance, ind = allowedTree.query(connectionPoint, k =1) positions = numpy.zeros((arraySize,2)) positions[0] = allowedTree.data[ind] positionsTree = cKDTree(positions) axes = numpy.zeros((4,2)) axes[0] = weakAxis axes[1] = -weakAxis axes[2] = strongAxis axes[3] = -strongAxis index = 0 distances = numpy.array([weakDist,weakDist,strongDist,strongDist]) while len(positions) < arraySize: departurePt = positions[index] positions = searchOnAxes(distances,departurePt,axes,allowedTree,positions) numpy.concatenate((positions,newPositions)) index = index + 1 if len(positions) > arraySize: diff = len(positions) - arraySize for i in range(diff): r = random.randint(0,len(positions)) positions = numpy.delete(positions,r,0) return positions, allowedTree
def __init__(self,ncfile='woa13_decav_04v2.nc'):
nc = netCDF4.Dataset(ncfile)
sv = nc.variables
latslice=slice(173,262) # 45.-67.
lonslice=slice(320,455) # -20.-13.625
self.lon = sv['lon'][lonslice]
self.lat = sv['lat'][latslice]
self.d = -sv['depth'][:]
self.time = sv['time'][:]
self.timeunits = sv['time'].units
self.s = sv['s_mn'][:,:,latslice,lonslice]
self.t = sv['t_mn'][:,:,latslice,lonslice]
self.lon2,self.lat2 = meshgrid(self.lon,self.lat)
if True:
self.s_tree={}
self.t_tree={}
self.s_var={}
self.t_var={}
for tidx,time in enumerate(self.time):
print(' build trees for month=%d'%(tidx+1))
self.d3,self.lat3,self.lon3 = meshgrid(self.d,self.lat*100.,self.lon*100.,indexing='ij')
vlon3 = self.lon3[where(self.s.mask[tidx]==False)]
vlat3 = self.lat3[where(self.s.mask[tidx]==False)]
vd3 = self.d3[where(self.s.mask[tidx]==False)]
self.s_tree[tidx]=cKDTree(zip(vlon3,vlat3,vd3))
vlon3 = self.lon3[where(self.t.mask[tidx]==False)]
vlat3 = self.lat3[where(self.t.mask[tidx]==False)]
vd3 = self.d3[where(self.t.mask[tidx]==False)]
self.t_tree[tidx]=cKDTree(zip(vlon3,vlat3,vd3))
self.s_var[tidx] = self.s[tidx][where(self.s.mask[tidx]==False)].flatten()
self.t_var[tidx] = self.t[tidx][where(self.t.mask[tidx]==False)].flatten()
def setUp(self):
n = 50
m = 4
np.random.seed(0)
self.T1 = cKDTree(np.random.randn(n,m),leafsize=2)
self.T2 = cKDTree(np.random.randn(n,m),leafsize=2)
self.r = 0.3
def max_neighbor(self, in_lon, in_lat, radius=0.05):
"""
Finds the largest value within a given radius of a point on the interpolated grid.
Args:
in_lon: 2D array of longitude values
in_lat: 2D array of latitude values
radius: radius of influence for largest neighbor search in degrees
Returns:
Array of interpolated data
"""
out_data = np.zeros((self.data.shape[0], in_lon.shape[0], in_lon.shape[1]))
in_tree = cKDTree(np.vstack((in_lat.ravel(), in_lon.ravel())).T)
out_indices = np.indices(out_data.shape[1:])
out_rows = out_indices[0].ravel()
out_cols = out_indices[1].ravel()
for d in range(self.data.shape[0]):
nz_points = np.where(self.data[d] > 0)
if len(nz_points[0]) > 0:
nz_vals = self.data[d][nz_points]
nz_rank = np.argsort(nz_vals)
original_points = cKDTree(np.vstack((self.lat[nz_points[0][nz_rank]], self.lon[nz_points[1][nz_rank]])).T)
all_neighbors = original_points.query_ball_tree(in_tree, radius, p=2, eps=0)
for n, neighbors in enumerate(all_neighbors):
if len(neighbors) > 0:
out_data[d, out_rows[neighbors], out_cols[neighbors]] = nz_vals[nz_rank][n]
return out_data
def bench_count_neighbors(self):
print()
print(' Count neighbors kd-tree')
print('====================================================================')
print(' dim | # points T1 | # points T2 | probe radius | KDTree | cKDTree')
for (m, n1, n2, repeat) in [(3,1000,1000,30),
(8,1000,1000,30),
(16,1000,1000,30)]:
data1 = np.concatenate((np.random.randn(n1//2,m),
np.random.randn(n1-n1//2,m)+np.ones(m)))
data2 = np.concatenate((np.random.randn(n2//2,m),
np.random.randn(n2-n2//2,m)+np.ones(m)))
T1 = KDTree(data1)
T2 = KDTree(data2)
cT1 = cKDTree(data1)
cT2 = cKDTree(data2)
for probe_radius in (0.2, 0.5):
print('%4s | %11s | %11s | %11.1f ' % (m, n1, n2, probe_radius), end=' ')
sys.stdout.flush()
print('| %6.3fs ' % (measure('T1.count_neighbors(T2, probe_radius)', 1) / 1), end=' ')
sys.stdout.flush()
print('| %6.3fs ' % (measure('cT1.count_neighbors(cT2, probe_radius)', repeat) / repeat), end=' ')
sys.stdout.flush()
print('')
def test_ckdtree_box_upper_bounds():
data = np.linspace(0, 2, 10).reshape(-1, 2)
data[:, 1] += 10
assert_raises(ValueError, cKDTree, data, leafsize=1, boxsize=1.0)
assert_raises(ValueError, cKDTree, data, leafsize=1, boxsize=(0.0, 2.0))
# skip a dimension.
cKDTree(data, leafsize=1, boxsize=(2.0, 0.0))
def bench_query(self):
print()
print(' Querying kd-tree')
print('===============================================================')
print(' dim | # points | # queries | KDTree | cKDTree | flat cKDTree')
for (m, n, r, repeat) in [(3,10000,1000,3),
(8,10000,1000,3),
(16,10000,1000,3)]:
print('%4s | %8s | %8s ' % (m, n, r), end=' ')
sys.stdout.flush()
data = np.concatenate((np.random.randn(n//2,m),
np.random.randn(n-n//2,m)+np.ones(m)))
queries = np.concatenate((np.random.randn(r//2,m),
np.random.randn(r-r//2,m)+np.ones(m)))
T1 = KDTree(data)
T2 = cKDTree(data)
T3 = cKDTree(data,leafsize=n)
print('| %6.3fs ' % (measure('T1.query(queries)', 1) / 1), end=' ')
sys.stdout.flush()
print('| %6.3fs' % (measure('T2.query(queries)', repeat) / repeat), end=' ')
sys.stdout.flush()
print('| %6.3fs' % (measure('T3.query(queries)', repeat) / repeat), end=' ')
sys.stdout.flush()
print('')
def bench_query_pairs(self):
print()
print(' Query pairs kd-tree')
print('==================================================================')
print(' dim | # points | probe radius | KDTree | cKDTree | flat cKDTree')
for (m, n, repeat) in [(3,1000,30),
(8,1000,30),
(16,1000,30)]:
for probe_radius in (0.2, 0.5):
print('%4s | %8s | %11.1f ' % (m, n, probe_radius), end=' ')
sys.stdout.flush()
data = np.concatenate((np.random.randn(n//2,m),
np.random.randn(n-n//2,m)+np.ones(m)))
T1 = KDTree(data)
T2 = cKDTree(data)
T3 = cKDTree(data,leafsize=n)
print('| %6.3fs ' % (measure('T1.query_pairs(probe_radius)', 1) / 1), end=' ')
sys.stdout.flush()
print('| %6.3fs' % (measure('T2.query_pairs(probe_radius)', repeat) / repeat), end=' ')
sys.stdout.flush()
print('| %6.3fs' % (measure('T3.query_pairs(probe_radius)', repeat) / repeat), end=' ')
sys.stdout.flush()
print('')
def search_tree(self, centers, radius):
"""
Searches all the pairs within `radius` between `centers`
and ``coords``
``coords`` are the already initialized coordinates in the tree
during :meth:`set_coords`.
``centers`` are wrapped around the primary unit cell
if PBC is desired. Minimum image convention (PBC) is
activated if the `box` argument is provided during
class initialization
Parameters
----------
centers: array_like (N,3)
coordinate array to search for neighbors
radius: float
maximum distance to search for neighbors.
Returns
-------
pairs : array
all the pairs between ``coords`` and ``centers``
Note
----
This method constructs another tree from the ``centers``
and queries the previously built tree (built in
:meth:`set_coords`)
"""
if not self._built:
raise RuntimeError('Unbuilt tree. Run tree.set_coords(...)')
centers = np.asarray(centers)
if centers.shape == (self.dim, ):
centers = centers.reshape((1, self.dim))
# Sanity check
if self.pbc:
if self.cutoff < radius:
raise RuntimeError('Set cutoff greater or equal to the radius.')
# Bring all query points to the central cell
wrapped_centers = apply_PBC(centers, self.box)
other_tree = cKDTree(wrapped_centers, leafsize=self.leafsize)
pairs = other_tree.query_ball_tree(self.ckdt, radius)
pairs = np.array([[i, j] for i, lst in enumerate(pairs) for j in lst],
dtype=np.int64)
if pairs.size > 0:
pairs[:, 1] = undo_augment(pairs[:, 1],
self.mapping,
len(self.coords))
else:
other_tree = cKDTree(centers, leafsize=self.leafsize)
pairs = other_tree.query_ball_tree(self.ckdt, radius)
pairs = np.array([[i, j] for i, lst in enumerate(pairs) for j in lst],
dtype=np.int64)
if pairs.size > 0:
pairs = unique_rows(pairs)
return pairs
def period_neighborhood_probability(self, radius, smoothing, threshold, stride,start_time,end_time):
"""
Calculate the neighborhood probability over the full period of the forecast
Args:
radius: circular radius from each point in km
smoothing: width of Gaussian smoother in km
threshold: intensity of exceedance
stride: number of grid points to skip for reduced neighborhood grid
Returns:
(neighborhood probabilities)
"""
neighbor_x = self.x[::stride, ::stride]
neighbor_y = self.y[::stride, ::stride]
neighbor_kd_tree = cKDTree(np.vstack((neighbor_x.ravel(), neighbor_y.ravel())).T)
neighbor_prob = np.zeros((self.data.shape[0], neighbor_x.shape[0], neighbor_x.shape[1]))
print('Forecast Hours: {0}-{1}'.format(start_time, end_time))
for m in range(len(self.members)):
period_max = self.data[m,start_time:end_time,:,:].max(axis=0)
valid_i, valid_j = np.where(period_max >= threshold)
print(self.members[m], len(valid_i))
if len(valid_i) > 0:
var_kd_tree = cKDTree(np.vstack((self.x[valid_i, valid_j], self.y[valid_i, valid_j])).T)
exceed_points = np.unique(np.concatenate(var_kd_tree.query_ball_tree(neighbor_kd_tree, radius))).astype(int)
exceed_i, exceed_j = np.unravel_index(exceed_points, neighbor_x.shape)
neighbor_prob[m][exceed_i, exceed_j] = 1
if smoothing > 0:
neighbor_prob[m] = gaussian_filter(neighbor_prob[m], smoothing,mode='constant')
return neighbor_prob
def period_neighborhood_probability(self, radius, smoothing, threshold, stride, x, y, dx):
"""
Calculate the neighborhood probability over the full period of the forecast
Args:
radius: circular radius from each point in km
smoothing: width of Gaussian smoother in km
threshold: intensity of exceedance
stride: number of grid points to skip for reduced neighborhood grid
x: x-coordinate array in km
y: y-coordinate array in km
dx: distance between grid points in km
Returns:
neighborhood probablities
"""
neighbor_x = x[::stride, ::stride]
neighbor_y = y[::stride, ::stride]
neighbor_kd_tree = cKDTree(np.vstack((neighbor_x.ravel(), neighbor_y.ravel())).T)
neighbor_prob = np.zeros((neighbor_x.shape[0], neighbor_x.shape[1]))
period_max = self.data.max(axis=0)
valid_i, valid_j = np.where(period_max >= threshold)
if len(valid_i) > 0:
var_kd_tree = cKDTree(np.vstack((x[valid_i, valid_j], y[valid_i, valid_j])).T)
exceed_points = np.unique(np.concatenate(var_kd_tree.query_ball_tree(neighbor_kd_tree, radius))).astype(int)
exceed_i, exceed_j = np.unravel_index(exceed_points, neighbor_x.shape)
neighbor_prob[exceed_i, exceed_j] = 1
if smoothing > 0:
neighbor_prob = gaussian_filter(neighbor_prob, int(smoothing / dx / stride))
return neighbor_prob
def set_coords(self, coords):
"""Unclear what this does. James??"""
# Normalize coordinates 0-1
if np.any(coords.max(0) > 1) or np.any(coords.min(0) < 0):
coords -= coords.min(0)
coords /= coords.max(0)
# Renormalize coordinates to shape of svg
self.coords = coords * self.svgshape
# Update of scipy (0.16+) means that cKDTree hangs / takes absurdly long to compute with new default
# balanced_tree=True. Seems only to be true on Mac OS, for whatever reason. Possibly a broken
# C library, unclear. Setting balanced_tree=False seems to resolve the issue, thus going with that for now
# See http://stackoverflow.com/questions/31819778/scipy-spatial-ckdtree-running-slowly
try:
# not compatible with scipy version < 0.16
self.kdt = cKDTree(self.coords, balanced_tree=False)
except:
# Older call signature
self.kdt = cKDTree(self.coords)
for layer in self:
for name in layer.labels.elements:
for element in layer.labels.elements[name]:
x, y = float(element.get("x")), float(element.get("y"))
dist, idx = self.kdt.query((x, self.svgshape[1]-y))
if idx >= len(self.kdt.data):
idx = 0
element.attrib['data-ptidx'] = str(idx)
def init_element_tree(self,latlon=True):
"""
build element tree for xy or latlon coordinates
(default: latlon=True)
"""
from scipy.spatial import cKDTree
#print(' schism.py: build element tree')
if self.element_depth == None:
self.element_depth={}
calc_depths = True
else:
calc_depths = False
if latlon:
self.element_lon={}
self.element_lat={}
for el in self.nvdict:
self.element_lon[el] = sum([self.londict[idx] for idx in self.nvdict[el]])/len(self.nvdict[el])
self.element_lat[el] = sum([self.latdict[idx] for idx in self.nvdict[el]])/len(self.nvdict[el])
if calc_depths:
self.element_depth[el] = sum([self.depthsdict[idx] for idx in self.nvdict[el]])/len(self.nvdict[el])
self.element_tree_latlon = cKDTree(list(zip(self.element_lon.values(),self.element_lat.values())))
self.element_tree_ids = self.element_lon.keys()
else:
self.element_x={}
self.element_y={}
for el in self.nvdict:
self.element_x[el] = sum([self.xdict[idx] for idx in self.nvdict[el]])/len(self.nvdict[el])
self.element_y[el] = sum([self.ydict[idx] for idx in self.nvdict[el]])/len(self.nvdict[el])
if calc_depths:
self.element_depth[el] = sum([self.depthsdict[idx] for idx in self.nvdict[el]])/len(self.nvdict[el])
self.element_tree_xy = cKDTree(list(zip(self.element_x.values(),self.element_y.values())))
self.element_tree_ids = self.element_x.keys()
def kldiv(x,xp,k=3,base=2):
""" KL Divergence between p and q for x~p(x),xp~q(x)
x,xp should be a list of vectors, e.g. x = [[1.3],[3.7],[5.1],[2.4]]
if x is a one-dimensional scalar and we have four samples
"""
assert k <= len(x) - 1, "Set k smaller than num. samples - 1"
assert k <= len(xp) - 1, "Set k smaller than num. samples - 1"
assert len(x[0]) == len(xp[0]), "Two distributions must have same dim."
d = len(x[0])
n = len(x)
m = len(xp)
const = log(m) - log(n-1)
tree = ss.cKDTree(x)
treep = ss.cKDTree(xp)
nn_old = [tree.query(point,k+1,p=float('inf'))[0][k] for point in x]
nnp_old = [treep.query(point,k,p=float('inf'))[0][k-1] for point in x]
# This isn't right!
nn = []
nnp = []
for val in nn_old:
if val > 0:
nn.append(val)
for val in nnp_old:
if val > 0:
nnp.append(val)
return (const + d*np.mean(map(log,nnp))-d*np.mean(map(log,nn)))/log(base)
def init_node_tree(self,latlon=True):
#print(' build node tree')
from scipy.spatial import cKDTree
if latlon:
self.node_tree_latlon = cKDTree(list(zip(self.lon,self.lat)))
else:
self.node_tree_xy = cKDTree(list(zip(self.x,self.y)))
def get_ctmmap(subject, **kwargs):
"""
Parameters
----------
subject : str
Subject name
Returns
-------
lnew :
rnew :
"""
from scipy.spatial import cKDTree
from . import brainctm
jsfile = get_ctmpack(subject, **kwargs)
ctmfile = os.path.splitext(jsfile)[0]+".ctm"
try:
left, right = db.get_surf(subject, "pia")
except IOError:
left, right = db.get_surf(subject, "fiducial")
lmap, rmap = cKDTree(left[0]), cKDTree(right[0])
left, right = brainctm.read_pack(ctmfile)
lnew = lmap.query(left[0])[1]
rnew = rmap.query(right[0])[1]
return lnew, rnew
def fitDataRigidScaleNoCorr_TwoSurfaces( X1,X2,data1,data2, xtol=1e-5, maxfev=0, t0=None ): """ fit list of points X to list of points data by minimising least squares distance between each point in X and closest neighbour in data """ if t0==None: t0 = scipy.array([0.0,0.0,0.0,0.0,0.0,0.0,1.0]) data1Tree = cKDTree( data1 ) data2Tree = cKDTree( data2 ) X1 = scipy.array(X1) X2 = scipy.array(X2) def obj( t ): x1R = transformRigid3D( X1, t[:6] ) x1RS = transformScale3D( x1R, scipy.ones(3)*t[6] ) d1 = data1Tree.query( list(x1RS) )[0] x2R = transformRigid3D( X2, t[:6] ) x2RS = transformScale3D( x2R, scipy.ones(3)*t[6] ) d2 = data2Tree.query( list(x2RS) )[0] d=concatenate((d1,d2),0) return d*d tOpt = leastsq( obj, t0, xtol=xtol, maxfev=maxfev )[0] X1Opt = transformRigid3D( X1, tOpt[:6] ) X2Opt = transformRigid3D( X2, tOpt[:6] ) X1Opt = transformScale3D( X1Opt, tOpt[6:] ) X2Opt = transformScale3D( X2Opt, tOpt[6:] ) return tOpt, X1Opt,X2Opt
def revised_mi(x,y,k=5,q=float('inf')):
'''
Estimate the mutual information I(X;Y) of X and Y from samples {x_i, y_i}_{i=1}^N
Using *REVISED* KSG mutual information estimator (see arxiv.org/abs/1604.03006)
Input: x: 2D list of size N*d_x
y: 2D list of size N*d_y
k: k-nearest neighbor parameter
q: l_q norm used to decide k-nearest distance
Output: one number of I(X;Y)
'''
assert len(x)==len(y), "Lists should have same length"
assert k <= len(x)-1, "Set k smaller than num. samples - 1"
N = len(x)
dx = len(x[0])
dy = len(y[0])
data = np.concatenate((x,y),axis=1)
tree_xy = ss.cKDTree(data)
tree_x = ss.cKDTree(x)
tree_y = ss.cKDTree(y)
knn_dis = [tree_xy.query(point,k+1,p=q)[0][k] for point in data]
ans_xy = -digamma(k) + log(N) + vd(dx+dy,q)
ans_x = log(N) + vd(dx,q)
ans_y = log(N) + vd(dy,q)
for i in range(N):
ans_xy += (dx+dy)*log(knn_dis[i])/N
ans_x += -log(len(tree_x.query_ball_point(x[i],knn_dis[i]+1e-15,p=q))-1)/N+dx*log(knn_dis[i])/N
ans_y += -log(len(tree_y.query_ball_point(y[i],knn_dis[i]+1e-15,p=q))-1)/N+dy*log(knn_dis[i])/N
return ans_x+ans_y-ans_xy
def map_cells(input_cells, output_cells, max_distance=5, min_overlap=0.001):
input_position_array = np.array([cell.center.coords[0] for cell in input_cells])
output_position_array = np.array([cell.center.coords[0] for cell in output_cells])
input_tree = cKDTree(input_position_array)
output_tree = cKDTree(output_position_array)
# retrieve lists of (output) neighbors around input cells
print '>> Fetching neighbors for the', len(input_cells), 'input cells'
neighbors = input_tree.query_ball_tree(output_tree, max_distance)
overlaps = []
print '>> Computing overlaps'
for i,cells in enumerate(neighbors):
input_cell = input_cells[i]
if len(cells)==0:
raise RuntimeError('Cannot match input cell to any output cell')
intersection_candidates = [output_cells[j] for j in cells]
areas = []
for intersection_candidate in intersection_candidates:
intersection = input_cell.vertices.intersection(intersection_candidate.vertices)
areas.append(intersection.area)
areas = np.array(areas)
areas /= np.sum(areas)
cells = np.array(cells)
intersection_indices = areas>min_overlap
intersection_cells = cells[intersection_indices]
intersection_sum = np.sum(areas[intersection_indices])
overlaps.append(zip(intersection_cells, areas[intersection_indices]/intersection_sum))
overlaps_dict = {}
for cell,overlap in zip(input_cells,overlaps):
overlaps_dict[cell.id] = []
for output_cell, area in overlap:
overlaps_dict[cell.id].append((output_cells[output_cell].id, area))
return overlaps, overlaps_dict
def match_radec(r1, d1, r2, d2, rad=1./60./60., nneighbor=0, notself=False):
"""Match r1, d1, to r2, d2, within radius rad."""
if notself and nneighbor > 0:
nneighbor += 1
uv1 = lb2uv(r1, d1)
uv2 = lb2uv(r2, d2)
from scipy.spatial import cKDTree
tree = cKDTree(uv2)
dub = 2*numpy.sin(numpy.radians(rad)/2)
if nneighbor > 0:
d12, m2 = tree.query(uv1, nneighbor, distance_upper_bound=dub)
if nneighbor > 1:
m2 = m2.reshape(-1)
d12 = d12.reshape(-1)
m1 = numpy.arange(len(r1)*nneighbor, dtype='i4') // nneighbor
d12 = 2*numpy.arcsin(numpy.clip(d12 / 2, 0, 1))*180/numpy.pi
m = m2 < len(r2)
else:
tree1 = cKDTree(uv1)
res = tree.query_ball_tree(tree1, dub)
lens = [len(r) for r in res]
m2 = numpy.repeat(numpy.arange(len(r2), dtype='i4'), lens)
m1 = numpy.concatenate([r for r in res if len(r) > 0])
d12 = gc_dist(r1[m1], d1[m1], r2[m2], d2[m2])
m = numpy.ones(len(m1), dtype='bool')
if notself:
m = m & (m1 != m2)
return m1[m], m2[m], d12[m]
def test_ckdtree_box_lower_bounds():
data = np.linspace(-1, 1, 10)
try:
cKDTree(data, leafsize=1, boxsize=1.0)
except ValueError:
return
raise AssertionError("ValueError is not raised")
def set_coords(self, coords, cutoff=None):
"""Constructs KDTree from the coordinates
Wrapping of coordinates to the primary unit cell is enforced
before any distance evaluations. If periodic boundary conditions
are enabled, then duplicate particles are generated in the
vicinity of the box. An additional array `mapping` is also
generated which can be later used to trace the origin of
duplicate particle coordinates.
For non-periodic calculations, cutoff should not be provided
the parameter is only required for periodic calculations.
Parameters
----------
coords: array_like
Coordinate array of shape ``(N, 3)`` for N atoms.
cutoff: float
Specified cutoff distance to create duplicate images
Typically equivalent to the desired search radius
or the maximum of the desired cutoff radius. Relevant images
corresponding to every atom which lies
within ``cutoff`` distance from either of the box boundary
will be generated.
See Also
--------
MDAnalysis.lib.distances.augment_coordinates
"""
# If no cutoff distance is provided but PBC aware
if self.pbc and (cutoff is None):
raise RuntimeError('Provide a cutoff distance'
' with tree.set_coords(...)')
# set coords dtype to float32
# augment coordinates will work only with float32
coords = np.asarray(coords, dtype=np.float32)
if self.pbc:
self.cutoff = cutoff
# Bring the coordinates in the central cell
self.coords = apply_PBC(coords, self.box)
# generate duplicate images
self.aug, self.mapping = augment_coordinates(self.coords,
self.box,
self.cutoff)
# Images + coords
self.all_coords = np.concatenate([self.coords, self.aug])
self.ckdt = cKDTree(self.all_coords, leafsize=self.leafsize)
else:
# if cutoff distance is provided for non PBC calculations
if cutoff is not None:
raise RuntimeError('Donot provide cutoff distance for'
' non PBC aware calculations')
self.coords = coords
self.ckdt = cKDTree(self.coords, self.leafsize)
self._built = True
else:
mesh = meshInput
print("Calculating active cells from topo")
# Compute active cells
activeCells = Utils.surface2ind_topo(mesh, topo, gridLoc='CC')
Mesh.TreeMesh.writeUBC(mesh,
outDir + 'OctreeMeshGlobal.msh',
models={outDir + 'ActiveSurface.act': activeCells})
if "adjust_clearance" in list(input_dict.keys()):
print("Forming cKDTree for clearance calculations")
tree = cKDTree(mesh.gridCC[activeCells, :])
# Get the layer of cells directly below topo
nC = int(activeCells.sum()) # Number of active cells
# Create active map to go from reduce set to full
activeCellsMap = Maps.InjectActiveCells(mesh, activeCells, ndv)
# Create identity map
if input_dict["inversion_type"].lower() == 'mvi':
wrGlobal = np.zeros(3 * nC)
else:
idenMap = Maps.IdentityMap(nP=nC)
wrGlobal = np.zeros(nC)
# Loop over different tile size and break problem until
from transfer.frontends.swiftvr import SWIFTSnapshotData
from scipy.spatial import cKDTree
import numpy
final = SWIFTSnapshotData(
'/cosma6/data/dp004/dc-borr1/snap7/new_randomness_runs/adiabatic/Run_0/eagle_0007.hdf5',
'/cosma6/data/dp004/dc-borr1/snap7/new_randomness_runs/adiabatic/Run_0/halo_0007.properties',
)
boxsize = final.boxsize
boxsize.convert_to_units('Mpc')
dm_coords = final.dark_matter.coordinates
halo_coords = final.halo_coordinates
dm_coords.convert_to_units('Mpc')
halo_coords.convert_to_units('Mpc')
tree = cKDTree(halo_coords, boxsize=boxsize.value)
d, i = tree.query(dm_coords, k=1, n_jobs=-1)
numpy.savetxt('/cosma5/data/durham/dc-murr1/adi_dm_nearest_halo_distance.txt',
d)
numpy.savetxt('/cosma5/data/durham/dc-murr1/adi_dm_nearest_halos.txt', i)
def pars(kx,
ky,
k,
sens,
tx=None,
ty=None,
kernel_size=25,
kernel_radius=None,
coil_axis=-1):
'''Parallel MRI with adaptive radius in k‐space.
Parameters
----------
kx, ky : array_like
Sample points in kspace corresponding to measurements k.
kx, kx are 1D arrays.
k : array_like
Complex kspace coil measurements corresponding to points
(kx, ky).
sens : array_like
Coil sensitivity maps with shape of desired reconstruction.
tx, ty : array_like
Sample points in kspace defining the grid of ifft2(sens).
If None, then tx, ty will be generated from a meshgrid with
endpoints [min(kx), max(kx), min(ky), max(ky)].
kernel_size : int, optional
Number of nearest neighbors to use when interpolating kspace.
kernel_radius : float, optional
Raidus in kspace (units same as (kx, ky)) to select neighbors
when training kernels.
coil_axis : int, optional
Dimension holding coil data.
Returns
-------
res : array_like
Reconstructed image space on a Cartesian grid with the same
shape as sens.
Notes
-----
Implements the algorithm described in [1]_.
Using kernel_radius seems to perform better than kernel_size.
References
----------
.. [1] Yeh, Ernest N., et al. "3Parallel magnetic resonance
imaging with adaptive radius in k‐space (PARS):
Constrained image reconstruction using k‐space locality in
radiofrequency coil encoded data." Magnetic Resonance in
Medicine: An Official Journal of the International Society
for Magnetic Resonance in Medicine 53.6 (2005): 1383-1392.
'''
# Move coil axis to the back
k = np.moveaxis(k, coil_axis, -1)
sens = np.moveaxis(sens, coil_axis, -1)
kxy = np.concatenate((kx[:, None], ky[:, None]), axis=-1)
# Oversample the sensitivity maps by a factor of 2
t0 = time()
sensr = zoom(sens.real, (2, 2, 1), order=1)
sensi = zoom(sens.imag, (2, 2, 1), order=1)
sens = sensr + 1j * sensi
print('Took %g seconds to upsample sens' % (time() - t0))
# We want to resample onto a Cartesian grid
sx, sy, nc = sens.shape[:]
if tx is None or ty is None:
tx, ty = np.meshgrid(np.linspace(np.min(kx), np.max(kx), sx),
np.linspace(np.min(ky), np.max(ky), sy))
tx, ty = tx.flatten(), ty.flatten()
txy = np.concatenate((tx[:, None], ty[:, None]), axis=-1)
# Make a kd-tree and find all point around targets
t0 = time()
kdtree = cKDTree(kxy)
if kernel_radius is None:
_, idx = kdtree.query(txy, k=kernel_size)
else:
idx = kdtree.query_ball_point(txy, r=kernel_radius)
print('Took %g seconds to find nearest neighbors' % (time() - t0))
# Scale kspace coordinates to be within [-.5, .5]
kxy0 = np.concatenate(
(kx[:, None] / np.max(kx), ky[:, None] / np.max(ky)), axis=-1) / 2
txy0 = np.concatenate(
(tx[:, None] / np.max(tx), ty[:, None] / np.max(ty)), axis=-1) / 2
# Encoding matrix is much too large to invert, so we'll go
# kernel by kernel to grid/reconstruct kspace
sens = np.reshape(sens, (-1, nc))
res = np.zeros(sens.shape, dtype=sens.dtype)
t0 = time()
for ii, idx0 in tqdm(enumerate(idx),
total=idx.shape[0],
leave=False,
desc='PARS'):
dk = kxy0[idx0, :]
r = txy0[ii, :]
# Create local encoding matrix and train weights
E = np.exp(1j * (-dk @ r))
E = E[:, None] * sens[ii, :]
W = sens[ii, :] @ E.conj().T @ np.linalg.pinv(E @ E.conj().T)
# Grid the sample:
res[ii, :] = W @ k[idx0, :]
print('Took %g seconds to regrid' % (time() - t0))
# Return image at correct resolution with coil axis in right place
ax = (0, 1)
sx4 = int(sx / 4)
return np.moveaxis(
np.fft.fftshift(np.fft.ifft2(np.fft.ifftshift(np.reshape(
res, (sx, sy, nc), 'F'),
axes=ax),
axes=ax),
axes=ax)[sx4:-sx4, sx4:-sx4, :], -1, coil_axis)
# Transform centers
reduced_centers = pca_model.transform(centers)
# Step size of mesh for plotting
h = .02
# Plot the decision boundary. For that, we will assign a color to each
x_min, x_max = reduced_data[:, 0].min() - 1, reduced_data[:, 0].max() + 1
y_min, y_max = reduced_data[:, 1].min() - 1, reduced_data[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
# Get k-means classifications for the grid points
xx_pt = list(xx.ravel())
yy_pt = list(yy.ravel())
xy_pts = np.array([[x, y] for x, y in zip(xx_pt, yy_pt)])
mytree = cKDTree(reduced_centers)
dist, indexes = mytree.query(xy_pts)
# Put the result into a color plot
indexes = indexes.reshape(xx.shape)
plt.figure(1)
plt.clf()
plt.imshow(indexes,
interpolation='nearest',
extent=(xx.min(), xx.max(), yy.min(), yy.max()),
cmap=plt.cm.Paired,
aspect='auto',
origin='lower')
# Plot each of the true iris data groups
symbols = ['o', '^', 'D']
def process_frame(u, residues, prot, notH):
'''Calculate distance descriptors for all requested residues of the current
frame of a molecular dynamics simulation. Provide selections of the protein atoms and all notH atoms'''
bb_atoms = ["N", "O"]
res = prot.atoms.residues
# Load this frame into a spatial cKDTree for fast nearest-neighbor search
tree = spatial.cKDTree(np.array([x.pos for x in notH.atoms]))
ret = dict() #indexed by resid
# Simultaneously iterate through each residue and corresponding open file
for r in residues:
i = r.id - 1
assert res[i] == r
######################################################
## Process the N dome
######################################################
# Original atom selection
# orig_cloud = u.select_atoms("around 5.0 atom SYSTEM " + str(res[i].id) + " H")
cloud = []
processed_cloud = []
atom_pattern_N = ''
atom_pattern_dist = []
current_res_H_pos = r.H.pos
x = r.N.pos - current_res_H_pos
res_i_O_pos = res[i].O.pos
res_i_m1_N_pos = res[i - 1].N.pos
res_i_p1_N_pos = res[i + 1].N.pos
res_i_m2_O_pos = res[i - 2].O.pos
# First round of processing from enhanced atom selection
for j in tree.query_ball_point(current_res_H_pos, 5.0):
atom = notH.atoms[j]
atom_pos = atom.pos
ref_name = atom_to_pattern(atom)
# Check to ensure backbone atoms within (+/-) 2 residues are not included
if atom.name in bb_atoms and abs(r.id - atom.resid) < 3:
continue
else:
if ref_name:
# Variables for calculating the vector angles of each atom
v = atom_pos - current_res_H_pos
# Eliminate any atoms that have vector angles less than 90.0
if abs(vangle(x, v)) < 90.0:
continue
else:
######################################################
## Compute distance between atoms based on types
## Aromatic atoms: within 5.0 A
## Carbon atoms: within 4.0 A
## Other atoms: wtihin 2.5 A
######################################################
dist = distance(atom_pos, current_res_H_pos)
if ref_name == "R":
cloud.append([atom, dist])
elif ref_name == "C":
if dist <= 4.0:
cloud.append([atom, dist])
elif dist <= 2.5:
cloud.append([atom, dist])
# Second round of processing
if cloud is None:
atom_pattern_N = "Z:"
atom_pattern_dist += [0.0, 0.0, 0.0, 0.0, 0.0]
else:
processed_cloud = find_closest_atom(cloud, res[i].H)
if processed_cloud == None:
atom_pattern_N = "Z:"
atom_pattern_dist += [0.0, 0.0, 0.0, 0.0, 0.0]
else:
# Rank atoms based on the atom type preferences noted above
processed_cloud.sort(key=lambda atom: (atom_ranking[
atom_to_pattern(atom[0])], atom[1]))
# Generate the atom pattern with their corresponding distances
for atom in processed_cloud:
atom_pattern_N += atom_to_pattern(atom[0]) + ':'
apos = atom[0].pos
atom_pattern_dist += [
distance(current_res_H_pos, apos),
distance(res_i_m1_N_pos, apos),
distance(res_i_m2_O_pos, apos),
distance(res_i_O_pos, apos),
distance(res_i_p1_N_pos, apos)
]
# Write the output to the open file
nvals = [
atom_pattern_N, u.trajectory.frame,
distance(res_i_m1_N_pos, current_res_H_pos),
distance(res_i_m2_O_pos, current_res_H_pos),
distance(res[i - 1].O.pos, current_res_H_pos),
distance(res_i_O_pos, current_res_H_pos),
distance(res[i + 1].N.pos, current_res_H_pos)
] + atom_pattern_dist
# Intentionally offset the oxygen output
o = i - 1
######################################################
## Process the O dome
######################################################
# Original atom selection
# orig_cloud = u.select_atoms("around 3.9 atom SYSTEM " + str(res[o].id) + " O")
cloud = []
processed_cloud = []
atom_pattern_O = ''
atom_pattern_dist = []
current_res_O_pos = res[o].O.pos
x = res[o + 1].N.pos - current_res_O_pos
res_o_N_pos = res[o].N.pos
res_o_m1_O_pos = res[o - 1].O.pos
res_o_p1_O_pos = res[o + 1].O.pos
res_o_p2_N_pos = res[o + 2].N.pos
# First round of processing from enhanced atom selection
for j in tree.query_ball_point(current_res_O_pos, 3.9):
atom = notH.atoms[j]
atom_pos = atom.pos
ref_name = atom_to_pattern(atom)
# Check to ensure backbone atoms within (+/-) 2 residues are not included
if atom.name in bb_atoms and abs(res[o].id - atom.resid) < 3:
continue
else:
if ref_name:
# Eliminate all Carbon atoms
if "C" == ref_name:
continue
else:
# Variables for calculating the vector angles of each atom
v = atom_pos - current_res_O_pos
# Eliminate any atoms that have vector angles less than 90.0
if abs(vangle(x, v)) < 90.0:
continue
elif ref_name:
cloud.append(
[atom,
distance(atom_pos, current_res_O_pos)])
# Second round of processing
if cloud is None:
atom_pattern_O = "Z:"
atom_pattern_dist += [0.0, 0.0, 0.0, 0.0, 0.0]
else:
processed_cloud = find_closest_atom(cloud, res[o].O)
if processed_cloud == None or len(processed_cloud) < 1:
atom_pattern_O = "Z:"
atom_pattern_dist += [0.0, 0.0, 0.0, 0.0, 0.0]
else:
# Rank atoms based on the atom type preferences noted above
processed_cloud.sort(key=lambda atom: (atom_ranking[
atom_to_pattern(atom[0])], atom[1]))
# Generate the atom pattern with their corresponding distances
for atom in processed_cloud:
atom_pattern_O += atom_to_pattern(atom[0]) + ":"
apos = atom[0].pos
atom_pattern_dist += [
distance(current_res_O_pos, apos),
distance(res_o_N_pos, apos),
distance(res_o_m1_O_pos, apos),
distance(res_o_p1_O_pos, apos),
distance(res_o_p2_N_pos, apos)
]
# Write the output to the open file
ovals = [
atom_pattern_O, u.trajectory.frame,
distance(res_o_N_pos, current_res_O_pos),
distance(res_o_m1_O_pos, current_res_O_pos),
distance(res[o + 1].N.pos, current_res_O_pos),
distance(res_o_p1_O_pos, current_res_O_pos),
distance(res_o_p2_N_pos, current_res_O_pos)
] + atom_pattern_dist
ret[i] = (nvals, ovals)
return ret
for record in disFile:
inXYT.append([
float(record.split(" ")[1]),
float(record.split(" ")[2]),
float(record.split(" ")[3])
])
disFile.close()
#number of points
numPts = len(inXYT)
#sort dataset temporally
inXYT_s = np.array(sorted(inXYT, key=lambda a: a[2]))
#index dataset
sTree = spatial.cKDTree(inXYT_s[:, :2])
stNeighIndex = []
#maximum number of neighbors
mNN = np.inf
#current number of neighbors
NN = numPts
# output folder
outDir = 'outputs/ST_MC_1/sim_' + sim
if not os.path.exists(outDir):
os.makedirs(outDir)
outFile = open(outDir + os.sep + 'bandwidths_' + str(neighThres) + '.txt', 'w')
def set_hbonds_in_selection_and_water_around(self,
around_radius,
not_water_water=False):
sfilter = _np.array([(ids.split('-')[3].startswith('S'))
for ids in self._all_ids_atomwise])
result = {}
frame_count = 0
frames = self.nb_frames
for ts in self._universe.trajectory[self._trajectory_slice]:
water_coordinates = self._water.positions
selection_coordinates = self._da_selection.positions
selection_tree = _sp.cKDTree(selection_coordinates)
if self._nb_acceptors > 0 and self._nb_donors > 0:
d_tree = _sp.cKDTree(self._donors.positions)
a_tree = _sp.cKDTree(self._acceptors.positions)
hydrogen_coordinates = self._hydrogen.positions
water_tree = _sp.cKDTree(water_coordinates, leafsize=32)
local_water_index = []
[
local_water_index.extend(l)
for l in water_tree.query_ball_point(selection_coordinates,
float(around_radius))
]
local_water_index = _np.unique(local_water_index)
local_water_coordinates = water_coordinates[local_water_index]
local_water_tree = _sp.cKDTree(local_water_coordinates)
local_pairs = [(i, local_water_index[j] + self._first_water_id)
for i, bla in enumerate(
selection_tree.query_ball_tree(
local_water_tree, self.distance))
for j in bla]
water_pairs = [(local_water_index[p[0]] + self._first_water_id,
local_water_index[p[1]] + self._first_water_id)
for p in local_water_tree.query_pairs(self.distance)
]
if self._nb_acceptors > 0 and self._nb_donors > 0:
da_pairs = _np.array([[i, j] for i, donors in enumerate(
a_tree.query_ball_tree(d_tree, self.distance))
for j in donors])
da_pairs[:, 0] += self._nb_donors
else:
da_pairs = []
if da_pairs != []:
da_pairs = da_pairs[_np.logical_not(
_np.all(sfilter[da_pairs], axis=1))]
if self.check_angle:
all_coordinates = _np.vstack(
(selection_coordinates, water_coordinates))
if not_water_water:
hbonds = _hf.check_angle(
list(da_pairs) + local_pairs, self.heavy2hydrogen,
all_coordinates, hydrogen_coordinates, self.cut_angle)
else:
hbonds = _hf.check_angle(
list(da_pairs) + water_pairs + local_pairs,
self.heavy2hydrogen, all_coordinates,
hydrogen_coordinates, self.cut_angle)
else:
if not_water_water:
hbonds = list(da_pairs) + local_pairs
else:
hbonds = list(da_pairs) + water_pairs + local_pairs
hbonds = _np.array(hbonds)
sorted_bonds = _np.sort(hbonds)
check = self._resids[sorted_bonds]
check = check[:, 0] < check[:, 1]
if self.residuewise:
frame_res = [
self._all_ids[i] + ':' +
self._all_ids[j] if check[ii] else self._all_ids[j] + ':' +
self._all_ids[i] for ii, (i, j) in enumerate(sorted_bonds)
]
else:
frame_res = [
self._all_ids_atomwise[i] + ':' + self._all_ids_atomwise[j]
if check[ii] else self._all_ids_atomwise[j] + ':' +
self._all_ids_atomwise[i]
for ii, (i, j) in enumerate(sorted_bonds)
]
for bond in frame_res:
a, b = bond.split(':')
if a == b: continue
try:
result[bond][frame_count] = True
except:
result[bond] = _np.zeros(frames, dtype=bool)
result[bond][frame_count] = True
frame_count += 1
if self.progress_callback is not None:
self.progress_callback.emit(
'Computing H bonds in frame {}/{}'.format(
frame_count, self.nb_frames))
self._set_results(result)
def rebuildKDT(self):
"""
Force the internal KDTree to be rebuilt.
"""
self.kdt = cKDTree(self.data[:, :-1], leafsize=self.leafsize)
self.rebuild_tree = False
def point2plane(self,
steps=1,
neigh=10,
inside=True,
subset=None,
scale=None,
smooth=1,
fixBrim=False,
error='norm'):
r"""
Point to Plane method for registration between the two meshes
Parameters
----------
steps: int, default 1
Number of iterations
int, default 10
Number of nearest neighbours to interrogate for each baseline point
inside: bool, default True
If True, a barycentric centre check is made to ensure the registered
point lines within the target triangle
subset: array_like, default None
Indicies of the baseline nodes to include in the registration, default is none so
all are used
scale: float, default None
If not None scale the baseline mesh to match the target mesh in the z-direction,
the value of scale will be used as a plane from which the nodes are scaled.
Nodes with a higher z value will not be scaled.
smooth: int, default 1
Indicate number of Laplacian smooth steps in between the steps
fixBrim: bool, default False
If True, the nodes on the brim line will not be included in the smooth
error: bool, default False
If True, the polarity will be included when calculating the distance
between the target and baseline mesh
"""
# Calc FaceCentroids
fC = self.t.vert[self.t.faces].mean(axis=1)
# Construct knn tree
tTree = spatial.cKDTree(fC)
bData = dict(
zip(['vert', 'faces', 'values'],
[self.b.vert, self.b.faces, self.b.values]))
regData = copy.deepcopy(bData)
self.reg = AmpObject(regData, stype='reg')
self.disp = AmpObject({
'vert': np.zeros(self.reg.vert.shape),
'faces': self.reg.faces,
'values': self.reg.values
})
if scale is not None:
tmin = self.t.vert.min(axis=0)[2]
rmin = self.reg.vert.min(axis=0)[2]
SF = ((tmin - scale) / (rmin - scale)) - 1
logic = self.reg.vert[:, 2] < scale
d = (self.reg.vert[logic, 2] - scale) * SF
self.disp.vert[logic, 2] += d
self.reg.vert = self.b.vert + self.disp.vert
normals = np.cross(
self.t.vert[self.t.faces[:, 1]] - self.t.vert[self.t.faces[:, 0]],
self.t.vert[self.t.faces[:, 2]] - self.t.vert[self.t.faces[:, 0]])
mag = (normals**2).sum(axis=1)
for step in np.arange(steps, 0, -1, dtype=float):
# Index of 10 centroids nearest to each baseline vertex
ind = tTree.query(self.reg.vert, neigh)[1]
# Define normals for faces of nearest faces
norms = normals[ind]
# Get a point on each face
fPoints = self.t.vert[self.t.faces[ind, 0]]
# Calculate dot product between point on face and normals
d = np.einsum('ijk, ijk->ij', norms, fPoints)
t = (d - np.einsum('ijk, ik->ij', norms, self.reg.vert)) / mag[ind]
# Calculate the vector from old point to new point
G = self.reg.vert[:, None, :] + np.einsum('ijk, ij->ijk', norms, t)
# Ensure new points lie inside points otherwise set to 99999
# Find smallest distance from old to new point
if inside is False:
G = G - self.reg.vert[:, None, :]
GMag = np.sqrt(np.einsum('ijk, ijk->ij', G, G))
GInd = GMag.argmin(axis=1)
else:
G, GInd = self.__calcBarycentric(self.reg.vert, G, ind)
# Define vector from baseline point to intersect point
D = G[np.arange(len(G)), GInd, :]
# rVert += D/step
self.disp.vert += D / step
if smooth > 0 and step > 1:
self.disp.lp_smooth(smooth, brim=fixBrim)
self.reg.vert = self.b.vert + self.disp.vert
else:
self.reg.vert = self.b.vert + self.disp.vert
self.reg.calcNorm()
self.reg.calcStruct(vNorm=True)
self.reg.values[:] = self.calcError(error)
setup = schism_setup()
xmin = min(setup.x)
xmax = max(setup.x)
ymin = min(setup.y)
ymax = max(setup.y)
# make new grid
dx=2000.
dy=2000.
xn = arange(xmin-dx,xmax+2*dx,dx)
yn = arange(ymin-dy,ymax+2*dy,dy)
XN,YN = meshgrid(xn,yn)
# build tree
print(' build tree')
tree = cKDTree(zip(x,y))
# find weights and distances
print(' query weights')
dist,inds = tree.query(zip(XN.flat[:],YN.flat[:]), k=100)
weights = 1.0 / dist**2
# do interpolation
print(' interpolate')
depths = np.min(d[inds],axis=1)
#depths = np.sum( weights*d[inds],axis=1) / np.sum(weights,axis=1)
# create size field
maxlength=4000.
minlength=500.
maxdepth=50.
def config():
"""
Starts the program up with all necessary things. Reads the inputs,
creates the Singleton objects properly, sets up the heightmap for later,
makes sure all Vertices in the axiom have the correct neighbor. Could
need a rework in which the Singletons are unified and not broken as they
are now.
Returns
-------
variables : Variables object
Singleton with all numeric values which are not to be changed at runtime
singleton.global_lists : singleton.global_lists object
Singleton with the Global Lists which will be altered at runtime
"""
import json
from collections import namedtuple
import os
import procedural_city_generation
from procedural_city_generation.additional_stuff.Singleton import Singleton
path = os.path.dirname(procedural_city_generation.__file__)
singleton = Singleton("roadmap")
#Creates Singleton-Variables object from namedtuple
from procedural_city_generation.roadmap.Vertex import Vertex, set_plotbool
#Creates Vertex objects from coordinates
singleton.axiom = [
Vertex(np.array([float(x[0]), float(x[1])])) for x in singleton.axiom
]
singleton.border = np.array([singleton.border_x, singleton.border_y])
set_plotbool(singleton.plot)
#Finds the longest possible length of a connection between to vertices
singleton.maxLength = max(singleton.radiallMax, singleton.gridlMax,
singleton.organiclMax, singleton.minor_roadlMax,
singleton.seedlMax)
import os
from procedural_city_generation.roadmap.config_functions.input_image_setup import input_image_setup
singleton.img, singleton.img2 = input_image_setup(
singleton.rule_image_name, singleton.density_image_name)
with open(path + "/temp/" + singleton.output_name + "_densitymap.txt",
'w') as f:
f.write(singleton.density_image_name.split(".")[0] + "diffused.png")
from procedural_city_generation.roadmap.config_functions.find_radial_centers import find_radial_centers
singleton.center = find_radial_centers(singleton)
singleton.center = [
np.array([
singleton.border[0] * ((x[1] / singleton.img.shape[1]) - 0.5) * 2,
singleton.border[1] *
(((singleton.img.shape[0] - x[0]) / singleton.img.shape[0]) - 0.5)
* 2
]) for x in singleton.center
]
from procedural_city_generation.roadmap.config_functions.setup_heightmap import setup_heightmap
setup_heightmap(singleton, path)
singleton.global_lists = Global_Lists()
singleton.global_lists.vertex_list.extend(singleton.axiom)
singleton.global_lists.coordsliste = [
x.coords for x in singleton.global_lists.vertex_list
]
def setNeighbours(vertex):
""" Correctly Sets up the neighbors for a vertex from the axiom.
Parameters
----------
vertex : vertex Object
"""
d = np.inf
neighbour = None
for v in singleton.axiom:
if v is not vertex:
dneu = np.linalg.norm(v.coords - vertex.coords)
if dneu < d:
d = dneu
neighbour = v
vertex.neighbours = [neighbour]
for k in singleton.axiom:
setNeighbours(k)
from scipy.spatial import cKDTree
singleton.global_lists.tree = cKDTree(singleton.global_lists.coordsliste,
leafsize=160)
return singleton
rest_pose[0, 0] = np.pi / 2
rest_pose[1, 2] = .15
rest_pose[2, 2] = -.15
smpl_path = os.path.dirname(os.path.abspath(__file__)) + '/../Model/smpl/'
SMPL = {
0: SMPLModel(smpl_path + 'model_f.pkl', rest_pose),
1: SMPLModel(smpl_path + 'model_m.pkl', rest_pose)
}
# get sample list
samples = os.listdir(SRC)
N = len(samples)
# for each sample
for i, sample in enumerate(samples):
print(str(i + 1) + " / " + str(N))
src = SRC + sample + '/'
src_pre = SRC_PREPROCESS + sample + '/'
dst = src_pre + 'weights.npy'
# load info
info = loadInfo(src + 'info.mat')
shape = info['shape']
gender = info['gender']
# load template outfit
T = readPC2(src_pre + 'rest.pc16', True)['V'][0]
# compute SMPL in rest pose
_, B = SMPL[gender].set_params(pose=rest_pose, beta=shape, with_body=True)
# nearest neighbour
tree = cKDTree(B)
_, idx = tree.query(T)
W_prior = SMPL[gender].weights[idx]
np.save(dst, W_prior)
def __init__(self, data):
cart_data = self.spherical2cartesian(data)
self.data = data
self.kd_tree = cKDTree(data=cart_data, leafsize=10)
def __init__(self, data, leafsize = 10):
self.real_tree = cKDTree(data.T, leafsize)
try:
if i > 2:
values[i] = float(v)
if int(v) == 99999:
values[i] = None
except:
continue
lat, lon = values[6], values[7]
if lat and lon:
# Some stations have no lat-long; this isn't useful
stations.append(IntegratedSurfaceDatabaseStation(*values))
_latlongs.append((lat, lon))
_latlongs = np.array(_latlongs)
station_count = len(stations)
kd_tree = cKDTree(
_latlongs) # _latlongs must be unchanged as data is not copied
def get_closest_station(latitude,
longitude,
minumum_recent_data=20140000,
match_max=100):
"""Query function to find the nearest weather station to a particular set of
coordinates. Optionally allows for a recent date by which the station is
required to be still active at.
Parameters
----------
latitude : float
Latitude to search for nearby weather stations at, [degrees]
longitude : float
def _calculate_novelty(self, bc, _archive):
kd = spatial.cKDTree(_archive)
distances, idxs = kd.query(bc, k=self.k)
distances = distances[distances < float('inf')]
novelty = np.sum(distances) / np.linalg.norm(_archive)
return novelty
def do_kdtree(combined_x_y_arrays, points):
mytree = cKDTree(combined_x_y_arrays)
dist, indexes = mytree.query(points)
return dist, indexes
def adi(pts_est, pts_gt):
nn_index = spatial.cKDTree(pts_est)
nn_dists, _ = nn_index.query(pts_gt, k=1)
e = nn_dists.mean()
return e
def set_hbonds_in_selection(self):
sfilter = _np.array([(ids.split('-')[3].startswith('S'))
for ids in self._all_ids_atomwise])
frame_count = 0
frames = self.nb_frames
result = {}
#t0 = time.time()
for ts in self._universe.trajectory[self._trajectory_slice]:
selection_coordinates = self._da_selection.positions
d_tree = _sp.cKDTree(self._donors.positions)
a_tree = _sp.cKDTree(self._acceptors.positions)
hydrogen_coordinates = self._hydrogen.positions
da_pairs = _np.array([[i, j] for i, donors in enumerate(
a_tree.query_ball_tree(d_tree, self.distance))
for j in donors])
da_pairs[:, 0] += self._nb_donors
da_pairs = da_pairs[_np.logical_not(
_np.all(sfilter[da_pairs], axis=1))]
if self.check_angle:
all_coordinates = selection_coordinates
local_hbonds = _hf.check_angle(da_pairs, self.heavy2hydrogen,
all_coordinates,
hydrogen_coordinates,
self.cut_angle)
else:
local_hbonds = da_pairs
sorted_bonds = _np.sort(local_hbonds)
check = self._resids[sorted_bonds]
check = check[:, 0] < check[:, 1]
if self.residuewise:
frame_res = [
self._all_ids[i] + ':' +
self._all_ids[j] if check[ii] else self._all_ids[j] + ':' +
self._all_ids[i] for ii, (i, j) in enumerate(sorted_bonds)
]
else:
frame_res = [
self._all_ids_atomwise[i] + ':' + self._all_ids_atomwise[j]
if check[ii] else self._all_ids_atomwise[j] + ':' +
self._all_ids_atomwise[i]
for ii, (i, j) in enumerate(sorted_bonds)
]
for bond in frame_res:
a, b = bond.split(':')
if self.residuewise and (a.split('-')[:3] == b.split('-')[:3]):
continue
try:
result[bond][frame_count] = True
except:
result[bond] = _np.zeros(frames, dtype=bool)
result[bond][frame_count] = True
frame_count += 1
if self.progress_callback is not None:
self.progress_callback.emit(
'Computing H bonds in frame {}/{}'.format(
frame_count, self.nb_frames))
#print('Time to compute {} H-bonds: {}s'.format(len(result), _np.round(time.time()-t0,5)))
self._set_results(result)
def run(in_jsons: List[params.preprocessing_content_type],
out_dir: str,
resolution: float = 0.5,
min_elevation_offset: float = None,
max_elevation_offset: float = None,
epsg: int = None,
sigma: float = None,
dsm_radius: int = 1,
dsm_no_data: int = -32768,
msk_no_data: int = 65535,
color_no_data: int = 0,
corr_config: Dict = None,
output_stats: bool = False,
mode: str = "local_dask",
nb_workers: int = 4,
walltime: str = "00:59:00",
roi: Tuple[List[int], int] = None,
use_geoid_alt: bool = False,
use_sec_disp: bool = False,
snap_to_img1: bool = False,
align: bool = False,
cloud_small_components_filter: bool = True,
cloud_statistical_outliers_filter: bool = True,
epi_tile_size: int = None):
"""
Main function for the compute_dsm pipeline subcommand
It computes independent tiles of the final DSM, with the following steps:
1. Epipolar resampling (including mask)
2. Disparity map estimation
3. Triangulation of disparity map
4. Rasterization to DSM
:param in_jsons: Input pair dictionaries (as produced by cars prepare step)
:param out_dir: Computed raster and color image output directory
:param resolution: DSM resolution to produce
:param min_elevation_offset: Override minimum disparity
from prepare step with this offset in meters
:param max_elevation_offset: Override maximum disparity
from prepare step with this offset in meters
:param epsg: Output DSM Coordinate Reference System EPSG code
:param sigma: Rasterization width of gaussian weight
:param dsm_radius: Rasterization radius around a cell for gathering points
:param dsm_no_data: No data value to use in the final DSM file
:param color_no_data: No data value to use in the final colored image
:param msk_no_data: No data value to use in the final mask image
:param corr_config: Correlator configuration
:param output_stats: Ouput DSM associated quality statistics flag boolean
:param mode: Parallelization mode
:param nb_workers: Number of dask workers to use for the sift matching step
:param walltime: Walltime of the dask workers
:param roi: DSM Region Of Interest in final projection with EPSG reference
([xmin, ymin, xmax, ymax], roi_epsg))
(roi_epsg can be set to None if the ROI is in final projection)
:param use_geoid_alt: Geoid height reference for DSM altitude flag.
:param use_sec_disp: Secondary disparity map activation flag.
:param snap_to_img1: Force Img2 / Img1 Lines of Sight crossing flag.
:param align: If this is True, use the correction estimated during prepare
to align to lowres DEM (if available)
:param cloud_small_components_filter:
Activating the points cloud small components filtering.
The filter's parameters are set in static configuration json.
:param cloud_statistical_outliers_filter:
Activating the points cloud statistical outliers filtering.
The filter's parameters are set in static configuration json.
:param epi_tile_size: Force the size of epipolar tiles (None by default)
"""
out_dir = os.path.abspath(out_dir)
# Ensure that outdir exists
try:
os.makedirs(out_dir)
except OSError as exc:
if exc.errno == errno.EEXIST and os.path.isdir(out_dir):
pass
else:
raise
tmp_dir = os.path.join(out_dir, 'tmp')
utils.add_log_file(out_dir, 'compute_dsm')
logging.info("Received {} stereo pairs configurations".format(
len(in_jsons)))
# Retrieve static parameters (rasterization and cloud filtering)
static_params = static_cfg.get_cfg()
# Initiate ouptut json dictionary
out_json = {
params.stereo_inputs_section_tag: [],
params.stereo_section_tag: {
params.stereo_version_tag: utils.get_version(),
params.stereo_parameters_section_tag: {
params.resolution_tag: resolution,
params.sigma_tag: sigma,
params.dsm_radius_tag: dsm_radius
},
params.static_params_tag:
static_params[static_cfg.compute_dsm_tag],
params.stereo_output_section_tag: {}
}
}
if use_geoid_alt:
geoid_data = utils.read_geoid_file()
out_json[params.stereo_section_tag][params.stereo_output_section_tag][
params.alt_reference_tag] = 'geoid'
else:
geoid_data = None
out_json[params.stereo_section_tag][params.stereo_output_section_tag][
params.alt_reference_tag] = 'ellipsoid'
if epsg is not None:
out_json[params.stereo_section_tag][
params.stereo_parameters_section_tag][params.epsg_tag] = epsg
roi_epsg = None
if roi is not None:
(roi_xmin, roi_ymin, roi_xmax, roi_ymax), roi_epsg = roi
roi_poly = Polygon([(roi_xmin, roi_ymin), (roi_xmax, roi_ymin),
(roi_xmax, roi_ymax), (roi_xmin, roi_ymax),
(roi_xmin, roi_ymin)])
# set the timeout for each job in multiprocessing mode (in seconds)
per_job_timeout = 600
configurations_data = {}
config_idx = 1
ref_left_img = None
write_msk = False
for in_json in in_jsons:
# Build config id
config_id = "config_{}".format(config_idx)
# Check configuration with respect to schema
configuration = utils.check_json(in_json,
params.preprocessing_content_schema)
# retrieve masks classes usages
mask1_classes = configuration[params.input_section_tag].get(
params.mask1_classes_tag, None)
mask2_classes = configuration[params.input_section_tag].get(
params.mask2_classes_tag, None)
classes_usage = dict()
if mask1_classes is not None:
mask1_classes_dict = mask_classes.read_mask_classes(mask1_classes)
classes_usage[params.mask1_ignored_by_corr_tag] =\
mask1_classes_dict.get(mask_classes.ignored_by_corr_tag, None)
classes_usage[params.mask1_set_to_ref_alt_tag] = \
mask1_classes_dict.get(mask_classes.set_to_ref_alt_tag, None)
if mask2_classes is not None:
mask2_classes_dict = mask_classes.read_mask_classes(mask2_classes)
classes_usage[params.mask2_ignored_by_corr_tag] = \
mask2_classes_dict.get(mask_classes.ignored_by_corr_tag, None)
classes_usage[params.mask2_set_to_ref_alt_tag] = \
mask2_classes_dict.get(mask_classes.set_to_ref_alt_tag, None)
# Append input configuration to output json
out_json_config = {
params.stereo_input_tag: configuration,
}
if mask1_classes is not None or mask2_classes is not None:
out_json_config[
params.stereo_mask_classes_usage_tag] = classes_usage
out_json[params.stereo_inputs_section_tag].append(out_json_config)
configurations_data[config_id] = {}
configurations_data[config_id]['configuration'] = configuration
# Get local conf left image for this in_json iteration
conf_left_img = configuration[params.input_section_tag][
params.img1_tag]
# Check left image and raise a warning
# if different left images are used along with snap_to_img1 mode
if ref_left_img is None:
ref_left_img = conf_left_img
else:
if snap_to_img1 and ref_left_img != conf_left_img:
logging.warning(
"--snap_to_left_image mode is used but input "
"configurations have different images as their "
"left image in pair. This may result in "
"increasing registration discrepencies between pairs")
# If mask1 and/or mask2 are set in the prepare input configuration json
# then the DSM rasterized mask will be written alongside the DSM
# TODO : Mask 2 ?
mask1 = \
configuration[params.input_section_tag].get(params.mask1_tag, None)
if mask1 is not None:
write_msk = True
# Get Preprocessing output config
preprocessing_output_config = configuration[
params.preprocessing_section_tag][
params.preprocessing_output_section_tag]
# Get largest epipolar regions from configuration file
largest_epipolar_region =\
[0, 0, preprocessing_output_config[params.epipolar_size_x_tag],
preprocessing_output_config[params.epipolar_size_y_tag]]
configurations_data[config_id]['largest_epipolar_region'] =\
largest_epipolar_region
disp_min = preprocessing_output_config[params.minimum_disparity_tag]
disp_max = preprocessing_output_config[params.maximum_disparity_tag]
disp_to_alt_ratio = preprocessing_output_config[
params.disp_to_alt_ratio_tag]
# Check if we need to override disp_min
if min_elevation_offset is not None:
user_disp_min = min_elevation_offset / disp_to_alt_ratio
if user_disp_min > disp_min:
logging.warning(
('Overriden disparity minimum = {:.3f} pix. (= {:.3f} m.) '
'is greater than disparity minimum estimated '
'in prepare step = {:.3f} pix. (or {:.3f} m.) '
'for configuration {}').format(
user_disp_min, min_elevation_offset, disp_min,
disp_min * disp_to_alt_ratio, config_id))
disp_min = user_disp_min
# Check if we need to override disp_max
if max_elevation_offset is not None:
user_disp_max = max_elevation_offset / disp_to_alt_ratio
if user_disp_max < disp_max:
logging.warning((
'Overriden disparity maximum = {:.3f} pix. (or {:.3f} m.) '
'is lower than disparity maximum estimated '
'in prepare step = {:.3f} pix. (or {:.3f} m.) '
'for configuration {}').format(
user_disp_max, max_elevation_offset, disp_max,
disp_max * disp_to_alt_ratio, config_id))
disp_max = user_disp_max
logging.info(
'Disparity range for config {}: [{:.3f} pix., {:.3f} pix.] '
'(or [{:.3f} m., {:.3f} m.])'.format(config_id, disp_min, disp_max,
disp_min * disp_to_alt_ratio,
disp_max * disp_to_alt_ratio))
configurations_data[config_id]['disp_min'] = disp_min
configurations_data[config_id]['disp_max'] = disp_max
origin = [
preprocessing_output_config[params.epipolar_origin_x_tag],
preprocessing_output_config[params.epipolar_origin_y_tag]
]
spacing = [
preprocessing_output_config[params.epipolar_spacing_x_tag],
preprocessing_output_config[params.epipolar_spacing_y_tag]
]
configurations_data[config_id]['origin'] = origin
configurations_data[config_id]['spacing'] = spacing
logging.info(
"Size of epipolar image: {}".format(largest_epipolar_region))
logging.debug("Origin of epipolar grid: {}".format(origin))
logging.debug("Spacing of epipolar grid: {}".format(spacing))
# Warning if align is set but correction is missing
param_lowres_tag = params.lowres_dem_splines_fit_tag
if align and param_lowres_tag not in preprocessing_output_config:
logging.warning(
('Align with low resolution DSM option is set but splines '
'correction file is not available for configuration {}. '
'Correction will not be applied for this configuration'
).format(config_id))
# Numpy array with corners of largest epipolar region.
# Order does not matter here,
# since it will be passed to stereo.compute_epipolar_grid_min_max
corners = np.array(
[[[largest_epipolar_region[0], largest_epipolar_region[1]],
[largest_epipolar_region[0], largest_epipolar_region[3]]],
[[largest_epipolar_region[2], largest_epipolar_region[3]],
[largest_epipolar_region[2], largest_epipolar_region[1]]]],
dtype=np.float64)
# get UTM zone with the middle point of terrain_min if epsg is None
if epsg is None:
# Compute epipolar image terrain position corners
# for min and max disparity
terrain_dispmin, terrain_dispmax =\
stereo.compute_epipolar_grid_min_max(
corners, 4326, configuration, disp_min, disp_max)
epsg = rasterization.get_utm_zone_as_epsg_code(
*np.mean(terrain_dispmin, axis=0))
logging.info("EPSG code: {}".format(epsg))
# Compute terrain min and max again, this time using estimated epsg code
terrain_dispmin, terrain_dispmax =\
stereo.compute_epipolar_grid_min_max(
corners, epsg, configuration, disp_min, disp_max)
if roi_epsg is not None:
if roi_epsg != epsg:
roi_poly =\
projection.polygon_projection(roi_poly, roi_epsg, epsg)
# Compute bounds from epipolar image corners and dispmin/dispmax
terrain_bounds = np.stack((terrain_dispmin, terrain_dispmax), axis=0)
terrain_min = np.amin(terrain_bounds, axis=(0, 1))
terrain_max = np.amax(terrain_bounds, axis=(0, 1))
terrain_area =\
(terrain_max[0]-terrain_min[0])*(terrain_max[1]-terrain_min[1])
configurations_data[config_id]['terrain_area'] = terrain_area
logging.info(
"Terrain area covered: {} square meters (or square degrees)".
format(terrain_area))
# Retrieve bounding box of the ground intersection of the envelopes
inter_poly, inter_epsg = utils.read_vector(
preprocessing_output_config[params.envelopes_intersection_tag])
if epsg != inter_epsg:
inter_poly =\
projection.polygon_projection(inter_poly, inter_epsg, epsg)
(inter_xmin, inter_ymin, inter_xmax, inter_ymax) = inter_poly.bounds
# Align bounding box to integer resolution steps
xmin, ymin, xmax, ymax = tiling.snap_to_grid(inter_xmin, inter_ymin,
inter_xmax, inter_ymax,
resolution)
logging.info("Terrain bounding box : [{}, {}] x [{}, {}]".format(
xmin, xmax, ymin, ymax))
configurations_data[config_id]['terrain_bounding_box'] = [
xmin, ymin, xmax, ymax
]
if roi is not None:
if not roi_poly.intersects(inter_poly):
logging.warning("The pair composed of {} and {} "
"does not intersect the requested ROI".format(
configuration[params.input_section_tag][
params.img1_tag],
configuration[params.input_section_tag][
params.img2_tag]))
# Get optimal tile size
if epi_tile_size is not None:
opt_epipolar_tile_size = epi_tile_size
else:
opt_epipolar_tile_size =\
stereo.optimal_tile_size_pandora_plugin_libsgm(
disp_min, disp_max,
margin=static_cfg.get_epi_tile_margin_percent())
logging.info(
"Optimal tile size for epipolar regions: "
"{size}x{size} pixels".format(size=opt_epipolar_tile_size))
configurations_data[config_id]['opt_epipolar_tile_size'] =\
opt_epipolar_tile_size
# Split epipolar image in pieces
epipolar_regions = tiling.split(
0, 0, preprocessing_output_config[params.epipolar_size_x_tag],
preprocessing_output_config[params.epipolar_size_y_tag],
opt_epipolar_tile_size, opt_epipolar_tile_size)
epipolar_regions_grid = tiling.grid(
0, 0, preprocessing_output_config[params.epipolar_size_x_tag],
preprocessing_output_config[params.epipolar_size_y_tag],
opt_epipolar_tile_size, opt_epipolar_tile_size)
configurations_data[config_id]['epipolar_regions'] = epipolar_regions
configurations_data[config_id]['epipolar_regions_grid'] =\
epipolar_regions_grid
logging.info("Epipolar image will be processed in {} splits".format(
len(epipolar_regions)))
# Increment config index
config_idx += 1
xmin, ymin, xmax, ymax = tiling.union([
conf['terrain_bounding_box']
for config_id, conf in configurations_data.items()
])
if roi is not None:
# terrain bounding box polygon
terrain_poly = Polygon([(xmin, ymin), (xmax, ymin), (xmax, ymax),
(xmin, ymax), (xmin, ymin)])
if not roi_poly.intersects(terrain_poly):
raise Exception(
'None of the input pairs intersect the requested ROI')
# Show ROI if valid (no exception raised) :
logging.info('Setting terrain bounding box to the requested ROI')
xmin, ymin, xmax, ymax = roi_poly.bounds
xmin, ymin, xmax, ymax = tiling.snap_to_grid(xmin, ymin, xmax, ymax,
resolution)
logging.info("Total terrain bounding box : [{}, {}] x [{}, {}]".format(
xmin, xmax, ymin, ymax))
# Compute optimal terrain tile size
optimal_terrain_tile_widths = []
for config_id, conf in configurations_data.items():
# Compute terrain area covered by a single epipolar tile
terrain_area_covered_by_epipolar_tile = conf["terrain_area"] / len(
epipolar_regions)
# Compute tile width in pixels
optimal_terrain_tile_widths.append(
math.sqrt(terrain_area_covered_by_epipolar_tile))
# In case of multiple json configuration, take the average optimal size,
# and align to multiple of resolution
optimal_terrain_tile_width = int(
math.ceil(
np.mean(optimal_terrain_tile_widths) / resolution)) * resolution
logging.info("Optimal terrain tile size: {}x{} pixels".format(
int(optimal_terrain_tile_width / resolution),
int(optimal_terrain_tile_width / resolution)))
# Split terrain bounding box in pieces
terrain_grid = tiling.grid(xmin, ymin, xmax, ymax,
optimal_terrain_tile_width,
optimal_terrain_tile_width)
number_of_terrain_splits = (terrain_grid.shape[0] -
1) * (terrain_grid.shape[1] - 1)
logging.info("Terrain bounding box will be processed in {} splits".format(
number_of_terrain_splits))
# Start dask cluster
cluster = None
client = None
# Use dask
use_dask = {"local_dask": True, "pbs_dask": True, "mp": False}
if mode not in use_dask.keys():
raise NotImplementedError('{} mode is not implemented'.format(mode))
if use_dask[mode]:
if mode == "local_dask":
cluster, client = start_local_cluster(nb_workers)
else:
cluster, client = start_cluster(nb_workers, walltime, out_dir)
# Add plugin to monitor memory of workers
plugin = ComputeDSMMemoryLogger(out_dir)
client.register_worker_plugin(plugin)
geoid_data_futures = None
if geoid_data is not None:
# Broadcast geoid data to all dask workers
geoid_data_futures = client.scatter(geoid_data, broadcast=True)
# Retrieve the epsg code which will be used
# for the triangulation's output points clouds
# (ecef if filters are activated)
if cloud_small_components_filter or cloud_statistical_outliers_filter:
stereo_out_epsg = 4978
else:
stereo_out_epsg = epsg
# Submit all epipolar regions to be processed as delayed tasks, and
# project terrain grid to epipolar
for config_id, conf in configurations_data.items():
# This list will hold the different epipolar tiles to be
# processed as points cloud
delayed_point_clouds = []
if use_dask[mode]:
# Use Dask delayed
for region in conf['epipolar_regions']:
delayed_point_clouds.append(
dask.delayed(stereo.images_pair_to_3d_points)(
conf['configuration'],
region,
corr_config,
disp_min=conf['disp_min'],
disp_max=conf['disp_max'],
geoid_data=geoid_data_futures,
out_epsg=stereo_out_epsg,
use_sec_disp=use_sec_disp,
snap_to_img1=snap_to_img1,
align=align,
add_msk_info=write_msk))
logging.info("Submitted {} epipolar delayed tasks to dask "
"for stereo configuration {}".format(
len(delayed_point_clouds), config_id))
else:
# Use multiprocessing module
# create progress bar with an update callback
pbar = tqdm(total=len(conf['epipolar_regions']))
def update(args): #pylint: disable=unused-argument
pbar.update()
# create a thread pool
pool = multiprocessing.Pool(nb_workers)
# launch several 'write_3d_points()' to process each epipolar region
for region in conf['epipolar_regions']:
delayed_point_clouds.append(
pool.apply_async(write_3d_points,
args=(conf['configuration'], region,
corr_config, tmp_dir, config_id),
kwds={
'disp_min': conf['disp_min'],
'disp_max': conf['disp_max'],
'geoid_data': geoid_data,
'out_epsg': stereo_out_epsg,
'use_sec_disp': use_sec_disp,
"add_msk_info": write_msk
},
callback=update))
# Wait computation results (timeout in seconds) and replace the
# async objects by the actual output of write_3d_points(), meaning
# the paths to cloud files
delayed_point_clouds =\
[delayed_pc.get(timeout=per_job_timeout)
for delayed_pc in delayed_point_clouds]
# closing thread pool when computation is done
pool.close()
pool.join()
configurations_data[config_id]['delayed_point_clouds'] =\
delayed_point_clouds
# build list of epipolar region hashes
configurations_data[config_id]['epipolar_regions_hash'] = [
region_hash_string(k) for k in conf['epipolar_regions']
]
# Compute disp_min and disp_max location for epipolar grid
epipolar_grid_min, epipolar_grid_max = \
stereo.compute_epipolar_grid_min_max(
conf['epipolar_regions_grid'],
epsg, conf["configuration"],
conf['disp_min'], conf['disp_max'])
epipolar_regions_grid_flat = conf['epipolar_regions_grid'].reshape(
-1, conf['epipolar_regions_grid'].shape[-1])
# in the following code a factor is used to increase the precision
spatial_ref = osr.SpatialReference()
spatial_ref.ImportFromEPSG(epsg)
if spatial_ref.IsGeographic():
precision_factor = 1000.0
else:
precision_factor = 1.0
# Build delaunay_triangulation
delaunay_min = Delaunay(epipolar_grid_min * precision_factor)
delaunay_max = Delaunay(epipolar_grid_max * precision_factor)
# Build kdtrees
tree_min = cKDTree(epipolar_grid_min * precision_factor)
tree_max = cKDTree(epipolar_grid_max * precision_factor)
# Look-up terrain_grid with Delaunay
s_min = tsearch(delaunay_min, terrain_grid * precision_factor)
s_max = tsearch(delaunay_max, terrain_grid * precision_factor)
points_disp_min =\
epipolar_regions_grid_flat[delaunay_min.simplices[s_min]]
points_disp_max =\
epipolar_regions_grid_flat[delaunay_max.simplices[s_max]]
nn_disp_min =\
epipolar_regions_grid_flat[
tree_min.query(terrain_grid*precision_factor)[1]]
nn_disp_max =\
epipolar_regions_grid_flat[
tree_max.query(terrain_grid*precision_factor)[1]]
points_disp_min_min = np.min(points_disp_min, axis=2)
points_disp_min_max = np.max(points_disp_min, axis=2)
points_disp_max_min = np.min(points_disp_max, axis=2)
points_disp_max_max = np.max(points_disp_max, axis=2)
# Use either Delaunay search or NN search
# if delaunay search fails (point outside triangles)
points_disp_min_min = np.where(
np.stack((s_min, s_min), axis=-1) != -1, points_disp_min_min,
nn_disp_min)
points_disp_min_max = np.where(
np.stack((s_min, s_min), axis=-1) != -1, points_disp_min_max,
nn_disp_min)
points_disp_max_min = np.where(
np.stack((s_max, s_max), axis=-1) != -1, points_disp_max_min,
nn_disp_max)
points_disp_max_max = np.where(
np.stack((s_max, s_max), axis=-1) != -1, points_disp_max_max,
nn_disp_max)
points = np.stack((points_disp_min_min, points_disp_min_max,
points_disp_max_min, points_disp_max_max),
axis=0)
points_min = np.min(points, axis=0)
points_max = np.max(points, axis=0)
configurations_data[config_id]['epipolar_points_min'] = points_min
configurations_data[config_id]['epipolar_points_max'] = points_max
# Retrieve number of bands
if params.color1_tag in configuration[params.input_section_tag]:
nb_bands = utils.rasterio_get_nb_bands(
configuration[params.input_section_tag][params.color1_tag])
else:
logging.info(
'No color image has been given in input, '
'{} will be used as the color image'.format(
configuration[params.input_section_tag][params.img1_tag]))
nb_bands = utils.rasterio_get_nb_bands(
configuration[params.input_section_tag][params.img1_tag])
logging.info("Number of bands in color image: {}".format(nb_bands))
rank = []
# This list will contained the different raster tiles to be written by cars
delayed_dsm_tiles = []
number_of_epipolar_tiles_per_terrain_tiles = []
if not use_dask[mode]:
# create progress bar with update callback
pbar = tqdm(
total=number_of_terrain_splits,
desc="Finding correspondences between terrain and epipolar tiles")
def update(args): #pylint: disable=unused-argument
pbar.update()
# initialize a thread pool for multiprocessing mode
pool = multiprocessing.Pool(nb_workers)
# Loop on terrain regions and derive dependency to epipolar regions
for terrain_region_dix in tqdm(range(number_of_terrain_splits),
total=number_of_terrain_splits,
desc="Delaunay look-up"):
j = int(terrain_region_dix / (terrain_grid.shape[1] - 1))
i = terrain_region_dix % (terrain_grid.shape[1] - 1)
logging.debug("Processing tile located at {},{} in tile grid".format(
i, j))
terrain_region = [
terrain_grid[j, i, 0], terrain_grid[j, i, 1],
terrain_grid[j + 1, i + 1, 0], terrain_grid[j + 1, i + 1, 1]
]
logging.debug(
"Corresponding terrain region: {}".format(terrain_region))
# This list will hold the required points clouds for this terrain tile
required_point_clouds = []
# For each stereo configuration
for config_id, conf in configurations_data.items():
epipolar_points_min = conf['epipolar_points_min']
epipolar_points_max = conf['epipolar_points_max']
tile_min = np.minimum(
np.minimum(
np.minimum(epipolar_points_min[j, i],
epipolar_points_min[j + 1, i]),
np.minimum(epipolar_points_min[j + 1, i + 1],
epipolar_points_min[j, i + 1])),
np.minimum(
np.minimum(epipolar_points_max[j, i],
epipolar_points_max[j + 1, i]),
np.minimum(epipolar_points_max[j + 1, i + 1],
epipolar_points_max[j, i + 1])))
tile_max = np.maximum(
np.maximum(
np.maximum(epipolar_points_min[j, i],
epipolar_points_min[j + 1, i]),
np.maximum(epipolar_points_min[j + 1, i + 1],
epipolar_points_min[j, i + 1])),
np.maximum(
np.maximum(epipolar_points_max[j, i],
epipolar_points_max[j + 1, i]),
np.maximum(epipolar_points_max[j + 1, i + 1],
epipolar_points_max[j, i + 1])))
# Bouding region of corresponding cell
epipolar_region_minx = tile_min[0]
epipolar_region_miny = tile_min[1]
epipolar_region_maxx = tile_max[0]
epipolar_region_maxy = tile_max[1]
# This mimics the previous code that was using
# transform_terrain_region_to_epipolar
epipolar_region = [
epipolar_region_minx, epipolar_region_miny,
epipolar_region_maxx, epipolar_region_maxy
]
# Crop epipolar region to largest region
epipolar_region = tiling.crop(epipolar_region,
conf['largest_epipolar_region'])
logging.debug(
"Corresponding epipolar region: {}".format(epipolar_region))
# Check if the epipolar region contains any pixels to process
if tiling.empty(epipolar_region):
logging.debug("Skipping terrain region "
"because corresponding epipolar region is empty")
else:
# Loop on all epipolar tiles covered by epipolar region
for epipolar_tile in tiling.list_tiles(
epipolar_region, conf['largest_epipolar_region'],
conf['opt_epipolar_tile_size']):
cur_hash = region_hash_string(epipolar_tile)
# Look for corresponding hash in delayed point clouds
# dictionnary
if cur_hash in conf['epipolar_regions_hash']:
# If hash can be found, append it to the required
# clouds to compute for this terrain tile
pos = conf['epipolar_regions_hash'].index(cur_hash)
required_point_clouds.append(
conf['delayed_point_clouds'][pos])
# start and size parameters for the rasterization function
xstart, ystart, xsize, ysize = tiling.roi_to_start_and_size(
terrain_region, resolution)
# cloud filtering params
if cloud_small_components_filter:
small_cpn_filter_params =\
static_cfg.get_small_components_filter_params()
else:
small_cpn_filter_params = None
if cloud_statistical_outliers_filter:
statistical_filter_params =\
static_cfg.get_statistical_outliers_filter_params()
else:
statistical_filter_params = None
# rasterization grid division factor
rasterization_params = static_cfg.get_rasterization_params()
grid_points_division_factor =\
getattr(
rasterization_params, static_cfg.grid_points_division_factor_tag)
if len(required_point_clouds) > 0:
logging.debug(
"Number of clouds to process for this terrain tile: {}".format(
len(required_point_clouds)))
if use_dask[mode]:
# Delayed call to rasterization operations using all required
# point clouds
rasterized = dask.delayed(rasterization_wrapper)(
required_point_clouds,
resolution,
epsg,
xstart=xstart,
ystart=ystart,
xsize=xsize,
ysize=ysize,
radius=dsm_radius,
sigma=sigma,
dsm_no_data=dsm_no_data,
color_no_data=color_no_data,
msk_no_data=msk_no_data,
small_cpn_filter_params=small_cpn_filter_params,
statistical_filter_params=statistical_filter_params,
grid_points_division_factor=grid_points_division_factor)
# Keep track of delayed raster tiles
delayed_dsm_tiles.append(rasterized)
rank.append(i * i + j * j)
else:
# Launch asynchrone job for write_dsm_by_tile()
delayed_dsm_tiles.append(
pool.apply_async(
write_dsm_by_tile,
args=(required_point_clouds, resolution, epsg, tmp_dir,
nb_bands, static_cfg.get_color_image_encoding(),
output_stats, write_msk),
kwds={
'xstart': xstart,
'ystart': ystart,
'xsize': xsize,
'ysize': ysize,
'radius': dsm_radius,
'sigma': sigma,
'dsm_no_data': dsm_no_data,
'color_no_data': color_no_data,
'small_cpn_filter_params': small_cpn_filter_params,
'statistical_filter_params':
statistical_filter_params,
'grid_points_division_factor':
grid_points_division_factor,
'msk_no_data': msk_no_data
},
callback=update))
number_of_epipolar_tiles_per_terrain_tiles.append(
len(required_point_clouds))
logging.info(
"Average number of epipolar tiles "
"for each terrain tile: {}".format(
int(np.round(
np.mean(number_of_epipolar_tiles_per_terrain_tiles)))))
logging.info("Max number of epipolar tiles "
"for each terrain tile: {}".format(
np.max(number_of_epipolar_tiles_per_terrain_tiles)))
bounds = (xmin, ymin, xmax, ymax)
# Derive output image files parameters to pass to rasterio
xsize, ysize = tiling.roi_to_start_and_size([xmin, ymin, xmax, ymax],
resolution)[2:]
out_dsm = os.path.join(out_dir, "dsm.tif")
out_clr = os.path.join(out_dir, "clr.tif")
out_msk = None
if write_msk:
out_msk = os.path.join(out_dir, "msk.tif")
out_dsm_mean = os.path.join(out_dir, "dsm_mean.tif")
out_dsm_std = os.path.join(out_dir, "dsm_std.tif")
out_dsm_n_pts = os.path.join(out_dir, "dsm_n_pts.tif")
out_dsm_points_in_cell = os.path.join(out_dir, "dsm_pts_in_cell.tif")
if use_dask[mode]:
# Sort tiles according to rank
delayed_dsm_tiles = [
delayed for _, delayed in sorted(zip(rank, delayed_dsm_tiles),
key=lambda pair: pair[0])
]
logging.info("Submitting {} tasks to dask".format(
len(delayed_dsm_tiles)))
# Transform all delayed raster tiles to futures (computation starts
# immediatly on workers, assynchronously)
future_dsm_tiles = client.compute(delayed_dsm_tiles)
logging.info("DSM output image size: {}x{} pixels".format(
xsize, ysize))
readwrite.write_geotiff_dsm(
future_dsm_tiles,
out_dir,
xsize,
ysize,
bounds,
resolution,
epsg,
nb_bands,
dsm_no_data,
color_no_data,
color_dtype=static_cfg.get_color_image_encoding(),
write_color=True,
write_stats=output_stats,
write_msk=write_msk,
msk_no_data=msk_no_data)
# stop cluster
stop_cluster(cluster, client)
else:
logging.info("Computing DSM tiles ...")
# Wait for asynchrone jobs (timeout in seconds) and replace them by
# write_dsm_by_tile() output
delayed_dsm_tiles = [
delayed_tile.get(timeout=per_job_timeout)
for delayed_tile in delayed_dsm_tiles
]
# closing the tread pool after computation
pool.close()
pool.join()
# vrt to tif
logging.info("Building VRT")
vrt_options = gdal.BuildVRTOptions(resampleAlg='nearest')
def vrt_mosaic(tiles_glob, vrt_name, vrt_options, output):
vrt_file = os.path.join(out_dir, vrt_name)
tiles_list = glob(os.path.join(out_dir, 'tmp', tiles_glob))
gdal.BuildVRT(vrt_file, tiles_list, options=vrt_options)
vrt_file_descriptor = gdal.Open(vrt_file)
vrt_file_descriptor = gdal.Translate(output, vrt_file_descriptor)
vrt_file_descriptor = None
vrt_mosaic('*_dsm.tif', 'dsm.vrt', vrt_options, out_dsm)
vrt_mosaic('*_clr.tif', 'clr.vrt', vrt_options, out_clr)
if write_msk:
vrt_mosaic('*_msk.tif', 'msk.vrt', vrt_options, out_msk)
if output_stats:
vrt_mosaic('*_dsm_mean.tif', 'dsm_mean.vrt', vrt_options,
out_dsm_mean)
vrt_mosaic('*_dsm_std.tif', 'dsm_std.vrt', vrt_options,
out_dsm_std)
vrt_mosaic('*_dsm_n_pts.tif', 'dsm_n_pts.vrt', vrt_options,
out_dsm_n_pts)
vrt_mosaic('*_pts_in_cell.tif', 'dsm_pts_in_cell.vrt', vrt_options,
out_dsm_points_in_cell)
# Fill output json file
out_json[params.stereo_section_tag][params.stereo_output_section_tag][
params.epsg_tag] = epsg
out_json[params.stereo_section_tag][params.stereo_output_section_tag][
params.dsm_tag] = out_dsm
out_json[params.stereo_section_tag][params.stereo_output_section_tag][
params.dsm_no_data_tag] = float(dsm_no_data)
out_json[params.stereo_section_tag][params.stereo_output_section_tag][
params.color_no_data_tag] = float(color_no_data)
out_json[params.stereo_section_tag][params.stereo_output_section_tag][
params.color_tag] = out_clr
if write_msk:
out_json[params.stereo_section_tag][params.stereo_output_section_tag][
params.msk_tag] = out_msk
if output_stats:
out_json[params.stereo_section_tag][params.stereo_output_section_tag][
params.dsm_mean_tag] = out_dsm_mean
out_json[params.stereo_section_tag][params.stereo_output_section_tag][
params.dsm_std_tag] = out_dsm_std
out_json[params.stereo_section_tag][params.stereo_output_section_tag][
params.dsm_n_pts_tag] = out_dsm_n_pts
out_json[params.stereo_section_tag][params.stereo_output_section_tag][
params.dsm_points_in_cell_tag] = out_dsm_points_in_cell
# Write the output json
out_json_path = os.path.join(out_dir, "content.json")
try:
utils.check_json(out_json, params.stereo_content_schema)
except CheckerError as check_error:
logging.warning(
"content.json does not comply with schema: {}".format(check_error))
params.write_stereo_content_file(out_json, out_json_path)
def get_subsampling_index2(data_process, standard_scale = True, cutoff_sig = 0.02, rate = 0.3, \
method = "pykdtree", verbose = 1):
"""
Using Nearest-Neighbor search based algorithm, find the list of indices of the subsampled dataset
Parameters
-------------
data_process: List. the list of datapoints, with selected features
standard_scale [True]: Boolean. Whether to apply standard scaler to the dataset prior to subsampling
cutoff_sig [0.02]: Float. cutoff significance. the cutoff distance equals to the Euclidean
norm of the standard deviations in all dimensions of the data points
rate [0.3]: Float. possibility of deletion
method ["pykdtree"]: String. which backend nearest neighbour model to use.
possible choices: ["pykdtree", "nmslib", "sklearn", "scipy", "annoy", "flann"]
verbose [1]: integer. level of verbosity
Return
-------------
overall_keep_list: The list of indices of the final subsampled entries
"""
if verbose >=1:
print("Started NN-subsampling, original length: {}".format(len(data_process)))
method = method.lower()
start = time.time()
if method == "flann":
if verbose >=1:
print("use flann backend")
elif method == "pykdtree":
if verbose >=1:
print("use pykdtree backend")
elif method == "sklearn":
if verbose >=1:
print("use slearn nearest neighbors backend")
elif method == "scipy":
if verbose >=1:
print("use scipy cKDTree backend")
elif method == "annoy":
if verbose >=1:
print("use annoy backend")
elif method == "nmslib":
if verbose >=1:
print("use nmslib backend")
else:
print("method {} not impletemented".format(method))
raise NotImplemented
# apply standard scaling
if standard_scale:
if verbose >= 2:
print("Subample with standard scaled data")
data_process = StandardScaler().fit_transform(np.asarray(data_process).copy())
else:
if verbose >= 2:
print("Subample with original data")
data_process = np.asarray(data_process).copy()
#set cutoff distance
list_of_descs = zip(*data_process)
sum_std2 = 0.
for descs in list_of_descs:
temp_std = np.std(descs)
sum_std2 += temp_std**2
cutoff = cutoff_sig * np.sqrt(sum_std2)
#initialize the index
overall_keep_list = np.arange(len(data_process)).tolist()
keep_going = True
iter_count = 1
while keep_going:
if verbose >= 2:
print('start iteration {}, total length: {}'.format(iter_count, len(overall_keep_list)))
start_cycle = time.time()
temp_data_process = get_array_based_on_index(data_process.copy(), overall_keep_list)
#build and query nearest neighbour model
if method == "flann":
flann = FLANN()
indices, distances = flann.nn(temp_data_process, temp_data_process, 2, algorithm="kmeans")
elif method == "scipy":
kd_tree = cKDTree(temp_data_process)
distances, indices = kd_tree.query(temp_data_process, k=2)
elif method == "pykdtree":
kd_tree = KDTree(temp_data_process,leafsize=6)
distances, indices = kd_tree.query(temp_data_process, k=2)
elif method == "sklearn":
nbrs = NearestNeighbors(n_neighbors=2, algorithm='kd_tree',n_jobs=-1).fit(temp_data_process)
distances, indices = nbrs.kneighbors(temp_data_process)
elif method == "annoy":
annoy = AnnoyIndex(len(temp_data_process[0]),metric='euclidean')
for i in range(len(temp_data_process)):
annoy.add_item(i,temp_data_process[i])
annoy.build(1)
distances = []
indices = []
for i in range(len(temp_data_process)):
temp_index, temp_dist = annoy.get_nns_by_vector(temp_data_process[i], 2, include_distances=True)
indices.append([i, temp_index[1]])
distances.append([0.0, temp_dist[1]])
elif method == "nmslib":
index = nmslib.init(method='hnsw', space='l2')
index.addDataPointBatch(temp_data_process)
index.createIndex( print_progress=False)
neighbours = index.knnQueryBatch(temp_data_process, k=2)
distances = []
indices = []
for item in neighbours:
indices.append(item[0])
distances.append(item[1])
else:
raise NotImplemented
# if distance between each point and its nearest neighbor is below cutoff distance,
# add the nearest neighbout to the candidate removal list
remove_index_li = []
index_li = []
for index, distance in zip(indices, distances):
index_li.append(index[0])
if distance[1] <= cutoff:
remove_index_li.append(index[1])
# randomly select datapoints in the candidate removal list (based on rate)
# and form the final removal list of this iteration
# stop the cycle if the final removal list is empty
temp_num = int(ceil(float(len(remove_index_li))*rate))
if temp_num == 0:
keep_going = False
remove_index_li = random_subsampling(remove_index_li,temp_num)
temp_keep_list = remove_list_from_list(index_li, remove_index_li)
overall_keep_list = [overall_keep_list[i] for i in temp_keep_list ]
if verbose >= 2:
print('end iteration {}. length: {}\t time:{}'.format(iter_count, len(overall_keep_list), time.time()-start_cycle))
iter_count += 1
if verbose >= 1:
print('end NN-subsampling. length: {}\t time:{}'.format(len(overall_keep_list), time.time()-start))
return overall_keep_list
def execute(self,
style,
xpoints,
ypoints,
mask=None,
backend='vectorized',
n_closest_points=None):
"""Calculates a kriged grid and the associated variance.
This is now the method that performs the main kriging calculation.
Note that currently measurements (i.e., z values) are considered
'exact'. This means that, when a specified coordinate for interpolation
is exactly the same as one of the data points, the variogram evaluated
at the point is forced to be zero. Also, the diagonal of the kriging
matrix is also always forced to be zero. In forcing the variogram
evaluated at data points to be zero, we are effectively saying that
there is no variance at that point (no uncertainty,
so the value is 'exact').
In the future, the code may include an extra 'exact_values' boolean
flag that can be adjusted to specify whether to treat the measurements
as 'exact'. Setting the flag to false would indicate that the
variogram should not be forced to be zero at zero distance
(i.e., when evaluated at data points). Instead, the uncertainty in
the point will be equal to the nugget. This would mean that the
diagonal of the kriging matrix would be set to
the nugget instead of to zero.
Parameters
----------
style : str
Specifies how to treat input kriging points. Specifying 'grid'
treats xpoints and ypoints as two arrays of x and y coordinates
that define a rectangular grid. Specifying 'points' treats
xpoints and ypoints as two arrays that provide coordinate pairs
at which to solve the kriging system. Specifying 'masked'
treats xpoints and ypoints as two arrays of x and y coordinates
that define a rectangular grid and uses mask to only evaluate
specific points in the grid.
xpoints : array_like, shape (N,) or (N, 1)
If style is specific as 'grid' or 'masked',
x-coordinates of MxN grid. If style is specified as 'points',
x-coordinates of specific points at which to solve
kriging system.
ypoints : array_like, shape (M,) or (M, 1)
If style is specified as 'grid' or 'masked',
y-coordinates of MxN grid. If style is specified as 'points',
y-coordinates of specific points at which to solve kriging
system. Note that in this case, xpoints and ypoints must have
the same dimensions (i.e., M = N).
mask : bool, array_like, shape (M, N), optional
Specifies the points in the rectangular grid defined
by xpoints and ypoints that are to be excluded in the
kriging calculations. Must be provided if style is specified
as 'masked'. False indicates that the point should not be
masked, so the kriging system will be solved at the point.
True indicates that the point should be masked, so the kriging
system should will not be solved at the point.
backend : str, optional
Specifies which approach to use in kriging.
Specifying 'vectorized' will solve the entire kriging problem
at once in a vectorized operation. This approach is faster but
also can consume a significant amount of memory for large grids
and/or large datasets. Specifying 'loop' will loop through each
point at which the kriging system is to be solved.
This approach is slower but also less memory-intensive.
Specifying 'C' will utilize a loop in Cython.
Default is 'vectorized'.
n_closest_points : int, optional
For kriging with a moving window, specifies the number of
nearby points to use in the calculation. This can speed up the
calculation for large datasets, but should be used
with caution. As Kitanidis notes, kriging with a moving window
can produce unexpected oddities if the variogram model
is not carefully chosen.
Returns
-------
zvalues : ndarray, shape (M, N) or (N, 1)
Z-values of specified grid or at the specified set of points.
If style was specified as 'masked', zvalues will
be a numpy masked array.
sigmasq : ndarray, shape (M, N) or (N, 1)
Variance at specified grid points or at the specified
set of points. If style was specified as 'masked', sigmasq
will be a numpy masked array.
"""
if self.verbose:
print("Executing Ordinary Kriging...\n")
if style != 'grid' and style != 'masked' and style != 'points':
raise ValueError("style argument must be 'grid', "
"'points', or 'masked'")
if n_closest_points is not None and n_closest_points <= 1:
# If this is not checked, nondescriptive errors emerge
# later in the code.
raise ValueError("n_closest_points has to be at least two!")
xpts = np.atleast_1d(np.squeeze(np.array(xpoints, copy=True)))
ypts = np.atleast_1d(np.squeeze(np.array(ypoints, copy=True)))
n = self.X_ADJUSTED.shape[0]
nx = xpts.size
ny = ypts.size
a = self._get_kriging_matrix(n)
if style in ['grid', 'masked']:
if style == 'masked':
if mask is None:
raise IOError("Must specify boolean masking array "
"when style is 'masked'.")
if mask.shape[0] != ny or mask.shape[1] != nx:
if mask.shape[0] == nx and mask.shape[1] == ny:
mask = mask.T
else:
raise ValueError("Mask dimensions do not match "
"specified grid dimensions.")
mask = mask.flatten()
npt = ny * nx
grid_x, grid_y = np.meshgrid(xpts, ypts)
xpts = grid_x.flatten()
ypts = grid_y.flatten()
elif style == 'points':
if xpts.size != ypts.size:
raise ValueError("xpoints and ypoints must have "
"same dimensions when treated as "
"listing discrete points.")
npt = nx
else:
raise ValueError("style argument must be 'grid', "
"'points', or 'masked'")
if self.coordinates_type == 'euclidean':
xpts, ypts = _adjust_for_anisotropy(
np.vstack((xpts, ypts)).T, [self.XCENTER, self.YCENTER],
[self.anisotropy_scaling], [self.anisotropy_angle]).T
# Prepare for cdist:
xy_data = np.concatenate(
(self.X_ADJUSTED[:, np.newaxis], self.Y_ADJUSTED[:,
np.newaxis]),
axis=1)
xy_points = np.concatenate(
(xpts[:, np.newaxis], ypts[:, np.newaxis]), axis=1)
elif self.coordinates_type == 'geographic':
# In spherical coordinates, we do not correct for anisotropy.
# Also, we don't use scipy.spatial.cdist, so we do not have to
# format the input data accordingly.
pass
if style != 'masked':
mask = np.zeros(npt, dtype='bool')
c_pars = None
if backend == 'C':
try:
from .lib.c*k import _c_exec_loop, _c_exec_loop_moving_window
except ImportError:
print('Warning: failed to load Cython extensions.\n'
' See https://github.com/bsmurphy/PyKrige/issues/8 \n'
' Falling back to a pure python backend...')
backend = 'loop'
except:
raise RuntimeError("Unknown error in trying to "
"load Cython extension.")
c_pars = {
key: getattr(self, key)
for key in [
'Z', 'eps', 'variogram_model_parameters',
'variogram_function'
]
}
if n_closest_points is not None:
if self.coordinates_type == 'geographic':
# To make use of the KDTree, we have to convert the
# spherical coordinates into three dimensional Euclidean
# coordinates, since the standard KDTree cannot handle
# the periodicity.
# Do the conversion just for the step involving the KDTree:
lon_d = self.X_ADJUSTED[:, np.newaxis] * np.pi / 180.0
lat_d = self.Y_ADJUSTED[:, np.newaxis] * np.pi / 180.0
xy_data = np.concatenate(
(np.cos(lon_d) * np.cos(lat_d),
np.sin(lon_d) * np.cos(lat_d), np.sin(lat_d)),
axis=1)
lon_p = xpts[:, np.newaxis] * np.pi / 180.0
lat_p = ypts[:, np.newaxis] * np.pi / 180.0
xy_points = np.concatenate(
(np.cos(lon_p) * np.cos(lat_p),
np.sin(lon_p) * np.cos(lat_p), np.sin(lat_p)),
axis=1)
from scipy.spatial import cKDTree
tree = cKDTree(xy_data)
bd, bd_idx = tree.query(xy_points, k=n_closest_points, eps=0.0)
if self.coordinates_type == 'geographic':
# Between the nearest neighbours from Euclidean search,
# calculate the great circle distance using the standard method:
x_points = np.tile(xpts[:, np.newaxis], (1, n_closest_points))
y_points = np.tile(ypts[:, np.newaxis], (1, n_closest_points))
bd = core.great_circle_distance(x_points, y_points,
self.X_ADJUSTED[bd_idx],
self.Y_ADJUSTED[bd_idx])
if backend == 'loop':
zvalues, sigmasq = \
self._exec_loop_moving_window(a, bd, mask, bd_idx)
elif backend == 'C':
zvalues, sigmasq = \
_c_exec_loop_moving_window(a, bd, mask.astype('int8'),
bd_idx, self.X_ADJUSTED.shape[0],
c_pars)
else:
raise ValueError('Specified backend {} for a moving window '
'is not supported.'.format(backend))
else:
if self.coordinates_type == 'euclidean':
bd = cdist(xy_points, xy_data, 'euclidean')
elif self.coordinates_type == 'geographic':
bd = core.great_circle_distance(xpts[:, np.newaxis],
ypts[:, np.newaxis],
self.X_ADJUSTED,
self.Y_ADJUSTED)
if backend == 'vectorized':
zvalues, sigmasq = self._exec_vector(a, bd, mask)
elif backend == 'loop':
zvalues, sigmasq = self._exec_loop(a, bd, mask)
elif backend == 'C':
zvalues, sigmasq = _c_exec_loop(a, bd, mask.astype('int8'),
self.X_ADJUSTED.shape[0],
c_pars)
else:
raise ValueError('Specified backend {} is not supported for '
'2D ordinary kriging.'.format(backend))
if style == 'masked':
zvalues = np.ma.array(zvalues, mask=mask)
sigmasq = np.ma.array(sigmasq, mask=mask)
if style in ['masked', 'grid']:
zvalues = zvalues.reshape((ny, nx))
sigmasq = sigmasq.reshape((ny, nx))
return zvalues, sigmasq
def count_i(land_name):
data = land_name.split('/')[-1].split('.')[0]
#file1 = open('txt_file_m18_m/'+data+'.txt','w')
maxNum = 0
maxName = ''
for name in glob(feature_db + '*'):
#print (name)
name2 = name.split('/')
number = name2[-1].split('.')[0]
# Read features.
locations_1, _, descriptors_1, _, _ = feature_io.ReadFromFile(
land_name)
#cmd_args.features_1_path)
num_features_1 = locations_1.shape[0]
#tf.logging.info("Loaded image 1's %d features" % num_features_1)
locations_2, _, descriptors_2, _, _ = feature_io.ReadFromFile(name)
#cmd_args.features_2_path)
num_features_2 = locations_2.shape[0]
#tf.logging.info("Loaded image 2's %d features" % num_features_2)
# Find nearest-neighbor matches using a KD tree.
d1_tree = cKDTree(descriptors_1)
_, indices = d1_tree.query(descriptors_2,
distance_upper_bound=_DISTANCE_THRESHOLD)
# Select feature locations for putative matches.
locations_2_to_use = np.array([
locations_2[i, ] for i in range(num_features_2)
if indices[i] != num_features_1
])
locations_1_to_use = np.array([
locations_1[indices[i], ] for i in range(num_features_2)
if indices[i] != num_features_1
])
# Perform geometric verification using RANSAC.
try:
_, inliers = ransac((locations_1_to_use, locations_2_to_use),
AffineTransform,
min_samples=3,
residual_threshold=20,
max_trials=1000)
#print(sum(inliers))
#print(maxNum)
#"""
#tf.logging.info('Found %d inliers' % sum(inliers))
#print(sum(inliers))
#print(maxNum)
score = int(sum(inliers))
#file1.write(name+'\t'+str(sum(inliers))+'\n')
#maxName = num2name[number]
num = name.split('/')[-1].split('.')[0]
if score > 35 and score > maxNum:
maxNum = score
maxName = num2name[number]
print(maxName)
print(maxNum)
#break
#"""
except:
#print('fail')
a = 0
#break
#print (maxName)
#result_dic[data] = maxName
return maxName
def make_pos_neg_dataset(tensor_first_view, tensor_second_view, ss, C, view1_id, view2_id, default_sigma=0.8,size=5, save_tiff_files=False, find_negative=True):
tensor_first_view = tensor_first_view.astype(np.float32)
tensor_second_view = tensor_second_view.astype(np.float32)
H, W, D = ss.info['Height'], ss.info['Width'], ss.info['Depth']
patchlen = (1 + 2 * size) ** 3
margin = ss.plist['Margin'] / 2
c_array = np.array([[c.x, c.y, c.z] for c in C])
trunc=1.5
fixed_radiiassigned={c:default_sigma for c in C if inside_margin(c,ss) > 0 }
cccc=fixed_radiiassigned.copy()
kdt = cKDTree(c_array)
dist,ind = kdt.query(c_array,k=c_array.shape[0],distance_upper_bound=(2*default_sigma*trunc)+2.)
for c,id_c in zip(C,xrange(c_array.shape[0])):
if inside_margin(c, ss) > 0:
for j in xrange(2,c_array.shape[0]):
if not np.isinf(dist[id_c][j]):
if inside_margin(C[ind[id_c][j]], ss) > 0:
if trunc*(fixed_radiiassigned[c]+fixed_radiiassigned[C[ind[id_c][j]]]) > dist[id_c][j] - 2.:
new_sigma=((dist[id_c][j]-1.)/trunc)*fixed_radiiassigned[c]/(fixed_radiiassigned[c]+fixed_radiiassigned[C[ind[id_c][j]]])
fixed_radiiassigned[C[ind[id_c][j]]]=(new_sigma*fixed_radiiassigned[C[ind[id_c][j]]])/fixed_radiiassigned[c]
fixed_radiiassigned[c]=new_sigma
target_tensor_3d = np.zeros(tensor_first_view.shape,dtype=np.uint8)
for c in C:
if inside_margin(c, ss) > 0:
target_tensor_3d_tmp = np.zeros(tensor_first_view.shape)
target_tensor_3d_tmp[c.z, c.y, c.x] = 1
sigma=fixed_radiiassigned[c]
target_tensor_3d_tmp = gfilter.gaussian_filter(target_tensor_3d_tmp, sigma,
mode='constant', cval=0.0,
truncate=trunc) #sigma=3.5,mode='constant',cval=0.0,truncate=1.5
target_tensor_3d_tmp = (target_tensor_3d_tmp.astype(np.float32) / np.max(target_tensor_3d_tmp))
target_tensor_3d = np.maximum(np.array(target_tensor_3d_tmp * 255.0, dtype=np.uint8), target_tensor_3d)
if save_tiff_files:
debug_path='/tmp/debug/sigma_'+str(default_sigma)
mkdir_p(debug_path)
minz = 0
imtensor.save_tensor_as_tif(target_tensor_3d, debug_path+'/'+ss.substack_id+'_'+view1_id+'_'+view2_id, minz)
print('num markers =',len(C))
target_tensor_3d = (target_tensor_3d.astype(np.float32) / np.max(target_tensor_3d))
tensor_first_view = tensor_first_view.astype(np.float32) / 255.0
tensor_second_view = tensor_second_view.astype(np.float32) / 255.0
nrej_intensity = 0
num_negative_targets = 0
num_positive_targets = 0
step_x = 1
step_y = 1
step_z = 1
num_iterations = (W - 2*margin +1) * (H- 2*margin) * (D - 2*margin) / (step_x * step_y * step_z)
pbar = ProgressBar(widgets=['Making positive and negative examples for %d points: ' % num_iterations, Percentage()],
maxval=num_iterations).start()
X_positive = np.zeros((num_iterations, patchlen*2), dtype=np.float32)
y_positive = np.zeros((num_iterations, patchlen), dtype=np.float32)
X_negative = []
y_negative = []
if find_negative == True:
X_negative = np.zeros((num_iterations, patchlen*2), dtype=np.float32)
y_negative = np.zeros((num_iterations, patchlen), dtype=np.float32)
pbi = 0
for x0 in range(margin, W - margin, step_x):
for y0 in range(margin, H - margin, step_y):
for z0 in range(margin, D - margin, step_z):
if inside_patch(c_array, x0, y0, z0, size, offset=2.0):
patch_left = tensor_first_view[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1]
patch_right = tensor_second_view[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1]
X_positive[num_positive_targets, 0:patchlen] = np.reshape(patch_left, (patchlen,))
X_positive[num_positive_targets, patchlen:2*patchlen] = np.reshape(patch_right, (patchlen,))
ypatch = target_tensor_3d[z0 - size:z0 + size + 1, y0 - size:y0 + size + 1,
x0 - size:x0 + size + 1]
y_positive[num_positive_targets, :] = np.reshape(ypatch, (patchlen,))
num_positive_targets += 1
elif find_negative == True:
patch_left = tensor_first_view[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1]
patch_right = tensor_second_view[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1]
if np.mean(patch_left * 255) > 5 or np.mean(patch_right * 255) > 5:
X_negative[num_negative_targets, 0:patchlen] = np.reshape(patch_left, (patchlen,))
X_negative[num_negative_targets, patchlen:2*patchlen] = np.reshape(patch_right, (patchlen,))
ypatch = target_tensor_3d[z0 - size: z0 + size + 1, y0 - size: y0 + size + 1, x0 - size: x0 + size + 1]
y_negative[num_negative_targets, :] = np.reshape(ypatch, (patchlen,))
num_negative_targets += 1
else:
nrej_intensity += 1
pbar.update(pbi+ 1)
pbi += 1
pbar.finish()
X_positive = X_positive[0: num_positive_targets]
y_positive = y_positive[0: num_positive_targets]
print('Total positive examples for substack (', ss.substack_id, '):', num_positive_targets)
if find_negative == True:
print('Total negative examples for substack (', ss.substack_id, '):', num_negative_targets)
print('Rejected by intensity ', nrej_intensity)
X_negative = X_negative[0: num_negative_targets]
y_negative = y_negative[0: num_negative_targets]
if (num_negative_targets > num_positive_targets):
print('Shuffling negative examples...')
perm = np.random.permutation(len(X_negative)).astype(int)
X_negative = X_negative[perm]
y_negative = y_negative[perm]
print('Shuffling done')
X_negative = X_negative[0: num_positive_targets]
y_negative = y_negative[0: num_positive_targets]
return X_positive, y_positive, X_negative, y_negative
def processPlumes(options):
eps = 1e-5
#=============================== Local functions
def checkConduitContinuity(nnodes):
maxVrShallow = -1e20
maxVrDeep = -1e20
selectedNodeShallow = -1
selectedNodeDeep = -1
for n in nnodes:
if ((cdata[n, 5] > maxVrShallow) and (cdata[n, 9] == 0)):
maxVrShallow = cdata[n, 5]
selectedNodeShallow = n
#end if
if ((cdata[n, 11] > maxVrDeep) and (cdata[n, 15] == 0)):
maxVrDeep = cdata[n, 11]
selectedNodeDeep = n
#end if
#end for
# Check for continuity of identified plume-region at depth
if (cdata[selectedNodeShallow, 9] == cdata[selectedNodeDeep, 15]):
return selectedNodeShallow, selectedNodeDeep, True,
else:
return selectedNodeShallow, selectedNodeDeep, False
#end if
#end func
def isPlumeEruption(plumeIds, theta, phi):
mindist = 1e20
for p in plumeIds:
dist = haversine(cdata[p, 2], cdata[p, 1], phi, theta)
if (dist < mindist): mindist = dist
#end for
if (mindist > MIN_PLUME_DIST): return True
else: return False
#end func
def shouldAdd(plist, pid):
nxyz = rtp2xyz(cdata[pid,0]*RADIUS, \
(90-cdata[pid,1])*math.pi/180, \
cdata[pid,2]*math.pi/180)
for epid in plist:
exyz = rtp2xyz(cdata[epid,0]*RADIUS, \
(90-cdata[epid,1])*math.pi/180, \
cdata[epid,2]*math.pi/180)
dist = (nxyz[0] - exyz[0])**2 + \
(nxyz[1] - exyz[1])**2 + \
(nxyz[2] - exyz[2])**2
dist = math.sqrt(dist)
if (dist < MIN_PLUME_DIST): return 0
#end for
return 1
#end func
def getNeighbours(nbFileName):
nf = open(nbFileName)
neighbours = defaultdict(list)
for line in nf:
vals = line.split(' ')
for i in range(len(vals)):
if (i > 0): neighbours[int(vals[0])].append(int(vals[i]))
#end for
#end for
nf.close()
return neighbours
#end func
def getNeighbouringNodes(tree, xyz, radius):
l = tree.query_ball_point(xyz, radius)
return l
#end func
#=============================== Begin processing
def key_func(x):
return int(x.split('.')[-2])
fileNames = glob.glob('%s*.txt' % (options.baseName))
fileNames = sorted(fileNames, key=key_func)
minNablaV = 1e30
maxNablaV = -1e30
cdataList = []
for fileName in fileNames:
''' Data-format=========================================
r, theta, phi, time, magGradVr, v_r, T, Eta, data cid magGradVrVD, v_rVD, TVD, EtaVD, dataVD cidVD tracComp
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 */
====================================================='''
# Load clusteredData
cdata = np.loadtxt(fileName)
cdataList.append(cdata)
if (np.min(cdata[:, 4]) < minNablaV): minNablaV = np.min(cdata[:, 4])
if (np.max(cdata[:, 4]) > maxNablaV): maxNablaV = np.max(cdata[:, 4])
#end for
nodes = []
tree = None
plumeIdsList = [] # List of identified plumes for each time-slice
outFile = open(options.outFileName, 'w')
for counter, fileName in enumerate(fileNames):
print 'Processing: %s..' % (fileName)
currTime = int(fileName.split('.')[-2])
dirName = os.path.dirname(options.outFileName)
if (len(dirName) == 0): dirName = './'
baseName = os.path.basename(fileName)
suppDirName = '%s/supp/' % (dirName)
# create supplimentary directory if it doesn't exist
if (not os.path.exists(suppDirName)):
os.makedirs(suppDirName)
#end if
cdata = cdataList[counter]
if (counter == 0):
for i in range(np.shape(cdata)[0]):
xyz = rtp2xyz(RADIUS,
math.pi / 2 - cdata[i, 1] * math.pi / 180,
cdata[i, 2] * math.pi / 180)
nodes.append(xyz)
#end for
tree = spatial.cKDTree(nodes, 16)
#end if
dataOut = np.zeros((361, 181))
xOut = np.zeros((361, 181))
yOut = np.zeros((361, 181))
for i in range(361):
for j in range(181):
theta = float(j)
phi = float(i)
xOut[i, j] = phi * math.pi / 180
yOut[i, j] = theta * math.pi / 180
xyz = rtp2xyz(RADIUS, yOut[i, j], xOut[i, j])
(d, l) = tree.query(xyz)
dataOut[i,
j] = cdata[l, 4] * THERM_DIFF / RADIUS #magGradVrDepth
#dataOut[i,j] = cdata[l, 9] #cidDepth
#dataOut[i,j] = cdata[l, 15] #cidValidationDepth
#end for
#end for
#=======================================Compute fluxes
numNodes = np.shape(cdata)[0]
plumeIds = []
meanTemp = np.mean(cdata[:, 6], axis=0)
meanBgTemp = 0.
hitCount = 0.
for i in range(numNodes):
if (cdata[i, 6] > meanTemp):
meanBgTemp += cdata[i, 6]
hitCount += 1.
#end if
#end for
meanBgTemp /= hitCount
print 'mean: %f, meanBg: %f' % (meanTemp, meanBgTemp)
for i in range(numNodes):
if ((cdata[i, 9] == 0)):
nn = np.array(
getNeighbouringNodes(tree, nodes[i], MAX_PLUME_OFFSET))
selectedNodeShallow, selectedNodeDeep, conduitIsContinuous = checkConduitContinuity(
nn)
if (conduitIsContinuous == False): continue
# Check if plume-conduit meets the minimum radial velocity criteria
if (cdata[selectedNodeShallow, 5] * VSCALE <
options.radialVelocityMin):
continue
if (shouldAdd(plumeIds, selectedNodeShallow)):
plumeIds.append(selectedNodeShallow)
#end if
#end if
#end for
# Save plume-ids list
plumeIdsList.append(plumeIds)
flux = np.zeros(len(plumeIds))
area = np.zeros(len(plumeIds))
avgConduitTemp = np.zeros(len(plumeIds))
avgConduitConcentration = np.zeros(len(plumeIds))
for i, pid in enumerate(plumeIds):
#print "plume %d: T(%d) = %f"%(i, pid, cdata[pid,6])
suppFigName = '%s/%s.plume%d.png' % (suppDirName, baseName, i)
nn = np.array(
getNeighbouringNodes(tree, nodes[pid], PLUME_PLOT_RADIUS))
p = Plume(
options,
np.array((cdata[nn, 2], 90 - cdata[nn, 1], cdata[nn, 6],
cdata[nn, 5], cdata[nn, 16])).T, meanBgTemp,
suppFigName)
result = p.ComputeBuoyancyFlux()
flux[i] = result['flux']
area[i] = result['area']
avgConduitTemp[i] = result['avgConduitTemp']
avgConduitConcentration[i] = result['avgConduitConcentration']
# Report plume=======
mlatp = 90 - p.mlat
mlonp = p.mlon
if (mlonp > 180): mlonp = mlonp - 360
print '\tplume(%d) loc: (%f %f)' % (i, mlatp, mlonp)
#end for
#================================== Done computing fluxes
#================================== Prepare data and axes
dataOutFlipped = np.zeros(np.shape(dataOut))
dataOutFlipped[0:181:, :] = dataOut[180:361, :]
dataOutFlipped[181:361:, :] = dataOut[0:180, :]
fig, gax = plt.subplots(nrows=1, ncols=1)
fig.set_size_inches(11.69, 11.69)
m = Basemap(ax=gax, projection='moll', lon_0=0, resolution='c')
#=======================================Plot flux
pxOut, pyOut = m(xOut * 180 / math.pi - 180, 90 - yOut * 180 / math.pi)
cmap = cm.get_cmap(name='jet', lut=None)
cbInfo = m.pcolormesh(pxOut, pyOut, dataOutFlipped, cmap=cmap)
m.colorbar(cbInfo, "bottom", size="5%", pad="15%")
legList = []
labels = []
for i, pid in enumerate(plumeIds):
#print "plume %d: T(%d) = %f"%(i, pid, cdata[pid,6])
#coords = circle(m, cdata[pid,2], cdata[pid,1], 50*flux[i])
#casa = SPolygon(coords)
#cpatch = PolygonPatch(casa, fc='magenta', ec='#333333', lw=0, alpha=.25, zorder=2)
#gax.add_patch(cpatch)
if (flux[i] > 0):
plumeEruption = 0
if (counter):
plumeEruption = int(
isPlumeEruption(plumeIdsList[counter - 1],
cdata[pid, 1], cdata[pid, 2]))
#end if
#(time, theta, phi, T, Vr, Flux, Area, Eruption, meanBgTemp, avgConduitTemp, concentration)
outFile.write('%f %f %f %f %f %f %f %f %f %f %f\n' %
(currTime, cdata[pid, 1], cdata[pid, 2],
cdata[pid, 6], cdata[pid, 5], flux[i], area[i],
plumeEruption, meanBgTemp, avgConduitTemp[i],
avgConduitConcentration[i]))
px, py = m(cdata[pid, 2], cdata[pid, 1])
#m.scatter(px,py,flux[i]*SYMBOL_SCALE, marker='o',color='green', edgecolors='none', alpha=0.5)
m.scatter(px,
py,
flux[i] * SYMBOL_SCALE,
marker='o',
color='none',
edgecolors='white',
lw=0.5,
alpha=1)
leg = m.scatter([], [], s=0, edgecolors='black', color='none')
legList.append(leg)
labels.append('(%d) : %0.1f' % (i, flux[i]))
phil = cdata[pid, 2]
thetal = cdata[pid, 1]
pxl, pyl = m(phil, thetal)
gax.text(pxl, pyl, '%d' % (i), color='white', alpha=0.5)
#gax.text(px,py,'%.1f Mg/s'%(flux[i]),color='white')
#gax.text(px,py,'%d: %.0f km2'%(i, area[i]),color='white')
#gax.text(px,py,'%.3f'%(cdata[pid, 6]),color='white') # temperature
else:
print 'Negative flux: %f detected for plume %d..' % (flux[i],
i)
#end if
#end for
#=======================================Done plotting flux
#==================================================Plot reconstruction
#m.drawparallels(np.arange(-90.,210.,30.))
#m.drawmeridians(np.arange(-180.,240.,60.))
age = options.startAge - int(fileName.split('.')[-2])
if (age < 0): age = 0
if 1:
sfName = '%s/reconstructed_%d.00Ma.shp' % (options.reconsPath, age)
sf = fiona.open(sfName)
features = []
for feature in sf:
#print feature['geometry']['type']
features.append(shape(feature['geometry']))
#end for
patches = []
for feature in features:
if (isinstance(feature, Polygon)):
polygon = feature
x, y = polygon.exterior.coords.xy
px, py = m(x, y)
ppolygon = Polygon(zip(px, py))
patches.append(
PolygonPatch(ppolygon,
fill=False,
edgecolor=(0.5, 0.5, 0.5),
alpha=0.5))
else:
multiPolugon = feature
for polygon in multiPolugon:
x, y = polygon.exterior.coords.xy
px, py = m(x, y)
ppolygon = Polygon(zip(px, py))
patches.append(
PolygonPatch(ppolygon,
fill=False,
edgecolor=(0.5, 0.5, 0.5),
alpha=0.5))
#end for
#end for
gax.add_collection(PatchCollection(patches, match_original=True))
#end if
m.drawmapboundary()
#=======================================Done plotting reconstruction
leg = plt.legend(legList,
labels,
ncol=6,
frameon=False,
fontsize=12,
handlelength=1,
loc='lower center',
borderaxespad=-13,
handletextpad=0.2,
title='Flux / (Mg/s)',
scatterpoints=1,
labelspacing=0.2)
gax.set_title('Plumes Identified at %d Ma' % (age))
gax.set_xlabel('$\\Vert\\nabla V_r \\Vert [s^{-1}]$')
plt.savefig('%s/%s.png' % (dirName, os.path.basename(fileName)),
dpi=300)
plt.close()
#end for
outFile.close()
def idw_serie_temporal():
"""
Método de interpolación inverse distance weighted con distance elevada a
una potencia, normalmente 2
Interpola la variable en una serie de puntos dados en 3D. Si no tienes
valor de Z pon en todos los puntos un valor cte, por ej 0
Los datos son una serie temporal; los puntos con datos varían en el
tiempo
"""
from datetime import date, timedelta
from os.path import join
from time import time
import interpolation_param as par
time_steps = ('diaria', 'day', 'mensual', 'month', 'anual', 'year')
cstrAemet = r'DRIVER={Microsoft Access Driver (*.mdb, *.accdb)}; ' +\
f'DBQ={par.dbMeteoro};'
PRINT_INTERVAL = 5.
if par.time_step in time_steps[:2]:
tstep = 1
time_step = timedelta(days=1)
datei = date(par.year1, par.month1, par.day1)
datefin = date(par.year2, par.month2, par.day2)
else:
raise ValueError(f'valor de time_step {par.time_step} ' +\
'no implementado')
start_time = time()
# datos donde hay que interpolar
rows = points2interpolate_get(join(par.PATHIN, par.FPOINTS),
par.skip_lines)
fidi = rows[:, 0] # array con los id de los puntos
if rows.shape[1] == 3: # 2D
xi = rows[:, [1, 2]].astype(np.float32) # array con las coordenadas
elif rows.shape[1] == 4: #3D
xi = rows[:, [1, 2, 3]].astype(np.float32)
else:
raise ValueError('El núm de columnas en el fichero de puntos debe ' +\
'3 o 4: id del punto, x, y, (z)')
rows = None
z = np.empty((len(xi), par.kidw), np.float32)
zi = np.empty((len(xi)), np.float32) # array con los valores interpolados
# datos para hacer las interpolaciones
con = pyodbc.connect(cstrAemet)
cur = con.cursor()
# fichero de salida
dst = file_name_out(par.PATHOUT, par.FPOINTS, par.SHORT_NAME_4_VARIABLE,
'idw')
f = open(dst, 'w')
f.write('fid\tfecha\tvalor\n')
t0 = PRINT_INTERVAL
while datei <= datefin:
if tstep == 1:
dateStr = datei.strftime('%d/%m/%Y')
if time() - t0 > PRINT_INTERVAL:
t0 = time()
print(dateStr)
cur.execute(par.SELECT1, (datei,))
data = [row for row in cur]
if xi.shape[1] == 2:
data = np.array([[row.X, row.Y, row.v] for row in data])
tree = spatial.cKDTree(data[:, [0, 1]])
else:
data = np.array([[row.X, row.Y, row.Z, row.v] for row in data])
tree = spatial.cKDTree(data[:, [0, 1, 2]])
dist, ii = tree.query(xi, k=par.kidw)
dist = np.where(dist<=par.epsidw, par.mindistidw, dist)
idwcore(data, xi.shape[1], dist, ii, par.poweridw, z, zi)
for i in range(len(fidi)):
f.write(f'{fidi[i]}\t{dateStr}\t{zi[i]:0.1f}\n')
datei = datei + time_step
elapsed_time = time() - start_time
print(f'La interpolación tardó {elapsed_time:0.1f} s')
# fichero de metadatos
dst = file_name_out(par.PATHOUT, par.FPOINTS, par.SHORT_NAME_4_VARIABLE,
'idw_matadata')
with open(dst, 'w') as f:
f.write(f'idw\n')
f.write(f'potencia a la que se eleva la dist, {par.poweridw:f}\n')
f.write(f'núm datos por interpolación, {par.kidw:d}\n')
f.write('distancia máx. a la que se considera que las ' +\
'localizaciones de una estación y un punto coinciden, ' +\
f'{par.epsidw:f}\n')
f.write('corrección de la distancia de un punto para evitar ' +\
f'infinitos en 1/dist, {par.mindistidw:f}\n')
f.write(f'db de los datos, {par.dbMeteoro}\n')
f.write(f'número de puntos interpolados {fidi.size:d}\n')
f.write(f'tiempo transcurrido, {elapsed_time:0.1f} s\n')
_ = input(MSG_FIN_PROCESO)
def adi(self):
kd_tree = spatial.cKDTree(self.transformed_pts_est)
kd_tree_distances, ids = kd_tree.query(self.transformed_pts_gt, k=1)
adi_error = kd_tree_distances.mean()
return adi_error
def convexHull(vertices, tol=1e-6):
"""Compute the 3D convex hull of a set of vertices and merge coplanar faces.
Args:
vertices (list): List of (x, y, z) coordinates
tol (float): Floating point tolerance for merging coplanar faces
Returns an array of vertices and a list of faces (vertex
indices) for the convex hull of the given set of vertice.
.. note::
This method uses scipy's quickhull wrapper and therefore requires scipy.
"""
from scipy.spatial import cKDTree, ConvexHull
from scipy.sparse.csgraph import connected_components
hull = ConvexHull(vertices)
# Triangles in the same face will be defined by the same linear equalities
dist = cKDTree(hull.equations)
trianglePairs = dist.query_pairs(tol)
connectivity = np.zeros((len(hull.simplices), len(hull.simplices)),
dtype=np.int32)
for (i, j) in trianglePairs:
connectivity[i, j] = connectivity[j, i] = 1
# connected_components returns (number of faces, cluster index for each input)
(_, joinTarget) = connected_components(connectivity, directed=False)
faces = defaultdict(list)
norms = defaultdict(list)
for (idx, target) in enumerate(joinTarget):
faces[target].append(idx)
norms[target] = hull.equations[idx][:3]
# a list of sets of all vertex indices in each face
faceVerts = [
set(hull.simplices[faces[faceIndex]].flat)
for faceIndex in sorted(faces)
]
# normal vector for each face
faceNorms = [norms[faceIndex] for faceIndex in sorted(faces)]
# polygonal faces
polyFaces = []
for (norm, faceIndices) in zip(faceNorms, faceVerts):
face = np.array(list(faceIndices), dtype=np.uint32)
N = len(faceIndices)
r = hull.points[face]
rcom = np.mean(r, axis=0)
# plane_{a, b}: basis vectors in the plane
plane_a = r[0] - rcom
plane_a /= np.sqrt(np.sum(plane_a**2))
plane_b = np.cross(norm, plane_a)
dr = r - rcom[np.newaxis, :]
thetas = np.arctan2(dr.dot(plane_b), dr.dot(plane_a))
sortidx = np.argsort(thetas)
face = face[sortidx]
polyFaces.append(face)
return (hull.points, polyFaces)
#print(VSXdata.iloc[1,1])
listdatara = datara.tolist()
listdatadec = datadec.tolist()
radectemp = []
for i in range(0, len(listdatara)):
RA = np.float32(listdatara[i])
DEC = np.float32(listdatadec[i])
radectemp.append(RA)
radectemp.append(DEC)
VSXradec = np.float32(radectemp).reshape(-1, 2)
kdt = cKDTree(radecname[:, 0:2])
dist, indices = kdt.query(VSXradec)
temp = []
for i in range(len(indices)):
index = indices[i]
temp.append(radecname[index])
nptemp = np.array(temp)
VSXradec = np.column_stack((dist, VSXradec))
allradec = np.column_stack((VSXradec, nptemp))
lista = ['distance', 'VSXRA', 'VSXDEC', 'cg20RA', 'cg20DEC', 'cg20name']
dfallradec = pd.DataFrame(allradec, columns=lista)
df4 = [VSXdata, dfallradec]
lon_rad = np.radians(lon)
lat_rad = np.radians(lat)
# convert to cartesian coordinates
r_n = A / (np.sqrt(1 - E2 * (np.sin(lat_rad)**2)))
x = r_n * np.cos(lat_rad) * np.cos(lon_rad)
y = r_n * np.cos(lat_rad) * np.sin(lon_rad)
z = r_n * (1 - E2) * np.sin(lat_rad)
return x, y, z
xs, ys, zs = lon_lat_to_cartesian(lon_source.values.flatten(),
lat_source.values.flatten())
xt, yt, zt = lon_lat_to_cartesian(lon_target2d.flatten(),
lat_target2d.flatten())
tree = cKDTree(np.column_stack((xs, ys, zs)))
d, inds = tree.query(np.column_stack((xt, yt, zt)), k=10)
def interpolate(source, target, d, inds):
nt = source["time"].shape[0]
tmp = []
for t in range(0, nt):
w = 1.0 / d**2
air_idw = np.sum(w * source.tmax[t].values.flatten()[inds],
axis=1) / np.sum(w, axis=1)
air_idw.shape = target.shape
tmp.append(air_idw)
return tmp
