def profile(points, radius=4, leafsize=4):
points = points[:1024]
closest = closest_neighbors(KDTree(points))
points *= 2 / closest
# points += np.random.normal(scale=1e-2 * (2 / closest))
tree = KDTree(points, leafsize=leafsize)
k = np.arange(2, 40)
times = []
for k0 in tqdm(k):
times.append(
timeit(lambda: recursive_query_ball_point(tree, points, radius, k0),
number=20))
_, mask = recursive_query_ball_point(tree, points, radius, 10)
spt = spatial.cKDTree(points, leafsize=leafsize) # pylint: disable=no-member
t = timeit(lambda: spt.query_ball_tree(spt, radius), number=20)
times = np.array(times) / t
neighbors = np.count_nonzero(mask, axis=1)
ax = plt.gca()
ax.plot(k, times)
# ax.plot([k[0], k[-1]], [t, t], linestyle='dashed')
ax.plot([k[0], k[-1]], [0, 0], linestyle='dashed', color='k')
ax.plot([k[0], k[-1]], [1, 1], linestyle='dashed', color='k')
ax.set_xlabel('k')
ax.set_ylabel('t')
ax.set_title('mean = {}, max = {}'.format(np.mean(neighbors),
mask.shape[1]))
print(times[14] / np.min(times))
# ax.set_yscale('log')
plt.show()
def relief(X, Y):
"""
Funcion para calcular los pesos mediante el algoritmo RELIEF.
:param X: Conjunto de vectores de caracteristicas.
:param Y: Conjunto de etiquetas.
:return Devuelve los pesos w que ponderan las caracteristicas.
"""
# Obtener numero de elementos de la muestra
N = Y.shape[0]
# Inicializar w a [0, 0, ... , 0]
w = np.zeros((X.shape[1], ), np.float64)
# Recorrer todos los elementos de la muestra
# Para cada elemento, obtener su amigo mas cercano y su enemigo
# mas cercano
# Actualizar w con la nueva informacion
for i in range(N):
# Obtener el elemento y su etiqueta
x, y = X[i], Y[i]
# Obtener la lista de amigos y enemigos
allies = X[Y == y]
enemies = X[Y != y]
# Construir los arboles de amigos y enemigos
kdtree_allies = KDTree(allies)
kdtree_enemies = KDTree(enemies)
# Obtener el aliado mas cercano mediante la tecnica de leave-one-out
# Se escoge el segundo vecino mas cercano a x, ya que el primero es
# el mismo (un indice al segundo vecino mas cercano)
closest_ally = kdtree_allies.query(x.reshape(1, -1), k=2)[1][0, 1]
# Obtener el enemigo mas cercano a x (se obtiene un indice)
closest_enemy = kdtree_enemies.query(x.reshape(1, -1), k=1)[1][0]
# Obtener el aliado y el enemigo correspondientes
ally = allies[closest_ally]
enemy = enemies[closest_enemy]
# Actualizar w
w += abs(x - enemy) - abs(x - ally)
# Obtener el maximo de w
w_max = w.max()
# Hacer que los elementos inferiores a 0 tengan como valor 0
w[w < 0.0] = 0.0
# Normalizar w con el maximo
w = w / w_max
return w
def _cleanDiscarded(self, discarded_X, discarded_y):
stm_tree = KDTree(np.array(self.STMX))
STMy = np.array(self.STMy)
clean_mask = np.zeros(discarded_X.shape[0], dtype=bool)
for x,y in zip(self.STMX, self.STMy):
# searching for points from the stm in the stm will also return that points, so we query one more to get k neighbours
dist, ind = stm_tree.query(np.array([x]), k=self.n_neighbors +1)
dist = dist[0]
ind = ind[0]
"""
find weighted maximum difference and max distance among next n neighbours in STM
"""
dist_max = np.amax(dist)
w_diff = self._clean_metric(STMy[ind] - y, dist, dist_max)
w_diff_max = np.amax(w_diff)
# print("w_diff_max:", w_diff_max)
"""
Query all points among the discarded that lie inside the maximum distance. Delete every point that has a greater weighted difference than the previously gathered maximum distance.
"""
discarded_tree = KDTree(discarded_X)
dist, ind = discarded_tree.query(
np.array([x]),
k=len(discarded_X),
distance_upper_bound=dist_max)
keep = ind < len(discarded_X)
ind = ind[keep]
dist = dist[keep]
disc_w_diff = self._clean_metric(discarded_y[ind] - y, dist, dist_max)
clean = ind[disc_w_diff < w_diff_max]
"""
We create a mask which us used to drop all values in the discarded
set whose weighted difference is to far from __all__ points.
E.g. it does not appear in neighbourhood of any other point or
is too different if it does.
"""
clean_mask[clean] = True
discarded_X = discarded_X[clean_mask]
discarded_y = discarded_y[clean_mask]
return discarded_X, discarded_y
def make_bud_neighbor_df(manual_bud_indices, auto_bud_indices, **kwargs):
"""
auto_bud_indices should be made using find_peaks from scipy.signal
as follows:
peak_indices, peak_dict = find_peaks(new_df.dsred_mean_local_mean_norm_meanfilt,
distance=min_cycle_frames)
"""
collection_interval = kwargs.get('collection_interval', 10)
death_cutoff_hr = (np.max(manual_bud_indices)*collection_interval)/60
df1 = pd.DataFrame({'manual_timepoint': manual_bud_indices[:]})
df2 = pd.DataFrame({'auto_timepoint': auto_bud_indices[auto_bud_indices <= death_cutoff_hr*6]})
comparison_df = pd.concat([df1, df2], ignore_index=True, axis=1)
# Find nearest manual frame for each auto frame
X = np.array(comparison_df.loc[:, 0])
Y = np.array(comparison_df.loc[:, 1])
tree = KDTree(X)
nearest_manual_dists, neighbor_indices = tree.query(Y)
nearest_manual_frame = []
for i in neighbor_indices:
try:
nearest_manual_frame.append(X[i])
except:
nearest_manual_frame.append(np.NaN)
# Find nearest auto frame for each manual frame
X = np.array(comparison_df.loc[:, 0]) # X is manual
Y = np.array(comparison_df.loc[:, 1]) # Y is auto
tree = KDTree(Y)
nearest_auto_dists, neighbor_indices = tree.query(X)
nearest_auto_frame = []
for i in neighbor_indices:
try:
nearest_auto_frame.append(Y[i])
except:
nearest_auto_frame.append(np.NaN)
neighbor_df = pd.DataFrame({'manual_bud_frame': X,
'auto_bud_frame': Y,
'nearest_manual_frame': nearest_manual_frame,
'nearest_auto_frame': nearest_auto_frame,
'nearest_auto_dist': nearest_auto_dists,
'nearest_manual_dist': nearest_manual_dists})
return neighbor_df
def _build_kdtree(self, coords):
rect_center = coords.copy()
rect_center[:, :-1] += coords[:, 1:]
rect_center[:-1, :] += coords[1:, :]
rect_center[:-1, :-1] += coords[1:, 1:]
rect_center *= 0.25
return KDTree(rect_center[:-1, :-1].reshape(-1, 2))
def calcEt(original_data):
point_kd_tree = KDTree(original_data['coords'])
vertices = pd.read_pickle('./tmp/burned-vertices.pickle')
def getRadii(row):
index = row['index']
if index % 500 == 0:
print 'et:', index
here = row.values[:3].reshape([1, 3]).astype(np.float32)
neighbor, idx = point_kd_tree.query(here, k=3)
radii = np.linalg.norm(here - neighbor)
return radii
def getEt(row):
return row['time'] - row['radii']
vertices.loc[:, 'radii'] = vertices.apply(getRadii, axis=1)
vertices.loc[:, 'et'] = vertices.apply(getEt, axis=1)
vertices.to_pickle('./tmp/et-vertices.pickle')
def findInCloud(self, cloud, cloudNormals):
sceneTree = KDTree(cloud)
indexes = list(range(len(cloud)))
r = self.Model.Radius
iterations = 0
while True:
iterations += 1
if iterations % 100 == 0:
print(iterations)
i = np.random.choice(indexes, 1)[0]
p1 = cloud[i]
n1 = cloudNormals[i]
neighborIdx = [
ii for ii in sceneTree.query(
p1.reshape((1, 3)), 50, distance_upper_bound=2 * r)[1][0]
if ii < len(cloud) and ii != i
]
j, k = np.random.choice(neighborIdx, 2, replace=False)
p2, p3 = cloud[j], cloud[k]
n2, n3 = cloudNormals[j], cloudNormals[k]
if np.linalg.cond(np.column_stack((n1, n2, n3))) > 1e5:
continue
for pose in self.Model.getPose(np.column_stack((p1, p2, p3)),
np.column_stack((n1, n2, n3))):
yield pose
return None
def match_cris_viirs(crisLos, crisPos, viirsPos, viirsMask):
"""
Match crisLos with viirsPos using the method by Wang et al. (2016)
Wang, L., D. A. Tremblay, B. Zhang, and Y. Han, 2016: Fast and Accurate
Collocation of the Visible Infrared Imaging Radiometer Suite
Measurements and Cross-track Infrared Sounder Measurements.
Remote Sensing, 8, 76; doi:10.3390/rs8010076.
"""
# Derive Satellite Postion
crisSat = crisPos - crisLos
# using KD-tree to find best matched points
# build kdtree to find match index
pytree_los = KDTree(viirsPos.reshape(viirsPos.size / 3, 3))
dist_los, idx_los = pytree_los.query(crisPos.reshape(crisPos.size / 3, 3),
sqr_dists=False)
my, mx = np.unravel_index(idx_los, viirsPos.shape[0:2])
idy, idx = find_match_index(crisLos.reshape(crisLos.size/3, 3),\
crisSat.reshape(crisSat.size/3, 3),\
viirsPos, viirsMask, mx, my)
idy = np.array(idy).reshape(crisLos.shape[0:crisLos.ndim - 1])
idx = np.array(idx).reshape(crisLos.shape[0:crisLos.ndim - 1])
return idy, idx
def __init__(self, fields: Volume_T, max_dist: Number_T = 0.1):
# Process data type
self.dtype = get_dtype(fields[0])
# Process time
t_arr = np.array(
[time.mktime(i.scan_time.timetuple()) for i in fields])
if (t_arr.max() - t_arr.min()) / 60 > 10:
raise RadarCalculationError(
"Time difference of input data should not exceed 10 minutes")
mean_time = t_arr.mean()
mean_dtime = datetime.datetime(*time.localtime(int(mean_time))[:6])
time_increment = 10
time_rest = mean_dtime.minute % time_increment
if time_rest > time_increment / 2:
mean_dtime += datetime.timedelta(minutes=(time_increment -
time_rest))
else:
mean_dtime -= datetime.timedelta(minutes=time_rest)
self.scan_time = mean_dtime
self.lon_ravel = np.hstack(
[i["longitude"].values.ravel() for i in fields])
self.lat_ravel = np.hstack(
[i["latitude"].values.ravel() for i in fields])
self.data_ravel = np.ma.hstack(
[i[self.dtype].values.ravel() for i in fields])
self.dist_ravel = np.hstack([
np.broadcast_to(i["distance"], i["longitude"].shape).ravel()
for i in fields
])
self.tree = KDTree(np.dstack((self.lon_ravel, self.lat_ravel))[0])
self.md = max_dist
self.attr = fields[0].attrs.copy()
def _create_resample_kdtree(source_lons,
source_lats,
valid_input_index,
nprocs=1):
"""Set up kd tree on input."""
source_lons_valid = source_lons[valid_input_index]
source_lats_valid = source_lats[valid_input_index]
if nprocs > 1:
cartesian = _spatial_mp.Cartesian_MP(nprocs)
else:
cartesian = _spatial_mp.Cartesian()
input_coords = cartesian.transform_lonlats(source_lons_valid,
source_lats_valid)
if input_coords.size == 0:
raise EmptyResult('No valid data points in input data')
# Build kd-tree on input
if nprocs > 1:
resample_kdtree = _spatial_mp.cKDTree_MP(input_coords, nprocs=nprocs)
else:
resample_kdtree = KDTree(input_coords)
return resample_kdtree
def space_nearest_join(points: np.array, source='pykdtree', leafsize=8) -> np.array:
"""采用空间最近邻匹配的方式将提取的特征数据集匹配到位置点上, scipy
parameter
--------
points: np.array, as [1, 1]
source: str, ['pykdtree', ckdtree]
tar_points: np.array, as [1, 1]
return
------
ind: np.array, as [1]
``
call = space_nearest_join(points)
ind = call(tar_points)
``
"""
if source == 'pykdtree':
kd_tree = KDTree(points, leafsize=leafsize)
else:
kd_tree = cKDTree(points, leafsize=leafsize)
def _call(tar_points: np.array):
_, ind = kd_tree.query(tar_points, k=1)
return ind
return _call
def _create_resample_kdtree(self, source_lons, source_lats,
valid_input_index):
"""Set up kd tree on input"""
"""
if not isinstance(source_geo_def, geometry.BaseDefinition):
raise TypeError('source_geo_def must be of geometry type')
#Get reduced cartesian coordinates and flatten them
source_cartesian_coords = source_geo_def.get_cartesian_coords(nprocs=nprocs)
input_coords = geometry._flatten_cartesian_coords(source_cartesian_coords)
input_coords = input_coords[valid_input_index]
"""
vii = valid_input_index.compute().ravel()
source_lons_valid = source_lons.ravel()[vii]
source_lats_valid = source_lats.ravel()[vii]
input_coords = self.transform_lonlats(source_lons_valid,
source_lats_valid)
if input_coords.size == 0:
raise EmptyResult('No valid data points in input data')
# Build kd-tree on input
input_coords = input_coords.astype(np.float)
if kd_tree_name == 'pykdtree':
resample_kdtree = KDTree(input_coords.compute())
else:
resample_kdtree = sp.cKDTree(input_coords.compute())
return resample_kdtree
def construct_tree(ds):
from pykdtree.kdtree import KDTree
X, Y = np.meshgrid(ds.x.values, ds.y.values)
pts = np.asarray([X.reshape(-1), Y.reshape(-1)]).T
tree = KDTree(pts)
return tree
def _makeSparseness(self, features):
""" Sparseness is the mean dist to K nearest neighbors. """
feature_arr = np.array(features)
tree = KDTree(np.vstack((np.array(self.archive), feature_arr)))
dists, _ = tree.query(feature_arr, k=self.ns_K + 1)
sparseness = np.mean(dists[:, 1:], axis=1)
return sparseness
def distance_p2p(points_src, normals_src, points_tgt, normals_tgt):
"""Computes minimal distances of each point in points_src to points_tgt.
Arguments:
----------
points_src (numpy array): source points
normals_src (numpy array): source normals
points_tgt (numpy array): target points
normals_tgt (numpy array): target normals
"""
kdtree = KDTree(points_tgt)
dist, idx = kdtree.query(points_src)
if normals_src is not None and normals_tgt is not None:
normals_src = \
normals_src / np.linalg.norm(normals_src, axis=-1, keepdims=True)
normals_tgt = \
normals_tgt / np.linalg.norm(normals_tgt, axis=-1, keepdims=True)
normals_dot_product = (normals_tgt[idx] * normals_src).sum(axis=-1)
# Handle normals that point into wrong direction gracefully
# (mostly due to mehtod not caring about this in generation)
normals_dot_product = np.abs(normals_dot_product)
else:
normals_dot_product = np.array([np.nan] * points_src.shape[0],
dtype=np.float32)
return dist, normals_dot_product
def closest_node_idx_to_sample_idx(
df,
axes_limits,
array_shape,
verbose = False):
"""
Purpose: To get the index of the closest node
point to a coordinate in the sampling
"""
limits_coords_by_axis = ctcu.axes_limits_coordinates(axes_limits,array_shape = array_shape)
from pykdtree.kdtree import KDTree
xi,yi,zi = np.meshgrid(*limits_coords_by_axis,indexing="ij")
limits_coords = np.vstack([k.ravel() for k in [xi,yi,zi]]).T
# if verbose:
# print(f"limits_coords=\n{limits_coords}")
limits_coords_kd = KDTree(df[["x","y","z"]].to_numpy())
dist,closest_nodes = limits_coords_kd.query(limits_coords)
if verbose:
print(f"closest_nodes = {closest_nodes}")
print(f"dist = {dist}")
return closest_nodes
def test127d_ok():
pts = 2
dims = 127
data_pts = np.arange(pts * dims).reshape(pts, dims)
kdtree = KDTree(data_pts)
dist, idx = kdtree.query(data_pts)
assert np.all(dist == 0)
def __find_index(self, lat, lon, latvar, lonvar, n=1):
if self._kdt.get(latvar.name) is None:
latvals = latvar[:] * RAD_FACTOR
lonvals = lonvar[:] * RAD_FACTOR
clat, clon = np.cos(latvals), np.cos(lonvals)
slat, slon = np.sin(latvals), np.sin(lonvals)
triples = np.array(
[np.ravel(clat * clon),
np.ravel(clat * slon),
np.ravel(slat)]).transpose()
self._kdt[latvar.name] = KDTree(triples)
del clat, clon
del slat, slon
del triples
if not hasattr(lat, "__len__"):
lat = [lat]
lon = [lon]
lat = np.array(lat)
lon = np.array(lon)
lat_rad = lat * RAD_FACTOR
lon_rad = lon * RAD_FACTOR
clat, clon = np.cos(lat_rad), np.cos(lon_rad)
slat, slon = np.sin(lat_rad), np.sin(lon_rad)
q = np.array([clat * clon, clat * slon, slat]).transpose()
dist_sq_min, minindex_1d = self._kdt[latvar.name].query(np.float32(q),
k=n)
iy_min, ix_min = np.unravel_index(minindex_1d, latvar.shape)
return iy_min, ix_min, dist_sq_min * EARTH_RADIUS
def test3d_8n():
query_pts = np.array([[787014.438, -340616.906, 6313018.],
[751763.125, -59925.969, 6326205.5],
[769957.188, -202418.125, 6321069.5]])
kdtree = KDTree(data_pts_real)
dist, idx = kdtree.query(query_pts, k=8)
exp_dist = np.array([[
0.00000000e+00, 4.05250235e+03, 4.07389794e+03, 8.08201128e+03,
8.17063009e+03, 1.20904577e+04, 1.22902057e+04, 1.60775136e+04
],
[
1.73205081e+00, 2.70216896e+03, 2.71431274e+03,
5.39537066e+03, 5.43793210e+03, 8.07855631e+03,
8.17119970e+03, 1.07513693e+04
],
[
1.41424892e+02, 3.25500021e+03, 3.44284958e+03,
6.58019346e+03, 6.81038455e+03, 9.89140135e+03,
1.01918659e+04, 1.31892516e+04
]])
exp_idx = np.array([[7, 8, 6, 9, 5, 10, 4, 11],
[93, 94, 92, 95, 91, 96, 90, 97],
[45, 46, 44, 47, 43, 48, 42, 49]])
assert np.array_equal(idx, exp_idx)
assert np.allclose(dist, exp_dist)
def fitness(self):
pos = np.asarray(self.node_pos[self.n_nodes])
kd_tree = KDTree(pos)
dist, _ = kd_tree.query(pos, k=30)
fitness = np.mean(dist)
return fitness
def test3d_8n_ub_eps():
query_pts = np.array([[787014.438, -340616.906, 6313018.],
[751763.125, -59925.969, 6326205.5],
[769957.188, -202418.125, 6321069.5]])
kdtree = KDTree(data_pts_real)
dist, idx = kdtree.query(query_pts,
k=8,
eps=0.1,
distance_upper_bound=10e3,
sqr_dists=False)
exp_dist = np.array([[
0.00000000e+00, 4.05250235e+03, 4.07389794e+03, 8.08201128e+03,
8.17063009e+03, np.Inf, np.Inf, np.Inf
],
[
1.73205081e+00, 2.70216896e+03, 2.71431274e+03,
5.39537066e+03, 5.43793210e+03, 8.07855631e+03,
8.17119970e+03, np.Inf
],
[
1.41424892e+02, 3.25500021e+03, 3.44284958e+03,
6.58019346e+03, 6.81038455e+03, 9.89140135e+03,
np.Inf, np.Inf
]])
n = 100
exp_idx = np.array([[7, 8, 6, 9, 5, n, n, n],
[93, 94, 92, 95, 91, 96, 90, n],
[45, 46, 44, 47, 43, 48, n, n]])
assert np.array_equal(idx, exp_idx)
assert np.allclose(dist, exp_dist)
def filt_stdev(coords, k=3, std_dev=2):
kDTree = KDTree(coords, leafsize = 5)
if pykdtree==1:
dx, idx_knn = kDTree.query(coords[:, :], k = k)
else:
dx, idx_knn = kDTree.query(coords[:, :], k = k, n_jobs=-1)
dx, idx_knn = dx[:,1:], idx_knn[:,1:]
distances = np.sum(dx, axis=1)/(k - 1.0)
valid_distances = np.shape(distances)[0]
#Estimate the mean and the standard deviation of the distance vector
sum = np.sum(distances)
sq_sum = np.sum(distances**2)
mean = sum / float(valid_distances)
variance = (sq_sum - sum * sum / float(valid_distances)) / (float(valid_distances) - 1)
stddev = np.sqrt (variance)
# a distance that is bigger than this signals an outlier
distance_threshold = mean + std_dev * stddev
idx = np.nonzero(distances < distance_threshold)
return idx, np.copy(coords[idx])
def _create_resample_kdtree(self):
"""Set up kd tree on input"""
# Get input information
valid_input_index, source_lons, source_lats = \
_get_valid_input_index_dask(self.source_geo_def,
self.target_geo_def,
self.reduce_data,
self.radius_of_influence,
nprocs=self.nprocs)
# FIXME: Is dask smart enough to only compute the pixels we end up
# using even with this complicated indexing
input_coords = lonlat2xyz(source_lons, source_lats)
valid_input_index = da.ravel(valid_input_index)
input_coords = input_coords[valid_input_index, :]
input_coords = input_coords.compute()
# Build kd-tree on input
input_coords = input_coords.astype(np.float)
valid_input_index, input_coords = da.compute(valid_input_index,
input_coords)
if kd_tree_name == 'pykdtree':
resample_kdtree = KDTree(input_coords)
else:
resample_kdtree = sp.cKDTree(input_coords)
return valid_input_index, resample_kdtree
def __init__(self, fields: Volume_T, max_dist: Number_T = 0.1):
# Process data type
if len(set([i.dtype for i in fields])) > 1:
raise RadarCalculationError("All input data should have same data type")
self.dtype = fields[0].dtype
# Process time
t_arr = np.array([time.mktime(i.scantime.timetuple()) for i in fields])
if (t_arr.max() - t_arr.min()) / 60 > 10:
raise RadarCalculationError(
"Time difference of input data should not exceed 10 minutes"
)
mean_time = t_arr.mean()
mean_dtime = datetime.datetime(*time.localtime(int(mean_time))[:6])
time_increment = 10
time_rest = mean_dtime.minute % time_increment
if time_rest > time_increment / 2:
mean_dtime += datetime.timedelta(minutes=(time_increment - time_rest))
else:
mean_dtime -= datetime.timedelta(minutes=time_rest)
self.scan_time = mean_dtime
self.lon_ravel = np.hstack([i.lon.ravel() for i in fields])
self.lat_ravel = np.hstack([i.lat.ravel() for i in fields])
self.data_ravel = np.ma.hstack([i.data.ravel() for i in fields])
self.dist_ravel = np.hstack(
[np.broadcast_to(i.dist, i.lon.shape).ravel() for i in fields]
)
self.tree = KDTree(np.dstack((self.lon_ravel, self.lat_ravel))[0])
self.md = max_dist
def populate_indices(self):
"""Pre-populate guesses of particle xi/yi indices using a kdtree.
This is only intended for curvilinear grids, where the initial index search
may be quite expensive.
"""
if self.fieldset is None:
# we need to be attached to a fieldset to have a valid
# gridset to search for indices
return
if KDTree is None:
return
else:
for i, grid in enumerate(self.fieldset.gridset.grids):
if not isinstance(grid, CurvilinearGrid):
continue
tree_data = np.stack((grid.lon.flat, grid.lat.flat), axis=-1)
tree = KDTree(tree_data)
# stack all the particle positions for a single query
pts = np.stack((self._collection.data['lon'],
self._collection.data['lat']),
axis=-1)
# query datatype needs to match tree datatype
_, idx = tree.query(pts.astype(tree_data.dtype))
yi, xi = np.unravel_index(idx, grid.lon.shape)
self._collection.data['xi'][:, i] = xi
self._collection.data['yi'][:, i] = yi
def __init__(self, ncfile, latvarname, lonvarname):
"""Initialization function
Arguments:
latvar -- the netCDF latitude variable
lonvar -- the netCDF longitude variable
"""
self.latvar = ncfile.variables[latvarname]
self.lonvar = ncfile.variables[lonvarname]
self.time_var = utils.get_time_var(ncfile)
self.kdt = _data_cache.get(ncfile.filepath())
if self.kdt is None:
rad_factor = pi / 180.0
latvals = self.latvar[:] * rad_factor
lonvals = self.lonvar[:] * rad_factor
clat, clon = np.cos(latvals), np.cos(lonvals)
slat, slon = np.sin(latvals), np.sin(lonvals)
triples = np.array(
[np.ravel(clat * clon),
np.ravel(clat * slon),
np.ravel(slat)]).transpose()
self.kdt = KDTree(triples)
_data_cache[ncfile.filepath()] = self.kdt
self._shape = ncfile.variables[latvarname].shape
def distance_p2p(pointcloud_pred, pointcloud_gt, normals_pred, normals_gt):
''' Computes minimal distances of each point in points_src to points_tgt.
Args:
points_src (numpy array): source points
normals_src (numpy array): source normals
points_tgt (numpy array): target points
normals_tgt (numpy array): target normals
'''
kdtree = KDTree(pointcloud_gt)
dist, idx = kdtree.query(pointcloud_pred)
if normals_pred is None:
return dist, None
normals_pred = normals_pred / np.linalg.norm(
normals_pred, axis=-1, keepdims=True)
normals_gt = normals_gt / np.linalg.norm(
normals_gt, axis=-1, keepdims=True)
normals_dot_product = (normals_gt[idx] * normals_pred).sum(axis=-1)
# Handle normals that point into wrong direction gracefully
# (mostly due to mehtod not caring about this in generation)
normals_dot_product = np.abs(normals_dot_product)
return dist, normals_dot_product
def match_surface_precip(self, input_data):
"""
Match surface precipitation from .sim file to points in xarray
dataset.
Args:
input_data: xarray dataset containing the input data from
the preprocessor.
Return:
The input dataset but with the surface_precip field added.
"""
n_scans = input_data.scans.size
dx = 40
i_c = 110
ix_start = i_c - dx // 2
ix_end = i_c + 1 + dx // 2
lats_1c = input_data["latitude"][:,
ix_start:ix_end].data.reshape(-1, 1)
lons_1c = input_data["longitude"][:,
ix_start:ix_end].data.reshape(-1, 1)
z = np.zeros_like(lats_1c)
coords_1c = pyproj.transform(_LLA,
_ECEF,
lons_1c,
lats_1c,
z,
radians=False)
coords_1c = np.concatenate(coords_1c, axis=1)
lats = self.data["latitude"].reshape(-1, 1)
lons = self.data["longitude"].reshape(-1, 1)
z = np.zeros_like(lats)
coords_sim = pyproj.transform(_LLA,
_ECEF,
lons,
lats,
z,
radians=False)
coords_sim = np.concatenate(coords_sim, 1)
kdtree = KDTree(coords_1c)
dists, indices = kdtree.query(coords_sim)
print(dists, indices)
surface_precip = np.zeros(n_scans * (dx + 1))
surface_precip[:] = np.nan
surface_precip[indices] = self.data["surface_precip"]
surface_precip = surface_precip.reshape(n_scans, dx + 1)
surface_precip_full = np.zeros(input_data["latitude"].shape,
dtype=np.float32)
surface_precip_full[:] = np.nan
surface_precip_full[:, ix_start:ix_end] = surface_precip
input_data["surface_precip"] = (("scans", "pixels"),
surface_precip_full)
return input_data
def gll_2_exodus(gll_model,
exodus_model,
gll_order=4,
dimensions=3,
nelem_to_search=20,
parameters="TTI",
model_path="MODEL/data",
coordinates_path="MODEL/coordinates",
gradient=False):
"""
Interpolate parameters from gll file to exodus model. This will mostly be
used to interpolate gradients to begin with.
:param gll_model: path to gll_model
:param exodus_model: path_to_exodus_model
:param parameters: Currently not used but will be fixed later
"""
with h5py.File(gll_model, 'r') as gll_model:
gll_points = np.array(gll_model[coordinates_path][:], dtype=np.float64)
gll_data = gll_model[model_path][:]
params = gll_model[model_path].attrs.get(
"DIMENSION_LABELS")[1].decode()
parameters = params[2:-2].replace(" ", "").split("|")
centroids = _find_gll_centroids(gll_points, dimensions)
print("centroids", np.shape(centroids))
# Build a KDTree of the centroids to look for nearest elements
print("Building KDTree")
centroid_tree = KDTree(centroids)
print("Read in mesh")
exodus = Exodus(exodus_model, mode="a")
# Find nearest elements
print("Querying the KDTree")
print(exodus.points.shape)
# if exodus.points.shape[1] == 3:
# exodus.points = exodus.points[:, :-1]
_, nearest_element_indices = centroid_tree.query(exodus.points,
k=nelem_to_search)
npoints = exodus.npoint
# parameters = utils.pick_parameters(parameters)
values = np.zeros(shape=[npoints, len(parameters)])
print(parameters)
s = 0
for point in exodus.points:
if s == 0 or (s + 1) % 1000 == 0:
print(f"Now I'm looking at point number:"
f"{s+1}{len(exodus.points)}")
element, ref_coord = _check_if_inside_element(
gll_points, nearest_element_indices[s, :], point, dimensions)
coeffs = get_coefficients(4, 4, 0, ref_coord, dimensions)
values[s, :] = np.sum(gll_data[element, :, :] * coeffs, axis=1)
s += 1
i = 0
for param in parameters:
exodus.attach_field(param, np.zeros_like(values[:, i]))
exodus.attach_field(param, values[:, i])
i += 1
def __init__(
self, data: np.ndarray, x: np.ndarray, y: np.ndarray, roi: Number_T = 0.02
):
x_ravel = x.ravel()
y_ravel = y.ravel()
self.tree = KDTree(np.dstack((x_ravel, y_ravel))[0])
self.data = data
self.roi = roi
