def build(self):
logger.debug(
"Building maps for conversion between real and image space")
self.imagetree = {}
self.imagemap = {}
self.realtree = KDTree([x.realcoords for x in self.patches])
self.realmapping = {}
for num, patch in enumerate(self.patches):
if num % 1000 == 0:
logger.debug("Built maps for {0} of {1} patches".format(
num, len(self.patches)))
resp = {}
for _, projection in self.projections.items():
inimage = patch.project(projection)
if projection.id not in self.imagetree:
self.imagetree[projection.id] = []
self.imagemap[projection.id] = {}
self.imagetree[projection.id].append(inimage)
self.imagemap[projection.id][tuple(inimage)] = patch.realcoords
resp[projection.id] = inimage
self.realmapping[tuple(patch.realcoords)] = resp
logger.debug("Done building maps for {0} patches".format(
len(self.patches)))
logger.debug("Building image KD tree")
for key, imagecoords in self.imagetree.items():
self.imagetree[key] = KDTree(imagecoords)
def compute_graph(samples,
weights=None,
p_norm=2,
max_degree=10,
max_distance=INF,
approximate_eps=0.):
vertices = list(range(len(samples)))
edges = set()
if not vertices:
return vertices, edges
if weights is None:
weights = np.ones(len(samples[0]))
embed_fn = get_embed_fn(weights)
embedded = list(map(embed_fn, samples))
kd_tree = KDTree(embedded)
for v1 in vertices:
# TODO: could dynamically compute distances
distances, neighbors = kd_tree.query(embedded[v1],
k=max_degree + 1,
eps=approximate_eps,
p=p_norm,
distance_upper_bound=max_distance)
for d, v2 in zip(distances, neighbors):
if (d < max_distance) and (v1 != v2):
edges.update([(v1, v2), (v2, v1)])
# print(time.time() - start_time, len(edges), float(len(edges))/len(samples))
return vertices, edges
def get_nearest(lasfile):
""" return a nearest neighbor kdtree query """
dataset = np.vstack([lasfile.x, lasfile.y, lasfile.z]).transpose()
tree = KDTree(dataset)
query = tree.query(dataset[100,], k=5)
return query
def get_interpolation_weights(self, source_lons = None, source_lats = None,
dest_lons = None, dest_lats = None, nneighbours = 4
):
"""
get the interpolation array M (nx*ny, nneighbors) and negibor indices G:
DEST_FIELD = M * SOURCE_FIELD.flatten()[G]
"""
source_lons_1d, source_lats_1d = source_lons.flatten(), source_lats.flatten()
[x,y,z] = lat_lon.lon_lat_to_cartesian_normalized(source_lons_1d, source_lats_1d)
kdtree = KDTree(zip(x, y, z))
[xi, yi, zi] = lat_lon.lon_lat_to_cartesian_normalized(dest_lons.flatten(), dest_lats.flatten())
[distances, indices] = kdtree.query( zip( xi, yi, zi ) , k = nneighbours )
if len(distances.shape) == 2:
weights = 1.0 / distances ** 2
norm = weights.sum(axis = 1)
norm = np.array( [norm] * nneighbours ).transpose()
weights /= norm
else:
weights = np.ones(distances.shape)
return weights, indices
def __init__(self,
ax,
basemap,
tmin_3d,
tmax_3d,
lons2d,
lats2d,
levelheights=None):
"""
Plots a vertical profile at the point nearest to the clicked one
:type ax: Axes
"""
assert isinstance(ax, Axes)
self.basemap = basemap
assert isinstance(self.basemap, Basemap)
self.tmin_3d = tmin_3d
self.tmax_3d = tmax_3d
self.lons2d = lons2d
self.lats2d = lats2d
self.T0 = 273.15
self.lons_flat = lons2d.flatten()
self.lats_flat = lats2d.flatten()
self.level_heights = levelheights
self.counter = 0
self.ax = ax
x, y, z = lat_lon.lon_lat_to_cartesian(lons2d.flatten(),
lats2d.flatten())
self.kdtree = KDTree(list(zip(x, y, z)))
ax.figure.canvas.mpl_connect("button_press_event", self)
pass
def add_clusters(df_cells, neighbor_dist=50):
"""Assigns -1 to clusters with only one cell.
"""
from scipy.spatial.kdtree import KDTree
import networkx as nx
x = df_cells[GLOBAL_X] + df_cells[POSITION_J]
y = df_cells[GLOBAL_Y] + df_cells[POSITION_I]
barcodes = df_cells[BARCODE_0]
barcodes = np.array(barcodes)
kdt = KDTree(np.array([x, y]).T)
num_cells = len(df_cells)
print('searching for clusters among %d cells' % num_cells)
pairs = kdt.query_pairs(neighbor_dist)
pairs = np.array(list(pairs))
x = barcodes[pairs]
y = x[:, 0] == x[:, 1]
G = nx.Graph()
G.add_edges_from(pairs[y])
clusters = list(nx.connected_components(G))
cluster_index = np.zeros(num_cells, dtype=int) - 1
for i, c in enumerate(clusters):
cluster_index[list(c)] = i
df_cells[CLUSTER] = cluster_index
return df_cells
def __init__(self, ax, basemap, lons2d, lats2d, ncVarDict, times,
start_date, end_date):
"""
Plots a vertical profile at the point nearest to the clicked one
:type ax: Axes
"""
assert isinstance(ax, Axes)
self.basemap = basemap
assert isinstance(self.basemap, Basemap)
self.lons_flat = lons2d.flatten()
self.lats_flat = lats2d.flatten()
self.ncVarDict = ncVarDict
self.lons2d = lons2d
self.lats2d = lats2d
self.counter = 0
self.ax = ax
x, y, z = lat_lon.lon_lat_to_cartesian(lons2d.flatten(),
lats2d.flatten())
self.kdtree = KDTree(list(zip(x, y, z)))
self.sel_time_indices = np.where(
[start_date <= t <= end_date for t in times])[0]
self.times = times[self.sel_time_indices]
ax.figure.canvas.mpl_connect("button_press_event", self)
def locally_extreme_points(coords,
data,
neighbourhood,
lookfor='max',
p_norm=2.):
# Description
# Find local maxima of points in a pointcloud. Ties result in both points passing through the filter.
#
# Not to be used for high-dimensional data. It will be slow.
#
# coords: A shape (n_points, n_dims) array of point locations
# data: A shape (n_points, ) vector of point values
# neighbourhood: The (scalar) size of the neighbourhood in which to search.
# lookfor: Either 'max', or 'min', depending on whether you want local maxima or minima
# p_norm: The p-norm to use for measuring distance (e.g. 1=Manhattan, 2=Euclidian)
#
# returns
# filtered_coords: The coordinates of locally extreme points
# filtered_data: The values of these points
assert coords.shape[0] == data.shape[
0], 'You must have one coordinate per data point'
extreme_fcn = {'min': np.min, 'max': np.max}[lookfor]
kdtree = KDTree(coords)
neighbours = kdtree.query_ball_tree(kdtree, r=neighbourhood, p=p_norm)
i_am_extreme = [
data[i] == extreme_fcn(data[n]) for i, n in enumerate(neighbours)
]
extrema, = np.nonzero(
i_am_extreme) # This line just saves time on indexing
return coords[extrema], data[extrema]
def estimatemeans(self):
feature_positions = []
for i in range(len(self.edges[0]) - 1):
for j in range(len(self.edges[1]) - 1):
if len(self.points[i, j]) > 0:
feature_positions.append([i, j])
tree = KDTree(feature_positions)
for i in range(len(self.edges[0]) - 1):
for j in range(len(self.edges[1]) - 1):
if self.points[i, j] == []:
radius, neighbour = tree.query(x=np.array([i, j]), k=1)
if radius > feature_resolution[0] / 10:
self.estmeans[i, j] = np.nan
self.means[i, j] = np.nan
else:
nearby = tree.data[tree.query_ball_point(
x=np.array([i, j]), r=radius)]
#print([self.trumeans[i] for i in nearby])
estmean = np.nanmean(
[self.trumeans[i.astype(int)] for i in nearby])
self.estmeans[i, j] = estmean
self.means[i, j] = estmean
else:
self.means[i, j] = self.trumeans[i, j]
class NeighborsFinder:
def __init__(self, data):
"""Find neigbors
Args:
data (pd.DataFrame or dict): data with at columns id,pos,and eventually range
"""
if data is not None:
self.data = data
# Safety checks
assert isinstance(data, pd.DataFrame)
assert "pos" in data.columns
# Initialize KDTree finder from scipy
self.tree = KDTree(self.data["pos"].tolist())
def find_in_range(self, obj, search_range):
# TODO
# Will not work with double colliders
# Can be made using colliders in PyGame ?
# Safety check
assert hasattr(self, "tree")
# Get position from dataset
pos = obj.pos
# Find neighbors in range
# We remove the first one which is the identity object
idx = self.tree.query_ball_point(pos, search_range)
# Return filtered data
ids = self.data.iloc[idx].index.tolist()
assert obj.id not in ids
return ids
def find_closest(self, obj, k=1):
# Safety check
assert hasattr(self, "tree")
# Get object position from which we want to find neighbors
pos = obj.pos
# Get position from dataset
distances, idx = self.tree.query(pos, k=k)
if k == 1:
distances = [distances]
idx = [idx]
# Get ids from dataset
# Safe check object is not in neighbors, which would mean above we remove another overlapping objects with [1:]
ids = self.data.iloc[idx].index.tolist()
assert obj.id not in ids
return distances, ids
def warp_keypoints(self, keypoints, grid_unnormalized):
from scipy.spatial.kdtree import KDTree
warp_grid = grid_unnormalized.reshape(-1, 2)
regular_grid = self.grid_pixels_unnormalized.reshape(-1, 2)
kd = KDTree(warp_grid)
dists, idxs = kd.query(keypoints)
new_keypoints = regular_grid[idxs]
return new_keypoints
def mutual_knn(points, n=10, distance=radial_kernel()):
knn = {}
kt = KDTree(points)
for i, point in enumerate(points):
# cannot use euclidean distance directly
for neighbour in kt.query(point, n + 1)[1]:
if i != neighbour:
knn.setdefault(i, []).append((distance(point, points[neighbour]), neighbour))
return knn
def compute_minimax_distance(path1, path2):
overall_distance = 0.
for path, other in permutations([path1, path2]):
tree = KDTree(other)
for q1 in path:
#closest_distance = min(get_distance(q1, q2) for q2 in other)
closest_distance, closest_index = tree.query(q1, k=1, eps=0.)
overall_distance = max(overall_distance, closest_distance)
return overall_distance
def make_adjacency_matrix(data, k):
kt = KDTree(data)
out = zeros((len(data),len(data)))
for i, points in enumerate(data):
distance, neighbors = kt.query_ball_point(points, k)
for j in neighbors[1:]:
out[i][j] = 1;
out[j][i] = 1; #to ensure symmetry, one can imagine the the nearest neighbors is not necessarily associative
return out
def mutual_knn(points, n=10, distance=radial_kernel()):
knn = {}
kt = KDTree(points)
for i, point in enumerate(points):
# cannot use euclidean distance directly
for neighbour in kt.query(point, n + 1)[1]:
if i != neighbour:
knn.setdefault(i, []).append(
(distance(point, points[neighbour]), neighbour))
return knn
def __init__(self, filepath):
start = time.time()
self.file = File(filepath, mode="r")
self.scale = self.file.header.scale[0]
self.offset = self.file.header.offset[0]
self.tree = KDTree(
np.vstack([self.file.x, self.file.y, self.file.z]).transpose())
self.time = self.file.header.get_date()
end = time.time() - start
print("Time Elapsed: {}".format(end))
def constructKnnGraph(points, numNeighbor):
def euclidean_kernel(a, b):
d = np.linalg.norm(a-b)
return d
knn = {}
kt = KDTree(points)
for i, point in enumerate(points):
for neighbour in kt.query(point, numNeighbor + 1)[1]:
if i != neighbour:
knn.setdefault(i, []).append((euclidean_kernel(point, points[neighbour]), neighbour))
return knn
def waypoints_cb(self, msg):
# self.waypoints = waypoints
# rospy.loginfo("TL_DETECTOR: waypoints_cb called...")
self.base_waypoints = msg.waypoints
if self.base_waypoints is not None:
# populate the Lane msg header as well (not really required, but to be consistent)
# self.lane_msg_header = msg.header
# get the 2d (x, y) waypoints
self.base_waypoints_2d = [[wp.pose.pose.position.x, wp.pose.pose.position.y] for wp in self.base_waypoints]
# build a KDTree for efficient search of nearest waypoint
self.base_waypoints_kdtree = KDTree(self.base_waypoints_2d)
def interpolate_data_to_model_grid(self, model_lons_2d, model_lats_2d,
data_obs):
x0, y0, z0 = lat_lon.lon_lat_to_cartesian(model_lons_2d.flatten(),
model_lats_2d.flatten())
x, y, z = lat_lon.lon_lat_to_cartesian(self.lons2d.flatten(),
self.lats2d.flatten())
if self.ktree is None:
self.ktree = KDTree(list(zip(x, y, z)))
d, i = self.ktree.query(list(zip(x0, y0, z0)))
return data_obs.flatten()[i].reshape(model_lons_2d.shape)
def build(self):
logger.debug("Building maps for conversion between real and image space")
self.imagetree = {}
self.imagemap = {}
self.realtree = KDTree([x.realcoords for x in self.patches])
self.realmapping = {}
for num, patch in enumerate(self.patches):
if num % 1000 == 0:
logger.debug("Built maps for {0} of {1} patches".format(num, len(self.patches)))
resp = {}
for _, projection in self.projections.items():
inimage = patch.project(projection)
if projection.id not in self.imagetree:
self.imagetree[projection.id] = []
self.imagemap[projection.id] = {}
self.imagetree[projection.id].append(inimage)
self.imagemap[projection.id][tuple(inimage)] = patch.realcoords
resp[projection.id] = inimage
self.realmapping[tuple(patch.realcoords)] = resp
logger.debug("Done building maps for {0} patches".format(len(self.patches)))
logger.debug("Building image KD tree")
for key, imagecoords in self.imagetree.items():
self.imagetree[key] = KDTree(imagecoords)
def updatemeans(self, newpoint, model, fr):
'''For use when self = meanmap
Calculates the mean observed value for each niche and populates
the current map with those mean values.
'''
#Update the new point first
new_index = tuple(self.nichefinder(newpoint[2]))
i, j = new_index[0], new_index[1]
obsmean = self.trumeans[i, j]
#estmean = self.estimatemean( i,j)
#self.estmeans[ i,j ] = estmean
#self.means[i,j] = np.mean([estmean, obsmean])
self.means[i, j] = obsmean
## Now update points that will be affected by the current point
feature_positions = []
for i in range(len(self.edges[0]) - 1):
for j in range(len(self.edges[1]) - 1):
if self.points[i, j] != []:
feature_positions.append([i, j])
#print(feature_positions)
tree = KDTree(feature_positions)
checkindex = [new_index]
# Explore all the points around it.
end = False
checked = []
while not end:
newcheck = []
for index in checkindex:
newcheck, checked = self.pointsearch(
model, index[0], index[1], fr, tree, newpoint, checked)
checkindex += newcheck
if len(newcheck) == 0:
end = True
def solvent_surface(molecule, num_points_per_atom=140, delta=0.1):
radii = get_solvent_radii(molecule.atomic_numbers)
N = len(molecule)
grid = load_grid_num_points(num_points_per_atom)
num_points_per_atom = grid.shape[0]
axes = molecule.axes()
positions = molecule.positions_in_molecular_axis_frame()
surface = np.empty((N * num_points_per_atom, 4))
atom_idx = np.empty(N * num_points_per_atom, dtype=np.int32)
for i in range(N):
r = radii[i] + delta
l, u = num_points_per_atom * i, num_points_per_atom * (i + 1)
surface[l:u, 3] = grid[:, 3] * 4 * np.pi * radii[i] * radii[i]
surface[l:u, :3] = grid[:, :3] * r
surface[l:u, :3] += positions[i, :]
atom_idx[l:u] = i
mask = np.ones_like(atom_idx, dtype=bool)
tree = KDTree(surface[:, :3])
for i in range(N):
p = positions[i]
radius = radii[i] + delta
idxs = tree.query_ball_point(p, radius - 1e-12)
mask[idxs] = False
surface = surface[mask, :]
surface[:, :3] = (np.dot(surface[:, :3], axes) +
molecule.center_of_mass[np.newaxis, :])
atom_idx = atom_idx[mask]
return surface
def knnClassify(trData, trLabels, testData, nClasses):
#run the knn classifier using the raw features
(nDocs, _) = testData.shape
knnLabels = numpy.zeros((nDocs, nClasses))
leafsize = 5
kdtree = KDTree(trData.tolist(), leafsize)
(_, idxs) = kdtree.query(testData.tolist(), 3)
for d in range(nDocs):
testPointIdxs = idxs[d]
votes = [trLabels[l] for l in testPointIdxs]
classVotes = [0] * nClasses
for v in votes:
classVotes[v] += 1
knnLabels[d, classVotes.index(max(classVotes))] = 1
return (knnLabels, kdtree)
def _init_kd_tree(self):
"""
Init KDTree used for interpolation
"""
x0, y0, z0 = lat_lon.lon_lat_to_cartesian(self.lon2d.flatten(),
self.lat2d.flatten())
self.kdtree = KDTree(list(zip(x0, y0, z0)))
def __init__(self, data):
"""Find neigbors
Args:
data (pd.DataFrame or dict): data with at columns id,pos,and eventually range
"""
if data is not None:
self.data = data
# Safety checks
assert isinstance(data, pd.DataFrame)
assert "pos" in data.columns
# Initialize KDTree finder from scipy
self.tree = KDTree(self.data["pos"].tolist())
def __init__(self, name_to_date_to_field, basemap, lons2d, lats2d,
ax = None, cell_area = None, cell_manager = None, data_manager = None):
self.gwdi_mean_field = None
self.traf_mean_field = None
self.tdra_mean_field = None
self.upin_mean_field = None
self.basemap = basemap
self.date_to_stfl_field = name_to_date_to_field["STFL"]
self.date_to_traf_field = name_to_date_to_field["TRAF"]
self.date_to_tdra_field = name_to_date_to_field["TDRA"]
self.date_to_pr_field = name_to_date_to_field["PR"]
self.date_to_swe_field = name_to_date_to_field["I5"]
self.date_to_swst_field = name_to_date_to_field["SWST"]
#self.date_to_imav_field = name_to_date_to_field["IMAV"]
self.acc_area_km2 = name_to_date_to_field["FACC"]
#:type : CellManager
self.cell_manager = cell_manager
assert isinstance(self.cell_manager, CellManager)
self.cell_area = cell_area
x, y, z = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten())
self.kdtree = KDTree(list(zip(x,y,z)))
ax.figure.canvas.mpl_connect("button_press_event", self)
self.ax = ax
self.lons2d = lons2d
self.lats2d = lats2d
self.data_manager = data_manager
assert isinstance(self.data_manager, Crcm5ModelDataManager)
self.x_pr, self.y_pr = basemap(lons2d, lats2d)
self.lons_flat = lons2d.flatten()
self.lats_flat = lats2d.flatten()
self.dates_sorted = list( sorted(list(name_to_date_to_field.items())[0][1].keys()) )
self.counter = 0
self.date_to_swsr_field = name_to_date_to_field["SWSR"]
self.date_to_swsl_field = name_to_date_to_field["SWSL"]
#self.date_to_gwdi_field = name_to_date_to_field["GWDI"]
self.date_to_upin_field = name_to_date_to_field["UPIN"]
#static fields
self.slope = name_to_date_to_field["SLOP"]
self.channel_length = name_to_date_to_field["LENG"]
self.lake_outlet = name_to_date_to_field["LKOU"]
self.coef_bf = -np.ones(self.slope.shape)
good_points = self.slope >= 0
self.coef_bf[good_points] = (self.slope[good_points]) ** 0.5 / ((self.channel_length[good_points]) ** (4.0/3.0) * data_manager.manning_bf[good_points] )
def nearest_neighbor_fill(ball, mask):
# Fill flag holes with nearest neighbor
ff = np.asarray(ball).copy()
x, y = np.mgrid[0:ff.shape[0], 0:ff.shape[1]]
xygood = np.array((x[~mask], y[~mask])).T
xybad = np.array((x[mask], y[mask])).T
ff[mask] = ff[~mask][KDTree(xygood).query(xybad)[1]]
return Image.fromarray(ff)
def build_tree(laser_file):
# build a KD-tree over the point clouds to find nearest neighbors.
# input: the point cloud file that is going to be searched.
# output: the tree file that includes indices ready to be searched.
data_ready = np.vstack([laser_file.X, laser_file.Y,
laser_file.Z]).transpose()
tree = KDTree(data_ready)
return tree
def _merge(self, roi):
points = [ e[2] for e in roi ]
tree = KDTree(points)
pp = tree.query_pairs(4)
#pp = sorted( pp , key=lambda e: max(roi[e[0]] , roi[e[1]]), reverse=True)
skips = set()
for i,j in pp:
if self.Debug:
print '%s(%.2f),%s(%.2f)'%(roi[i][0],roi[i][1], roi[j][0], roi[j][1] )
if i in skips or j in skips: continue
skip = i if roi[i][1]<roi[j][1] else j
skips.add(skip)
rs = ( e for i, e in enumerate(roi) if not i in skips )
rs = sorted( rs , key=lambda e: e[1] )[-4:]
return sorted( rs, key=lambda e: e[2][1] )
def _merge(self, roi):
points = [e[2] for e in roi]
tree = KDTree(points)
pp = tree.query_pairs(4)
#pp = sorted( pp , key=lambda e: max(roi[e[0]] , roi[e[1]]), reverse=True)
skips = set()
for i, j in pp:
if self.Debug:
print '%s(%.2f),%s(%.2f)' % (roi[i][0], roi[i][1], roi[j][0],
roi[j][1])
if i in skips or j in skips: continue
skip = i if roi[i][1] < roi[j][1] else j
skips.add(skip)
rs = (e for i, e in enumerate(roi) if not i in skips)
rs = sorted(rs, key=lambda e: e[1])[-4:]
return sorted(rs, key=lambda e: e[2][1])
def test_evol_for_point():
lon = -100
lat = 60
path = "/home/huziy/skynet1_rech3/cordex/for_Samira/b1/tmp_era40_b1.nc"
ds = Dataset(path)
data = ds.variables["tmp"][:]
years = ds.variables["year"][:]
coord_file = "/skynet1_rech3/huziy/cordex/CORDEX_DIAG/NorthAmerica_0.44deg_CanESM_B1"
coord_file = os.path.join(coord_file, "pmNorthAmerica_0.44deg_CanHisto_B1_195801_moyenne")
#coord_file = os.path.join(data_folder, "pmNorthAmerica_0.44deg_ERA40-Int2_195801_moyenne")
b, lons2d, lats2d = draw_regions.get_basemap_and_coords(file_path=coord_file)
sel_lons = [lon]
sel_lats = [lat]
xo,yo,zo = lat_lon.lon_lat_to_cartesian(sel_lons, sel_lats)
xi, yi, zi = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten())
ktree = KDTree(list(zip(xi,yi,zi)))
dists, indexes = ktree.query(list(zip(xo,yo,zo)))
print(len(indexes))
print(indexes)
idx = indexes[0]
import matplotlib.pyplot as plt
plt.figure()
data_to_show = []
for i, y in enumerate(years):
data_to_show.append(data[i,:,:].flatten()[idx])
plt.plot(years, data_to_show, "-s", lw = 3)
plt.grid()
plt.show()
pass
def get_kd_tree(self):
"""
:rtype : KDTree
for interpolation purposes
"""
if self._kdtree is None:
x, y, z = lat_lon.lon_lat_to_cartesian(self.lons.flatten(),
self.lats.flatten())
self._kdtree = KDTree(list(zip(x, y, z)))
return self._kdtree
def locally_extreme_points(coords,
data,
radius,
smoothing_radius=None,
min_neighbourhood_size=-1,
lookfor='max',
p_norm=2):
'''
Find local maxima of points in a pointcloud. Ties result in both points passing through the filter.
Not to be used for high-dimensional data. It will be slow.
coords: A shape (n_points, n_dims) array of point locations
data: A shape (n_points, ) vector of point values
radius: The (scalar) size of the neighbourhood in which to search.
min_neighbourhood_size: Extrema whose neighbourhood has less than min_neighbourhood_size points are discarded
lookfor: Either 'max', or 'min', depending on whether you want local maxima or minima
p_norm: The p-norm to use for measuring distance (e.g. 1=Manhattan, 2=Euclidian)
returns
filtered_coords: The coordinates of locally extreme points
filtered_data: The values of these points
'''
assert coords.shape[0] == data.shape[
0], 'You must have one coordinate per data point'
extreme_fcn = {'min': np.min, 'max': np.max}[lookfor]
kdtree = KDTree(coords)
if smoothing_radius is not None:
smoothing_neighbours = kdtree.query_ball_tree(kdtree,
r=smoothing_radius,
p=p_norm)
smooth_data = np.array(
[np.mean(data[n]) for n in smoothing_neighbours])
else:
smooth_data = data
neighbours = kdtree.query_ball_tree(kdtree, r=radius, p=p_norm)
i_am_extreme = [
(np.nonzero(np.equal(n, i))[0][0] == np.argmax(smooth_data[n])) &
(len(n) > min_neighbourhood_size) for i, n in enumerate(neighbours)
]
extrema, = np.nonzero(
i_am_extreme) # This line just saves time on indexing
return coords[extrema], data[extrema]
def add_clusters(df_cells,
barcode_col=BARCODE_0,
radius=50,
verbose=True,
ij=(POSITION_I, POSITION_J)):
"""Assigns -1 to clusters with only one cell.
"""
from scipy.spatial.kdtree import KDTree
import networkx as nx
I, J = ij
x = df_cells[GLOBAL_X] + df_cells[J]
y = df_cells[GLOBAL_Y] + df_cells[I]
barcodes = df_cells[barcode_col]
barcodes = np.array(barcodes)
kdt = KDTree(np.array([x, y]).T)
num_cells = len(df_cells)
if verbose:
message = 'searching for clusters among {} {} objects'
print(message.format(num_cells, barcode_col))
pairs = kdt.query_pairs(radius)
pairs = np.array(list(pairs))
x = barcodes[pairs]
y = x[:, 0] == x[:, 1]
G = nx.Graph()
G.add_edges_from(pairs[y])
clusters = list(nx.connected_components(G))
cluster_index = np.zeros(num_cells, dtype=int) - 1
for i, c in enumerate(clusters):
cluster_index[list(c)] = i
df_cells = df_cells.copy()
df_cells[CLUSTER] = cluster_index
df_cells[CLUSTER_SIZE] = (
df_cells.groupby(CLUSTER)[barcode_col].transform('size'))
df_cells.loc[df_cells[CLUSTER] == -1, CLUSTER_SIZE] = 1
return df_cells
def interpolate_data_to_model_grid(self, model_lons_2d, model_lats_2d, data_obs):
x0, y0, z0 = lat_lon.lon_lat_to_cartesian(model_lons_2d.flatten(), model_lats_2d.flatten())
x, y, z = lat_lon.lon_lat_to_cartesian(self.lons2d.flatten(), self.lats2d.flatten())
if self.ktree is None:
self.ktree = KDTree(list(zip(x, y, z)))
d, i = self.ktree.query(list(zip(x0, y0, z0)))
return data_obs.flatten()[i].reshape(model_lons_2d.shape)
def __init__(self, xml_annot):
self._data = []
self.ref_by_pages = {}
idx = 0
for annotation in xml_annot.findall(
".//ANNOTATION/ACTION[@type='goto']/.."):
dest = annotation.find("ACTION/DEST")
try:
page_dest = dest.get("page")
x_dest = dest.get("x")
y_dest = dest.get("y")
except:
# Maybe change that, see how much we loose
continue
page_num = annotation.get("pagenum")
page_n = int(page_num)
quadpoints = annotation.findall("QUADPOINTS/QUADRILATERAL/POINT")
min_h, min_v, max_h, max_v = None, None, None, None
for point in quadpoints:
h, v = float(point.get("HPOS")), float(point.get("VPOS"))
if min_h is None:
min_h, max_h = h, h
min_v, max_v = v, v
else:
min_h = min(min_h, h)
max_h = max(max_h, h)
min_v = min(min_v, v)
max_v = max(max_v, v)
self._data.append({
"page_dest": page_dest,
"x_dest": x_dest,
"y_dest": y_dest,
"bbx": BBX(page_num, min_h, min_v, max_h, max_v)
})
"""
Add a new point in the QDTree of the page *page_n*
"""
if page_n not in self.ref_by_pages:
self.ref_by_pages[page_n] = {"idx": [], "pos": []}
self.ref_by_pages[page_n]["idx"].append(idx)
self.ref_by_pages[page_n]["pos"].append([(min_v + max_v) / 2,
(min_h + max_h) / 2])
idx += 1
# Convert list to QDTree
for page in self.ref_by_pages:
self.ref_by_pages[page]["tree"] = KDTree(
self.ref_by_pages[page]["pos"])
def create_main_grid(points, extend, img_size=32, values=[]):
"""Create grid and caluclate features
:param points: Array Vstack [x, y, z, classification] [m]
:type points: float
:param extend: Array [minX minY, maxX maxY]
:type extend: float
:param img_size: Spatial size of feature area Default 32. Should be 2 to power of n
:type img_size: int
:param values: Values for Feature Stats, if non is passed height is used
:type values: float
"""
tree = KDTree(points[:, 0:2])
buff = int(img_size / 2)
if values:
points[:, 2] = values
minX = extend[0, 0] - buff
minY = extend[0, 1] - buff
maxX = extend[1, 0] + buff
maxY = extend[1, 1] + buff
gridX = np.linspace(int(minX), int(maxX), int(maxX - minX + 1))
gridY = np.linspace(int(minY), int(maxY), int(maxY - minY + 1))
f1 = np.zeros((len(gridX), len(gridY)))
f2 = np.zeros((len(gridX), len(gridY)))
f3 = np.zeros((len(gridX), len(gridY)))
for x, i in zip(gridX, range(0, len(gridX))):
for y, j in zip(gridY, range(0, len(gridY))):
list = tree.query_ball_point([x, y], 1.4)
cell_ext = np.array([[x - 0.5, y - 0.5], [x + 0.5, y + 0.5]])
cell_points = clip(points[list], cell_ext)
if cell_points.any():
f1[i, j] = np.mean(cell_points[:, 2])
f2[i, j] = np.min(cell_points[:, 2])
f3[i, j] = np.max(cell_points[:, 2])
return [f1, f2, f3]
def kdtree_density(points, radius, return_tree=False):
""" Returns the density calculated sing a Ball Tree.
Higher is denser.
Scales according to the dimension of the points.
see: https://stackoverflow.com/questions/14070565/calculating-point-density-using-python
If `return_tree` is `True` returns a tuple of the density and the KDTree.
"""
tree = KDTree(points)
n = points.shape[-1]
ball_trees = tree.query_ball_tree(tree, radius)
frequency = np.array([len(neighbours) for neighbours in ball_trees])
density = frequency / radius**n
if return_tree:
return np.mean(density), tree
return np.mean(density)
def get_edges(df1, df2):
from scipy.spatial.kdtree import KDTree
get_label = lambda x: tuple(int(y) for y in x[[2, 3]])
x1 = df1[['i', 'j', 'frame', 'label']].as_matrix()
x2 = df2[['i', 'j', 'frame', 'label']].as_matrix()
kdt = KDTree(df1[['i', 'j']])
points = df2[['i', 'j']]
result = kdt.query(points, 3)
edges = []
for i2, (ds, ns) in enumerate(zip(*result)):
end_node = get_label(x2[i2])
for d, i1 in zip(ds, ns):
start_node = get_label(x1[i1])
w = d
edges.append((start_node, end_node, w))
return edges
def _calculatePointResiduals(self, curve, tube_radius = None):
if tube_radius is None:
X = self._X
else:
within_tube_indices = self.calculateCoverageIndices(curve, tube_radius)
X = self._X.take(list(within_tube_indices), axis = 0)
if self._maxSegmentLength is None:
self._maxSegmentLength = self._calculateMaxSegmentLength(curve)
lpc_points = curve['save_xd']
num_lpc_points = len(lpc_points)
tree_lpc_points = KDTree(lpc_points)
residuals = empty(len(X))
residuals_lamb = empty(len(X))
path_length = curve['lamb']
for j, p in enumerate(X):
closest_lpc_point = tree_lpc_points.query(p)
candidate_radius = sqrt(closest_lpc_point[0]**2 + 0.25*self._maxSegmentLength**2)
candidate_segment_ends = tree_lpc_points.query_ball_point(p, candidate_radius)
candidate_segment_ends.sort()
current_min_segment_dist = (closest_lpc_point[0],0)
current_closest_index = closest_lpc_point[1]
last_index = None
for i, index in enumerate(candidate_segment_ends):
if index!=0 and last_index != index - 1:
prv_segment_dist = self._distancePointToLineSegment(lpc_points[index-1], lpc_points[index], p)
if prv_segment_dist[0] < current_min_segment_dist[0]:
current_min_segment_dist = prv_segment_dist
current_closest_index = index - 1
if index != num_lpc_points - 1:
prv_segment_dist = self._distancePointToLineSegment(lpc_points[index], lpc_points[index+1], p)
if prv_segment_dist[0] < current_min_segment_dist[0]:
current_min_segment_dist = prv_segment_dist
current_closest_index = index
last_index = index
residuals[j] = current_min_segment_dist[0]
residuals_lamb[j] = path_length[current_closest_index] + current_min_segment_dist[1]
lamb_order = argsort(residuals_lamb)
return (residuals_lamb[lamb_order], residuals[lamb_order])
def get_data_interpolated_to_points(self, dest_lons = None,
dest_lats = None,
source_lons = None,
source_lats = None,
data = None):
"""
Designed to interpolate all data to the AMNO domain
"""
if None not in [source_lons, source_lats]:
lons1d = source_lons.flatten()
lats1d = source_lats.flatten()
points = lat_lon.lon_lat_to_cartesian_normalized(lons1d, lats1d)
points = np.array(points).transpose()
point_tree = KDTree(points)
else:
point_tree = self.kd_tree
assert source_lons.shape == source_lats.shape == data.shape
[xi, yi, zi] = lat_lon.lon_lat_to_cartesian_normalized(dest_lons.flatten(), dest_lats.flatten())
pointsi = np.array([xi, yi, zi]).transpose()
data1d = data.flatten()
#distances dimensions = (n_points, n_neigbours)
[distances, indices] = point_tree.query(pointsi, k = 4)
weights = 1.0 / distances ** 2
norm = [np.sum(weights, axis = 1)] * weights.shape[1]
norm = np.array(norm).transpose()
weights /= norm
result = []
for i in xrange(pointsi.shape[0]):
w = weights[i, :]
d = data1d[indices[i, :]]
result.append(np.sum(w * d))
return np.array(result).reshape( dest_lons.shape )
def make_edge_graph(data, k, ball = True):
kt = KDTree(data)
out = [set() for i in data]
k_max = 0
k_min = np.infty
for i, points in enumerate(data):
if(ball):
neighbors = kt.query_ball_point(points, k)
else:
distance, neighbors = kt.query(points,int(k+1))
neighbors = neighbors[1:]
for j in neighbors:
if(j != i):
out[i].add(j)
if(len(out[i])>k_max):
k_max = len(out[i])
out[j].add(i)
if(len(out[j])>k_max):
k_max = len(out[j])
if(len(out[i])<k_min):
k_min = len(out[i])
return (k_min, k_max, out)
def upscale(manager_in, manager_out, swe_in, nneighbours = 25):
assert isinstance(manager_in, Crcm5ModelDataManager)
assert isinstance(manager_out, Crcm5ModelDataManager)
lons_in_1d = manager_in.lons2D.flatten()
lats_in_1d = manager_in.lats2D.flatten()
x0, y0, z0 = lat_lon.lon_lat_to_cartesian(lons_in_1d, lats_in_1d)
kdtree = KDTree(list(zip(x0,y0,z0)))
lons_out_1d = manager_out.lons2D.flatten()
lats_out_1d = manager_out.lats2D.flatten()
x1, y1, z1 = lat_lon.lon_lat_to_cartesian(lons_out_1d, lats_out_1d)
dd, ii = kdtree.query(list(zip(x1, y1, z1)), k=nneighbours)
print(ii.shape)
swe_in_1d = swe_in.flatten()
return np.mean(swe_in_1d[ii], axis=1).reshape(manager_out.lons2D.shape)
pass
def _init_kd_tree(self):
"""
Has to be called after self._init_date_to_path_dict
"""
if not len(self.date_to_path):
print("You should call {0} first".format("self._init_date_to_path_dict"))
raise Exception()
for d, path in self.date_to_path.items():
ds = Dataset(path)
lons1d = ds.variables["g0_lon_1"][:]
lats1d = ds.variables["g0_lat_0"][:]
self.lats2d, self.lons2d = np.meshgrid(lats1d, lons1d)
x, y, z = lat_lon.lon_lat_to_cartesian(self.lons2d.flatten(), self.lats2d.flatten())
self.kdtree = KDTree(list(zip(x, y, z)))
return
pass
def __init__(self, ax, basemap, lons2d, lats2d, ncVarDict, times, start_date, end_date):
"""
Plots a vertical profile at the point nearest to the clicked one
:type ax: Axes
"""
assert isinstance(ax, Axes)
self.basemap = basemap
assert isinstance(self.basemap, Basemap)
self.lons_flat = lons2d.flatten()
self.lats_flat = lats2d.flatten()
self.ncVarDict = ncVarDict
self.lons2d = lons2d
self.lats2d = lats2d
self.counter = 0
self.ax = ax
x, y, z = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten())
self.kdtree = KDTree(list(zip(x,y,z)))
self.sel_time_indices = np.where([start_date <= t <= end_date for t in times])[0]
self.times = times[self.sel_time_indices]
ax.figure.canvas.mpl_connect("button_press_event", self)
def __init__(self, ax , basemap, tmin_3d, tmax_3d, lons2d, lats2d, levelheights = None):
"""
Plots a vertical profile at the point nearest to the clicked one
:type ax: Axes
"""
assert isinstance(ax, Axes)
self.basemap = basemap
assert isinstance(self.basemap, Basemap)
self.tmin_3d = tmin_3d
self.tmax_3d = tmax_3d
self.lons2d = lons2d
self.lats2d = lats2d
self.T0 = 273.15
self.lons_flat = lons2d.flatten()
self.lats_flat = lats2d.flatten()
self.level_heights = levelheights
self.counter = 0
self.ax = ax
x, y, z = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten())
self.kdtree = KDTree(list(zip(x,y,z)))
ax.figure.canvas.mpl_connect("button_press_event", self)
pass
class TimeseriesPlotter:
def __init__(self, name_to_date_to_field, basemap, lons2d, lats2d,
ax = None, cell_area = None, cell_manager = None, data_manager = None):
self.gwdi_mean_field = None
self.traf_mean_field = None
self.tdra_mean_field = None
self.upin_mean_field = None
self.basemap = basemap
self.date_to_stfl_field = name_to_date_to_field["STFL"]
self.date_to_traf_field = name_to_date_to_field["TRAF"]
self.date_to_tdra_field = name_to_date_to_field["TDRA"]
self.date_to_pr_field = name_to_date_to_field["PR"]
self.date_to_swe_field = name_to_date_to_field["I5"]
self.date_to_swst_field = name_to_date_to_field["SWST"]
#self.date_to_imav_field = name_to_date_to_field["IMAV"]
self.acc_area_km2 = name_to_date_to_field["FACC"]
#:type : CellManager
self.cell_manager = cell_manager
assert isinstance(self.cell_manager, CellManager)
self.cell_area = cell_area
x, y, z = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten())
self.kdtree = KDTree(list(zip(x,y,z)))
ax.figure.canvas.mpl_connect("button_press_event", self)
self.ax = ax
self.lons2d = lons2d
self.lats2d = lats2d
self.data_manager = data_manager
assert isinstance(self.data_manager, Crcm5ModelDataManager)
self.x_pr, self.y_pr = basemap(lons2d, lats2d)
self.lons_flat = lons2d.flatten()
self.lats_flat = lats2d.flatten()
self.dates_sorted = list( sorted(list(name_to_date_to_field.items())[0][1].keys()) )
self.counter = 0
self.date_to_swsr_field = name_to_date_to_field["SWSR"]
self.date_to_swsl_field = name_to_date_to_field["SWSL"]
#self.date_to_gwdi_field = name_to_date_to_field["GWDI"]
self.date_to_upin_field = name_to_date_to_field["UPIN"]
#static fields
self.slope = name_to_date_to_field["SLOP"]
self.channel_length = name_to_date_to_field["LENG"]
self.lake_outlet = name_to_date_to_field["LKOU"]
self.coef_bf = -np.ones(self.slope.shape)
good_points = self.slope >= 0
self.coef_bf[good_points] = (self.slope[good_points]) ** 0.5 / ((self.channel_length[good_points]) ** (4.0/3.0) * data_manager.manning_bf[good_points] )
def _get_closest_ij(self, event):
lon, lat = self.basemap(event.xdata, event.ydata, inverse = True)
x0, y0, z0 = lat_lon.lon_lat_to_cartesian(lon, lat)
dist, i = self.kdtree.query((x0,y0,z0))
lon0, lat0 = self.lons_flat[i], self.lats_flat[i]
ind = np.where((self.lons2d == lon0) & (self.lats2d == lat0))
ix = ind[0][0]
jy = ind[1][0]
return ix, jy
def __call__(self,event):
if event.button != 3:
return
i,j = self._get_closest_ij( event )
vals = [
self.date_to_stfl_field[d][i,j] for d in self.dates_sorted
]
plt.figure()
plt.plot(self.dates_sorted, vals, label = "STFL")
mask = self.cell_manager.get_mask_of_cells_connected_with(self.cell_manager.cells[i][j])
print("sum(mask) = ", np.sum(mask))
vals1 = [
np.sum( self.date_to_traf_field[d][mask == 1] ) for d in self.dates_sorted
]
plt.plot(self.dates_sorted, vals1, label = "TRAF")
vals2 = [
np.sum( self.date_to_tdra_field[d][mask == 1] ) for d in self.dates_sorted
]
plt.plot(self.dates_sorted, vals2, label = "TDRA")
vals3 = [
np.sum( self.date_to_pr_field[d][mask == 1] ) for d in self.dates_sorted
]
plt.plot(self.dates_sorted, vals3, label = "PR")
#vals4 = [
# np.sum( self.date_to_gwdi_field[d][mask == 1] ) for d in self.dates_sorted
#]
#plt.plot(self.dates_sorted, vals4, label = "GWDI")
vals5 = [
np.sum( self.date_to_upin_field[d][i,j] ) for d in self.dates_sorted
]
plt.plot(self.dates_sorted, vals5, label = "UPIN")
if self.upin_mean_field is None:
self.upin_mean_field = np.mean(list(self.date_to_upin_field.values()), axis = 0)
plt.legend()
plt.title("{0}: acc={1} km**2".format(self.counter, self.acc_area_km2[i, j]))
# plt.figure()
# ax1 = plt.gca()
# to_plot_2d = np.ma.masked_where(mask < 0.5, self.upin_mean_field)
# img = self.basemap.pcolormesh(self.x_pr, self.y_pr, to_plot_2d, ax = ax1)
# plt.colorbar(img, ax = ax1)
# self.basemap.drawcoastlines(ax = ax1)
# plt.title("min-max: {0};{1}".format(to_plot_2d.min(), to_plot_2d.max()))
#
# self.ax.annotate(str(self.counter), (event.xdata, event.ydata), font_properties =
# FontProperties(size=10), bbox=dict(boxstyle="round", fc="w"))
# self.ax.redraw_in_frame()
#
# plt.figure()
# ax1 = plt.gca()
# to_plot_2d = np.ma.masked_where(mask < 0.5, self.data_manager.cbf)
# img = self.basemap.pcolormesh(self.x_pr, self.y_pr, to_plot_2d, ax = ax1)
# plt.colorbar(img, ax = ax1)
# self.basemap.drawcoastlines(ax = ax1)
#
#
# plt.title("CBF, {0:g}: v= {1}, min={2}, max={3}".format(self.counter, to_plot_2d[i,j], to_plot_2d.min(), to_plot_2d.max()))
#
#
# plt.figure()
# ax1 = plt.gca()
# to_plot_2d = np.ma.masked_where(mask < 0.5, self.data_manager.bankfull_storage_m3)
# img = self.basemap.pcolormesh(self.x_pr, self.y_pr, to_plot_2d, ax = ax1)
# plt.colorbar(img, ax = ax1)
# self.basemap.drawcoastlines(ax = ax1)
# plt.title("STBM, {0}: v= {1}".format(self.counter, to_plot_2d[i,j]))
#
#
# plt.figure()
# ax1 = plt.gca()
# mbf = self.data_manager.manning_bf
# to_plot_2d = np.ma.masked_where(mask < 0.5, mbf)
# img = self.basemap.pcolormesh(self.x_pr, self.y_pr, to_plot_2d, ax = ax1)
# plt.colorbar(img, ax = ax1)
# self.basemap.drawcoastlines(ax = ax1)
# plt.title("MABF, {0}: v= {1}, min={2}, max={3}".format(self.counter, to_plot_2d[i,j], to_plot_2d.min(), to_plot_2d.max()))
#
# plt.figure()
# ax1 = plt.gca()
# to_plot_2d = np.ma.masked_where(mask < 0.5, self.slope)
# img = self.basemap.pcolormesh(self.x_pr, self.y_pr, to_plot_2d, ax = ax1)
# plt.colorbar(img, ax = ax1)
# self.basemap.drawcoastlines(ax = ax1)
# plt.title("SLOPe, {0}: v= {1}, min={2}, max={3}".format(self.counter, to_plot_2d[i,j], to_plot_2d.min(), to_plot_2d.max()))
#
#
#
#
# plt.figure()
# ax1 = plt.gca()
# to_plot_2d = np.ma.masked_where(mask < 0.5, self.data_manager.lake_area)
# img = self.basemap.pcolormesh(self.x_pr, self.y_pr, to_plot_2d, ax = ax1)
# plt.colorbar(img, ax = ax1)
# self.basemap.drawcoastlines(ax = ax1)
# plt.title("lake area, {0}: v= {1}".format(self.counter, to_plot_2d[i,j]))
#
# plt.figure()
# ax1 = plt.gca()
# to_plot_2d = np.ma.masked_where(mask < 0.5, self.coef_bf)
# img = self.basemap.pcolormesh(self.x_pr, self.y_pr, to_plot_2d, ax = ax1)
# plt.colorbar(img, ax = ax1)
# self.basemap.drawcoastlines(ax = ax1)
# plt.title("coef_bf, {0}: v= {1:.1g}, min={2:.1g}, max={3:.1g}".format(self.counter, to_plot_2d[i,j], to_plot_2d.min(), to_plot_2d.max()))
#
plt.figure()
#snow
vals6 = [
np.sum( self.date_to_swe_field[d][mask == 1] ) for d in self.dates_sorted
]
plt.plot(self.dates_sorted, vals6, label = "SWE")
vals4 = [
np.sum( self.date_to_swe_field[d][i,j] ) for d in self.dates_sorted
]
plt.plot(self.dates_sorted, vals4, label = "GWST")
vals5 = [
np.sum( self.date_to_swsr_field[d][i,j] ) for d in self.dates_sorted
]
plt.plot(self.dates_sorted, vals5, label = "SWSR")
vals5 = [
np.sum( self.date_to_swsl_field[d][i,j] ) for d in self.dates_sorted
]
plt.plot(self.dates_sorted, vals5, label = "SWSL")
plt.legend()
plt.title("{0}, lkfr = {1}".format(self.counter, self.data_manager.lake_fraction[i,j]))
fName = "route_params_{0}_{1}.bin".format(i, j)
info = {}
#traf -> dict( date -> value in m**3/s )
traf_dict = dict(list(zip(self.dates_sorted,
[self.date_to_traf_field[d][i,j] for d in self.dates_sorted])) )
traf_dict = {"TRAF": traf_dict}
info.update(traf_dict)
upin_dict = dict(list(zip(self.dates_sorted,
[self.date_to_upin_field[d][i,j] for d in self.dates_sorted])) )
upin_dict = {"UPIN": upin_dict}
info.update(upin_dict)
# gwdi_dict = dict(zip(self.dates_sorted,
# [self.date_to_gwdi_field[d][i,j] for d in self.dates_sorted]) )
# gwdi_dict = {"GWDI": gwdi_dict}
# info.update(gwdi_dict)
swsr_dict = dict(list(zip(self.dates_sorted,
[self.date_to_swsr_field[d][i,j] for d in self.dates_sorted])) )
swsr_dict = {"SWSR": swsr_dict }
info.update(swsr_dict)
swsl_dict = dict(list(zip(self.dates_sorted,
[self.date_to_swsl_field[d][i,j] for d in self.dates_sorted])) )
swsl_dict = {"SWSL": swsl_dict }
info.update(swsl_dict)
stfl_dict = dict(list(zip(self.dates_sorted,
[self.date_to_stfl_field[d][i,j] for d in self.dates_sorted])) )
stfl_dict = {"STFL": stfl_dict }
info.update(stfl_dict)
swst_dict = dict(list(zip(self.dates_sorted,
[self.date_to_swst_field[d][i,j] for d in self.dates_sorted])) )
swst_dict = {"SWST": swst_dict }
info.update(swst_dict)
info["SBFM"] = self.data_manager.bankfull_storage_m3[i,j]
info["CBF"] = self.data_manager.cbf[i,j]
info["LKFR"] = self.data_manager.lake_fraction[i,j]
info["LKAR"] = self.data_manager.lake_area[i,j]
info["LKOU"] = self.lake_outlet[i,j]
pickle.dump(info, open(fName, mode="w"))
self.counter += 1
plt.show()
pass
def interpolate_to_amno(data_folder, start_year=1970, end_year=1999, rcm="", gcm="", out_folder=""):
print "data_folder: {0}".format(data_folder)
#check if the result file already exists
sim_folder = os.path.join(out_folder, "{0}-{1}_{2}-{3}".format(gcm, rcm, start_year, end_year))
#create a folder for each simulation
if not os.path.isdir(sim_folder):
os.mkdir(sim_folder)
out_path = os.path.join(sim_folder, "narccap_runoff_{0}-{1}_{2}-{3}.nc".format(start_year, end_year, gcm, rcm))
if os.path.isfile(out_path):
print("{0} already exists, remove if you want to recreate.".format(out_path))
return
srof_pattern = os.path.join(data_folder, "mrros_*_*_*.nc")
trof_pattern = os.path.join(data_folder, "mrro_*_*_*.nc")
srof_ds = MFDataset(srof_pattern)
trof_ds = MFDataset(trof_pattern)
lon_in = srof_ds.variables["lon"][:]
lat_in = srof_ds.variables["lat"][:]
x_in, y_in, z_in = lat_lon.lon_lat_to_cartesian(lon_in.flatten(), lat_in.flatten())
tree = KDTree(zip(x_in, y_in, z_in))
lon_out, lat_out = polar_stereographic.lons.flatten(), polar_stereographic.lats.flatten()
x_out, y_out, z_out = lat_lon.lon_lat_to_cartesian(lon_out, lat_out)
distances, indices = tree.query(zip(x_out, y_out, z_out))
time_var = srof_ds.variables["time"]
time_in_units = time_var[:]
times = num2date(time_in_units, time_var.units)
time_indices = np.where(
np.array(map(lambda x: start_year <= x.year <= end_year, times), dtype=np.bool)
)[0]
srof_sub = srof_ds.variables["mrros"][time_indices, :, :]
trof_sub = trof_ds.variables["mrro"][time_indices, :, :]
times_sub = itertools.ifilter(lambda x: start_year <= x.year <= end_year, times)
print("selected time window data")
#writing result to netcdf
out_nc = Dataset(out_path, "w")
out_nc.createDimension("x", polar_stereographic.lons.shape[0])
out_nc.createDimension("y", polar_stereographic.lats.shape[1])
out_nc.createDimension("time")
srof_var = out_nc.createVariable("mrros", "f4", dimensions=("time", "x", "y"))
trof_var = out_nc.createVariable("mrro", "f4", dimensions=("time", "x", "y"))
assert isinstance(srof_var, Variable)
srof_in_var = srof_ds.variables["mrros"]
for attr_name in srof_in_var.ncattrs():
print attr_name
srof_var.setncattr(attr_name, getattr(srof_in_var, attr_name))
trof_in_var = trof_ds.variables["mrro"]
for attr_name in trof_in_var.ncattrs():
print attr_name
trof_var.setncattr(attr_name, getattr(trof_in_var, attr_name))
t_var = out_nc.createVariable("time", "f4", dimensions=("time",))
lon_var = out_nc.createVariable("longitude", "f4", dimensions=( "x", "y"))
lat_var = out_nc.createVariable("latitude", "f4", dimensions=("x", "y"))
t_var.units = time_var.units
print("interpolating and saving data to netcdf file")
nrows, ncols = polar_stereographic.lons.shape
#interpolate in time if necessary
n_interps = 0
for i, t in enumerate(times_sub):
sr_slice = srof_sub[i, :, :].flatten()
tr_slice = trof_sub[i, :, :].flatten()
trof1 = tr_slice[indices].reshape(nrows, ncols)
srof1 = sr_slice[indices].reshape(nrows, ncols)
if hasattr(trof1, "mask") and np.all(trof1.mask):
trof1 = trof_var[i - 1, :, :]
n_interps += 1
if hasattr(srof1, "mask") and np.all(srof1.mask):
srof1 = srof_var[i - 1, :, :]
trof_var[i, :, :] = trof1
srof_var[i, :, :] = srof1
t_var[i] = date2num(t, time_var.units)
print "Number of interpolations in time: {0}".format(n_interps)
lon_var[:] = polar_stereographic.lons
lat_var[:] = polar_stereographic.lats
out_nc.close()
class GrdcDataManager:
def __init__(self, path_tofolder = "/home/huziy/skynet3_exec1/grdc_global"):
self.path_to_annual_rof = os.path.join(path_tofolder, "obs_ro.grd")
self.lons2d = None
self.lats2d = None
self.ncols = None
self.nrows = None
self.xll = None
self.yll = None
self.cellsize = None
self.nodata_value = None
self.ktree = None
pass
def _get_lon_lats(self):
if self.lons2d is None:
lons = [ self.xll + i * self.cellsize for i in range(self.ncols) ]
lats = [ self.yll + i * self.cellsize for i in range(self.nrows) ]
self.lats2d, self.lons2d = np.meshgrid(lats, lons)
return self.lons2d, self.lats2d
def interpolate_data_to_model_grid(self, model_lons_2d, model_lats_2d, data_obs):
x0, y0, z0 = lat_lon.lon_lat_to_cartesian(model_lons_2d.flatten(), model_lats_2d.flatten())
x, y, z = lat_lon.lon_lat_to_cartesian(self.lons2d.flatten(), self.lats2d.flatten())
if self.ktree is None:
self.ktree = KDTree(list(zip(x, y, z)))
d, i = self.ktree.query(list(zip(x0, y0, z0)))
return data_obs.flatten()[i].reshape(model_lons_2d.shape)
def _read_data_from_file(self, path):
f = open(path)
vals = []
for line in f:
line = line.strip()
if line == "": continue
if line.startswith("ncols"):
self.ncols = int(line.split()[1].strip())
elif line.startswith("nrows"):
self.nrows = int(line.split()[1].strip())
elif line.startswith("xllcorner"):
self.xll = float(line.split()[1].strip())
elif line.startswith("yllcorner"):
self.yll = float(line.split()[1].strip())
elif line.startswith("cellsize"):
self.cellsize = float(line.split()[1].strip())
elif line.startswith("NODATA"):
self.nodata_value = int(line.split()[1].strip())
else:
vals.append( list(map(float, [s.strip() for s in line.split()])) )
#print len(vals), self.ncols * self.nrows
vals = np.array( vals[::-1] )#reverse row order
vals = vals.transpose()
return vals
def get_mean_annual_runoff_in_mm_per_s(self):
vals = self._read_data_from_file(self.path_to_annual_rof)
#vals = np.ma.masked_where(vals.astype(int) == self.nodata_value, vals)
vals = np.ma.masked_where(vals < 0, vals)
print(self.nodata_value, np.min(vals), self.nodata_value == np.min(vals))
vals /= 365 * 24 * 60 * 60 #convert to mm/s
self._get_lon_lats()
return self.lons2d, self.lats2d, vals
def compare_alt():
#obs
stations = get_station_list()
#model data
alts_model = [ active_layer_thickness.get_alt_for_year(the_year)
for the_year in range(1991, 2001)
]
alt_mean = np.mean(alts_model, axis=0)
b, lons2d, lats2d = draw_regions.get_basemap_and_coords()
permafrost_kinds = draw_regions.get_permafrost_mask(lons2d, lats2d)
permafrost_kinds_flat = permafrost_kinds.flatten()
lons2d[lons2d > 180] -= 360
#find corresponding indices on the model grid
x, y, z = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten())
kdtree = KDTree(list(zip(x, y, z)))
alt_mean_flat = alt_mean.flatten()
h_mod = []
h_obs = []
station_lons = []
station_lats = []
for the_station in stations:
x0, y0, z0 = lat_lon.lon_lat_to_cartesian(the_station.lon, the_station.lat)
d, i = kdtree.query([x0, y0, z0])
if permafrost_kinds_flat[i] not in (1,2):
continue
print(d, i)
print(the_station.mean_alt_m, alt_mean_flat[i])
h_mod.append(alt_mean_flat[i])
h_obs.append(the_station.mean_alt_m)
station_lons.append(the_station.lon)
station_lats.append(the_station.lat)
plot_utils.apply_plot_params(width_pt=None, height_cm=20, width_cm=16, font_size=12)
fig = plt.figure()
gs = gridspec.GridSpec(2,1)
ax = fig.add_subplot(gs[0,0])
ax.plot(h_obs, h_mod, 'o')
ax.set_xlabel("Obs.")
ax.set_ylabel("Mod.")
upper_lim = max(np.max(h_mod), np.max(h_obs))
ax.set_xlim(0, upper_lim + 0.1 * upper_lim)
ax.set_ylim(0, upper_lim + 0.1 * upper_lim)
ax = fig.add_subplot(gs[1,0])
min_lon, max_lon = min(station_lons), max(station_lons)
min_lat, max_lat = min(station_lats), max(station_lats)
dx = (max_lon - min_lon) * 0.1
dy = (max_lat - min_lat) * 0.6
min_lon -= dx
max_lon += dx
min_lat -= dy
max_lat += dy
lon1 = -97
lat1 = 47.50
lon2 = -7
lat2 = 0
b_zoom = Basemap(projection="omerc", resolution="l",
llcrnrlon=min_lon, llcrnrlat=min_lat,
urcrnrlon=max_lon, urcrnrlat=max_lat,
lat_1=lat1, lon_1=lon1, lat_2=lat2, lon_2=lon2, no_rot=True
)
s_x, s_y = b_zoom(station_lons, station_lats)
b_zoom.scatter(s_x, s_y, c = "r", ax = ax, marker = "*", s = 30, linewidths = 0.1, zorder = 2)
b_zoom.drawcoastlines(ax = ax, linewidth = 0.5)
fig.savefig("pf_validate.png")
pass
def plot_current_alts():
from . import plot_dpth_to_bdrck
bdrck_field = plot_dpth_to_bdrck.get_depth_to_bedrock()
start_year = 1980
end_year = 1996
sim_data_folder = "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/era40_driven_b1"
coord_file = os.path.join(sim_data_folder, "pmNorthAmerica_0.44deg_ERA40-Int_B1_200812_moyenne")
#coord_file = "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/NA_1.0deg_soil_spinup2/pmNA_1.0deg_soil_spinup2_228006_moyenne"
sim_names = ["ERA40", "MPI","CanESM"]
simname_to_path = {
#"ERA40": "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/era40_driven_b1",
"ERA40": "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/NorthAmerica_0.44deg_ERA40-Int_old_snow_cond",
#"ERA40" : "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/NA_1.0deg_soil_spinup2",
"MPI": "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/NorthAmerica_0.44deg_MPI_B1",
"CanESM": "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/NorthAmerica_0.44deg_CanESM_B1"
}
basemap, lons2d, lats2d = draw_regions.get_basemap_and_coords(resolution="c",
file_path = coord_file, llcrnrlat=45.0, llcrnrlon=-145, urcrnrlon=-20, urcrnrlat=74,
anchor="W"
)
assert isinstance(basemap, Basemap)
#basemap.transform_scalar()
#basemap = Basemap()
lons2d[lons2d > 180] -= 360
x, y = basemap(lons2d, lats2d)
#x = (x[1:,1:] + x[:-1, :-1]) /2.0
permafrost_mask = draw_regions.get_permafrost_mask(lons2d, lats2d)
mask_cond = (permafrost_mask <= 0) | (permafrost_mask >= 2)
# plot_utils.apply_plot_params(width_pt=None, width_cm=20, height_cm=40, font_size=16)
fig = plt.figure()
assert isinstance(fig, Figure)
h_max = 10
bounds = [0,0.1,0.5,1,2,3,4,5]
cmap = my_colormaps.get_lighter_jet_cmap(ncolors=len(bounds) - 1) #cm.get_cmap("jet",10)
#cmap = my_colormaps.get_cmap_wo_red(ncolors=len(bounds) - 1)
norm = BoundaryNorm(boundaries=bounds,ncolors=len(bounds), clip=True)
cmap.set_over(cmap(1.0))
clevels = np.arange(0,h_max+1,1)
gs = gridspec.GridSpec(3,2, width_ratios=[1,0.06], hspace=0, wspace=0.0,
left=0.05, bottom = 0.02, top=0.95)
all_axes = []
all_img = []
i = 0
hc_list = []
for name in sim_names:
path = simname_to_path[name]
dm = CRCMDataManager(data_folder=path)
hc0, t3d_min, t3d_max = dm.get_alt_using_monthly_mean_climatology(list(range(start_year,end_year+1)))
hc_list.append(hc0)
ax = fig.add_subplot(gs[i,0])
#cp = SoundingPlotter(ax, basemap, t3d_min, t3d_max, lons2d, lats2d, levelheights=dm.level_heights)
assert isinstance(ax, Axes)
hc = np.ma.masked_where(mask_cond | (hc0 < 0), hc0)
#hc = np.ma.masked_where( (hc0 < 0), hc0)
hc5 = np.ma.masked_where((hc0 <= 15) | hc.mask, hc)
img = basemap.pcolormesh(x, y, hc, cmap = cmap, vmax = h_max, norm=norm)
if not i:
ax.set_title("ALT ({0} - {1}) \n".format(start_year, end_year))
i += 1
ax.set_ylabel("CRCM ({0})".format(name))
all_axes.append(ax)
all_img.append(img)
print(np.ma.min(hc), np.ma.max(hc))
#hc5 = np.ma.masked_where((hc0 <= 6) | hc.mask, hc)
#print "Number of cells with alt > 5 is {0}, and the range is {1} ... {2}".format(hc5.count(), hc5.min(), hc5.max())
#bdrck_field5 = np.ma.masked_where(hc5.mask, bdrck_field)
#print "Bedrock ranges for those points: {0} ... {1}".format(bdrck_field5.min(), bdrck_field5.max())
ind = np.where(~hc5.mask)
xs = ind[0]
ys = ind[1]
all_months, all_temps = dm.get_monthly_mean_soiltemps(year_range=range(start_year,end_year+1))
all_months_ord = date2num(all_months)
# mpl.rcParams['contour.negative_linestyle'] = 'solid'
#
# for the_i, the_j in zip(xs,ys):
# #plot profile
# plt.figure()
# plt.plot(t3d_max[the_i, the_j, :] - dm.T0, dm.level_heights, color = "r")
# plt.plot(t3d_min[the_i, the_j, :] - dm.T0, dm.level_heights, color = "b")
# plt.plot([0 , 0], [dm.level_heights[0], dm.level_heights[-1]], color = "k")
#
# x1, x2 = plt.xlim()
# #plt.plot( [x1, x2], [bdrck_field[the_i, the_j], bdrck_field[the_i, the_j]], color = "k", lw = 3 )
# #plt.title(str(i) + ", dpth_to_bedrock = {0} m".format(bdrck_field[the_i, the_j]))
# plt.title(str(i))
# plt.gca().invert_yaxis()
# plt.savefig("prof{0}.png".format(i))
# ax.annotate(str(i), (x[the_i, the_j], y[the_i, the_j]), font_properties =
# FontProperties(size=10))
#
#
# #plot vertical temp cross-section
# plt.figure()
# plt.title(str(i) + ", ({0} - {1})".format(start_year, end_year))
#
# levs2d, times2d = np.meshgrid(dm.level_heights, all_months_ord)
# clevs = [-25,-20,-10,-5,-1,0,1,5,10,20,25]
# norm = BoundaryNorm(boundaries=clevs, ncolors=len(clevs) - 1)
# cmap = cm.get_cmap("jet", len(clevs) - 1)
#
# img = plt.contourf(times2d, levs2d, all_temps[:,the_i, the_j, :] - dm.T0, levels = clevs, cmap = cmap, norm = norm)
# #plt.contour(times2d, levs2d, all_temps[:,the_i, the_j, :] - dm.T0, levels = clevs, colors = "k", linewidth = 1)
# the_ax = plt.gca()
# assert isinstance(the_ax, Axes)
# the_ax.invert_yaxis()
# the_ax.xaxis.set_major_formatter(FuncFormatter(
# lambda x, pos: num2date(float(x)).strftime("%Y")
# ))
#
# print "i = {0}; lon, lat = {1}, {2}".format(i, lons2d[the_i, the_j], lats2d[the_i, the_j])
#
# plt.colorbar(img, ticks = clevs)
#
# plt.savefig("temp_section_{0}.png".format(i))
#
# i += 1
print("lons = [{0}]".format(",".join([str(x) for x in lons2d[np.array(xs), np.array(ys)]])))
print("lats = [{0}]".format(",".join([str(x) for x in lats2d[np.array(xs), np.array(ys)]])))
# draw barplot with numbers of alt in given ranges
# plt.figure()
# alt_ranges = xrange(0,18)
# numbers = []
# for the_alt in alt_ranges:
# hci = np.ma.masked_where( (hc0 < the_alt) | (hc0 > the_alt + 1) | hc.mask ,hc)
# numbers.append(hci.count())
# plt.bar(alt_ranges, numbers, width=1)
# plt.xlabel("ALT (m)")
# plt.ylabel("Number of cells")
# plt.savefig("numbers_in_range.png")
i = 0
axs_to_hide = []
#zones and coastlines
for the_ax, the_img in zip(all_axes, all_img):
# divider = make_axes_locatable(the_ax)
# cax = divider.append_axes("right", "5%", pad="3%")
assert isinstance(the_ax, Axes)
basemap.drawcoastlines(ax = the_ax, linewidth=0.5)
basemap.readshapefile("data/pf_4/permafrost8_wgs84/permaice", name="zone",
ax=the_ax, linewidth=1.5, drawbounds=False)
for nshape,seg in enumerate(basemap.zone):
if basemap.zone_info[nshape]["EXTENT"] not in ["C"]: continue
poly = mpl.patches.Polygon(seg,edgecolor = "k", facecolor="none", zorder = 10, lw = 1.5)
the_ax.add_patch(poly)
# if i != 1:
# axs_to_hide.append(cax)
i += 1
cax = fig.add_subplot(gs[:,1])
cax.set_anchor("W")
cax.set_aspect(35)
formatter = FuncFormatter(
lambda x, pos: "{0: <6}".format(x)
)
cb = fig.colorbar(all_img[0], ax = cax, cax = cax, extend = "max", ticks = bounds, format = formatter)
cax.set_title("m")
#fig.tight_layout(h_pad=0)
# for the_ax in axs_to_hide:
# the_ax.set_visible(False)
fig.savefig("alt_from_climatology_current.png")
#print ALT for selected points
site_names = ["S","K","T"]
sel_lons = [-75.646, -65.92, -69.95]
sel_lats = [62.197, 58.709, 58.67]
xo,yo,zo = lat_lon.lon_lat_to_cartesian(sel_lons, sel_lats)
xi, yi, zi = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten())
ktree = KDTree(list(zip(xi,yi,zi)))
dists, indexes = ktree.query(list(zip(xo,yo,zo)))
for name, data in zip(sim_names, hc_list):
print(name)
flat_data = data.flatten()
for p_name, ind in zip(site_names, indexes):
print(p_name, "{0} m".format(flat_data[ind]))
print("--" * 10)
pass
def plot_current_alts_nyear_rule(nyear = 2):
start_year = 1981
end_year = 2008
sim_data_folder = "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/era40_driven_b1"
sim_names = ["ERA40", "MPI","CanESM"]
all_data_f = "/home/huziy/skynet1_rech3/cordex/for_Samira"
simname_to_path = {
"ERA40": os.path.join(all_data_f, "alt_era_b1_yearly.nc"),
"MPI": os.path.join(all_data_f, "alt_mpi_b1_yearly.nc"),
"CanESM": os.path.join(all_data_f, "alt_canesm_b1_yearly.nc")
}
coord_file = os.path.join(sim_data_folder, "pmNorthAmerica_0.44deg_ERA40-Int_B1_200812_moyenne")
basemap, lons2d, lats2d = draw_regions.get_basemap_and_coords(resolution="c",
file_path = coord_file, llcrnrlat=40.0, llcrnrlon=-145, urcrnrlon=-20, urcrnrlat=74
)
assert isinstance(basemap, Basemap)
#basemap.transform_scalar()
#basemap = Basemap()
lons2d[lons2d > 180] -= 360
x, y = basemap(lons2d, lats2d)
#x = (x[1:,1:] + x[:-1, :-1]) /2.0
permafrost_mask = draw_regions.get_permafrost_mask(lons2d, lats2d)
mask_cond = (permafrost_mask <= 0) | (permafrost_mask >= 3)
# plot_utils.apply_plot_params(width_pt=None, width_cm=20, height_cm=40, font_size=25)
fig = plt.figure()
assert isinstance(fig, Figure)
h_max = 10
cmap = my_colormaps.get_lighter_jet_cmap(ncolors=10) #cm.get_cmap("jet",10)
bounds = [0,0.1,0.5,1,2,3,5,8,9,10,11]
norm = BoundaryNorm(boundaries=bounds,ncolors=len(bounds), clip=True)
cmap.set_over(cmap(1.0))
clevels = np.arange(0,h_max+1,1)
gs = gridspec.GridSpec(3,1)
all_axes = []
all_img = []
i = 0
hc_list = []
hct_list = []
for name in sim_names:
path = simname_to_path[name]
#select data and needed alt
ds = Dataset(path)
years = ds.variables["year"][:]
hct = ds.variables["alt"][(years >= start_year) & (years <= end_year),:,:]
hct_list.append(hct)
print("hct.shape = ", hct.shape)
#hc = get_alt_using_nyear_rule(hct, nyears = nyear)
hc = np.mean(hct, axis = 0)
hc_list.append(hc)
ax = fig.add_subplot(gs[i,0])
assert isinstance(ax, Axes)
hc = np.ma.masked_where(mask_cond | (np.min(hct, axis = 0) < 0), hc)
#hc = np.ma.masked_where( (hc < 0), hc)
img = basemap.pcolormesh(x, y, hc, cmap = cmap, vmax = h_max, norm=norm)
if not i:
ax.set_title("ALT, mean ({0} - {1}) \n".format(start_year, end_year))
i += 1
ax.set_ylabel(name)
all_axes.append(ax)
all_img.append(img)
i = 0
axs_to_hide = []
#zones and coastlines
for the_ax, the_img in zip(all_axes, all_img):
assert isinstance(the_ax, Axes)
basemap.drawcoastlines(ax = the_ax, linewidth=0.5)
basemap.readshapefile("data/pf_4/permafrost8_wgs84/permaice", name="zone",
ax=the_ax, linewidth=1.5)
divider = make_axes_locatable(the_ax)
cax = divider.append_axes("right", "5%", pad="3%")
cb = fig.colorbar(the_img, cax = cax, extend = "max", ticks = bounds)
cax.set_title("m \n")
if i != 2:
axs_to_hide.append(cax)
i += 1
fig.tight_layout(w_pad=0.0)
for the_ax in axs_to_hide:
the_ax.set_visible(False)
fig.savefig("alt_mean_current.png")
#print ALT for selected points
site_names = ["S","K","T"]
sel_lons = [-75.646, -65.92, -69.95]
sel_lats = [62.197, 58.709, 58.67]
xo,yo,zo = lat_lon.lon_lat_to_cartesian(sel_lons, sel_lats)
xi, yi, zi = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten())
ktree = KDTree(list(zip(xi,yi,zi)))
dists, indexes = ktree.query(list(zip(xo,yo,zo)))
for name, data, the_hct in zip(sim_names, hc_list, hct_list):
print(name)
flat_data = data.flatten()
for p_name, ind in zip(site_names, indexes):
in_data = []
for t in range(the_hct.shape[0]):
in_data.append(the_hct[t,:,:].flatten()[ind])
print(",".join(["{0:.1f}".format(float(x)) for x in in_data]))
print(p_name, "{0:.1f} m".format(float(flat_data[ind])))
print("--" * 10)
class RealWorldMap(object):
def __init__(self, patches, projections):
self.patches = patches
self.projections = projections
self.build()
def build(self):
logger.debug("Building maps for conversion between real and image space")
self.imagetree = {}
self.imagemap = {}
self.realtree = KDTree([x.realcoords for x in self.patches])
self.realmapping = {}
for num, patch in enumerate(self.patches):
if num % 1000 == 0:
logger.debug("Built maps for {0} of {1} patches".format(num, len(self.patches)))
resp = {}
for _, projection in self.projections.items():
inimage = patch.project(projection)
if projection.id not in self.imagetree:
self.imagetree[projection.id] = []
self.imagemap[projection.id] = {}
self.imagetree[projection.id].append(inimage)
self.imagemap[projection.id][tuple(inimage)] = patch.realcoords
resp[projection.id] = inimage
self.realmapping[tuple(patch.realcoords)] = resp
logger.debug("Done building maps for {0} patches".format(len(self.patches)))
logger.debug("Building image KD tree")
for key, imagecoords in self.imagetree.items():
self.imagetree[key] = KDTree(imagecoords)
def realtoimages(self, coords):
_, nearestindex = self.realtree.query(coords)
nearest = self.realtree.data[nearestindex]
return self.realmapping[tuple(nearest)]
def realregiontoimages(self, coords):
_, nearestindices = self.realtree.query(coords)
resp = {}
for nearestindex in nearestindices:
nearest = self.realtree.data[nearestindex]
points = self.realmapping[tuple(nearest)]
for k, v in points.iteritems():
if k not in resp:
resp[k] = []
resp[k].append(v)
return resp
def imagetoreal(self, projection, coords):
try:
projection = projection.id
except:
pass
_, nearestindex = self.imagetree[projection].query(coords)
nearest = self.imagetree[projection].data[nearestindex]
return self.imagemap[projection][tuple(nearest)]
def convert(inPath, lonlats):
ds = gdal.Open(inPath, gdal.GA_ReadOnly)
assert isinstance(ds, Dataset)
(Xul, deltaX, rotation, Yul, rotation, deltaY) = ds.GetGeoTransform()
print(dir(ds))
print(ds.GetMetadata_Dict())
print(ds.GetDescription())
srs_wkt = ds.GetProjection()
Nx = ds.RasterXSize
Ny = ds.RasterYSize
print(ds.RasterCount)
nxToRead = Nx / 2
nyToRead = int(Ny / 1.5)
data = ds.GetRasterBand(1).ReadAsArray(0, 0, nxToRead, nyToRead).transpose()
print(srs_wkt)
print(data.shape)
#plt.imshow(data)
#plt.show()
ds = None
print(Xul, Yul, deltaX, deltaY, rotation)
x1d = np.arange(Xul, Xul + deltaX * nxToRead, deltaX)
y1d = np.arange(Yul, Yul + deltaY * nyToRead, deltaY)
assert len(x1d) == nxToRead
assert len(y1d) == nyToRead
y, x = np.meshgrid(y1d, x1d)
fieldName = os.path.basename(inPath).split("_")[0].lower()
coef = name_to_mult[fieldName]
no_data = name_to_nodata_value[fieldName]
usable = (data != no_data)
print(x.shape, usable.shape)
x0 = x[usable]
y0 = y[usable]
cartx, carty, cartz = lat_lon.lon_lat_to_cartesian(x0, y0)
data_1d = data[usable]
print("useful data points : {0}".format(len(x0)))
tree = KDTree(list(zip(cartx, carty, cartz)))
print("constructed the kdtree")
xi, yi, zi = lat_lon.lon_lat_to_cartesian(lonlats[:,0], lonlats[:,1])
dists, inds = tree.query(list(zip(xi, yi, zi)), k = AGGR_SIZE)
npoints = dists.shape[0]
interp_data = np.zeros((npoints, ))
for i in range(npoints):
the_dists = dists[i,:]
the_inds = inds[i,:]
good_pts = (the_dists < LIMIT_DIST)
if len(the_dists[good_pts]) < 0.25 * AGGR_SIZE: #if there is no usable points in the vicinity, then set the value to no_data
interp_data[i] = -1
continue
the_dists = the_dists[good_pts]
the_inds = the_inds[good_pts]
interp_coefs = 1.0 / the_dists ** 2
interp_data[i] = np.sum( interp_coefs * data_1d[the_inds] ) / np.sum(interp_coefs)
interp_data[interp_data >= 0] *= coef
print("completed interpolation")
return interp_data
class GldasManager():
def __init__(self, folder_path="/home/huziy/skynet3_exec1/gldas_data"):
"""
Data access interface to the folder of netcdf files
runoff units: kg/m^2/s = mm/s
"""
self.data_folder = folder_path
self.surface_rof_varname = "Qs_GDS0_SFC_ave4h"
self.subsurface_rof_varname = "Qsb_GDS0_SFC_ave4h"
self.date_format = "%m/%d/%Y (%H:%M)"
self._init_date_to_path_dict()
self._init_kd_tree()
pass
def plot_subsrof_ts(self, i=0, j=0):
all_dates = list(sorted(self.date_to_path.keys()))
vals = [self.get_field_for_date(x, var_name=self.subsurface_rof_varname)[i, j] for x in all_dates]
vals1 = [self.get_field_for_date(x, var_name=self.surface_rof_varname)[i, j] for x in all_dates]
print(min(vals), max(vals))
dates_num = date2num(all_dates)
print(min(dates_num), max(dates_num))
import matplotlib.pyplot as plt
plt.figure()
plt.plot(dates_num, vals, label="subsurf rof")
plt.plot(dates_num, vals1, label="surf rof")
plt.legend()
#plt.xticks(rotation='vertical')
plt.show()
def _init_date_to_path_dict(self):
self.date_to_path = {}
for fName in os.listdir(self.data_folder):
if not fName.endswith(".nc"): continue #regard only nectdf files
path = os.path.join(self.data_folder, fName)
ds = Dataset(path)
srofVar = ds.variables[self.surface_rof_varname]
date = datetime.strptime(srofVar.initial_time, self.date_format)
self.date_to_path[date] = path
ds.close()
def _init_kd_tree(self):
"""
Has to be called after self._init_date_to_path_dict
"""
if not len(self.date_to_path):
print("You should call {0} first".format("self._init_date_to_path_dict"))
raise Exception()
for d, path in self.date_to_path.items():
ds = Dataset(path)
lons1d = ds.variables["g0_lon_1"][:]
lats1d = ds.variables["g0_lat_0"][:]
self.lats2d, self.lons2d = np.meshgrid(lats1d, lons1d)
x, y, z = lat_lon.lon_lat_to_cartesian(self.lons2d.flatten(), self.lats2d.flatten())
self.kdtree = KDTree(list(zip(x, y, z)))
return
pass
def get_field_for_month_and_year(self, var_name="", month=None, year=None):
d1 = datetime(year=year, month=month, day=1)
path = self.date_to_path[d1]
ds = Dataset(path)
return ds.variables[var_name][:]
def get_field_for_date(self, the_date, var_name=""):
path = self.date_to_path[the_date]
ds = Dataset(path)
data = ds.variables[var_name][:].transpose() # transpose because I allways use (lon, lat) order of coordinates
ds.close()
return data
def get_srof_spat_integrals_over_points_in_time(self, lons2d_target, lats2d_target, mask,
areas2d, start_date=None, end_date=None):
return self._get_spatial_integrals_over_points_in_time(lons2d_target, lats2d_target, mask, areas2d,
start_date=start_date, end_date=end_date,
var_name=self.surface_rof_varname)
def get_subsrof_spat_integrals_over_points_in_time(self, lons2d_target, lats2d_target, mask,
areas2d, start_date=None, end_date=None):
return self._get_spatial_integrals_over_points_in_time(lons2d_target, lats2d_target, mask, areas2d,
start_date=start_date, end_date=end_date,
var_name=self.subsurface_rof_varname)
def _get_spatial_integrals_over_points_in_time(self, lons2d_target, lats2d_target, mask,
areas2d, start_date=None, end_date=None, var_name=""):
"""
i) Interpolate to the grid (lons2d_target, lats2d_target)
ii) Apply the mask to the interpoated fields and sum with coefficients from areas2d
Note: the interpolation is done using nearest neighbor approach
returns a timeseries of {t -> sum(Ai[mask]*xi[mask])(t)}
"""
#interpolation
x1, y1, z1 = lat_lon.lon_lat_to_cartesian(lons2d_target.flatten(), lats2d_target.flatten())
dists, indices = self.kdtree.query(list(zip(x1, y1, z1)))
mask1d = mask.flatten().astype(int)
areas1d = areas2d.flatten()
result = {}
for the_date in list(self.date_to_path.keys()):
if start_date is not None:
if start_date > the_date: continue
if end_date is not None:
if end_date < the_date: continue
data = self.get_field_for_date(the_date, var_name=var_name)
result[the_date] = np.sum(data.flatten()[indices][mask1d == 1] * areas1d[mask1d == 1])
times = list(sorted(result.keys()))
values = [result[x] for x in times]
print("nvals, min, max", len(values), min(values), max(values))
return TimeSeries(time=times, data=values)
def main():
start_year = 1970
end_year = 1999
stations = cehq_station.read_station_data(folder="data/cehq_measure_data_all")
stations = list( itertools.ifilter( lambda s: s.is_natural, stations) )
for s in stations:
s.delete_data_after_year(end_year)
s.delete_data_before_year(start_year)
pass
stations = list( itertools.ifilter(lambda s: s.get_num_of_years_with_continuous_data() >= 10, stations) )
s = stations[0]
assert isinstance(s, Station)
#stations = list( itertools.ifilter(lambda s: s.is_natural, stations) )
x, y = polar_stereographic.lons, polar_stereographic.lats
basemap = polar_stereographic.basemap
x, y = basemap(x,y)
sx = [s.longitude for s in stations]
sy = [s.latitude for s in stations]
sx, sy = basemap(sx, sy)
#read model data
model_file_path = "data/streamflows/hydrosheds_euler9/aex_discharge_1970_01_01_00_00.nc"
acc_area = data_select.get_field_from_file(path=model_file_path,
field_name="drainage")
i_indices, j_indices = data_select.get_indices_from_file(path=model_file_path)
lons_1d = data_select.get_field_from_file(path=model_file_path, field_name="longitude")
lats_1d = data_select.get_field_from_file(path=model_file_path, field_name="latitude")
x1d, y1d, z1d = lat_lon.lon_lat_to_cartesian(lons_1d, lats_1d)
kdtree = KDTree(zip(x1d, y1d, z1d))
print "Id: 4 DA (km2) <-> 4 dist (km) <-> 4 (i,j)"
#basemap.scatter(sx, sy, c = "r", zorder = 5)
for s, isx, isy in zip( stations, sx, sy ):
assert isinstance(s, Station)
plt.annotate(s.id, xy=(isx, isy),
bbox = dict(facecolor = 'white'), weight = "bold", font_properties = FontProperties(size=0.5))
#get model drainaige areas for the four closest gridcells to the station
x0, y0, z0 = lat_lon.lon_lat_to_cartesian(s.longitude, s.latitude)
dists, indices = kdtree.query([x0, y0, z0], k = 4)
dists /= 1000
print("{0}: {1:.1f}; {2:.1f}; {3:.1f}; {4:.1f} <-> {5:.1f}; {6:.1f}; {7:.1f}; {8:.1f} <-> {9};{10};{11};{12}".format(
"{0} (S_DA = {1:.1f})".format(s.id, s.drainage_km2),
float(acc_area[indices[0]]),
float(acc_area[indices[1]]),
float(acc_area[indices[2]]),
float(acc_area[indices[3]]),
float( dists[0] ),
float(dists[1]),
float(dists[2]),
float(dists[3]),
"({0}, {1})".format(i_indices[indices[0]] + 1, j_indices[indices[0]] + 1),
"({0}, {1})".format(i_indices[indices[1]] + 1, j_indices[indices[1]] + 1),
"({0}, {1})".format(i_indices[indices[2]] + 1, j_indices[indices[2]] + 1),
"({0}, {1})".format(i_indices[indices[3]] + 1, j_indices[indices[3]] + 1)
))
basemap.drawcoastlines(linewidth=0.5)
xmin, xmax = min(sx), max(sx)
ymin, ymax = min(sy), max(sy)
marginx = (xmax - xmin) * 0.1
marginy = (ymax - ymin) * 0.1
xmin -= marginx * 1.5
xmax += marginx * 2
ymin -= marginy
ymax += marginy * 2
plt.xlim(xmin, xmax)
plt.ylim(ymin, ymax)
plt.tight_layout()
basin_boundaries.plot_basin_boundaries_from_shape(basemap=basemap, plotter=plt, linewidth=1)
plt.savefig("10yr_cont_stations_natural_fs0.5.pdf")
#plt.show()
pass
class SoundingPlotter:
def __init__(self, ax , basemap, tmin_3d, tmax_3d, lons2d, lats2d, levelheights = None):
"""
Plots a vertical profile at the point nearest to the clicked one
:type ax: Axes
"""
assert isinstance(ax, Axes)
self.basemap = basemap
assert isinstance(self.basemap, Basemap)
self.tmin_3d = tmin_3d
self.tmax_3d = tmax_3d
self.lons2d = lons2d
self.lats2d = lats2d
self.T0 = 273.15
self.lons_flat = lons2d.flatten()
self.lats_flat = lats2d.flatten()
self.level_heights = levelheights
self.counter = 0
self.ax = ax
x, y, z = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten())
self.kdtree = KDTree(list(zip(x,y,z)))
ax.figure.canvas.mpl_connect("button_press_event", self)
pass
def _get_closest_ij(self, event):
lon, lat = self.basemap(event.xdata, event.ydata, inverse = True)
x0, y0, z0 = lat_lon.lon_lat_to_cartesian(lon, lat)
dist, i = self.kdtree.query((x0,y0,z0))
lon0, lat0 = self.lons_flat[i], self.lats_flat[i]
ind = np.where((self.lons2d == lon0) & (self.lats2d == lat0))
ix = ind[0][0]
jy = ind[1][0]
return ix, jy
def _plot_sounding(self, ax, ix, jy):
ax.plot(self.tmax_3d[ix, jy, :] - self.T0, self.level_heights, color = "r")
ax.plot(self.tmin_3d[ix, jy, :] - self.T0, self.level_heights, color = "b")
ax.plot([0 , 0], [self.level_heights[0], self.level_heights[-1]], color = "k")
ax.set_title(str(self.counter))
assert isinstance(ax, Axes)
ax.invert_yaxis()
def __call__(self, event):
print(event.xdata, event.ydata)
print(event.button)
if event.button != 3:
return
ix, jy = self._get_closest_ij(event)
fig = plt.figure()
sounding_ax = fig.add_subplot(1,1,1)
self._plot_sounding(sounding_ax, ix, jy)
self.ax.annotate(str(self.counter), (event.xdata, event.ydata), font_properties =
FontProperties(size=10))
self.ax.redraw_in_frame()
self.counter += 1
assert isinstance(fig, Figure)
plt.show()
pass
class TimeSeriesPlotter:
def __init__(self, ax, basemap, lons2d, lats2d, ncVarDict, times, start_date, end_date):
"""
Plots a vertical profile at the point nearest to the clicked one
:type ax: Axes
"""
assert isinstance(ax, Axes)
self.basemap = basemap
assert isinstance(self.basemap, Basemap)
self.lons_flat = lons2d.flatten()
self.lats_flat = lats2d.flatten()
self.ncVarDict = ncVarDict
self.lons2d = lons2d
self.lats2d = lats2d
self.counter = 0
self.ax = ax
x, y, z = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten())
self.kdtree = KDTree(list(zip(x,y,z)))
self.sel_time_indices = np.where([start_date <= t <= end_date for t in times])[0]
self.times = times[self.sel_time_indices]
ax.figure.canvas.mpl_connect("button_press_event", self)
def _get_closest_ij(self, event):
lon, lat = self.basemap(event.xdata, event.ydata, inverse = True)
x0, y0, z0 = lat_lon.lon_lat_to_cartesian(lon, lat)
dist, i = self.kdtree.query((x0,y0,z0))
lon0, lat0 = self.lons_flat[i], self.lats_flat[i]
ind = np.where((self.lons2d == lon0) & (self.lats2d == lat0))
ix = ind[0][0]
jy = ind[1][0]
return ix, jy
def _plot_timeseries(self, ax, ix, jy):
fig_daily = plt.figure()
ax_daily = plt.gca()
fig_monthly = plt.figure()
ax_monthly = plt.gca()
for varName, ncVar in self.ncVarDict.items():
sel_values = ncVar[self.sel_time_indices, 0, ix, jy]
ax.plot(self.times, sel_values, label = varName)
#calculate and plot daily means
ts = pd.TimeSeries(index = self.times, data = sel_values)
ts = ts.resample("D", how = "mean")
ax_daily.plot(ts.index, ts.values, label = varName)
#calculate and plot monthly means
ts = ts.resample("M", how = "mean")
ax_monthly.plot(ts.index, ts.values, label = varName)
ax.legend()
ax.set_title(str(self.counter))
ax_daily.legend()
ax_daily.set_title(str(self.counter) + " - daily")
ax_monthly.legend()
ax_monthly.set_title(str(self.counter) + " - monthly")
assert isinstance(ax, Axes)
def __call__(self, event):
print(event.xdata, event.ydata)
print(event.button)
if event.button != 3:
return
ix, jy = self._get_closest_ij(event)
fig = plt.figure()
sounding_ax = fig.add_subplot(1,1,1)
self._plot_timeseries(sounding_ax, ix, jy)
self.ax.annotate(str(self.counter), (event.xdata, event.ydata), font_properties =
FontProperties(size=10))
self.ax.redraw_in_frame()
self.counter += 1
assert isinstance(fig, Figure)
plt.show()
