def build(self): logger.debug( "Building maps for conversion between real and image space") self.imagetree = {} self.imagemap = {} self.realtree = KDTree([x.realcoords for x in self.patches]) self.realmapping = {} for num, patch in enumerate(self.patches): if num % 1000 == 0: logger.debug("Built maps for {0} of {1} patches".format( num, len(self.patches))) resp = {} for _, projection in self.projections.items(): inimage = patch.project(projection) if projection.id not in self.imagetree: self.imagetree[projection.id] = [] self.imagemap[projection.id] = {} self.imagetree[projection.id].append(inimage) self.imagemap[projection.id][tuple(inimage)] = patch.realcoords resp[projection.id] = inimage self.realmapping[tuple(patch.realcoords)] = resp logger.debug("Done building maps for {0} patches".format( len(self.patches))) logger.debug("Building image KD tree") for key, imagecoords in self.imagetree.items(): self.imagetree[key] = KDTree(imagecoords)
def compute_graph(samples, weights=None, p_norm=2, max_degree=10, max_distance=INF, approximate_eps=0.): vertices = list(range(len(samples))) edges = set() if not vertices: return vertices, edges if weights is None: weights = np.ones(len(samples[0])) embed_fn = get_embed_fn(weights) embedded = list(map(embed_fn, samples)) kd_tree = KDTree(embedded) for v1 in vertices: # TODO: could dynamically compute distances distances, neighbors = kd_tree.query(embedded[v1], k=max_degree + 1, eps=approximate_eps, p=p_norm, distance_upper_bound=max_distance) for d, v2 in zip(distances, neighbors): if (d < max_distance) and (v1 != v2): edges.update([(v1, v2), (v2, v1)]) # print(time.time() - start_time, len(edges), float(len(edges))/len(samples)) return vertices, edges
def get_nearest(lasfile): """ return a nearest neighbor kdtree query """ dataset = np.vstack([lasfile.x, lasfile.y, lasfile.z]).transpose() tree = KDTree(dataset) query = tree.query(dataset[100,], k=5) return query
def get_interpolation_weights(self, source_lons = None, source_lats = None, dest_lons = None, dest_lats = None, nneighbours = 4 ): """ get the interpolation array M (nx*ny, nneighbors) and negibor indices G: DEST_FIELD = M * SOURCE_FIELD.flatten()[G] """ source_lons_1d, source_lats_1d = source_lons.flatten(), source_lats.flatten() [x,y,z] = lat_lon.lon_lat_to_cartesian_normalized(source_lons_1d, source_lats_1d) kdtree = KDTree(zip(x, y, z)) [xi, yi, zi] = lat_lon.lon_lat_to_cartesian_normalized(dest_lons.flatten(), dest_lats.flatten()) [distances, indices] = kdtree.query( zip( xi, yi, zi ) , k = nneighbours ) if len(distances.shape) == 2: weights = 1.0 / distances ** 2 norm = weights.sum(axis = 1) norm = np.array( [norm] * nneighbours ).transpose() weights /= norm else: weights = np.ones(distances.shape) return weights, indices
def __init__(self, ax, basemap, tmin_3d, tmax_3d, lons2d, lats2d, levelheights=None): """ Plots a vertical profile at the point nearest to the clicked one :type ax: Axes """ assert isinstance(ax, Axes) self.basemap = basemap assert isinstance(self.basemap, Basemap) self.tmin_3d = tmin_3d self.tmax_3d = tmax_3d self.lons2d = lons2d self.lats2d = lats2d self.T0 = 273.15 self.lons_flat = lons2d.flatten() self.lats_flat = lats2d.flatten() self.level_heights = levelheights self.counter = 0 self.ax = ax x, y, z = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten()) self.kdtree = KDTree(list(zip(x, y, z))) ax.figure.canvas.mpl_connect("button_press_event", self) pass
def add_clusters(df_cells, neighbor_dist=50): """Assigns -1 to clusters with only one cell. """ from scipy.spatial.kdtree import KDTree import networkx as nx x = df_cells[GLOBAL_X] + df_cells[POSITION_J] y = df_cells[GLOBAL_Y] + df_cells[POSITION_I] barcodes = df_cells[BARCODE_0] barcodes = np.array(barcodes) kdt = KDTree(np.array([x, y]).T) num_cells = len(df_cells) print('searching for clusters among %d cells' % num_cells) pairs = kdt.query_pairs(neighbor_dist) pairs = np.array(list(pairs)) x = barcodes[pairs] y = x[:, 0] == x[:, 1] G = nx.Graph() G.add_edges_from(pairs[y]) clusters = list(nx.connected_components(G)) cluster_index = np.zeros(num_cells, dtype=int) - 1 for i, c in enumerate(clusters): cluster_index[list(c)] = i df_cells[CLUSTER] = cluster_index return df_cells
def __init__(self, ax, basemap, lons2d, lats2d, ncVarDict, times, start_date, end_date): """ Plots a vertical profile at the point nearest to the clicked one :type ax: Axes """ assert isinstance(ax, Axes) self.basemap = basemap assert isinstance(self.basemap, Basemap) self.lons_flat = lons2d.flatten() self.lats_flat = lats2d.flatten() self.ncVarDict = ncVarDict self.lons2d = lons2d self.lats2d = lats2d self.counter = 0 self.ax = ax x, y, z = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten()) self.kdtree = KDTree(list(zip(x, y, z))) self.sel_time_indices = np.where( [start_date <= t <= end_date for t in times])[0] self.times = times[self.sel_time_indices] ax.figure.canvas.mpl_connect("button_press_event", self)
def locally_extreme_points(coords, data, neighbourhood, lookfor='max', p_norm=2.): # Description # Find local maxima of points in a pointcloud. Ties result in both points passing through the filter. # # Not to be used for high-dimensional data. It will be slow. # # coords: A shape (n_points, n_dims) array of point locations # data: A shape (n_points, ) vector of point values # neighbourhood: The (scalar) size of the neighbourhood in which to search. # lookfor: Either 'max', or 'min', depending on whether you want local maxima or minima # p_norm: The p-norm to use for measuring distance (e.g. 1=Manhattan, 2=Euclidian) # # returns # filtered_coords: The coordinates of locally extreme points # filtered_data: The values of these points assert coords.shape[0] == data.shape[ 0], 'You must have one coordinate per data point' extreme_fcn = {'min': np.min, 'max': np.max}[lookfor] kdtree = KDTree(coords) neighbours = kdtree.query_ball_tree(kdtree, r=neighbourhood, p=p_norm) i_am_extreme = [ data[i] == extreme_fcn(data[n]) for i, n in enumerate(neighbours) ] extrema, = np.nonzero( i_am_extreme) # This line just saves time on indexing return coords[extrema], data[extrema]
def estimatemeans(self): feature_positions = [] for i in range(len(self.edges[0]) - 1): for j in range(len(self.edges[1]) - 1): if len(self.points[i, j]) > 0: feature_positions.append([i, j]) tree = KDTree(feature_positions) for i in range(len(self.edges[0]) - 1): for j in range(len(self.edges[1]) - 1): if self.points[i, j] == []: radius, neighbour = tree.query(x=np.array([i, j]), k=1) if radius > feature_resolution[0] / 10: self.estmeans[i, j] = np.nan self.means[i, j] = np.nan else: nearby = tree.data[tree.query_ball_point( x=np.array([i, j]), r=radius)] #print([self.trumeans[i] for i in nearby]) estmean = np.nanmean( [self.trumeans[i.astype(int)] for i in nearby]) self.estmeans[i, j] = estmean self.means[i, j] = estmean else: self.means[i, j] = self.trumeans[i, j]
class NeighborsFinder: def __init__(self, data): """Find neigbors Args: data (pd.DataFrame or dict): data with at columns id,pos,and eventually range """ if data is not None: self.data = data # Safety checks assert isinstance(data, pd.DataFrame) assert "pos" in data.columns # Initialize KDTree finder from scipy self.tree = KDTree(self.data["pos"].tolist()) def find_in_range(self, obj, search_range): # TODO # Will not work with double colliders # Can be made using colliders in PyGame ? # Safety check assert hasattr(self, "tree") # Get position from dataset pos = obj.pos # Find neighbors in range # We remove the first one which is the identity object idx = self.tree.query_ball_point(pos, search_range) # Return filtered data ids = self.data.iloc[idx].index.tolist() assert obj.id not in ids return ids def find_closest(self, obj, k=1): # Safety check assert hasattr(self, "tree") # Get object position from which we want to find neighbors pos = obj.pos # Get position from dataset distances, idx = self.tree.query(pos, k=k) if k == 1: distances = [distances] idx = [idx] # Get ids from dataset # Safe check object is not in neighbors, which would mean above we remove another overlapping objects with [1:] ids = self.data.iloc[idx].index.tolist() assert obj.id not in ids return distances, ids
def warp_keypoints(self, keypoints, grid_unnormalized): from scipy.spatial.kdtree import KDTree warp_grid = grid_unnormalized.reshape(-1, 2) regular_grid = self.grid_pixels_unnormalized.reshape(-1, 2) kd = KDTree(warp_grid) dists, idxs = kd.query(keypoints) new_keypoints = regular_grid[idxs] return new_keypoints
def mutual_knn(points, n=10, distance=radial_kernel()): knn = {} kt = KDTree(points) for i, point in enumerate(points): # cannot use euclidean distance directly for neighbour in kt.query(point, n + 1)[1]: if i != neighbour: knn.setdefault(i, []).append((distance(point, points[neighbour]), neighbour)) return knn
def compute_minimax_distance(path1, path2): overall_distance = 0. for path, other in permutations([path1, path2]): tree = KDTree(other) for q1 in path: #closest_distance = min(get_distance(q1, q2) for q2 in other) closest_distance, closest_index = tree.query(q1, k=1, eps=0.) overall_distance = max(overall_distance, closest_distance) return overall_distance
def make_adjacency_matrix(data, k): kt = KDTree(data) out = zeros((len(data),len(data))) for i, points in enumerate(data): distance, neighbors = kt.query_ball_point(points, k) for j in neighbors[1:]: out[i][j] = 1; out[j][i] = 1; #to ensure symmetry, one can imagine the the nearest neighbors is not necessarily associative return out
def mutual_knn(points, n=10, distance=radial_kernel()): knn = {} kt = KDTree(points) for i, point in enumerate(points): # cannot use euclidean distance directly for neighbour in kt.query(point, n + 1)[1]: if i != neighbour: knn.setdefault(i, []).append( (distance(point, points[neighbour]), neighbour)) return knn
def __init__(self, filepath): start = time.time() self.file = File(filepath, mode="r") self.scale = self.file.header.scale[0] self.offset = self.file.header.offset[0] self.tree = KDTree( np.vstack([self.file.x, self.file.y, self.file.z]).transpose()) self.time = self.file.header.get_date() end = time.time() - start print("Time Elapsed: {}".format(end))
def constructKnnGraph(points, numNeighbor): def euclidean_kernel(a, b): d = np.linalg.norm(a-b) return d knn = {} kt = KDTree(points) for i, point in enumerate(points): for neighbour in kt.query(point, numNeighbor + 1)[1]: if i != neighbour: knn.setdefault(i, []).append((euclidean_kernel(point, points[neighbour]), neighbour)) return knn
def waypoints_cb(self, msg): # self.waypoints = waypoints # rospy.loginfo("TL_DETECTOR: waypoints_cb called...") self.base_waypoints = msg.waypoints if self.base_waypoints is not None: # populate the Lane msg header as well (not really required, but to be consistent) # self.lane_msg_header = msg.header # get the 2d (x, y) waypoints self.base_waypoints_2d = [[wp.pose.pose.position.x, wp.pose.pose.position.y] for wp in self.base_waypoints] # build a KDTree for efficient search of nearest waypoint self.base_waypoints_kdtree = KDTree(self.base_waypoints_2d)
def interpolate_data_to_model_grid(self, model_lons_2d, model_lats_2d, data_obs): x0, y0, z0 = lat_lon.lon_lat_to_cartesian(model_lons_2d.flatten(), model_lats_2d.flatten()) x, y, z = lat_lon.lon_lat_to_cartesian(self.lons2d.flatten(), self.lats2d.flatten()) if self.ktree is None: self.ktree = KDTree(list(zip(x, y, z))) d, i = self.ktree.query(list(zip(x0, y0, z0))) return data_obs.flatten()[i].reshape(model_lons_2d.shape)
def build(self): logger.debug("Building maps for conversion between real and image space") self.imagetree = {} self.imagemap = {} self.realtree = KDTree([x.realcoords for x in self.patches]) self.realmapping = {} for num, patch in enumerate(self.patches): if num % 1000 == 0: logger.debug("Built maps for {0} of {1} patches".format(num, len(self.patches))) resp = {} for _, projection in self.projections.items(): inimage = patch.project(projection) if projection.id not in self.imagetree: self.imagetree[projection.id] = [] self.imagemap[projection.id] = {} self.imagetree[projection.id].append(inimage) self.imagemap[projection.id][tuple(inimage)] = patch.realcoords resp[projection.id] = inimage self.realmapping[tuple(patch.realcoords)] = resp logger.debug("Done building maps for {0} patches".format(len(self.patches))) logger.debug("Building image KD tree") for key, imagecoords in self.imagetree.items(): self.imagetree[key] = KDTree(imagecoords)
def updatemeans(self, newpoint, model, fr): '''For use when self = meanmap Calculates the mean observed value for each niche and populates the current map with those mean values. ''' #Update the new point first new_index = tuple(self.nichefinder(newpoint[2])) i, j = new_index[0], new_index[1] obsmean = self.trumeans[i, j] #estmean = self.estimatemean( i,j) #self.estmeans[ i,j ] = estmean #self.means[i,j] = np.mean([estmean, obsmean]) self.means[i, j] = obsmean ## Now update points that will be affected by the current point feature_positions = [] for i in range(len(self.edges[0]) - 1): for j in range(len(self.edges[1]) - 1): if self.points[i, j] != []: feature_positions.append([i, j]) #print(feature_positions) tree = KDTree(feature_positions) checkindex = [new_index] # Explore all the points around it. end = False checked = [] while not end: newcheck = [] for index in checkindex: newcheck, checked = self.pointsearch( model, index[0], index[1], fr, tree, newpoint, checked) checkindex += newcheck if len(newcheck) == 0: end = True
def solvent_surface(molecule, num_points_per_atom=140, delta=0.1): radii = get_solvent_radii(molecule.atomic_numbers) N = len(molecule) grid = load_grid_num_points(num_points_per_atom) num_points_per_atom = grid.shape[0] axes = molecule.axes() positions = molecule.positions_in_molecular_axis_frame() surface = np.empty((N * num_points_per_atom, 4)) atom_idx = np.empty(N * num_points_per_atom, dtype=np.int32) for i in range(N): r = radii[i] + delta l, u = num_points_per_atom * i, num_points_per_atom * (i + 1) surface[l:u, 3] = grid[:, 3] * 4 * np.pi * radii[i] * radii[i] surface[l:u, :3] = grid[:, :3] * r surface[l:u, :3] += positions[i, :] atom_idx[l:u] = i mask = np.ones_like(atom_idx, dtype=bool) tree = KDTree(surface[:, :3]) for i in range(N): p = positions[i] radius = radii[i] + delta idxs = tree.query_ball_point(p, radius - 1e-12) mask[idxs] = False surface = surface[mask, :] surface[:, :3] = (np.dot(surface[:, :3], axes) + molecule.center_of_mass[np.newaxis, :]) atom_idx = atom_idx[mask] return surface
def knnClassify(trData, trLabels, testData, nClasses): #run the knn classifier using the raw features (nDocs, _) = testData.shape knnLabels = numpy.zeros((nDocs, nClasses)) leafsize = 5 kdtree = KDTree(trData.tolist(), leafsize) (_, idxs) = kdtree.query(testData.tolist(), 3) for d in range(nDocs): testPointIdxs = idxs[d] votes = [trLabels[l] for l in testPointIdxs] classVotes = [0] * nClasses for v in votes: classVotes[v] += 1 knnLabels[d, classVotes.index(max(classVotes))] = 1 return (knnLabels, kdtree)
def _init_kd_tree(self): """ Init KDTree used for interpolation """ x0, y0, z0 = lat_lon.lon_lat_to_cartesian(self.lon2d.flatten(), self.lat2d.flatten()) self.kdtree = KDTree(list(zip(x0, y0, z0)))
def __init__(self, data): """Find neigbors Args: data (pd.DataFrame or dict): data with at columns id,pos,and eventually range """ if data is not None: self.data = data # Safety checks assert isinstance(data, pd.DataFrame) assert "pos" in data.columns # Initialize KDTree finder from scipy self.tree = KDTree(self.data["pos"].tolist())
def __init__(self, name_to_date_to_field, basemap, lons2d, lats2d, ax = None, cell_area = None, cell_manager = None, data_manager = None): self.gwdi_mean_field = None self.traf_mean_field = None self.tdra_mean_field = None self.upin_mean_field = None self.basemap = basemap self.date_to_stfl_field = name_to_date_to_field["STFL"] self.date_to_traf_field = name_to_date_to_field["TRAF"] self.date_to_tdra_field = name_to_date_to_field["TDRA"] self.date_to_pr_field = name_to_date_to_field["PR"] self.date_to_swe_field = name_to_date_to_field["I5"] self.date_to_swst_field = name_to_date_to_field["SWST"] #self.date_to_imav_field = name_to_date_to_field["IMAV"] self.acc_area_km2 = name_to_date_to_field["FACC"] #:type : CellManager self.cell_manager = cell_manager assert isinstance(self.cell_manager, CellManager) self.cell_area = cell_area x, y, z = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten()) self.kdtree = KDTree(list(zip(x,y,z))) ax.figure.canvas.mpl_connect("button_press_event", self) self.ax = ax self.lons2d = lons2d self.lats2d = lats2d self.data_manager = data_manager assert isinstance(self.data_manager, Crcm5ModelDataManager) self.x_pr, self.y_pr = basemap(lons2d, lats2d) self.lons_flat = lons2d.flatten() self.lats_flat = lats2d.flatten() self.dates_sorted = list( sorted(list(name_to_date_to_field.items())[0][1].keys()) ) self.counter = 0 self.date_to_swsr_field = name_to_date_to_field["SWSR"] self.date_to_swsl_field = name_to_date_to_field["SWSL"] #self.date_to_gwdi_field = name_to_date_to_field["GWDI"] self.date_to_upin_field = name_to_date_to_field["UPIN"] #static fields self.slope = name_to_date_to_field["SLOP"] self.channel_length = name_to_date_to_field["LENG"] self.lake_outlet = name_to_date_to_field["LKOU"] self.coef_bf = -np.ones(self.slope.shape) good_points = self.slope >= 0 self.coef_bf[good_points] = (self.slope[good_points]) ** 0.5 / ((self.channel_length[good_points]) ** (4.0/3.0) * data_manager.manning_bf[good_points] )
def nearest_neighbor_fill(ball, mask): # Fill flag holes with nearest neighbor ff = np.asarray(ball).copy() x, y = np.mgrid[0:ff.shape[0], 0:ff.shape[1]] xygood = np.array((x[~mask], y[~mask])).T xybad = np.array((x[mask], y[mask])).T ff[mask] = ff[~mask][KDTree(xygood).query(xybad)[1]] return Image.fromarray(ff)
def build_tree(laser_file): # build a KD-tree over the point clouds to find nearest neighbors. # input: the point cloud file that is going to be searched. # output: the tree file that includes indices ready to be searched. data_ready = np.vstack([laser_file.X, laser_file.Y, laser_file.Z]).transpose() tree = KDTree(data_ready) return tree
def _merge(self, roi): points = [ e[2] for e in roi ] tree = KDTree(points) pp = tree.query_pairs(4) #pp = sorted( pp , key=lambda e: max(roi[e[0]] , roi[e[1]]), reverse=True) skips = set() for i,j in pp: if self.Debug: print '%s(%.2f),%s(%.2f)'%(roi[i][0],roi[i][1], roi[j][0], roi[j][1] ) if i in skips or j in skips: continue skip = i if roi[i][1]<roi[j][1] else j skips.add(skip) rs = ( e for i, e in enumerate(roi) if not i in skips ) rs = sorted( rs , key=lambda e: e[1] )[-4:] return sorted( rs, key=lambda e: e[2][1] )
def _merge(self, roi): points = [e[2] for e in roi] tree = KDTree(points) pp = tree.query_pairs(4) #pp = sorted( pp , key=lambda e: max(roi[e[0]] , roi[e[1]]), reverse=True) skips = set() for i, j in pp: if self.Debug: print '%s(%.2f),%s(%.2f)' % (roi[i][0], roi[i][1], roi[j][0], roi[j][1]) if i in skips or j in skips: continue skip = i if roi[i][1] < roi[j][1] else j skips.add(skip) rs = (e for i, e in enumerate(roi) if not i in skips) rs = sorted(rs, key=lambda e: e[1])[-4:] return sorted(rs, key=lambda e: e[2][1])
def test_evol_for_point(): lon = -100 lat = 60 path = "/home/huziy/skynet1_rech3/cordex/for_Samira/b1/tmp_era40_b1.nc" ds = Dataset(path) data = ds.variables["tmp"][:] years = ds.variables["year"][:] coord_file = "/skynet1_rech3/huziy/cordex/CORDEX_DIAG/NorthAmerica_0.44deg_CanESM_B1" coord_file = os.path.join(coord_file, "pmNorthAmerica_0.44deg_CanHisto_B1_195801_moyenne") #coord_file = os.path.join(data_folder, "pmNorthAmerica_0.44deg_ERA40-Int2_195801_moyenne") b, lons2d, lats2d = draw_regions.get_basemap_and_coords(file_path=coord_file) sel_lons = [lon] sel_lats = [lat] xo,yo,zo = lat_lon.lon_lat_to_cartesian(sel_lons, sel_lats) xi, yi, zi = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten()) ktree = KDTree(list(zip(xi,yi,zi))) dists, indexes = ktree.query(list(zip(xo,yo,zo))) print(len(indexes)) print(indexes) idx = indexes[0] import matplotlib.pyplot as plt plt.figure() data_to_show = [] for i, y in enumerate(years): data_to_show.append(data[i,:,:].flatten()[idx]) plt.plot(years, data_to_show, "-s", lw = 3) plt.grid() plt.show() pass
def get_kd_tree(self): """ :rtype : KDTree for interpolation purposes """ if self._kdtree is None: x, y, z = lat_lon.lon_lat_to_cartesian(self.lons.flatten(), self.lats.flatten()) self._kdtree = KDTree(list(zip(x, y, z))) return self._kdtree
def locally_extreme_points(coords, data, radius, smoothing_radius=None, min_neighbourhood_size=-1, lookfor='max', p_norm=2): ''' Find local maxima of points in a pointcloud. Ties result in both points passing through the filter. Not to be used for high-dimensional data. It will be slow. coords: A shape (n_points, n_dims) array of point locations data: A shape (n_points, ) vector of point values radius: The (scalar) size of the neighbourhood in which to search. min_neighbourhood_size: Extrema whose neighbourhood has less than min_neighbourhood_size points are discarded lookfor: Either 'max', or 'min', depending on whether you want local maxima or minima p_norm: The p-norm to use for measuring distance (e.g. 1=Manhattan, 2=Euclidian) returns filtered_coords: The coordinates of locally extreme points filtered_data: The values of these points ''' assert coords.shape[0] == data.shape[ 0], 'You must have one coordinate per data point' extreme_fcn = {'min': np.min, 'max': np.max}[lookfor] kdtree = KDTree(coords) if smoothing_radius is not None: smoothing_neighbours = kdtree.query_ball_tree(kdtree, r=smoothing_radius, p=p_norm) smooth_data = np.array( [np.mean(data[n]) for n in smoothing_neighbours]) else: smooth_data = data neighbours = kdtree.query_ball_tree(kdtree, r=radius, p=p_norm) i_am_extreme = [ (np.nonzero(np.equal(n, i))[0][0] == np.argmax(smooth_data[n])) & (len(n) > min_neighbourhood_size) for i, n in enumerate(neighbours) ] extrema, = np.nonzero( i_am_extreme) # This line just saves time on indexing return coords[extrema], data[extrema]
def add_clusters(df_cells, barcode_col=BARCODE_0, radius=50, verbose=True, ij=(POSITION_I, POSITION_J)): """Assigns -1 to clusters with only one cell. """ from scipy.spatial.kdtree import KDTree import networkx as nx I, J = ij x = df_cells[GLOBAL_X] + df_cells[J] y = df_cells[GLOBAL_Y] + df_cells[I] barcodes = df_cells[barcode_col] barcodes = np.array(barcodes) kdt = KDTree(np.array([x, y]).T) num_cells = len(df_cells) if verbose: message = 'searching for clusters among {} {} objects' print(message.format(num_cells, barcode_col)) pairs = kdt.query_pairs(radius) pairs = np.array(list(pairs)) x = barcodes[pairs] y = x[:, 0] == x[:, 1] G = nx.Graph() G.add_edges_from(pairs[y]) clusters = list(nx.connected_components(G)) cluster_index = np.zeros(num_cells, dtype=int) - 1 for i, c in enumerate(clusters): cluster_index[list(c)] = i df_cells = df_cells.copy() df_cells[CLUSTER] = cluster_index df_cells[CLUSTER_SIZE] = ( df_cells.groupby(CLUSTER)[barcode_col].transform('size')) df_cells.loc[df_cells[CLUSTER] == -1, CLUSTER_SIZE] = 1 return df_cells
def __init__(self, xml_annot): self._data = [] self.ref_by_pages = {} idx = 0 for annotation in xml_annot.findall( ".//ANNOTATION/ACTION[@type='goto']/.."): dest = annotation.find("ACTION/DEST") try: page_dest = dest.get("page") x_dest = dest.get("x") y_dest = dest.get("y") except: # Maybe change that, see how much we loose continue page_num = annotation.get("pagenum") page_n = int(page_num) quadpoints = annotation.findall("QUADPOINTS/QUADRILATERAL/POINT") min_h, min_v, max_h, max_v = None, None, None, None for point in quadpoints: h, v = float(point.get("HPOS")), float(point.get("VPOS")) if min_h is None: min_h, max_h = h, h min_v, max_v = v, v else: min_h = min(min_h, h) max_h = max(max_h, h) min_v = min(min_v, v) max_v = max(max_v, v) self._data.append({ "page_dest": page_dest, "x_dest": x_dest, "y_dest": y_dest, "bbx": BBX(page_num, min_h, min_v, max_h, max_v) }) """ Add a new point in the QDTree of the page *page_n* """ if page_n not in self.ref_by_pages: self.ref_by_pages[page_n] = {"idx": [], "pos": []} self.ref_by_pages[page_n]["idx"].append(idx) self.ref_by_pages[page_n]["pos"].append([(min_v + max_v) / 2, (min_h + max_h) / 2]) idx += 1 # Convert list to QDTree for page in self.ref_by_pages: self.ref_by_pages[page]["tree"] = KDTree( self.ref_by_pages[page]["pos"])
def create_main_grid(points, extend, img_size=32, values=[]): """Create grid and caluclate features :param points: Array Vstack [x, y, z, classification] [m] :type points: float :param extend: Array [minX minY, maxX maxY] :type extend: float :param img_size: Spatial size of feature area Default 32. Should be 2 to power of n :type img_size: int :param values: Values for Feature Stats, if non is passed height is used :type values: float """ tree = KDTree(points[:, 0:2]) buff = int(img_size / 2) if values: points[:, 2] = values minX = extend[0, 0] - buff minY = extend[0, 1] - buff maxX = extend[1, 0] + buff maxY = extend[1, 1] + buff gridX = np.linspace(int(minX), int(maxX), int(maxX - minX + 1)) gridY = np.linspace(int(minY), int(maxY), int(maxY - minY + 1)) f1 = np.zeros((len(gridX), len(gridY))) f2 = np.zeros((len(gridX), len(gridY))) f3 = np.zeros((len(gridX), len(gridY))) for x, i in zip(gridX, range(0, len(gridX))): for y, j in zip(gridY, range(0, len(gridY))): list = tree.query_ball_point([x, y], 1.4) cell_ext = np.array([[x - 0.5, y - 0.5], [x + 0.5, y + 0.5]]) cell_points = clip(points[list], cell_ext) if cell_points.any(): f1[i, j] = np.mean(cell_points[:, 2]) f2[i, j] = np.min(cell_points[:, 2]) f3[i, j] = np.max(cell_points[:, 2]) return [f1, f2, f3]
def kdtree_density(points, radius, return_tree=False): """ Returns the density calculated sing a Ball Tree. Higher is denser. Scales according to the dimension of the points. see: https://stackoverflow.com/questions/14070565/calculating-point-density-using-python If `return_tree` is `True` returns a tuple of the density and the KDTree. """ tree = KDTree(points) n = points.shape[-1] ball_trees = tree.query_ball_tree(tree, radius) frequency = np.array([len(neighbours) for neighbours in ball_trees]) density = frequency / radius**n if return_tree: return np.mean(density), tree return np.mean(density)
def get_edges(df1, df2): from scipy.spatial.kdtree import KDTree get_label = lambda x: tuple(int(y) for y in x[[2, 3]]) x1 = df1[['i', 'j', 'frame', 'label']].as_matrix() x2 = df2[['i', 'j', 'frame', 'label']].as_matrix() kdt = KDTree(df1[['i', 'j']]) points = df2[['i', 'j']] result = kdt.query(points, 3) edges = [] for i2, (ds, ns) in enumerate(zip(*result)): end_node = get_label(x2[i2]) for d, i1 in zip(ds, ns): start_node = get_label(x1[i1]) w = d edges.append((start_node, end_node, w)) return edges
def _calculatePointResiduals(self, curve, tube_radius = None): if tube_radius is None: X = self._X else: within_tube_indices = self.calculateCoverageIndices(curve, tube_radius) X = self._X.take(list(within_tube_indices), axis = 0) if self._maxSegmentLength is None: self._maxSegmentLength = self._calculateMaxSegmentLength(curve) lpc_points = curve['save_xd'] num_lpc_points = len(lpc_points) tree_lpc_points = KDTree(lpc_points) residuals = empty(len(X)) residuals_lamb = empty(len(X)) path_length = curve['lamb'] for j, p in enumerate(X): closest_lpc_point = tree_lpc_points.query(p) candidate_radius = sqrt(closest_lpc_point[0]**2 + 0.25*self._maxSegmentLength**2) candidate_segment_ends = tree_lpc_points.query_ball_point(p, candidate_radius) candidate_segment_ends.sort() current_min_segment_dist = (closest_lpc_point[0],0) current_closest_index = closest_lpc_point[1] last_index = None for i, index in enumerate(candidate_segment_ends): if index!=0 and last_index != index - 1: prv_segment_dist = self._distancePointToLineSegment(lpc_points[index-1], lpc_points[index], p) if prv_segment_dist[0] < current_min_segment_dist[0]: current_min_segment_dist = prv_segment_dist current_closest_index = index - 1 if index != num_lpc_points - 1: prv_segment_dist = self._distancePointToLineSegment(lpc_points[index], lpc_points[index+1], p) if prv_segment_dist[0] < current_min_segment_dist[0]: current_min_segment_dist = prv_segment_dist current_closest_index = index last_index = index residuals[j] = current_min_segment_dist[0] residuals_lamb[j] = path_length[current_closest_index] + current_min_segment_dist[1] lamb_order = argsort(residuals_lamb) return (residuals_lamb[lamb_order], residuals[lamb_order])
def get_data_interpolated_to_points(self, dest_lons = None, dest_lats = None, source_lons = None, source_lats = None, data = None): """ Designed to interpolate all data to the AMNO domain """ if None not in [source_lons, source_lats]: lons1d = source_lons.flatten() lats1d = source_lats.flatten() points = lat_lon.lon_lat_to_cartesian_normalized(lons1d, lats1d) points = np.array(points).transpose() point_tree = KDTree(points) else: point_tree = self.kd_tree assert source_lons.shape == source_lats.shape == data.shape [xi, yi, zi] = lat_lon.lon_lat_to_cartesian_normalized(dest_lons.flatten(), dest_lats.flatten()) pointsi = np.array([xi, yi, zi]).transpose() data1d = data.flatten() #distances dimensions = (n_points, n_neigbours) [distances, indices] = point_tree.query(pointsi, k = 4) weights = 1.0 / distances ** 2 norm = [np.sum(weights, axis = 1)] * weights.shape[1] norm = np.array(norm).transpose() weights /= norm result = [] for i in xrange(pointsi.shape[0]): w = weights[i, :] d = data1d[indices[i, :]] result.append(np.sum(w * d)) return np.array(result).reshape( dest_lons.shape )
def make_edge_graph(data, k, ball = True): kt = KDTree(data) out = [set() for i in data] k_max = 0 k_min = np.infty for i, points in enumerate(data): if(ball): neighbors = kt.query_ball_point(points, k) else: distance, neighbors = kt.query(points,int(k+1)) neighbors = neighbors[1:] for j in neighbors: if(j != i): out[i].add(j) if(len(out[i])>k_max): k_max = len(out[i]) out[j].add(i) if(len(out[j])>k_max): k_max = len(out[j]) if(len(out[i])<k_min): k_min = len(out[i]) return (k_min, k_max, out)
def upscale(manager_in, manager_out, swe_in, nneighbours = 25): assert isinstance(manager_in, Crcm5ModelDataManager) assert isinstance(manager_out, Crcm5ModelDataManager) lons_in_1d = manager_in.lons2D.flatten() lats_in_1d = manager_in.lats2D.flatten() x0, y0, z0 = lat_lon.lon_lat_to_cartesian(lons_in_1d, lats_in_1d) kdtree = KDTree(list(zip(x0,y0,z0))) lons_out_1d = manager_out.lons2D.flatten() lats_out_1d = manager_out.lats2D.flatten() x1, y1, z1 = lat_lon.lon_lat_to_cartesian(lons_out_1d, lats_out_1d) dd, ii = kdtree.query(list(zip(x1, y1, z1)), k=nneighbours) print(ii.shape) swe_in_1d = swe_in.flatten() return np.mean(swe_in_1d[ii], axis=1).reshape(manager_out.lons2D.shape) pass
def _init_kd_tree(self): """ Has to be called after self._init_date_to_path_dict """ if not len(self.date_to_path): print("You should call {0} first".format("self._init_date_to_path_dict")) raise Exception() for d, path in self.date_to_path.items(): ds = Dataset(path) lons1d = ds.variables["g0_lon_1"][:] lats1d = ds.variables["g0_lat_0"][:] self.lats2d, self.lons2d = np.meshgrid(lats1d, lons1d) x, y, z = lat_lon.lon_lat_to_cartesian(self.lons2d.flatten(), self.lats2d.flatten()) self.kdtree = KDTree(list(zip(x, y, z))) return pass
def __init__(self, ax, basemap, lons2d, lats2d, ncVarDict, times, start_date, end_date): """ Plots a vertical profile at the point nearest to the clicked one :type ax: Axes """ assert isinstance(ax, Axes) self.basemap = basemap assert isinstance(self.basemap, Basemap) self.lons_flat = lons2d.flatten() self.lats_flat = lats2d.flatten() self.ncVarDict = ncVarDict self.lons2d = lons2d self.lats2d = lats2d self.counter = 0 self.ax = ax x, y, z = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten()) self.kdtree = KDTree(list(zip(x,y,z))) self.sel_time_indices = np.where([start_date <= t <= end_date for t in times])[0] self.times = times[self.sel_time_indices] ax.figure.canvas.mpl_connect("button_press_event", self)
def __init__(self, ax , basemap, tmin_3d, tmax_3d, lons2d, lats2d, levelheights = None): """ Plots a vertical profile at the point nearest to the clicked one :type ax: Axes """ assert isinstance(ax, Axes) self.basemap = basemap assert isinstance(self.basemap, Basemap) self.tmin_3d = tmin_3d self.tmax_3d = tmax_3d self.lons2d = lons2d self.lats2d = lats2d self.T0 = 273.15 self.lons_flat = lons2d.flatten() self.lats_flat = lats2d.flatten() self.level_heights = levelheights self.counter = 0 self.ax = ax x, y, z = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten()) self.kdtree = KDTree(list(zip(x,y,z))) ax.figure.canvas.mpl_connect("button_press_event", self) pass
class TimeseriesPlotter: def __init__(self, name_to_date_to_field, basemap, lons2d, lats2d, ax = None, cell_area = None, cell_manager = None, data_manager = None): self.gwdi_mean_field = None self.traf_mean_field = None self.tdra_mean_field = None self.upin_mean_field = None self.basemap = basemap self.date_to_stfl_field = name_to_date_to_field["STFL"] self.date_to_traf_field = name_to_date_to_field["TRAF"] self.date_to_tdra_field = name_to_date_to_field["TDRA"] self.date_to_pr_field = name_to_date_to_field["PR"] self.date_to_swe_field = name_to_date_to_field["I5"] self.date_to_swst_field = name_to_date_to_field["SWST"] #self.date_to_imav_field = name_to_date_to_field["IMAV"] self.acc_area_km2 = name_to_date_to_field["FACC"] #:type : CellManager self.cell_manager = cell_manager assert isinstance(self.cell_manager, CellManager) self.cell_area = cell_area x, y, z = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten()) self.kdtree = KDTree(list(zip(x,y,z))) ax.figure.canvas.mpl_connect("button_press_event", self) self.ax = ax self.lons2d = lons2d self.lats2d = lats2d self.data_manager = data_manager assert isinstance(self.data_manager, Crcm5ModelDataManager) self.x_pr, self.y_pr = basemap(lons2d, lats2d) self.lons_flat = lons2d.flatten() self.lats_flat = lats2d.flatten() self.dates_sorted = list( sorted(list(name_to_date_to_field.items())[0][1].keys()) ) self.counter = 0 self.date_to_swsr_field = name_to_date_to_field["SWSR"] self.date_to_swsl_field = name_to_date_to_field["SWSL"] #self.date_to_gwdi_field = name_to_date_to_field["GWDI"] self.date_to_upin_field = name_to_date_to_field["UPIN"] #static fields self.slope = name_to_date_to_field["SLOP"] self.channel_length = name_to_date_to_field["LENG"] self.lake_outlet = name_to_date_to_field["LKOU"] self.coef_bf = -np.ones(self.slope.shape) good_points = self.slope >= 0 self.coef_bf[good_points] = (self.slope[good_points]) ** 0.5 / ((self.channel_length[good_points]) ** (4.0/3.0) * data_manager.manning_bf[good_points] ) def _get_closest_ij(self, event): lon, lat = self.basemap(event.xdata, event.ydata, inverse = True) x0, y0, z0 = lat_lon.lon_lat_to_cartesian(lon, lat) dist, i = self.kdtree.query((x0,y0,z0)) lon0, lat0 = self.lons_flat[i], self.lats_flat[i] ind = np.where((self.lons2d == lon0) & (self.lats2d == lat0)) ix = ind[0][0] jy = ind[1][0] return ix, jy def __call__(self,event): if event.button != 3: return i,j = self._get_closest_ij( event ) vals = [ self.date_to_stfl_field[d][i,j] for d in self.dates_sorted ] plt.figure() plt.plot(self.dates_sorted, vals, label = "STFL") mask = self.cell_manager.get_mask_of_cells_connected_with(self.cell_manager.cells[i][j]) print("sum(mask) = ", np.sum(mask)) vals1 = [ np.sum( self.date_to_traf_field[d][mask == 1] ) for d in self.dates_sorted ] plt.plot(self.dates_sorted, vals1, label = "TRAF") vals2 = [ np.sum( self.date_to_tdra_field[d][mask == 1] ) for d in self.dates_sorted ] plt.plot(self.dates_sorted, vals2, label = "TDRA") vals3 = [ np.sum( self.date_to_pr_field[d][mask == 1] ) for d in self.dates_sorted ] plt.plot(self.dates_sorted, vals3, label = "PR") #vals4 = [ # np.sum( self.date_to_gwdi_field[d][mask == 1] ) for d in self.dates_sorted #] #plt.plot(self.dates_sorted, vals4, label = "GWDI") vals5 = [ np.sum( self.date_to_upin_field[d][i,j] ) for d in self.dates_sorted ] plt.plot(self.dates_sorted, vals5, label = "UPIN") if self.upin_mean_field is None: self.upin_mean_field = np.mean(list(self.date_to_upin_field.values()), axis = 0) plt.legend() plt.title("{0}: acc={1} km**2".format(self.counter, self.acc_area_km2[i, j])) # plt.figure() # ax1 = plt.gca() # to_plot_2d = np.ma.masked_where(mask < 0.5, self.upin_mean_field) # img = self.basemap.pcolormesh(self.x_pr, self.y_pr, to_plot_2d, ax = ax1) # plt.colorbar(img, ax = ax1) # self.basemap.drawcoastlines(ax = ax1) # plt.title("min-max: {0};{1}".format(to_plot_2d.min(), to_plot_2d.max())) # # self.ax.annotate(str(self.counter), (event.xdata, event.ydata), font_properties = # FontProperties(size=10), bbox=dict(boxstyle="round", fc="w")) # self.ax.redraw_in_frame() # # plt.figure() # ax1 = plt.gca() # to_plot_2d = np.ma.masked_where(mask < 0.5, self.data_manager.cbf) # img = self.basemap.pcolormesh(self.x_pr, self.y_pr, to_plot_2d, ax = ax1) # plt.colorbar(img, ax = ax1) # self.basemap.drawcoastlines(ax = ax1) # # # plt.title("CBF, {0:g}: v= {1}, min={2}, max={3}".format(self.counter, to_plot_2d[i,j], to_plot_2d.min(), to_plot_2d.max())) # # # plt.figure() # ax1 = plt.gca() # to_plot_2d = np.ma.masked_where(mask < 0.5, self.data_manager.bankfull_storage_m3) # img = self.basemap.pcolormesh(self.x_pr, self.y_pr, to_plot_2d, ax = ax1) # plt.colorbar(img, ax = ax1) # self.basemap.drawcoastlines(ax = ax1) # plt.title("STBM, {0}: v= {1}".format(self.counter, to_plot_2d[i,j])) # # # plt.figure() # ax1 = plt.gca() # mbf = self.data_manager.manning_bf # to_plot_2d = np.ma.masked_where(mask < 0.5, mbf) # img = self.basemap.pcolormesh(self.x_pr, self.y_pr, to_plot_2d, ax = ax1) # plt.colorbar(img, ax = ax1) # self.basemap.drawcoastlines(ax = ax1) # plt.title("MABF, {0}: v= {1}, min={2}, max={3}".format(self.counter, to_plot_2d[i,j], to_plot_2d.min(), to_plot_2d.max())) # # plt.figure() # ax1 = plt.gca() # to_plot_2d = np.ma.masked_where(mask < 0.5, self.slope) # img = self.basemap.pcolormesh(self.x_pr, self.y_pr, to_plot_2d, ax = ax1) # plt.colorbar(img, ax = ax1) # self.basemap.drawcoastlines(ax = ax1) # plt.title("SLOPe, {0}: v= {1}, min={2}, max={3}".format(self.counter, to_plot_2d[i,j], to_plot_2d.min(), to_plot_2d.max())) # # # # # plt.figure() # ax1 = plt.gca() # to_plot_2d = np.ma.masked_where(mask < 0.5, self.data_manager.lake_area) # img = self.basemap.pcolormesh(self.x_pr, self.y_pr, to_plot_2d, ax = ax1) # plt.colorbar(img, ax = ax1) # self.basemap.drawcoastlines(ax = ax1) # plt.title("lake area, {0}: v= {1}".format(self.counter, to_plot_2d[i,j])) # # plt.figure() # ax1 = plt.gca() # to_plot_2d = np.ma.masked_where(mask < 0.5, self.coef_bf) # img = self.basemap.pcolormesh(self.x_pr, self.y_pr, to_plot_2d, ax = ax1) # plt.colorbar(img, ax = ax1) # self.basemap.drawcoastlines(ax = ax1) # plt.title("coef_bf, {0}: v= {1:.1g}, min={2:.1g}, max={3:.1g}".format(self.counter, to_plot_2d[i,j], to_plot_2d.min(), to_plot_2d.max())) # plt.figure() #snow vals6 = [ np.sum( self.date_to_swe_field[d][mask == 1] ) for d in self.dates_sorted ] plt.plot(self.dates_sorted, vals6, label = "SWE") vals4 = [ np.sum( self.date_to_swe_field[d][i,j] ) for d in self.dates_sorted ] plt.plot(self.dates_sorted, vals4, label = "GWST") vals5 = [ np.sum( self.date_to_swsr_field[d][i,j] ) for d in self.dates_sorted ] plt.plot(self.dates_sorted, vals5, label = "SWSR") vals5 = [ np.sum( self.date_to_swsl_field[d][i,j] ) for d in self.dates_sorted ] plt.plot(self.dates_sorted, vals5, label = "SWSL") plt.legend() plt.title("{0}, lkfr = {1}".format(self.counter, self.data_manager.lake_fraction[i,j])) fName = "route_params_{0}_{1}.bin".format(i, j) info = {} #traf -> dict( date -> value in m**3/s ) traf_dict = dict(list(zip(self.dates_sorted, [self.date_to_traf_field[d][i,j] for d in self.dates_sorted])) ) traf_dict = {"TRAF": traf_dict} info.update(traf_dict) upin_dict = dict(list(zip(self.dates_sorted, [self.date_to_upin_field[d][i,j] for d in self.dates_sorted])) ) upin_dict = {"UPIN": upin_dict} info.update(upin_dict) # gwdi_dict = dict(zip(self.dates_sorted, # [self.date_to_gwdi_field[d][i,j] for d in self.dates_sorted]) ) # gwdi_dict = {"GWDI": gwdi_dict} # info.update(gwdi_dict) swsr_dict = dict(list(zip(self.dates_sorted, [self.date_to_swsr_field[d][i,j] for d in self.dates_sorted])) ) swsr_dict = {"SWSR": swsr_dict } info.update(swsr_dict) swsl_dict = dict(list(zip(self.dates_sorted, [self.date_to_swsl_field[d][i,j] for d in self.dates_sorted])) ) swsl_dict = {"SWSL": swsl_dict } info.update(swsl_dict) stfl_dict = dict(list(zip(self.dates_sorted, [self.date_to_stfl_field[d][i,j] for d in self.dates_sorted])) ) stfl_dict = {"STFL": stfl_dict } info.update(stfl_dict) swst_dict = dict(list(zip(self.dates_sorted, [self.date_to_swst_field[d][i,j] for d in self.dates_sorted])) ) swst_dict = {"SWST": swst_dict } info.update(swst_dict) info["SBFM"] = self.data_manager.bankfull_storage_m3[i,j] info["CBF"] = self.data_manager.cbf[i,j] info["LKFR"] = self.data_manager.lake_fraction[i,j] info["LKAR"] = self.data_manager.lake_area[i,j] info["LKOU"] = self.lake_outlet[i,j] pickle.dump(info, open(fName, mode="w")) self.counter += 1 plt.show() pass
def interpolate_to_amno(data_folder, start_year=1970, end_year=1999, rcm="", gcm="", out_folder=""): print "data_folder: {0}".format(data_folder) #check if the result file already exists sim_folder = os.path.join(out_folder, "{0}-{1}_{2}-{3}".format(gcm, rcm, start_year, end_year)) #create a folder for each simulation if not os.path.isdir(sim_folder): os.mkdir(sim_folder) out_path = os.path.join(sim_folder, "narccap_runoff_{0}-{1}_{2}-{3}.nc".format(start_year, end_year, gcm, rcm)) if os.path.isfile(out_path): print("{0} already exists, remove if you want to recreate.".format(out_path)) return srof_pattern = os.path.join(data_folder, "mrros_*_*_*.nc") trof_pattern = os.path.join(data_folder, "mrro_*_*_*.nc") srof_ds = MFDataset(srof_pattern) trof_ds = MFDataset(trof_pattern) lon_in = srof_ds.variables["lon"][:] lat_in = srof_ds.variables["lat"][:] x_in, y_in, z_in = lat_lon.lon_lat_to_cartesian(lon_in.flatten(), lat_in.flatten()) tree = KDTree(zip(x_in, y_in, z_in)) lon_out, lat_out = polar_stereographic.lons.flatten(), polar_stereographic.lats.flatten() x_out, y_out, z_out = lat_lon.lon_lat_to_cartesian(lon_out, lat_out) distances, indices = tree.query(zip(x_out, y_out, z_out)) time_var = srof_ds.variables["time"] time_in_units = time_var[:] times = num2date(time_in_units, time_var.units) time_indices = np.where( np.array(map(lambda x: start_year <= x.year <= end_year, times), dtype=np.bool) )[0] srof_sub = srof_ds.variables["mrros"][time_indices, :, :] trof_sub = trof_ds.variables["mrro"][time_indices, :, :] times_sub = itertools.ifilter(lambda x: start_year <= x.year <= end_year, times) print("selected time window data") #writing result to netcdf out_nc = Dataset(out_path, "w") out_nc.createDimension("x", polar_stereographic.lons.shape[0]) out_nc.createDimension("y", polar_stereographic.lats.shape[1]) out_nc.createDimension("time") srof_var = out_nc.createVariable("mrros", "f4", dimensions=("time", "x", "y")) trof_var = out_nc.createVariable("mrro", "f4", dimensions=("time", "x", "y")) assert isinstance(srof_var, Variable) srof_in_var = srof_ds.variables["mrros"] for attr_name in srof_in_var.ncattrs(): print attr_name srof_var.setncattr(attr_name, getattr(srof_in_var, attr_name)) trof_in_var = trof_ds.variables["mrro"] for attr_name in trof_in_var.ncattrs(): print attr_name trof_var.setncattr(attr_name, getattr(trof_in_var, attr_name)) t_var = out_nc.createVariable("time", "f4", dimensions=("time",)) lon_var = out_nc.createVariable("longitude", "f4", dimensions=( "x", "y")) lat_var = out_nc.createVariable("latitude", "f4", dimensions=("x", "y")) t_var.units = time_var.units print("interpolating and saving data to netcdf file") nrows, ncols = polar_stereographic.lons.shape #interpolate in time if necessary n_interps = 0 for i, t in enumerate(times_sub): sr_slice = srof_sub[i, :, :].flatten() tr_slice = trof_sub[i, :, :].flatten() trof1 = tr_slice[indices].reshape(nrows, ncols) srof1 = sr_slice[indices].reshape(nrows, ncols) if hasattr(trof1, "mask") and np.all(trof1.mask): trof1 = trof_var[i - 1, :, :] n_interps += 1 if hasattr(srof1, "mask") and np.all(srof1.mask): srof1 = srof_var[i - 1, :, :] trof_var[i, :, :] = trof1 srof_var[i, :, :] = srof1 t_var[i] = date2num(t, time_var.units) print "Number of interpolations in time: {0}".format(n_interps) lon_var[:] = polar_stereographic.lons lat_var[:] = polar_stereographic.lats out_nc.close()
class GrdcDataManager: def __init__(self, path_tofolder = "/home/huziy/skynet3_exec1/grdc_global"): self.path_to_annual_rof = os.path.join(path_tofolder, "obs_ro.grd") self.lons2d = None self.lats2d = None self.ncols = None self.nrows = None self.xll = None self.yll = None self.cellsize = None self.nodata_value = None self.ktree = None pass def _get_lon_lats(self): if self.lons2d is None: lons = [ self.xll + i * self.cellsize for i in range(self.ncols) ] lats = [ self.yll + i * self.cellsize for i in range(self.nrows) ] self.lats2d, self.lons2d = np.meshgrid(lats, lons) return self.lons2d, self.lats2d def interpolate_data_to_model_grid(self, model_lons_2d, model_lats_2d, data_obs): x0, y0, z0 = lat_lon.lon_lat_to_cartesian(model_lons_2d.flatten(), model_lats_2d.flatten()) x, y, z = lat_lon.lon_lat_to_cartesian(self.lons2d.flatten(), self.lats2d.flatten()) if self.ktree is None: self.ktree = KDTree(list(zip(x, y, z))) d, i = self.ktree.query(list(zip(x0, y0, z0))) return data_obs.flatten()[i].reshape(model_lons_2d.shape) def _read_data_from_file(self, path): f = open(path) vals = [] for line in f: line = line.strip() if line == "": continue if line.startswith("ncols"): self.ncols = int(line.split()[1].strip()) elif line.startswith("nrows"): self.nrows = int(line.split()[1].strip()) elif line.startswith("xllcorner"): self.xll = float(line.split()[1].strip()) elif line.startswith("yllcorner"): self.yll = float(line.split()[1].strip()) elif line.startswith("cellsize"): self.cellsize = float(line.split()[1].strip()) elif line.startswith("NODATA"): self.nodata_value = int(line.split()[1].strip()) else: vals.append( list(map(float, [s.strip() for s in line.split()])) ) #print len(vals), self.ncols * self.nrows vals = np.array( vals[::-1] )#reverse row order vals = vals.transpose() return vals def get_mean_annual_runoff_in_mm_per_s(self): vals = self._read_data_from_file(self.path_to_annual_rof) #vals = np.ma.masked_where(vals.astype(int) == self.nodata_value, vals) vals = np.ma.masked_where(vals < 0, vals) print(self.nodata_value, np.min(vals), self.nodata_value == np.min(vals)) vals /= 365 * 24 * 60 * 60 #convert to mm/s self._get_lon_lats() return self.lons2d, self.lats2d, vals
def compare_alt(): #obs stations = get_station_list() #model data alts_model = [ active_layer_thickness.get_alt_for_year(the_year) for the_year in range(1991, 2001) ] alt_mean = np.mean(alts_model, axis=0) b, lons2d, lats2d = draw_regions.get_basemap_and_coords() permafrost_kinds = draw_regions.get_permafrost_mask(lons2d, lats2d) permafrost_kinds_flat = permafrost_kinds.flatten() lons2d[lons2d > 180] -= 360 #find corresponding indices on the model grid x, y, z = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten()) kdtree = KDTree(list(zip(x, y, z))) alt_mean_flat = alt_mean.flatten() h_mod = [] h_obs = [] station_lons = [] station_lats = [] for the_station in stations: x0, y0, z0 = lat_lon.lon_lat_to_cartesian(the_station.lon, the_station.lat) d, i = kdtree.query([x0, y0, z0]) if permafrost_kinds_flat[i] not in (1,2): continue print(d, i) print(the_station.mean_alt_m, alt_mean_flat[i]) h_mod.append(alt_mean_flat[i]) h_obs.append(the_station.mean_alt_m) station_lons.append(the_station.lon) station_lats.append(the_station.lat) plot_utils.apply_plot_params(width_pt=None, height_cm=20, width_cm=16, font_size=12) fig = plt.figure() gs = gridspec.GridSpec(2,1) ax = fig.add_subplot(gs[0,0]) ax.plot(h_obs, h_mod, 'o') ax.set_xlabel("Obs.") ax.set_ylabel("Mod.") upper_lim = max(np.max(h_mod), np.max(h_obs)) ax.set_xlim(0, upper_lim + 0.1 * upper_lim) ax.set_ylim(0, upper_lim + 0.1 * upper_lim) ax = fig.add_subplot(gs[1,0]) min_lon, max_lon = min(station_lons), max(station_lons) min_lat, max_lat = min(station_lats), max(station_lats) dx = (max_lon - min_lon) * 0.1 dy = (max_lat - min_lat) * 0.6 min_lon -= dx max_lon += dx min_lat -= dy max_lat += dy lon1 = -97 lat1 = 47.50 lon2 = -7 lat2 = 0 b_zoom = Basemap(projection="omerc", resolution="l", llcrnrlon=min_lon, llcrnrlat=min_lat, urcrnrlon=max_lon, urcrnrlat=max_lat, lat_1=lat1, lon_1=lon1, lat_2=lat2, lon_2=lon2, no_rot=True ) s_x, s_y = b_zoom(station_lons, station_lats) b_zoom.scatter(s_x, s_y, c = "r", ax = ax, marker = "*", s = 30, linewidths = 0.1, zorder = 2) b_zoom.drawcoastlines(ax = ax, linewidth = 0.5) fig.savefig("pf_validate.png") pass
def plot_current_alts(): from . import plot_dpth_to_bdrck bdrck_field = plot_dpth_to_bdrck.get_depth_to_bedrock() start_year = 1980 end_year = 1996 sim_data_folder = "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/era40_driven_b1" coord_file = os.path.join(sim_data_folder, "pmNorthAmerica_0.44deg_ERA40-Int_B1_200812_moyenne") #coord_file = "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/NA_1.0deg_soil_spinup2/pmNA_1.0deg_soil_spinup2_228006_moyenne" sim_names = ["ERA40", "MPI","CanESM"] simname_to_path = { #"ERA40": "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/era40_driven_b1", "ERA40": "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/NorthAmerica_0.44deg_ERA40-Int_old_snow_cond", #"ERA40" : "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/NA_1.0deg_soil_spinup2", "MPI": "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/NorthAmerica_0.44deg_MPI_B1", "CanESM": "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/NorthAmerica_0.44deg_CanESM_B1" } basemap, lons2d, lats2d = draw_regions.get_basemap_and_coords(resolution="c", file_path = coord_file, llcrnrlat=45.0, llcrnrlon=-145, urcrnrlon=-20, urcrnrlat=74, anchor="W" ) assert isinstance(basemap, Basemap) #basemap.transform_scalar() #basemap = Basemap() lons2d[lons2d > 180] -= 360 x, y = basemap(lons2d, lats2d) #x = (x[1:,1:] + x[:-1, :-1]) /2.0 permafrost_mask = draw_regions.get_permafrost_mask(lons2d, lats2d) mask_cond = (permafrost_mask <= 0) | (permafrost_mask >= 2) # plot_utils.apply_plot_params(width_pt=None, width_cm=20, height_cm=40, font_size=16) fig = plt.figure() assert isinstance(fig, Figure) h_max = 10 bounds = [0,0.1,0.5,1,2,3,4,5] cmap = my_colormaps.get_lighter_jet_cmap(ncolors=len(bounds) - 1) #cm.get_cmap("jet",10) #cmap = my_colormaps.get_cmap_wo_red(ncolors=len(bounds) - 1) norm = BoundaryNorm(boundaries=bounds,ncolors=len(bounds), clip=True) cmap.set_over(cmap(1.0)) clevels = np.arange(0,h_max+1,1) gs = gridspec.GridSpec(3,2, width_ratios=[1,0.06], hspace=0, wspace=0.0, left=0.05, bottom = 0.02, top=0.95) all_axes = [] all_img = [] i = 0 hc_list = [] for name in sim_names: path = simname_to_path[name] dm = CRCMDataManager(data_folder=path) hc0, t3d_min, t3d_max = dm.get_alt_using_monthly_mean_climatology(list(range(start_year,end_year+1))) hc_list.append(hc0) ax = fig.add_subplot(gs[i,0]) #cp = SoundingPlotter(ax, basemap, t3d_min, t3d_max, lons2d, lats2d, levelheights=dm.level_heights) assert isinstance(ax, Axes) hc = np.ma.masked_where(mask_cond | (hc0 < 0), hc0) #hc = np.ma.masked_where( (hc0 < 0), hc0) hc5 = np.ma.masked_where((hc0 <= 15) | hc.mask, hc) img = basemap.pcolormesh(x, y, hc, cmap = cmap, vmax = h_max, norm=norm) if not i: ax.set_title("ALT ({0} - {1}) \n".format(start_year, end_year)) i += 1 ax.set_ylabel("CRCM ({0})".format(name)) all_axes.append(ax) all_img.append(img) print(np.ma.min(hc), np.ma.max(hc)) #hc5 = np.ma.masked_where((hc0 <= 6) | hc.mask, hc) #print "Number of cells with alt > 5 is {0}, and the range is {1} ... {2}".format(hc5.count(), hc5.min(), hc5.max()) #bdrck_field5 = np.ma.masked_where(hc5.mask, bdrck_field) #print "Bedrock ranges for those points: {0} ... {1}".format(bdrck_field5.min(), bdrck_field5.max()) ind = np.where(~hc5.mask) xs = ind[0] ys = ind[1] all_months, all_temps = dm.get_monthly_mean_soiltemps(year_range=range(start_year,end_year+1)) all_months_ord = date2num(all_months) # mpl.rcParams['contour.negative_linestyle'] = 'solid' # # for the_i, the_j in zip(xs,ys): # #plot profile # plt.figure() # plt.plot(t3d_max[the_i, the_j, :] - dm.T0, dm.level_heights, color = "r") # plt.plot(t3d_min[the_i, the_j, :] - dm.T0, dm.level_heights, color = "b") # plt.plot([0 , 0], [dm.level_heights[0], dm.level_heights[-1]], color = "k") # # x1, x2 = plt.xlim() # #plt.plot( [x1, x2], [bdrck_field[the_i, the_j], bdrck_field[the_i, the_j]], color = "k", lw = 3 ) # #plt.title(str(i) + ", dpth_to_bedrock = {0} m".format(bdrck_field[the_i, the_j])) # plt.title(str(i)) # plt.gca().invert_yaxis() # plt.savefig("prof{0}.png".format(i)) # ax.annotate(str(i), (x[the_i, the_j], y[the_i, the_j]), font_properties = # FontProperties(size=10)) # # # #plot vertical temp cross-section # plt.figure() # plt.title(str(i) + ", ({0} - {1})".format(start_year, end_year)) # # levs2d, times2d = np.meshgrid(dm.level_heights, all_months_ord) # clevs = [-25,-20,-10,-5,-1,0,1,5,10,20,25] # norm = BoundaryNorm(boundaries=clevs, ncolors=len(clevs) - 1) # cmap = cm.get_cmap("jet", len(clevs) - 1) # # img = plt.contourf(times2d, levs2d, all_temps[:,the_i, the_j, :] - dm.T0, levels = clevs, cmap = cmap, norm = norm) # #plt.contour(times2d, levs2d, all_temps[:,the_i, the_j, :] - dm.T0, levels = clevs, colors = "k", linewidth = 1) # the_ax = plt.gca() # assert isinstance(the_ax, Axes) # the_ax.invert_yaxis() # the_ax.xaxis.set_major_formatter(FuncFormatter( # lambda x, pos: num2date(float(x)).strftime("%Y") # )) # # print "i = {0}; lon, lat = {1}, {2}".format(i, lons2d[the_i, the_j], lats2d[the_i, the_j]) # # plt.colorbar(img, ticks = clevs) # # plt.savefig("temp_section_{0}.png".format(i)) # # i += 1 print("lons = [{0}]".format(",".join([str(x) for x in lons2d[np.array(xs), np.array(ys)]]))) print("lats = [{0}]".format(",".join([str(x) for x in lats2d[np.array(xs), np.array(ys)]]))) # draw barplot with numbers of alt in given ranges # plt.figure() # alt_ranges = xrange(0,18) # numbers = [] # for the_alt in alt_ranges: # hci = np.ma.masked_where( (hc0 < the_alt) | (hc0 > the_alt + 1) | hc.mask ,hc) # numbers.append(hci.count()) # plt.bar(alt_ranges, numbers, width=1) # plt.xlabel("ALT (m)") # plt.ylabel("Number of cells") # plt.savefig("numbers_in_range.png") i = 0 axs_to_hide = [] #zones and coastlines for the_ax, the_img in zip(all_axes, all_img): # divider = make_axes_locatable(the_ax) # cax = divider.append_axes("right", "5%", pad="3%") assert isinstance(the_ax, Axes) basemap.drawcoastlines(ax = the_ax, linewidth=0.5) basemap.readshapefile("data/pf_4/permafrost8_wgs84/permaice", name="zone", ax=the_ax, linewidth=1.5, drawbounds=False) for nshape,seg in enumerate(basemap.zone): if basemap.zone_info[nshape]["EXTENT"] not in ["C"]: continue poly = mpl.patches.Polygon(seg,edgecolor = "k", facecolor="none", zorder = 10, lw = 1.5) the_ax.add_patch(poly) # if i != 1: # axs_to_hide.append(cax) i += 1 cax = fig.add_subplot(gs[:,1]) cax.set_anchor("W") cax.set_aspect(35) formatter = FuncFormatter( lambda x, pos: "{0: <6}".format(x) ) cb = fig.colorbar(all_img[0], ax = cax, cax = cax, extend = "max", ticks = bounds, format = formatter) cax.set_title("m") #fig.tight_layout(h_pad=0) # for the_ax in axs_to_hide: # the_ax.set_visible(False) fig.savefig("alt_from_climatology_current.png") #print ALT for selected points site_names = ["S","K","T"] sel_lons = [-75.646, -65.92, -69.95] sel_lats = [62.197, 58.709, 58.67] xo,yo,zo = lat_lon.lon_lat_to_cartesian(sel_lons, sel_lats) xi, yi, zi = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten()) ktree = KDTree(list(zip(xi,yi,zi))) dists, indexes = ktree.query(list(zip(xo,yo,zo))) for name, data in zip(sim_names, hc_list): print(name) flat_data = data.flatten() for p_name, ind in zip(site_names, indexes): print(p_name, "{0} m".format(flat_data[ind])) print("--" * 10) pass
def plot_current_alts_nyear_rule(nyear = 2): start_year = 1981 end_year = 2008 sim_data_folder = "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/era40_driven_b1" sim_names = ["ERA40", "MPI","CanESM"] all_data_f = "/home/huziy/skynet1_rech3/cordex/for_Samira" simname_to_path = { "ERA40": os.path.join(all_data_f, "alt_era_b1_yearly.nc"), "MPI": os.path.join(all_data_f, "alt_mpi_b1_yearly.nc"), "CanESM": os.path.join(all_data_f, "alt_canesm_b1_yearly.nc") } coord_file = os.path.join(sim_data_folder, "pmNorthAmerica_0.44deg_ERA40-Int_B1_200812_moyenne") basemap, lons2d, lats2d = draw_regions.get_basemap_and_coords(resolution="c", file_path = coord_file, llcrnrlat=40.0, llcrnrlon=-145, urcrnrlon=-20, urcrnrlat=74 ) assert isinstance(basemap, Basemap) #basemap.transform_scalar() #basemap = Basemap() lons2d[lons2d > 180] -= 360 x, y = basemap(lons2d, lats2d) #x = (x[1:,1:] + x[:-1, :-1]) /2.0 permafrost_mask = draw_regions.get_permafrost_mask(lons2d, lats2d) mask_cond = (permafrost_mask <= 0) | (permafrost_mask >= 3) # plot_utils.apply_plot_params(width_pt=None, width_cm=20, height_cm=40, font_size=25) fig = plt.figure() assert isinstance(fig, Figure) h_max = 10 cmap = my_colormaps.get_lighter_jet_cmap(ncolors=10) #cm.get_cmap("jet",10) bounds = [0,0.1,0.5,1,2,3,5,8,9,10,11] norm = BoundaryNorm(boundaries=bounds,ncolors=len(bounds), clip=True) cmap.set_over(cmap(1.0)) clevels = np.arange(0,h_max+1,1) gs = gridspec.GridSpec(3,1) all_axes = [] all_img = [] i = 0 hc_list = [] hct_list = [] for name in sim_names: path = simname_to_path[name] #select data and needed alt ds = Dataset(path) years = ds.variables["year"][:] hct = ds.variables["alt"][(years >= start_year) & (years <= end_year),:,:] hct_list.append(hct) print("hct.shape = ", hct.shape) #hc = get_alt_using_nyear_rule(hct, nyears = nyear) hc = np.mean(hct, axis = 0) hc_list.append(hc) ax = fig.add_subplot(gs[i,0]) assert isinstance(ax, Axes) hc = np.ma.masked_where(mask_cond | (np.min(hct, axis = 0) < 0), hc) #hc = np.ma.masked_where( (hc < 0), hc) img = basemap.pcolormesh(x, y, hc, cmap = cmap, vmax = h_max, norm=norm) if not i: ax.set_title("ALT, mean ({0} - {1}) \n".format(start_year, end_year)) i += 1 ax.set_ylabel(name) all_axes.append(ax) all_img.append(img) i = 0 axs_to_hide = [] #zones and coastlines for the_ax, the_img in zip(all_axes, all_img): assert isinstance(the_ax, Axes) basemap.drawcoastlines(ax = the_ax, linewidth=0.5) basemap.readshapefile("data/pf_4/permafrost8_wgs84/permaice", name="zone", ax=the_ax, linewidth=1.5) divider = make_axes_locatable(the_ax) cax = divider.append_axes("right", "5%", pad="3%") cb = fig.colorbar(the_img, cax = cax, extend = "max", ticks = bounds) cax.set_title("m \n") if i != 2: axs_to_hide.append(cax) i += 1 fig.tight_layout(w_pad=0.0) for the_ax in axs_to_hide: the_ax.set_visible(False) fig.savefig("alt_mean_current.png") #print ALT for selected points site_names = ["S","K","T"] sel_lons = [-75.646, -65.92, -69.95] sel_lats = [62.197, 58.709, 58.67] xo,yo,zo = lat_lon.lon_lat_to_cartesian(sel_lons, sel_lats) xi, yi, zi = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten()) ktree = KDTree(list(zip(xi,yi,zi))) dists, indexes = ktree.query(list(zip(xo,yo,zo))) for name, data, the_hct in zip(sim_names, hc_list, hct_list): print(name) flat_data = data.flatten() for p_name, ind in zip(site_names, indexes): in_data = [] for t in range(the_hct.shape[0]): in_data.append(the_hct[t,:,:].flatten()[ind]) print(",".join(["{0:.1f}".format(float(x)) for x in in_data])) print(p_name, "{0:.1f} m".format(float(flat_data[ind]))) print("--" * 10)
class RealWorldMap(object): def __init__(self, patches, projections): self.patches = patches self.projections = projections self.build() def build(self): logger.debug("Building maps for conversion between real and image space") self.imagetree = {} self.imagemap = {} self.realtree = KDTree([x.realcoords for x in self.patches]) self.realmapping = {} for num, patch in enumerate(self.patches): if num % 1000 == 0: logger.debug("Built maps for {0} of {1} patches".format(num, len(self.patches))) resp = {} for _, projection in self.projections.items(): inimage = patch.project(projection) if projection.id not in self.imagetree: self.imagetree[projection.id] = [] self.imagemap[projection.id] = {} self.imagetree[projection.id].append(inimage) self.imagemap[projection.id][tuple(inimage)] = patch.realcoords resp[projection.id] = inimage self.realmapping[tuple(patch.realcoords)] = resp logger.debug("Done building maps for {0} patches".format(len(self.patches))) logger.debug("Building image KD tree") for key, imagecoords in self.imagetree.items(): self.imagetree[key] = KDTree(imagecoords) def realtoimages(self, coords): _, nearestindex = self.realtree.query(coords) nearest = self.realtree.data[nearestindex] return self.realmapping[tuple(nearest)] def realregiontoimages(self, coords): _, nearestindices = self.realtree.query(coords) resp = {} for nearestindex in nearestindices: nearest = self.realtree.data[nearestindex] points = self.realmapping[tuple(nearest)] for k, v in points.iteritems(): if k not in resp: resp[k] = [] resp[k].append(v) return resp def imagetoreal(self, projection, coords): try: projection = projection.id except: pass _, nearestindex = self.imagetree[projection].query(coords) nearest = self.imagetree[projection].data[nearestindex] return self.imagemap[projection][tuple(nearest)]
def convert(inPath, lonlats): ds = gdal.Open(inPath, gdal.GA_ReadOnly) assert isinstance(ds, Dataset) (Xul, deltaX, rotation, Yul, rotation, deltaY) = ds.GetGeoTransform() print(dir(ds)) print(ds.GetMetadata_Dict()) print(ds.GetDescription()) srs_wkt = ds.GetProjection() Nx = ds.RasterXSize Ny = ds.RasterYSize print(ds.RasterCount) nxToRead = Nx / 2 nyToRead = int(Ny / 1.5) data = ds.GetRasterBand(1).ReadAsArray(0, 0, nxToRead, nyToRead).transpose() print(srs_wkt) print(data.shape) #plt.imshow(data) #plt.show() ds = None print(Xul, Yul, deltaX, deltaY, rotation) x1d = np.arange(Xul, Xul + deltaX * nxToRead, deltaX) y1d = np.arange(Yul, Yul + deltaY * nyToRead, deltaY) assert len(x1d) == nxToRead assert len(y1d) == nyToRead y, x = np.meshgrid(y1d, x1d) fieldName = os.path.basename(inPath).split("_")[0].lower() coef = name_to_mult[fieldName] no_data = name_to_nodata_value[fieldName] usable = (data != no_data) print(x.shape, usable.shape) x0 = x[usable] y0 = y[usable] cartx, carty, cartz = lat_lon.lon_lat_to_cartesian(x0, y0) data_1d = data[usable] print("useful data points : {0}".format(len(x0))) tree = KDTree(list(zip(cartx, carty, cartz))) print("constructed the kdtree") xi, yi, zi = lat_lon.lon_lat_to_cartesian(lonlats[:,0], lonlats[:,1]) dists, inds = tree.query(list(zip(xi, yi, zi)), k = AGGR_SIZE) npoints = dists.shape[0] interp_data = np.zeros((npoints, )) for i in range(npoints): the_dists = dists[i,:] the_inds = inds[i,:] good_pts = (the_dists < LIMIT_DIST) if len(the_dists[good_pts]) < 0.25 * AGGR_SIZE: #if there is no usable points in the vicinity, then set the value to no_data interp_data[i] = -1 continue the_dists = the_dists[good_pts] the_inds = the_inds[good_pts] interp_coefs = 1.0 / the_dists ** 2 interp_data[i] = np.sum( interp_coefs * data_1d[the_inds] ) / np.sum(interp_coefs) interp_data[interp_data >= 0] *= coef print("completed interpolation") return interp_data
class GldasManager(): def __init__(self, folder_path="/home/huziy/skynet3_exec1/gldas_data"): """ Data access interface to the folder of netcdf files runoff units: kg/m^2/s = mm/s """ self.data_folder = folder_path self.surface_rof_varname = "Qs_GDS0_SFC_ave4h" self.subsurface_rof_varname = "Qsb_GDS0_SFC_ave4h" self.date_format = "%m/%d/%Y (%H:%M)" self._init_date_to_path_dict() self._init_kd_tree() pass def plot_subsrof_ts(self, i=0, j=0): all_dates = list(sorted(self.date_to_path.keys())) vals = [self.get_field_for_date(x, var_name=self.subsurface_rof_varname)[i, j] for x in all_dates] vals1 = [self.get_field_for_date(x, var_name=self.surface_rof_varname)[i, j] for x in all_dates] print(min(vals), max(vals)) dates_num = date2num(all_dates) print(min(dates_num), max(dates_num)) import matplotlib.pyplot as plt plt.figure() plt.plot(dates_num, vals, label="subsurf rof") plt.plot(dates_num, vals1, label="surf rof") plt.legend() #plt.xticks(rotation='vertical') plt.show() def _init_date_to_path_dict(self): self.date_to_path = {} for fName in os.listdir(self.data_folder): if not fName.endswith(".nc"): continue #regard only nectdf files path = os.path.join(self.data_folder, fName) ds = Dataset(path) srofVar = ds.variables[self.surface_rof_varname] date = datetime.strptime(srofVar.initial_time, self.date_format) self.date_to_path[date] = path ds.close() def _init_kd_tree(self): """ Has to be called after self._init_date_to_path_dict """ if not len(self.date_to_path): print("You should call {0} first".format("self._init_date_to_path_dict")) raise Exception() for d, path in self.date_to_path.items(): ds = Dataset(path) lons1d = ds.variables["g0_lon_1"][:] lats1d = ds.variables["g0_lat_0"][:] self.lats2d, self.lons2d = np.meshgrid(lats1d, lons1d) x, y, z = lat_lon.lon_lat_to_cartesian(self.lons2d.flatten(), self.lats2d.flatten()) self.kdtree = KDTree(list(zip(x, y, z))) return pass def get_field_for_month_and_year(self, var_name="", month=None, year=None): d1 = datetime(year=year, month=month, day=1) path = self.date_to_path[d1] ds = Dataset(path) return ds.variables[var_name][:] def get_field_for_date(self, the_date, var_name=""): path = self.date_to_path[the_date] ds = Dataset(path) data = ds.variables[var_name][:].transpose() # transpose because I allways use (lon, lat) order of coordinates ds.close() return data def get_srof_spat_integrals_over_points_in_time(self, lons2d_target, lats2d_target, mask, areas2d, start_date=None, end_date=None): return self._get_spatial_integrals_over_points_in_time(lons2d_target, lats2d_target, mask, areas2d, start_date=start_date, end_date=end_date, var_name=self.surface_rof_varname) def get_subsrof_spat_integrals_over_points_in_time(self, lons2d_target, lats2d_target, mask, areas2d, start_date=None, end_date=None): return self._get_spatial_integrals_over_points_in_time(lons2d_target, lats2d_target, mask, areas2d, start_date=start_date, end_date=end_date, var_name=self.subsurface_rof_varname) def _get_spatial_integrals_over_points_in_time(self, lons2d_target, lats2d_target, mask, areas2d, start_date=None, end_date=None, var_name=""): """ i) Interpolate to the grid (lons2d_target, lats2d_target) ii) Apply the mask to the interpoated fields and sum with coefficients from areas2d Note: the interpolation is done using nearest neighbor approach returns a timeseries of {t -> sum(Ai[mask]*xi[mask])(t)} """ #interpolation x1, y1, z1 = lat_lon.lon_lat_to_cartesian(lons2d_target.flatten(), lats2d_target.flatten()) dists, indices = self.kdtree.query(list(zip(x1, y1, z1))) mask1d = mask.flatten().astype(int) areas1d = areas2d.flatten() result = {} for the_date in list(self.date_to_path.keys()): if start_date is not None: if start_date > the_date: continue if end_date is not None: if end_date < the_date: continue data = self.get_field_for_date(the_date, var_name=var_name) result[the_date] = np.sum(data.flatten()[indices][mask1d == 1] * areas1d[mask1d == 1]) times = list(sorted(result.keys())) values = [result[x] for x in times] print("nvals, min, max", len(values), min(values), max(values)) return TimeSeries(time=times, data=values)
def main(): start_year = 1970 end_year = 1999 stations = cehq_station.read_station_data(folder="data/cehq_measure_data_all") stations = list( itertools.ifilter( lambda s: s.is_natural, stations) ) for s in stations: s.delete_data_after_year(end_year) s.delete_data_before_year(start_year) pass stations = list( itertools.ifilter(lambda s: s.get_num_of_years_with_continuous_data() >= 10, stations) ) s = stations[0] assert isinstance(s, Station) #stations = list( itertools.ifilter(lambda s: s.is_natural, stations) ) x, y = polar_stereographic.lons, polar_stereographic.lats basemap = polar_stereographic.basemap x, y = basemap(x,y) sx = [s.longitude for s in stations] sy = [s.latitude for s in stations] sx, sy = basemap(sx, sy) #read model data model_file_path = "data/streamflows/hydrosheds_euler9/aex_discharge_1970_01_01_00_00.nc" acc_area = data_select.get_field_from_file(path=model_file_path, field_name="drainage") i_indices, j_indices = data_select.get_indices_from_file(path=model_file_path) lons_1d = data_select.get_field_from_file(path=model_file_path, field_name="longitude") lats_1d = data_select.get_field_from_file(path=model_file_path, field_name="latitude") x1d, y1d, z1d = lat_lon.lon_lat_to_cartesian(lons_1d, lats_1d) kdtree = KDTree(zip(x1d, y1d, z1d)) print "Id: 4 DA (km2) <-> 4 dist (km) <-> 4 (i,j)" #basemap.scatter(sx, sy, c = "r", zorder = 5) for s, isx, isy in zip( stations, sx, sy ): assert isinstance(s, Station) plt.annotate(s.id, xy=(isx, isy), bbox = dict(facecolor = 'white'), weight = "bold", font_properties = FontProperties(size=0.5)) #get model drainaige areas for the four closest gridcells to the station x0, y0, z0 = lat_lon.lon_lat_to_cartesian(s.longitude, s.latitude) dists, indices = kdtree.query([x0, y0, z0], k = 4) dists /= 1000 print("{0}: {1:.1f}; {2:.1f}; {3:.1f}; {4:.1f} <-> {5:.1f}; {6:.1f}; {7:.1f}; {8:.1f} <-> {9};{10};{11};{12}".format( "{0} (S_DA = {1:.1f})".format(s.id, s.drainage_km2), float(acc_area[indices[0]]), float(acc_area[indices[1]]), float(acc_area[indices[2]]), float(acc_area[indices[3]]), float( dists[0] ), float(dists[1]), float(dists[2]), float(dists[3]), "({0}, {1})".format(i_indices[indices[0]] + 1, j_indices[indices[0]] + 1), "({0}, {1})".format(i_indices[indices[1]] + 1, j_indices[indices[1]] + 1), "({0}, {1})".format(i_indices[indices[2]] + 1, j_indices[indices[2]] + 1), "({0}, {1})".format(i_indices[indices[3]] + 1, j_indices[indices[3]] + 1) )) basemap.drawcoastlines(linewidth=0.5) xmin, xmax = min(sx), max(sx) ymin, ymax = min(sy), max(sy) marginx = (xmax - xmin) * 0.1 marginy = (ymax - ymin) * 0.1 xmin -= marginx * 1.5 xmax += marginx * 2 ymin -= marginy ymax += marginy * 2 plt.xlim(xmin, xmax) plt.ylim(ymin, ymax) plt.tight_layout() basin_boundaries.plot_basin_boundaries_from_shape(basemap=basemap, plotter=plt, linewidth=1) plt.savefig("10yr_cont_stations_natural_fs0.5.pdf") #plt.show() pass
class SoundingPlotter: def __init__(self, ax , basemap, tmin_3d, tmax_3d, lons2d, lats2d, levelheights = None): """ Plots a vertical profile at the point nearest to the clicked one :type ax: Axes """ assert isinstance(ax, Axes) self.basemap = basemap assert isinstance(self.basemap, Basemap) self.tmin_3d = tmin_3d self.tmax_3d = tmax_3d self.lons2d = lons2d self.lats2d = lats2d self.T0 = 273.15 self.lons_flat = lons2d.flatten() self.lats_flat = lats2d.flatten() self.level_heights = levelheights self.counter = 0 self.ax = ax x, y, z = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten()) self.kdtree = KDTree(list(zip(x,y,z))) ax.figure.canvas.mpl_connect("button_press_event", self) pass def _get_closest_ij(self, event): lon, lat = self.basemap(event.xdata, event.ydata, inverse = True) x0, y0, z0 = lat_lon.lon_lat_to_cartesian(lon, lat) dist, i = self.kdtree.query((x0,y0,z0)) lon0, lat0 = self.lons_flat[i], self.lats_flat[i] ind = np.where((self.lons2d == lon0) & (self.lats2d == lat0)) ix = ind[0][0] jy = ind[1][0] return ix, jy def _plot_sounding(self, ax, ix, jy): ax.plot(self.tmax_3d[ix, jy, :] - self.T0, self.level_heights, color = "r") ax.plot(self.tmin_3d[ix, jy, :] - self.T0, self.level_heights, color = "b") ax.plot([0 , 0], [self.level_heights[0], self.level_heights[-1]], color = "k") ax.set_title(str(self.counter)) assert isinstance(ax, Axes) ax.invert_yaxis() def __call__(self, event): print(event.xdata, event.ydata) print(event.button) if event.button != 3: return ix, jy = self._get_closest_ij(event) fig = plt.figure() sounding_ax = fig.add_subplot(1,1,1) self._plot_sounding(sounding_ax, ix, jy) self.ax.annotate(str(self.counter), (event.xdata, event.ydata), font_properties = FontProperties(size=10)) self.ax.redraw_in_frame() self.counter += 1 assert isinstance(fig, Figure) plt.show() pass
class TimeSeriesPlotter: def __init__(self, ax, basemap, lons2d, lats2d, ncVarDict, times, start_date, end_date): """ Plots a vertical profile at the point nearest to the clicked one :type ax: Axes """ assert isinstance(ax, Axes) self.basemap = basemap assert isinstance(self.basemap, Basemap) self.lons_flat = lons2d.flatten() self.lats_flat = lats2d.flatten() self.ncVarDict = ncVarDict self.lons2d = lons2d self.lats2d = lats2d self.counter = 0 self.ax = ax x, y, z = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten()) self.kdtree = KDTree(list(zip(x,y,z))) self.sel_time_indices = np.where([start_date <= t <= end_date for t in times])[0] self.times = times[self.sel_time_indices] ax.figure.canvas.mpl_connect("button_press_event", self) def _get_closest_ij(self, event): lon, lat = self.basemap(event.xdata, event.ydata, inverse = True) x0, y0, z0 = lat_lon.lon_lat_to_cartesian(lon, lat) dist, i = self.kdtree.query((x0,y0,z0)) lon0, lat0 = self.lons_flat[i], self.lats_flat[i] ind = np.where((self.lons2d == lon0) & (self.lats2d == lat0)) ix = ind[0][0] jy = ind[1][0] return ix, jy def _plot_timeseries(self, ax, ix, jy): fig_daily = plt.figure() ax_daily = plt.gca() fig_monthly = plt.figure() ax_monthly = plt.gca() for varName, ncVar in self.ncVarDict.items(): sel_values = ncVar[self.sel_time_indices, 0, ix, jy] ax.plot(self.times, sel_values, label = varName) #calculate and plot daily means ts = pd.TimeSeries(index = self.times, data = sel_values) ts = ts.resample("D", how = "mean") ax_daily.plot(ts.index, ts.values, label = varName) #calculate and plot monthly means ts = ts.resample("M", how = "mean") ax_monthly.plot(ts.index, ts.values, label = varName) ax.legend() ax.set_title(str(self.counter)) ax_daily.legend() ax_daily.set_title(str(self.counter) + " - daily") ax_monthly.legend() ax_monthly.set_title(str(self.counter) + " - monthly") assert isinstance(ax, Axes) def __call__(self, event): print(event.xdata, event.ydata) print(event.button) if event.button != 3: return ix, jy = self._get_closest_ij(event) fig = plt.figure() sounding_ax = fig.add_subplot(1,1,1) self._plot_timeseries(sounding_ax, ix, jy) self.ax.annotate(str(self.counter), (event.xdata, event.ydata), font_properties = FontProperties(size=10)) self.ax.redraw_in_frame() self.counter += 1 assert isinstance(fig, Figure) plt.show()