def regenerate_station_to_gridcell_mapping(start_year, end_year, model_manager): """ should be called when grid or search algorithm change """ assert isinstance(model_manager, Crcm5ModelDataManager) ktree = model_manager.kdtree model_acc_area = model_manager.accumulation_area_km2 model_acc_area_1d = model_acc_area.flatten() # selected_ids = ["104001", "103715", # "093806", "093801", # "092715", # "081006", "040830"] selected_ids = None start_date = datetime(start_year, 1, 1) end_date = datetime(end_year, 12, 31) stations = cehq_station.read_station_data(selected_ids=selected_ids, start_date=start_date, end_date=end_date) station_to_grid_point = {} for s in stations: assert isinstance(s, Station) x, y, z = lat_lon.lon_lat_to_cartesian(s.longitude, s.latitude) dists, inds = ktree.query((x, y, z), k=8) deltaDaMin = np.min(np.abs(model_acc_area_1d[inds] - s.drainage_km2)) imin = np.where( np.abs(model_acc_area_1d[inds] - s.drainage_km2) == deltaDaMin)[0] deltaDa2D = np.abs(model_acc_area - s.drainage_km2) ij = np.where(deltaDa2D == deltaDaMin) mp = ModelPoint() mp.accumulation_area = model_acc_area[ij[0][0], ij[1][0]] mp.ix = ij[0][0] mp.jy = ij[1][0] mp.longitude = model_manager.lons2D[mp.ix, mp.jy] mp.latitude = model_manager.lats2D[mp.ix, mp.jy] #flow in mask mp.flow_in_mask = model_manager.get_mask_for_cells_upstream( mp.ix, mp.jy) station_to_grid_point[s] = mp print("da_diff (sel) = ", deltaDaMin) print("dist (sel) = ", dists[imin]) return station_to_grid_point
def get_model_points_of_outlets(self, lower_accumulation_index_limit=5): """ Does the same thing as self.get_oulet_mask_array, except the result is a list of ModelPoint objects :param lower_accumulation_index_limit: """ omask = self.get_outlet_mask_array(lower_accumulation_index_limit=lower_accumulation_index_limit) return [ModelPoint(ix=i, jy=j, longitude=self.lons2d[i, j], latitude=self.lats2d[i, j]) for i, j in zip(*np.where(omask))]
def get_dataless_model_points_for_stations(station_list, accumulation_area_km2_2d, model_lons2d, model_lats2d, i_array, j_array): """ returns a map {station => modelpoint} for comparison modeled streamflows with observed this uses exactly the same method for searching model points as one in diagnose_point (nc-version) """ lons = model_lons2d[i_array, j_array] lats = model_lats2d[i_array, j_array] model_acc_area_1d = accumulation_area_km2_2d[i_array, j_array] npoints = 1 result = {} x0, y0, z0 = lat_lon.lon_lat_to_cartesian(lons, lats) kdtree = cKDTree(list(zip(x0, y0, z0))) for s in station_list: # list of model points which could represent the station assert isinstance(s, Station) x, y, z = lat_lon.lon_lat_to_cartesian(s.longitude, s.latitude) dists, inds = kdtree.query((x, y, z), k=5) if npoints == 1: deltaDaMin = np.min( np.abs(model_acc_area_1d[inds] - s.drainage_km2)) # this returns a list of numpy arrays imin = np.where( np.abs(model_acc_area_1d[inds] - s.drainage_km2) == deltaDaMin)[0][0] selected_cell_index = inds[imin] # check if difference in drainage areas is not too big less than 10 % print(s.river_name, deltaDaMin / s.drainage_km2) # if deltaDaMin / s.drainage_km2 > 0.2: # continue mp = ModelPoint() mp.accumulation_area = model_acc_area_1d[selected_cell_index] mp.longitude = lons[selected_cell_index] mp.latitude = lats[selected_cell_index] mp.cell_index = selected_cell_index mp.distance_to_station = dists[imin] print("Distance to station: ", dists[imin]) print("Model accumulation area: ", mp.accumulation_area) print("Obs accumulation area: ", s.drainage_km2) result[s] = mp else: raise Exception("npoints = {0}, is not yet implemented ...") return result
def get_lake_model_points_for_stations(self, station_list, lake_fraction=None, nneighbours=8): """ For lake levels we have a bit different search algorithm since accumulation area is not a very sensible param to compare :return {station: list of corresponding model points} :param station_list: :param lake_fraction: :param drainaige_area_reldiff_limit: :param nneighbours: :return: :raise Exception: """ station_to_model_point_list = {} nx, ny = self.lons2d.shape i1d, j1d = list(range(nx)), list(range(ny)) j2d, i2d = np.meshgrid(j1d, i1d) i_flat, j_flat = i2d.flatten(), j2d.flatten() for s in station_list: mp_list = [] assert isinstance(s, Station) x, y, z = lat_lon.lon_lat_to_cartesian(s.longitude, s.latitude) dists, inds = self.kdtree.query((x, y, z), k=nneighbours) if nneighbours == 1: dists = [dists] inds = [inds] for d, i in zip(dists, inds): ix = i_flat[i] jy = j_flat[i] mp = ModelPoint(ix=ix, jy=jy) mp.longitude = self.lons2d[ix, jy] mp.latitude = self.lats2d[ix, jy] mp.distance_to_station = d if lake_fraction is not None: if lake_fraction[ix, jy] <= 0.001: # skip the model point if almost no lakes inisde continue mp.lake_fraction = lake_fraction[ix, jy] mp_list.append(mp) if lake_fraction is not None: lf = 0.0 for mp in mp_list: lf += mp.lake_fraction if lf <= 0.001: continue station_to_model_point_list[s] = mp_list print("Found model point for the station {0}".format(s)) return station_to_model_point_list
def get_dataless_model_points_for_stations(station_list, accumulation_area_km2_2d, model_lons2d, model_lats2d, i_array, j_array): """ returns a map {station => modelpoint} for comparison modeled streamflows with observed this uses exactly the same method for searching model points as one in diagnose_point (nc-version) """ lons = model_lons2d[i_array, j_array] lats = model_lats2d[i_array, j_array] model_acc_area_1d = accumulation_area_km2_2d[i_array, j_array] npoints = 1 result = {} x0, y0, z0 = lat_lon.lon_lat_to_cartesian(lons, lats) kdtree = cKDTree(list(zip(x0, y0, z0))) for s in station_list: # list of model points which could represent the station assert isinstance(s, Station) x, y, z = lat_lon.lon_lat_to_cartesian(s.longitude, s.latitude) dists, inds = kdtree.query((x, y, z), k=5) if npoints == 1: deltaDaMin = np.min(np.abs(model_acc_area_1d[inds] - s.drainage_km2)) # this returns a list of numpy arrays imin = np.where(np.abs(model_acc_area_1d[inds] - s.drainage_km2) == deltaDaMin)[0][0] selected_cell_index = inds[imin] # check if difference in drainage areas is not too big less than 10 % print(s.river_name, deltaDaMin / s.drainage_km2) # if deltaDaMin / s.drainage_km2 > 0.2: # continue mp = ModelPoint() mp.accumulation_area = model_acc_area_1d[selected_cell_index] mp.longitude = lons[selected_cell_index] mp.latitude = lats[selected_cell_index] mp.cell_index = selected_cell_index mp.distance_to_station = dists[imin] print("Distance to station: ", dists[imin]) print("Model accumulation area: ", mp.accumulation_area) print("Obs accumulation area: ", s.drainage_km2) result[s] = mp else: raise Exception("npoints = {0}, is not yet implemented ...") return result
def plot_basin_outlets(shape_file=BASIN_BOUNDARIES_FILE, bmp_info=None, directions=None, accumulation_areas=None, lake_fraction_field=None): assert isinstance(bmp_info, BasemapInfo) driver = ogr.GetDriverByName("ESRI Shapefile") print(driver) ds = driver.Open(shape_file, 0) assert isinstance(ds, ogr.DataSource) layer = ds.GetLayer() assert isinstance(layer, ogr.Layer) print(layer.GetFeatureCount()) latlong_proj = osr.SpatialReference() latlong_proj.ImportFromEPSG(4326) utm_proj = layer.GetSpatialRef() # create Coordinate Transformation coord_transform = osr.CoordinateTransformation(latlong_proj, utm_proj) utm_coords = coord_transform.TransformPoints( list(zip(bmp_info.lons.flatten(), bmp_info.lats.flatten()))) utm_coords = np.asarray(utm_coords) x_utm = utm_coords[:, 0].reshape(bmp_info.lons.shape) y_utm = utm_coords[:, 1].reshape(bmp_info.lons.shape) basin_mask = np.zeros_like(bmp_info.lons) cell_manager = CellManager(directions, accumulation_area_km2=accumulation_areas, lons2d=bmp_info.lons, lats2d=bmp_info.lats) index = 1 basins = [] basin_names = [] basin_name_to_mask = {} for feature in layer: assert isinstance(feature, ogr.Feature) # print feature["FID"] geom = feature.GetGeometryRef() assert isinstance(geom, ogr.Geometry) basins.append(ogr.CreateGeometryFromWkb(geom.ExportToWkb())) basin_names.append(feature["abr"]) accumulation_areas_temp = accumulation_areas.copy() lons_out, lats_out = [], [] basin_names_out = [] name_to_ij_out = OrderedDict() min_basin_area = min(b.GetArea() * 1.0e-6 for b in basins) while len(basins): fm = np.max(accumulation_areas_temp) i, j = np.where(fm == accumulation_areas_temp) i, j = i[0], j[0] p = ogr.CreateGeometryFromWkt("POINT ({} {})".format( x_utm[i, j], y_utm[i, j])) b_selected = None name_selected = None for name, b in zip(basin_names, basins): assert isinstance(b, ogr.Geometry) assert isinstance(p, ogr.Geometry) if b.Contains(p.Buffer(2000 * 2**0.5)): # Check if there is an upstream cell from the same basin the_mask = cell_manager.get_mask_of_upstream_cells_connected_with_by_indices( i, j) # Save the mask of the basin for future use basin_name_to_mask[name] = the_mask # if is_part_of_points_in(b, x_utm[the_mask == 1], y_utm[the_mask == 1]): # continue b_selected = b name_selected = name # basin_names_out.append(name) lons_out.append(bmp_info.lons[i, j]) lats_out.append(bmp_info.lats[i, j]) name_to_ij_out[name] = (i, j) basin_mask[the_mask == 1] = index index += 1 break if b_selected is not None: basins.remove(b_selected) basin_names.remove(name_selected) outlet_index_in_basin = 1 current_basin_name = name_selected while current_basin_name in basin_names_out: current_basin_name = name_selected + str(outlet_index_in_basin) outlet_index_in_basin += 1 basin_names_out.append(current_basin_name) print(len(basins), basin_names_out) accumulation_areas_temp[i, j] = -1 plot_utils.apply_plot_params(font_size=12, width_pt=None, width_cm=20, height_cm=20) gs = GridSpec(2, 2, width_ratios=[1.0, 0.5], wspace=0.01) fig = plt.figure() ax = fig.add_subplot(gs[1, 0]) xx, yy = bmp_info.get_proj_xy() bmp_info.basemap.drawcoastlines(linewidth=0.5, ax=ax) bmp_info.basemap.drawrivers(zorder=5, color="0.5", ax=ax) upstream_edges = cell_manager.get_upstream_polygons_for_points( model_point_list=[ ModelPoint(ix=i, jy=j) for (i, j) in name_to_ij_out.values() ], xx=xx, yy=yy) upstream_edges_latlon = cell_manager.get_upstream_polygons_for_points( model_point_list=[ ModelPoint(ix=i, jy=j) for (i, j) in name_to_ij_out.values() ], xx=bmp_info.lons, yy=bmp_info.lats) plot_utils.draw_upstream_area_bounds(ax, upstream_edges=upstream_edges, color="r", linewidth=0.6) plot_utils.save_to_shape_file(upstream_edges_latlon, in_proj=None) xs, ys = bmp_info.basemap(lons_out, lats_out) bmp_info.basemap.scatter(xs, ys, c="0.75", s=30, zorder=10) bmp_info.basemap.drawparallels(np.arange(-90, 90, 5), labels=[True, False, False, False], linewidth=0.5) bmp_info.basemap.drawmeridians(np.arange(-180, 180, 5), labels=[False, False, False, True], linewidth=0.5) cmap = cm.get_cmap("rainbow", index - 1) bn = BoundaryNorm(list(range(index + 1)), index - 1) # basin_mask = np.ma.masked_where(basin_mask < 0.5, basin_mask) # bmp_info.basemap.pcolormesh(xx, yy, basin_mask, norm=bn, cmap=cmap, ax=ax) xmin, xmax = ax.get_xlim() ymin, ymax = ax.get_ylim() print(xmin, xmax, ymin, ymax) dx = xmax - xmin dy = ymax - ymin step_y = 0.1 step_x = 0.12 y0_frac = 0.75 y0_frac_bottom = 0.02 x0_frac = 0.35 bname_to_text_coords = { "RDO": (xmin + x0_frac * dx, ymin + y0_frac_bottom * dy), "STM": (xmin + (x0_frac + step_x) * dx, ymin + y0_frac_bottom * dy), "SAG": (xmin + (x0_frac + 2 * step_x) * dx, ymin + y0_frac_bottom * dy), "BOM": (xmin + (x0_frac + 3 * step_x) * dx, ymin + y0_frac_bottom * dy), "MAN": (xmin + (x0_frac + 4 * step_x) * dx, ymin + y0_frac_bottom * dy), "MOI": (xmin + (x0_frac + 5 * step_x) * dx, ymin + y0_frac_bottom * dy), "ROM": (xmin + (x0_frac + 5 * step_x) * dx, ymin + (y0_frac_bottom + step_y) * dy), "NAT": (xmin + (x0_frac + 5 * step_x) * dx, ymin + (y0_frac_bottom + 2 * step_y) * dy), ###### "CHU": (xmin + (x0_frac + 5 * step_x) * dx, ymin + y0_frac * dy), "GEO": (xmin + (x0_frac + 5 * step_x) * dx, ymin + (y0_frac + step_y) * dy), "BAL": (xmin + (x0_frac + 5 * step_x) * dx, ymin + (y0_frac + 2 * step_y) * dy), "PYR": (xmin + (x0_frac + 4 * step_x) * dx, ymin + (y0_frac + 2 * step_y) * dy), "MEL": (xmin + (x0_frac + 3 * step_x) * dx, ymin + (y0_frac + 2 * step_y) * dy), "FEU": (xmin + (x0_frac + 2 * step_x) * dx, ymin + (y0_frac + 2 * step_y) * dy), "ARN": (xmin + (x0_frac + 1 * step_x) * dx, ymin + (y0_frac + 2 * step_y) * dy), ###### "CAN": (xmin + 0.1 * dx, ymin + 0.80 * dy), "GRB": (xmin + 0.1 * dx, ymin + (0.80 - step_y) * dy), "LGR": (xmin + 0.1 * dx, ymin + (0.80 - 2 * step_y) * dy), "RUP": (xmin + 0.1 * dx, ymin + (0.80 - 3 * step_y) * dy), "WAS": (xmin + 0.1 * dx, ymin + (0.80 - 4 * step_y) * dy), "BEL": (xmin + 0.1 * dx, ymin + (0.80 - 5 * step_y) * dy), } # bmp_info.basemap.readshapefile(".".join(BASIN_BOUNDARIES_FILE.split(".")[:-1]).replace("utm18", "latlon"), "basin", # linewidth=1.2, ax=ax, zorder=9) for name, xa, ya, lona, lata in zip(basin_names_out, xs, ys, lons_out, lats_out): ax.annotate(name, xy=(xa, ya), xytext=bname_to_text_coords[name], textcoords='data', ha='right', va='bottom', bbox=dict(boxstyle='round,pad=0.4', fc='white'), arrowprops=dict(arrowstyle='->', connectionstyle='arc3,rad=0', linewidth=0.25), font_properties=FontProperties(size=8), zorder=20) print(r"{} & {:.0f} \\".format( name, accumulation_areas[name_to_ij_out[name]])) # Plot zonally averaged lake fraction ax = fig.add_subplot(gs[1, 1]) ydata = range(lake_fraction_field.shape[1]) ax.plot(lake_fraction_field.mean(axis=0) * 100, ydata, lw=2) ax.fill_betweenx(ydata, lake_fraction_field.mean(axis=0) * 100, alpha=0.5) ax.set_xlabel("Lake fraction (%)") ax.set_ylim(min(ydata), max(ydata)) ax.xaxis.set_tick_params(direction='out', width=1) ax.yaxis.set_tick_params(direction='out', width=1) ax.xaxis.set_ticks_position("bottom") ax.yaxis.set_ticks_position("none") ax.spines["top"].set_visible(False) ax.spines["right"].set_visible(False) for tl in ax.yaxis.get_ticklabels(): tl.set_visible(False) # plot elevation, buffer zone, big lakes, grid cells ax = fig.add_subplot(gs[0, :]) geophy_file = "/RESCUE/skynet3_rech1/huziy/from_guillimin/geophys_Quebec_0.1deg_260x260_with_dd_v6" r = RPN(geophy_file) elev = r.get_first_record_for_name("ME") lkfr = r.get_first_record_for_name("LKFR") fldr = r.get_first_record_for_name("FLDR") params = r.get_proj_parameters_for_the_last_read_rec() lons, lats = r.get_longitudes_and_latitudes_for_the_last_read_rec() rll = RotatedLatLon(**params) bsmp = rll.get_basemap_object_for_lons_lats(lons2d=lons, lats2d=lats, resolution="l") xx, yy = bsmp(lons, lats) dx = (xx[0, 0] - xx[-1, 0]) / xx.shape[0] dy = (yy[0, 0] - yy[0, -1]) / yy.shape[1] xx_ll_crnrs = xx - dx / 2 yy_ll_crnrs = yy - dy / 2 xx_ur_crnrs = xx + dx / 2 yy_ur_crnrs = yy + dy / 2 ll_lon, ll_lat = bsmp(xx_ll_crnrs[0, 0], yy_ll_crnrs[0, 0], inverse=True) ur_lon, ur_lat = bsmp(xx_ur_crnrs[-1, -1], yy_ur_crnrs[-1, -1], inverse=True) crnr_lons = np.array([[ll_lon, ll_lon], [ur_lon, ur_lon]]) crnr_lats = np.array([[ll_lat, ll_lat], [ur_lat, ur_lat]]) bsmp = rll.get_basemap_object_for_lons_lats(lons2d=crnr_lons, lats2d=crnr_lats) # plot elevation levs = [0, 100, 200, 300, 500, 700, 1000, 1500, 2000, 2800] norm = BoundaryNorm(levs, len(levs) - 1) the_cmap = my_colormaps.get_cmap_from_ncl_spec_file( path="colormap_files/OceanLakeLandSnow.rgb", ncolors=len(levs) - 1) lons[lons > 180] -= 360 me_to_plot = maskoceans(lons, lats, elev, resolution="l") im = bsmp.contourf(xx, yy, me_to_plot, cmap=the_cmap, levels=levs, norm=norm, ax=ax) bsmp.colorbar(im) bsmp.drawcoastlines(linewidth=0.5, ax=ax) # show large lake points gl_lakes = np.ma.masked_where((lkfr < 0.6) | (fldr <= 0) | (fldr > 128), lkfr) gl_lakes[~gl_lakes.mask] = 1.0 bsmp.pcolormesh(xx, yy, gl_lakes, cmap=cm.get_cmap("Blues"), ax=ax, vmin=0, vmax=1, zorder=3) # show free zone border margin = 20 x1 = xx_ll_crnrs[margin, margin] x2 = xx_ur_crnrs[-margin, margin] y1 = yy_ll_crnrs[margin, margin] y2 = yy_ur_crnrs[margin, -margin] pol_corners = ((x1, y1), (x2, y1), (x2, y2), (x1, y2)) ax.add_patch(Polygon(xy=pol_corners, fc="none", ls="solid", lw=3, zorder=5)) # show blending zone border (with halo zone) margin = 10 x1 = xx_ll_crnrs[margin, margin] x2 = xx_ur_crnrs[-margin, margin] y1 = yy_ll_crnrs[margin, margin] y2 = yy_ur_crnrs[margin, -margin] pol_corners = ((x1, y1), (x2, y1), (x2, y2), (x1, y2)) ax.add_patch( Polygon(xy=pol_corners, fc="none", ls="dashed", lw=3, zorder=5)) # show the grid step = 20 xx_ll_crnrs_ext = np.zeros([n + 1 for n in xx_ll_crnrs.shape]) yy_ll_crnrs_ext = np.zeros([n + 1 for n in yy_ll_crnrs.shape]) xx_ll_crnrs_ext[:-1, :-1] = xx_ll_crnrs yy_ll_crnrs_ext[:-1, :-1] = yy_ll_crnrs xx_ll_crnrs_ext[:-1, -1] = xx_ll_crnrs[:, -1] yy_ll_crnrs_ext[-1, :-1] = yy_ll_crnrs[-1, :] xx_ll_crnrs_ext[-1, :] = xx_ur_crnrs[-1, -1] yy_ll_crnrs_ext[:, -1] = yy_ur_crnrs[-1, -1] bsmp.pcolormesh(xx_ll_crnrs_ext[::step, ::step], yy_ll_crnrs_ext[::step, ::step], np.ma.masked_all_like(xx_ll_crnrs_ext)[::step, ::step], edgecolors="0.6", ax=ax, linewidth=0.05, zorder=4, alpha=0.5) ax.set_title("Elevation (m)") # plt.show() fig.savefig("qc_basin_outlets_points.png", bbox_inches="tight") # plt.show() plt.close(fig) return name_to_ij_out, basin_name_to_mask
def get_model_points_for_stations(self, station_list, lake_fraction=None, drainaige_area_reldiff_limit=None, nneighbours=4): """ returns a map {station => modelpoint} for comparison modeled streamflows with observed :rtype dict """ # if drainaige_area_reldiff_limit is None: # drainaige_area_reldiff_limit = self.DEFAULT_DRAINAGE_AREA_RELDIFF_MIN # if nneighbours == 1: # raise Exception("Searching over 1 neighbor is not very secure and not implemented yet") station_to_model_point = {} model_acc_area = self.accumulation_area_km2 model_acc_area_1d = model_acc_area.flatten() grid = np.indices(model_acc_area.shape) for s in station_list: x, y, z = lat_lon.lon_lat_to_cartesian(s.longitude, s.latitude) if s.drainage_km2 is None or nneighbours == 1: # return the closest grid point dists, inds = self.kdtree.query((x, y, z), k=1) ix, jy = [g1.flatten()[inds] for g1 in grid] imin = 0 dists = [dists] if s.drainage_km2 is None: print("Using the closest grid point, since the station does not report its drainage area: {}".format(s)) else: if s.drainage_km2 < self.characteristic_distance ** 2 * 1e-12: print("skipping {0}, because drainage area is too small: {1} km**2".format(s.id, s.drainage_km2)) continue assert isinstance(s, Station) dists, inds = self.kdtree.query((x, y, z), k=nneighbours) deltaDaMin = np.min(np.abs(model_acc_area_1d[inds] - s.drainage_km2)) # this returns a list of numpy arrays imin = np.where(np.abs(model_acc_area_1d[inds] - s.drainage_km2) == deltaDaMin)[0][0] # deltaDa2D = np.abs(self.accumulation_area_km2 - s.drainage_km2) # ij = np.where(deltaDa2D == deltaDaMin) ix, jy = grid[0].flatten()[inds][imin], grid[1].flatten()[inds][imin] # check if it is not global lake cell (move downstream if it is) if lake_fraction is not None: while lake_fraction[ix, jy] >= infovar.GLOBAL_LAKE_FRACTION: di, dj = direction_and_value.flowdir_values_to_shift(self.flow_directions[ix, jy]) ix, jy = ix + di, jy + dj # check if the gridcell is not too far from the station # if dists[imin] > 2 * self.characteristic_distance: # continue # check if difference in drainage areas is not too big less than 10 % if drainaige_area_reldiff_limit is not None and deltaDaMin / s.drainage_km2 > drainaige_area_reldiff_limit: print("Drainage area relative difference is too high, skipping {}.".format(s.id)) print(deltaDaMin / s.drainage_km2, deltaDaMin, s.drainage_km2) continue mp = ModelPoint() mp.ix = ix mp.jy = jy mp.longitude = self.lons2d[ix, jy] mp.latitude = self.lats2d[ix, jy] mp.accumulation_area = self.accumulation_area_km2[ix, jy] try: mp.distance_to_station = dists[imin] except TypeError: mp.distance_to_station = float(dists) station_to_model_point[s] = mp print("mp.accumulation_area_km2={}; s.drainage_km2={}".format(mp.accumulation_area, s.drainage_km2)) print("Found model point for the station {0}".format(s)) return station_to_model_point