def test_land_type_fractions(self): """ LandTypeFractions describes how large parts of a cell is forest,glacier, lake ,reservoir, - the rest is unspecified The current cell algorithms like ptgsk uses this information to manipulate the response. e.g. precipitation that falls into the reservoir fraction goes directly to the response (the difference of lake and reservoir is that reservoir is a lake where we store water to the power-plants.) """ # constructor 1 :all in one: specify glacier_size,lake_size,reservoir_size,forest_size,unspecified_size a = api.LandTypeFractions(1000.0, 2000.0, 3000.0, 4000.0, 5000.0) # keyword arguments does not work ?? # constructor 2: create, and set (with possible exceptions) b = api.LandTypeFractions() b.set_fractions(glacier=1 / 15.0, lake=2 / 15.0, reservoir=3 / 15.0, forest=4 / 15.0) self.assertAlmostEqual(a.glacier(), b.glacier()) self.assertAlmostEqual(a.lake(), b.lake()) self.assertAlmostEqual(a.reservoir(), b.reservoir()) self.assertAlmostEqual(a.forest(), b.forest()) self.assertAlmostEqual(a.unspecified(), b.unspecified()) self.assertAlmostEqual(a.snow_storage(), 1.0 - a.lake() - a.reservoir()) try: b.set_fractions(glacier=0.9, forest=0.2, lake=0.0, reservoir=0.0) self.fail("expected exception, nothing raised") except: self.assertTrue(True, "If we reach here all is ok")
def _create_std_geo_cell_data(self): geo_point = api.GeoPoint(1, 2, 3) ltf = api.LandTypeFractions() ltf.set_fractions(0.2, 0.2, 0.1, 0.3) geo_cell_data = api.GeoCellData(geo_point, 1000.0**2, 0, 0.7, ltf) geo_cell_data.radiation_slope_factor = 0.7 return geo_cell_data
def test_create(self): p = api.GeoPoint(100, 200, 300) ltf = api.LandTypeFractions() ltf.set_fractions(glacier=0.1, lake=0.1, reservoir=0.1, forest=0.1) self.assertAlmostEqual(ltf.unspecified(), 0.6) gcd = api.GeoCellData(p, 1000000.0, 1, 0.9, ltf) self.assertAlmostEqual(gcd.area(), 1000000) self.assertAlmostEqual(gcd.catchment_id(), 1)
def test_geo_cell_data_vector(self): gcdv = api.GeoCellDataVector() for i in range(5): p = api.GeoPoint(100, 200, 300) ltf = api.LandTypeFractions() ltf.set_fractions(glacier=0.1, lake=0.1, reservoir=0.1, forest=0.1) gcd = api.GeoCellData(p, 1000000.0, i, 0.9, ltf) gcdv.append(gcd) self.assertEqual(len(gcdv), 5) for i in range(5): self.assertEqual(gcdv[i].catchment_id(), i)
def build_model(model_t, parameter_t, model_size, num_catchments=1): cell_area = 1000 * 1000 region_parameter = parameter_t() gcds = api.GeoCellDataVector() # creating models from geo_cell-data is easier and more flexible for i in range(model_size): gp = api.GeoPoint(500+ 1000.0*i,500.0, 500.0*i/model_size) cid = 0 if num_catchments > 1: cid = random.randint(1, num_catchments + 1) geo_cell_data = api.GeoCellData(gp, cell_area, cid, 0.9, api.LandTypeFractions(0.01, 0.05, 0.19, 0.3, 0.45)) geo_cell_data.land_type_fractions_info().set_fractions(glacier=0.01, lake=0.05, reservoir=0.19, forest=0.3) gcds.append(geo_cell_data) return model_t(gcds, region_parameter)
def test_create(self): p = api.GeoPoint(100, 200, 300) ltf = api.LandTypeFractions() ltf.set_fractions(glacier=0.1, lake=0.1, reservoir=0.1, forest=0.1) self.assertAlmostEqual(ltf.unspecified(), 0.6) routing_info = api.RoutingInfo(2, 12000.0) gcd = api.GeoCellData(p, 1000000.0, 1, 0.9, ltf, routing_info) self.assertAlmostEqual(gcd.area(), 1000000) self.assertAlmostEqual(gcd.catchment_id(), 1) self.assertAlmostEqual(gcd.routing_info.distance, 12000.0) self.assertAlmostEqual(gcd.routing_info.id, 2) gcd.routing_info.distance = 13000.0 gcd.routing_info.id = 3 self.assertAlmostEqual(gcd.routing_info.distance, 13000.0) self.assertAlmostEqual(gcd.routing_info.id, 3)
def build_model(model_t, parameter_t, model_size, num_catchments=1): cells = model_t.cell_t.vector_t() cell_area = 1000 * 1000 region_parameter = parameter_t() for i in range(model_size): loc = (10000 * random.random(2)).tolist() + (500 * random.random(1)).tolist() gp = api.GeoPoint(*loc) cid = 0 if num_catchments>1: cid = random.randint(1,num_catchments) geo_cell_data = api.GeoCellData(gp, cell_area,cid,0.9,api.LandTypeFractions(0.01, 0.05, 0.19, 0.3, 0.45)) # geo_cell_data.land_type_fractions_info().set_fractions(glacier=0.01, lake=0.05, reservoir=0.19, forest=0.3) cell = model_t.cell_t() cell.geo = geo_cell_data cells.append(cell) return model_t(cells, region_parameter)
def test_geo_cell_data_vector(self): gcdv = api.GeoCellDataVector() for i in range(5): p = api.GeoPoint(100, 200, 300) ltf = api.LandTypeFractions() ltf.set_fractions(glacier=0.1, lake=0.1, reservoir=0.1, forest=0.1) gcd = api.GeoCellData(p, 1000000.0, i, 0.9, ltf) gcdv.append(gcd) self.assertEqual(len(gcdv), 5) for i in range(5): self.assertEqual(gcdv[i].catchment_id(), i) g2 = api.GeoCellDataVector(gcdv) # copy construct a new self.assertTrue(g2 == gcdv) g2[0].set_catchment_id(10) self.assertTrue(g2 != gcdv) # serialize gcdv_s= gcdv.serialize() self.assertGreater(len(gcdv_s),2) gcdv_deserialized = api.GeoCellDataVector.deserialize(gcdv_s) self.assertIsNotNone(gcdv_deserialized) self.assertTrue(gcdv_deserialized == gcdv)
# get dimensions from netcdf file num_cells = cell_data.dimensions['cell'].size for i in range(num_cells): gp = api.GeoPoint(x[i], y[i], z[i]) cid = cell_data.variables['catchment_id'][i] cell_area = cell_data.variables['area'][i] glac = cell_data.variables['glacier-fraction'][i] lake = cell_data.variables['lake-fraction'][i] rsvr = cell_data.variables['reservoir-fraction'][i] frst = cell_data.variables['forest-fraction'][i] unsp = 1 - (glac + lake + rsvr + frst) land_cover_frac = api.LandTypeFractions(float(glac), float(lake), float(rsvr), float(frst), float(unsp)) rad_fx = 0.9 # note, for now we need to make sure we cast some types to pure python, not numpy geo_cell_data = api.GeoCellData(gp, float(cell_area), int(cid), rad_fx, land_cover_frac) cell_data_vector.append(geo_cell_data) # put it all together to initialize a model, we'll use PTGSK params = api.pt_gs_k.PTGSKParameter() model = api.pt_gs_k.PTGSKModel(cell_data_vector, params) re = api.ARegionEnvironment()
def get_region_model(self, region_id, catchments=None): """ Return a fully specified shyft api region_model for region_id, based on data found in netcdf dataset. Parameters ----------- region_id: string unique identifier of region in data catchments: list of unique integers catchment indices when extracting a region consisting of a subset of the catchments has attribs to construct params and cells etc. Returns ------- region_model: shyft.api type """ with Dataset(self._data_file) as dset: Vars = dset.variables c_ids = Vars["catchment_id"][:] xcoord = Vars['x'][:] ycoord = Vars['y'][:] m_catch = np.ones(len(c_ids), dtype=bool) if self._catch_ids is not None: m_catch = np.in1d(c_ids, self._catch_ids) xcoord_m = xcoord[m_catch] ycoord_m = ycoord[m_catch] dataset_epsg = None if 'crs' in Vars.keys(): dataset_epsg = Vars['crs'].epsg_code.split(':')[1] if not dataset_epsg: raise interfaces.InterfaceError( "netcdf: epsg attr not found in group elevation") #if dataset_epsg != self._epsg: target_cs = "+proj=utm +zone={} +ellps={} +datum={} +units=m +no_defs".format( int(self._epsg) - 32600, "WGS84", "WGS84") source_cs = "+proj=utm +zone={} +ellps={} +datum={} +units=m +no_defs".format( int(dataset_epsg) - 32600, "WGS84", "WGS84") # Construct bounding region box_fields = set(("lower_left_x", "lower_left_y", "step_x", "step_y", "nx", "ny", "EPSG")) if box_fields.issubset(self._rconf.domain()): tmp = self._rconf.domain() epsg = tmp["EPSG"] x_min = tmp["lower_left_x"] x_max = x_min + tmp["nx"] * tmp["step_x"] y_min = tmp["lower_left_y"] y_max = y_min + tmp["ny"] * tmp["step_y"] bounding_region = BoundingBoxRegion(np.array([x_min, x_max]), np.array([y_min, y_max]), epsg, self._epsg) else: bounding_region = BoundingBoxRegion(xcoord_m, ycoord_m, dataset_epsg, self._epsg) self.bounding_box = bounding_region.bounding_box(self._epsg) x, y, m_xy, _ = self._limit(xcoord, ycoord, source_cs, target_cs) mask = ((m_xy) & (m_catch)) areas = Vars['area'][mask] elevation = Vars["z"][mask] coordinates = np.dstack( (x[mask], y[mask], elevation)).reshape(-1, 3) c_ids = Vars["catchment_id"][mask] c_ids_unique = list(np.unique(c_ids)) c_indx = np.array([c_ids_unique.index(cid) for cid in c_ids]) ff = Vars["forest-fraction"][mask] lf = Vars["lake-fraction"][mask] rf = Vars["reservoir-fraction"][mask] gf = Vars["glacier-fraction"][mask] # Construct region parameter: name_map = { "gamma_snow": "gs", "priestley_taylor": "pt", "kirchner": "kirchner", "actual_evapotranspiration": "ae", "skaugen": "skaugen", "hbv_snow": "snow" } region_parameter = self._region_model.parameter_t() for p_type_name, value_ in iteritems(self._mconf.model_parameters()): if p_type_name in name_map: if hasattr(region_parameter, name_map[p_type_name]): sub_param = getattr(region_parameter, name_map[p_type_name]) for p, v in iteritems(value_): if hasattr(sub_param, p): setattr(sub_param, p, v) else: raise RegionConfigError( "Invalid parameter '{}' for parameter set '{}'" .format(p, p_type_name)) else: raise RegionConfigError( "Invalid parameter set '{}' for selected model '{}'". format(p_type_name, self._region_model.__name__)) elif p_type_name == "p_corr_scale_factor": region_parameter.p_corr.scale_factor = value_ else: raise RegionConfigError( "Unknown parameter set '{}'".format(p_type_name)) # TODO: Move into yaml file similar to p_corr_scale_factor radiation_slope_factor = 0.9 # Construct cells cell_vector = self._region_model.cell_t.vector_t() for pt, a, c_id, ff, lf, rf, gf in zip(coordinates, areas, c_indx, ff, lf, rf, gf): cell = self._region_model.cell_t() cell.geo = api.GeoCellData( api.GeoPoint(*pt), a, int(c_id), radiation_slope_factor, api.LandTypeFractions(gf, lf, rf, ff, 0.0)) cell_vector.append(cell) # Construct catchment overrides catchment_parameters = self._region_model.parameter_t.map_t() for k, v in iteritems(self._rconf.parameter_overrides()): if k in c_ids_unique: param = self._region_model.parameter_t(region_parameter) for p_type_name, value_ in iteritems(v): if p_type_name in name_map: sub_param = getattr(param, name_map[p_type_name]) for p, pv in iteritems(value_): setattr(sub_param, p, pv) elif p_type_name == "p_corr_scale_factor": param.p_corr.scale_factor = value_ else: # Avoid unknown params to go unadvertised raise RegionConfigError( "parameter {} is not in the set of allowed ones". format(p_type_name)) catchment_parameters[c_ids_unique.index(k)] = param region_model = self._region_model(cell_vector, region_parameter, catchment_parameters) region_model.bounding_region = bounding_region region_model.catchment_id_map = c_ids_unique return region_model
def get_region_model(self, region_id, catchments=None): """ Return a fully specified shyft api region_model for region_id, based on data found in netcdf dataset. Parameters ----------- region_id: string unique identifier of region in data catchments: list of unique integers catchment indices when extracting a region consisting of a subset of the catchments has attribs to construct params and cells etc. Returns ------- region_model: shyft.api type """ with Dataset(self._data_file) as dset: grp = dset.groups["elevation"] xcoord = grp.variables["xcoord"][:] ycoord = grp.variables["ycoord"][:] dataset_epsg = None if hasattr(grp, "epsg"): dataset_epsg = grp.epsg if hasattr(grp, "EPSG"): dataset_epsg = grp.EPSG if not dataset_epsg: raise interfaces.InterfaceError( "netcdf: epsg attr not found in group elevation") if dataset_epsg != self._epsg: source_cs = "+proj=utm +zone={} +ellps={} +datum={} +units=m +no_defs".format( int(self._epsg) - 32600, "WGS84", "WGS84") target_cs = "+proj=utm +zone={} +ellps={} +datum={} +units=m +no_defs".format( int(dataset_epsg) - 32600, "WGS84", "WGS84") source_proj = Proj(source_cs) target_proj = Proj(target_cs) mesh2d = np.dstack( transform(source_proj, target_proj, *np.meshgrid(xcoord, ycoord))).reshape(-1, 2) dx = xcoord[1] - xcoord[0] dy = ycoord[1] - ycoord[0] x_corners = np.empty(len(xcoord) + 1, dtype=xcoord.dtype) y_corners = np.empty(len(ycoord) + 1, dtype=ycoord.dtype) x_corners[1:] = xcoord + dx / 2.0 x_corners[0] = xcoord[0] - dx / 2.0 y_corners[1:] = ycoord + dy / 2.0 y_corners[0] = ycoord[0] - dy / 2.0 xc, yc = transform(source_proj, target_proj, *np.meshgrid(x_corners, y_corners)) areas = np.empty((len(xcoord), len(ycoord)), dtype=xcoord.dtype) for i in range(len(xcoord)): for j in range(len(ycoord)): pts = [(xc[j, i], yc[j, i]), (xc[j, i + 1], yc[j, i + 1]), (xc[j + 1, i + 1], yc[j + 1, i + 1]), (xc[j + 1, i], yc[j + 1, i])] areas[i, j] = Polygon(pts).area areas = areas.flatten()[self.mask] else: mesh2d = np.dstack(np.meshgrid(xcoord, ycoord)).reshape(-1, 2) areas = np.ones(len(xcoord) * len(ycoord), dtype=xcoord.dtype)[ self.mask] * (xcoord[1] - xcoord[0]) * (ycoord[1] - ycoord[0]) elevation = grp.variables["elevation"][:] coordinates = np.hstack((mesh2d, elevation.reshape(-1, 1)))[self.mask] catchments = dset.groups["catchments"].variables[ "catchments"][:].reshape(-1)[self.mask] c_ids = dset.groups["catchments"].variables["catchment_indices"][:] def frac_extract(name): g = dset.groups # Alias for readability return g[name].variables[name][:].reshape(-1)[self.mask] ff = frac_extract("forest-fraction") lf = frac_extract("lake-fraction") rf = frac_extract("reservoir-fraction") gf = frac_extract("glacier-fraction") # Construct bounding region box_fields = set(("upper_left_x", "upper_left_y", "step_x", "step_y", "nx", "ny", "EPSG")) if box_fields.issubset(self._rconf.domain()): tmp = self._rconf.domain() epsg = tmp["EPSG"] x_min = tmp["upper_left_x"] x_max = x_min + tmp["nx"] * tmp["step_x"] y_max = tmp["upper_left_x"] y_min = y_max - tmp["ny"] * tmp["step_y"] bounding_region = BoundingBoxRegion(np.array([x_min, x_max]), np.array([y_min, y_max]), epsg, self._epsg) else: bounding_region = BoundingBoxRegion(xcoord, ycoord, dataset_epsg, self._epsg) # Construct region parameter: name_map = { "gamma_snow": "gs", "priestley_taylor": "pt", "kirchner": "kirchner", "actual_evapotranspiration": "ae", "skaugen": "skaugen", "hbv_snow": "snow" } region_parameter = self._region_model.parameter_t() for p_type_name, value_ in iteritems(self._mconf.model_parameters()): if p_type_name in name_map: if hasattr(region_parameter, name_map[p_type_name]): sub_param = getattr(region_parameter, name_map[p_type_name]) for p, v in iteritems(value_): setattr(sub_param, p, v) elif p_type_name == "p_corr_scale_factor": region_parameter.p_corr.scale_factor = value_ # TODO: Move into yaml file similar to p_corr_scale_factor radiation_slope_factor = 0.9 # Construct cells cell_vector = self._region_model.cell_t.vector_t() for pt, a, c_id, ff, lf, rf, gf in zip(coordinates, areas, catchments, ff, lf, rf, gf): cell = self._region_model.cell_t() cell.geo = api.GeoCellData( api.GeoPoint(*pt), a, int(c_id), radiation_slope_factor, api.LandTypeFractions(gf, lf, rf, ff, 0.0)) cell_vector.append(cell) # Construct catchment overrides catchment_parameters = self._region_model.parameter_t.map_t() for k, v in iteritems(self._rconf.parameter_overrides()): if k in c_ids: param = self._region_model.parameter_t(region_parameter) for p_type_name, value_ in iteritems(v): if p_type_name in name_map: sub_param = getattr(param, name_map[p_type_name]) for p, pv in iteritems(value_): setattr(sub_param, p, pv) elif p_type_name == "p_corr_scale_factor": param.p_corr.scale_factor = value_ else: # Avoid unknown params to go unadvertised raise RegionConfigError( "parameter {} is not in the set of allowed ones". format(p_type_name)) catchment_parameters[k] = param region_model = self._region_model(cell_vector, region_parameter, catchment_parameters) def do_clone(x): clone = x.__class__(x) clone.bounding_region = bounding_region return clone region_model.bounding_region = bounding_region region_model.clone = do_clone return region_model
def get_region_model(self, region_id, catchments=None): """ Return a fully specified shyft api region_model for region_id. Parameters ----------- region_id: string unique identifier of region in data catchments: list of unique integers catchment id_list when extracting a region consisting of a subset of the catchments has attribs to construct params and cells etc. Returns ------- region_model: shyft.api type with .bounding_region = grid_specification - as fetched from the rm.config .catchment_id_map = array, where i'th item is the external catchment'id .gis_info = result from CellDataFetcher - used to fetch the grid spec (to help plot etc) { 'cell_data': {catchment_id:[{'cell':shapely-shapes,'elevation':moh,'glacier':area,'lake':area,'reservoir':area,'forest':area}]} 'catchment_land_types':{catchment_id:{'glacier':[shapelys..],'forest':[shapelys..],'lake':[shapelys..],'reservoir':[shapelys..]}} 'elevation_raster': np.array(dtype.float64) } """ rm = self._get_cell_data_info( region_id, catchments) # fetch region model info needed to fetch efficiently cell_info_service = CellDataFetcher(rm.catchment_regulated_type, rm.service_id_field_name, rm.grid_specification, rm.id_list) result = cell_info_service.fetch( ) # clumsy result, we can adjust this.. cell_info = result['cell_data'] # this is the part we need here cell_vector = rm.region_model_type.cell_t.vector_t() radiation_slope_factor = 0.9 # todo: get it from service layer catchment_id_map = [ ] # needed to build up the external c-id to shyft core internal 0-based c-ids for c_id, c_info_list in cell_info.items(): if not c_id == 0: # only cells with c_id different from 0 if not c_id in catchment_id_map: catchment_id_map.append(c_id) c_id_0 = len(catchment_id_map) - 1 else: c_id_0 = catchment_id_map.index(c_id) for c_info in c_info_list: shape = c_info[ 'cell'] # todo fetcher should return geopoint,area, ltf.. z = c_info['elevation'] geopoint = api.GeoPoint(shape.centroid.x, shape.centroid.y, z) area = shape.area ltf = api.LandTypeFractions() ltf.set_fractions(c_info.get('glacier', 0.0), c_info.get('lake', 0.0), c_info.get('reservoir', 0.0), c_info.get('forest', 0.0)) cell = rm.region_model_type.cell_t() cell.geo = api.GeoCellData(geopoint, area, c_id_0, radiation_slope_factor, ltf) cell_vector.append(cell) catchment_parameter_map = rm.region_model_type.parameter_t.map_t() # todo add catchment level parameters to map region_model = rm.region_model_type(cell_vector, rm.region_parameters, catchment_parameter_map) region_model.bounding_region = rm.grid_specification # mandatory for orchestration region_model.catchment_id_map = catchment_id_map # needed to map from externa c_id to 0-based c_id used internally in region_model.gis_info = result # opt:needed for internal statkraft use/presentation def do_clone(x): clone = x.__class__(x) clone.bounding_region = x.bounding_region clone.catchment_id_map = catchment_id_map clone.gis_info = result return clone region_model.clone = do_clone return region_model