def _create_std_geo_cell_data(self):
geo_point = api.GeoPoint(1, 2, 3)
ltf = api.LandTypeFractions()
ltf.set_fractions(0.2, 0.2, 0.1, 0.3)
geo_cell_data = api.GeoCellData(geo_point, 1000.0**2, 0, 0.7, ltf)
geo_cell_data.radiation_slope_factor = 0.7
return geo_cell_data
def test_create(self):
p = api.GeoPoint(100, 200, 300)
ltf = api.LandTypeFractions()
ltf.set_fractions(glacier=0.1, lake=0.1, reservoir=0.1, forest=0.1)
self.assertAlmostEqual(ltf.unspecified(), 0.6)
gcd = api.GeoCellData(p, 1000000.0, 1, 0.9, ltf)
self.assertAlmostEqual(gcd.area(), 1000000)
self.assertAlmostEqual(gcd.catchment_id(), 1)
def test_geo_cell_data_vector(self):
gcdv = api.GeoCellDataVector()
for i in range(5):
p = api.GeoPoint(100, 200, 300)
ltf = api.LandTypeFractions()
ltf.set_fractions(glacier=0.1, lake=0.1, reservoir=0.1, forest=0.1)
gcd = api.GeoCellData(p, 1000000.0, i, 0.9, ltf)
gcdv.append(gcd)
self.assertEqual(len(gcdv), 5)
for i in range(5):
self.assertEqual(gcdv[i].catchment_id(), i)
def build_model(model_t, parameter_t, model_size, num_catchments=1):
cell_area = 1000 * 1000
region_parameter = parameter_t()
gcds = api.GeoCellDataVector() # creating models from geo_cell-data is easier and more flexible
for i in range(model_size):
gp = api.GeoPoint(500+ 1000.0*i,500.0, 500.0*i/model_size)
cid = 0
if num_catchments > 1:
cid = random.randint(1, num_catchments + 1)
geo_cell_data = api.GeoCellData(gp, cell_area, cid, 0.9, api.LandTypeFractions(0.01, 0.05, 0.19, 0.3, 0.45))
geo_cell_data.land_type_fractions_info().set_fractions(glacier=0.01, lake=0.05, reservoir=0.19, forest=0.3)
gcds.append(geo_cell_data)
return model_t(gcds, region_parameter)
def test_create(self):
p = api.GeoPoint(100, 200, 300)
ltf = api.LandTypeFractions()
ltf.set_fractions(glacier=0.1, lake=0.1, reservoir=0.1, forest=0.1)
self.assertAlmostEqual(ltf.unspecified(), 0.6)
routing_info = api.RoutingInfo(2, 12000.0)
gcd = api.GeoCellData(p, 1000000.0, 1, 0.9, ltf, routing_info)
self.assertAlmostEqual(gcd.area(), 1000000)
self.assertAlmostEqual(gcd.catchment_id(), 1)
self.assertAlmostEqual(gcd.routing_info.distance, 12000.0)
self.assertAlmostEqual(gcd.routing_info.id, 2)
gcd.routing_info.distance = 13000.0
gcd.routing_info.id = 3
self.assertAlmostEqual(gcd.routing_info.distance, 13000.0)
self.assertAlmostEqual(gcd.routing_info.id, 3)
def build_model(model_t, parameter_t, model_size, num_catchments=1):
cells = model_t.cell_t.vector_t()
cell_area = 1000 * 1000
region_parameter = parameter_t()
for i in range(model_size):
loc = (10000 * random.random(2)).tolist() + (
500 * random.random(1)).tolist()
gp = api.GeoPoint(*loc)
geo_cell_data = api.GeoCellData(gp, cell_area,
random.randint(0, num_catchments))
geo_cell_data.land_type_fractions_info().set_fractions(
glacier=0.0, lake=0.0, reservoir=0.1, forest=0.1)
cell = model_t.cell_t()
cell.geo = geo_cell_data
cells.append(cell)
return model_t(cells, region_parameter)
def build_model(model_t, parameter_t, model_size, num_catchments=1):
cells = model_t.cell_t.vector_t()
cell_area = 1000 * 1000
region_parameter = parameter_t()
for i in range(model_size):
loc = (10000 * random.random(2)).tolist() + (500 * random.random(1)).tolist()
gp = api.GeoPoint(*loc)
cid = 0
if num_catchments>1:
cid = random.randint(1,num_catchments)
geo_cell_data = api.GeoCellData(gp, cell_area,cid,0.9,api.LandTypeFractions(0.01, 0.05, 0.19, 0.3, 0.45))
# geo_cell_data.land_type_fractions_info().set_fractions(glacier=0.01, lake=0.05, reservoir=0.19, forest=0.3)
cell = model_t.cell_t()
cell.geo = geo_cell_data
cells.append(cell)
return model_t(cells, region_parameter)
def test_geo_cell_data_vector(self):
gcdv = api.GeoCellDataVector()
for i in range(5):
p = api.GeoPoint(100, 200, 300)
ltf = api.LandTypeFractions()
ltf.set_fractions(glacier=0.1, lake=0.1, reservoir=0.1, forest=0.1)
gcd = api.GeoCellData(p, 1000000.0, i, 0.9, ltf)
gcdv.append(gcd)
self.assertEqual(len(gcdv), 5)
for i in range(5):
self.assertEqual(gcdv[i].catchment_id(), i)
g2 = api.GeoCellDataVector(gcdv) # copy construct a new
self.assertTrue(g2 == gcdv)
g2[0].set_catchment_id(10)
self.assertTrue(g2 != gcdv)
# serialize
gcdv_s= gcdv.serialize()
self.assertGreater(len(gcdv_s),2)
gcdv_deserialized = api.GeoCellDataVector.deserialize(gcdv_s)
self.assertIsNotNone(gcdv_deserialized)
self.assertTrue(gcdv_deserialized == gcdv)
cid = cell_data.variables['catchment_id'][i]
cell_area = cell_data.variables['area'][i]
glac = cell_data.variables['glacier-fraction'][i]
lake = cell_data.variables['lake-fraction'][i]
rsvr = cell_data.variables['reservoir-fraction'][i]
frst = cell_data.variables['forest-fraction'][i]
unsp = 1 - (glac + lake + rsvr + frst)
land_cover_frac = api.LandTypeFractions(float(glac), float(lake),
float(rsvr), float(frst),
float(unsp))
rad_fx = 0.9
# note, for now we need to make sure we cast some types to pure python, not numpy
geo_cell_data = api.GeoCellData(gp, float(cell_area), int(cid), rad_fx,
land_cover_frac)
cell_data_vector.append(geo_cell_data)
# put it all together to initialize a model, we'll use PTGSK
params = api.pt_gs_k.PTGSKParameter()
model = api.pt_gs_k.PTGSKModel(cell_data_vector, params)
re = api.ARegionEnvironment()
# map the variable names in the netcdf file to source types
source_map = {
'precipitation':
('precipitation.nc', api.PrecipitationSource, re.precipitation),
'global_radiation': ('radiation.nc', api.RadiationSource, re.radiation),
def get_region_model(self, region_id, catchments=None):
"""
Return a fully specified shyft api region_model for region_id, based on data found
in netcdf dataset.
Parameters
-----------
region_id: string
unique identifier of region in data
catchments: list of unique integers
catchment indices when extracting a region consisting of a subset
of the catchments has attribs to construct params and cells etc.
Returns
-------
region_model: shyft.api type
"""
with Dataset(self._data_file) as dset:
Vars = dset.variables
c_ids = Vars["catchment_id"][:]
xcoord = Vars['x'][:]
ycoord = Vars['y'][:]
m_catch = np.ones(len(c_ids), dtype=bool)
if self._catch_ids is not None:
m_catch = np.in1d(c_ids, self._catch_ids)
xcoord_m = xcoord[m_catch]
ycoord_m = ycoord[m_catch]
dataset_epsg = None
if 'crs' in Vars.keys():
dataset_epsg = Vars['crs'].epsg_code.split(':')[1]
if not dataset_epsg:
raise interfaces.InterfaceError(
"netcdf: epsg attr not found in group elevation")
#if dataset_epsg != self._epsg:
target_cs = "+proj=utm +zone={} +ellps={} +datum={} +units=m +no_defs".format(
int(self._epsg) - 32600, "WGS84", "WGS84")
source_cs = "+proj=utm +zone={} +ellps={} +datum={} +units=m +no_defs".format(
int(dataset_epsg) - 32600, "WGS84", "WGS84")
# Construct bounding region
box_fields = set(("lower_left_x", "lower_left_y", "step_x",
"step_y", "nx", "ny", "EPSG"))
if box_fields.issubset(self._rconf.domain()):
tmp = self._rconf.domain()
epsg = tmp["EPSG"]
x_min = tmp["lower_left_x"]
x_max = x_min + tmp["nx"] * tmp["step_x"]
y_min = tmp["lower_left_y"]
y_max = y_min + tmp["ny"] * tmp["step_y"]
bounding_region = BoundingBoxRegion(np.array([x_min, x_max]),
np.array([y_min, y_max]),
epsg, self._epsg)
else:
bounding_region = BoundingBoxRegion(xcoord_m, ycoord_m,
dataset_epsg, self._epsg)
self.bounding_box = bounding_region.bounding_box(self._epsg)
x, y, m_xy, _ = self._limit(xcoord, ycoord, source_cs, target_cs)
mask = ((m_xy) & (m_catch))
areas = Vars['area'][mask]
elevation = Vars["z"][mask]
coordinates = np.dstack(
(x[mask], y[mask], elevation)).reshape(-1, 3)
c_ids = Vars["catchment_id"][mask]
c_ids_unique = list(np.unique(c_ids))
c_indx = np.array([c_ids_unique.index(cid) for cid in c_ids])
ff = Vars["forest-fraction"][mask]
lf = Vars["lake-fraction"][mask]
rf = Vars["reservoir-fraction"][mask]
gf = Vars["glacier-fraction"][mask]
# Construct region parameter:
name_map = {
"gamma_snow": "gs",
"priestley_taylor": "pt",
"kirchner": "kirchner",
"actual_evapotranspiration": "ae",
"skaugen": "skaugen",
"hbv_snow": "snow"
}
region_parameter = self._region_model.parameter_t()
for p_type_name, value_ in iteritems(self._mconf.model_parameters()):
if p_type_name in name_map:
if hasattr(region_parameter, name_map[p_type_name]):
sub_param = getattr(region_parameter,
name_map[p_type_name])
for p, v in iteritems(value_):
if hasattr(sub_param, p):
setattr(sub_param, p, v)
else:
raise RegionConfigError(
"Invalid parameter '{}' for parameter set '{}'"
.format(p, p_type_name))
else:
raise RegionConfigError(
"Invalid parameter set '{}' for selected model '{}'".
format(p_type_name, self._region_model.__name__))
elif p_type_name == "p_corr_scale_factor":
region_parameter.p_corr.scale_factor = value_
else:
raise RegionConfigError(
"Unknown parameter set '{}'".format(p_type_name))
# TODO: Move into yaml file similar to p_corr_scale_factor
radiation_slope_factor = 0.9
# Construct cells
cell_vector = self._region_model.cell_t.vector_t()
for pt, a, c_id, ff, lf, rf, gf in zip(coordinates, areas, c_indx, ff,
lf, rf, gf):
cell = self._region_model.cell_t()
cell.geo = api.GeoCellData(
api.GeoPoint(*pt), a, int(c_id), radiation_slope_factor,
api.LandTypeFractions(gf, lf, rf, ff, 0.0))
cell_vector.append(cell)
# Construct catchment overrides
catchment_parameters = self._region_model.parameter_t.map_t()
for k, v in iteritems(self._rconf.parameter_overrides()):
if k in c_ids_unique:
param = self._region_model.parameter_t(region_parameter)
for p_type_name, value_ in iteritems(v):
if p_type_name in name_map:
sub_param = getattr(param, name_map[p_type_name])
for p, pv in iteritems(value_):
setattr(sub_param, p, pv)
elif p_type_name == "p_corr_scale_factor":
param.p_corr.scale_factor = value_
else:
# Avoid unknown params to go unadvertised
raise RegionConfigError(
"parameter {} is not in the set of allowed ones".
format(p_type_name))
catchment_parameters[c_ids_unique.index(k)] = param
region_model = self._region_model(cell_vector, region_parameter,
catchment_parameters)
region_model.bounding_region = bounding_region
region_model.catchment_id_map = c_ids_unique
return region_model
def get_region_model(self, region_id, catchments=None):
"""
Return a fully specified shyft api region_model for region_id, based on data found
in netcdf dataset.
Parameters
-----------
region_id: string
unique identifier of region in data
catchments: list of unique integers
catchment indices when extracting a region consisting of a subset
of the catchments has attribs to construct params and cells etc.
Returns
-------
region_model: shyft.api type
"""
with Dataset(self._data_file) as dset:
grp = dset.groups["elevation"]
xcoord = grp.variables["xcoord"][:]
ycoord = grp.variables["ycoord"][:]
dataset_epsg = None
if hasattr(grp, "epsg"):
dataset_epsg = grp.epsg
if hasattr(grp, "EPSG"):
dataset_epsg = grp.EPSG
if not dataset_epsg:
raise interfaces.InterfaceError(
"netcdf: epsg attr not found in group elevation")
if dataset_epsg != self._epsg:
source_cs = "+proj=utm +zone={} +ellps={} +datum={} +units=m +no_defs".format(
int(self._epsg) - 32600, "WGS84", "WGS84")
target_cs = "+proj=utm +zone={} +ellps={} +datum={} +units=m +no_defs".format(
int(dataset_epsg) - 32600, "WGS84", "WGS84")
source_proj = Proj(source_cs)
target_proj = Proj(target_cs)
mesh2d = np.dstack(
transform(source_proj, target_proj,
*np.meshgrid(xcoord, ycoord))).reshape(-1, 2)
dx = xcoord[1] - xcoord[0]
dy = ycoord[1] - ycoord[0]
x_corners = np.empty(len(xcoord) + 1, dtype=xcoord.dtype)
y_corners = np.empty(len(ycoord) + 1, dtype=ycoord.dtype)
x_corners[1:] = xcoord + dx / 2.0
x_corners[0] = xcoord[0] - dx / 2.0
y_corners[1:] = ycoord + dy / 2.0
y_corners[0] = ycoord[0] - dy / 2.0
xc, yc = transform(source_proj, target_proj,
*np.meshgrid(x_corners, y_corners))
areas = np.empty((len(xcoord), len(ycoord)),
dtype=xcoord.dtype)
for i in range(len(xcoord)):
for j in range(len(ycoord)):
pts = [(xc[j, i], yc[j, i]),
(xc[j, i + 1], yc[j, i + 1]),
(xc[j + 1, i + 1], yc[j + 1, i + 1]),
(xc[j + 1, i], yc[j + 1, i])]
areas[i, j] = Polygon(pts).area
areas = areas.flatten()[self.mask]
else:
mesh2d = np.dstack(np.meshgrid(xcoord, ycoord)).reshape(-1, 2)
areas = np.ones(len(xcoord) * len(ycoord), dtype=xcoord.dtype)[
self.mask] * (xcoord[1] - xcoord[0]) * (ycoord[1] -
ycoord[0])
elevation = grp.variables["elevation"][:]
coordinates = np.hstack((mesh2d, elevation.reshape(-1,
1)))[self.mask]
catchments = dset.groups["catchments"].variables[
"catchments"][:].reshape(-1)[self.mask]
c_ids = dset.groups["catchments"].variables["catchment_indices"][:]
def frac_extract(name):
g = dset.groups # Alias for readability
return g[name].variables[name][:].reshape(-1)[self.mask]
ff = frac_extract("forest-fraction")
lf = frac_extract("lake-fraction")
rf = frac_extract("reservoir-fraction")
gf = frac_extract("glacier-fraction")
# Construct bounding region
box_fields = set(("upper_left_x", "upper_left_y", "step_x", "step_y",
"nx", "ny", "EPSG"))
if box_fields.issubset(self._rconf.domain()):
tmp = self._rconf.domain()
epsg = tmp["EPSG"]
x_min = tmp["upper_left_x"]
x_max = x_min + tmp["nx"] * tmp["step_x"]
y_max = tmp["upper_left_x"]
y_min = y_max - tmp["ny"] * tmp["step_y"]
bounding_region = BoundingBoxRegion(np.array([x_min, x_max]),
np.array([y_min, y_max]), epsg,
self._epsg)
else:
bounding_region = BoundingBoxRegion(xcoord, ycoord, dataset_epsg,
self._epsg)
# Construct region parameter:
name_map = {
"gamma_snow": "gs",
"priestley_taylor": "pt",
"kirchner": "kirchner",
"actual_evapotranspiration": "ae",
"skaugen": "skaugen",
"hbv_snow": "snow"
}
region_parameter = self._region_model.parameter_t()
for p_type_name, value_ in iteritems(self._mconf.model_parameters()):
if p_type_name in name_map:
if hasattr(region_parameter, name_map[p_type_name]):
sub_param = getattr(region_parameter,
name_map[p_type_name])
for p, v in iteritems(value_):
setattr(sub_param, p, v)
elif p_type_name == "p_corr_scale_factor":
region_parameter.p_corr.scale_factor = value_
# TODO: Move into yaml file similar to p_corr_scale_factor
radiation_slope_factor = 0.9
# Construct cells
cell_vector = self._region_model.cell_t.vector_t()
for pt, a, c_id, ff, lf, rf, gf in zip(coordinates, areas, catchments,
ff, lf, rf, gf):
cell = self._region_model.cell_t()
cell.geo = api.GeoCellData(
api.GeoPoint(*pt), a, int(c_id), radiation_slope_factor,
api.LandTypeFractions(gf, lf, rf, ff, 0.0))
cell_vector.append(cell)
# Construct catchment overrides
catchment_parameters = self._region_model.parameter_t.map_t()
for k, v in iteritems(self._rconf.parameter_overrides()):
if k in c_ids:
param = self._region_model.parameter_t(region_parameter)
for p_type_name, value_ in iteritems(v):
if p_type_name in name_map:
sub_param = getattr(param, name_map[p_type_name])
for p, pv in iteritems(value_):
setattr(sub_param, p, pv)
elif p_type_name == "p_corr_scale_factor":
param.p_corr.scale_factor = value_
else:
# Avoid unknown params to go unadvertised
raise RegionConfigError(
"parameter {} is not in the set of allowed ones".
format(p_type_name))
catchment_parameters[k] = param
region_model = self._region_model(cell_vector, region_parameter,
catchment_parameters)
def do_clone(x):
clone = x.__class__(x)
clone.bounding_region = bounding_region
return clone
region_model.bounding_region = bounding_region
region_model.clone = do_clone
return region_model
def get_region_model(self, region_id, catchments=None):
"""
Return a fully specified shyft api region_model for region_id.
Parameters
-----------
region_id: string
unique identifier of region in data
catchments: list of unique integers
catchment id_list when extracting a region consisting of a subset
of the catchments
has attribs to construct params and cells etc.
Returns
-------
region_model: shyft.api type with
.bounding_region = grid_specification - as fetched from the rm.config
.catchment_id_map = array, where i'th item is the external catchment'id
.gis_info = result from CellDataFetcher - used to fetch the grid spec (to help plot etc)
{
'cell_data': {catchment_id:[{'cell':shapely-shapes,'elevation':moh,'glacier':area,'lake':area,'reservoir':area,'forest':area}]}
'catchment_land_types':{catchment_id:{'glacier':[shapelys..],'forest':[shapelys..],'lake':[shapelys..],'reservoir':[shapelys..]}}
'elevation_raster': np.array(dtype.float64)
}
"""
rm = self._get_cell_data_info(
region_id,
catchments) # fetch region model info needed to fetch efficiently
cell_info_service = CellDataFetcher(rm.catchment_regulated_type,
rm.service_id_field_name,
rm.grid_specification, rm.id_list)
result = cell_info_service.fetch(
) # clumsy result, we can adjust this..
cell_info = result['cell_data'] # this is the part we need here
cell_vector = rm.region_model_type.cell_t.vector_t()
radiation_slope_factor = 0.9 # todo: get it from service layer
catchment_id_map = [
] # needed to build up the external c-id to shyft core internal 0-based c-ids
for c_id, c_info_list in cell_info.items():
if not c_id == 0: # only cells with c_id different from 0
if not c_id in catchment_id_map:
catchment_id_map.append(c_id)
c_id_0 = len(catchment_id_map) - 1
else:
c_id_0 = catchment_id_map.index(c_id)
for c_info in c_info_list:
shape = c_info[
'cell'] # todo fetcher should return geopoint,area, ltf..
z = c_info['elevation']
geopoint = api.GeoPoint(shape.centroid.x, shape.centroid.y,
z)
area = shape.area
ltf = api.LandTypeFractions()
ltf.set_fractions(c_info.get('glacier', 0.0),
c_info.get('lake', 0.0),
c_info.get('reservoir', 0.0),
c_info.get('forest', 0.0))
cell = rm.region_model_type.cell_t()
cell.geo = api.GeoCellData(geopoint, area, c_id_0,
radiation_slope_factor, ltf)
cell_vector.append(cell)
catchment_parameter_map = rm.region_model_type.parameter_t.map_t()
# todo add catchment level parameters to map
region_model = rm.region_model_type(cell_vector, rm.region_parameters,
catchment_parameter_map)
region_model.bounding_region = rm.grid_specification # mandatory for orchestration
region_model.catchment_id_map = catchment_id_map # needed to map from externa c_id to 0-based c_id used internally in
region_model.gis_info = result # opt:needed for internal statkraft use/presentation
def do_clone(x):
clone = x.__class__(x)
clone.bounding_region = x.bounding_region
clone.catchment_id_map = catchment_id_map
clone.gis_info = result
return clone
region_model.clone = do_clone
return region_model
