def test_land_type_fractions(self):
"""
LandTypeFractions describes how large parts of a cell is
forest,glacier, lake ,reservoir, - the rest is unspecified
The current cell algorithms like ptgsk uses this information
to manipulate the response.
e.g. precipitation that falls into the reservoir fraction goes directly to
the response (the difference of lake and reservoir is that reservoir is a lake where
we store water to the power-plants.)
"""
# constructor 1 :all in one: specify glacier_size,lake_size,reservoir_size,forest_size,unspecified_size
a = api.LandTypeFractions(1000.0, 2000.0, 3000.0, 4000.0,
5000.0) # keyword arguments does not work ??
# constructor 2: create, and set (with possible exceptions)
b = api.LandTypeFractions()
b.set_fractions(glacier=1 / 15.0,
lake=2 / 15.0,
reservoir=3 / 15.0,
forest=4 / 15.0)
self.assertAlmostEqual(a.glacier(), b.glacier())
self.assertAlmostEqual(a.lake(), b.lake())
self.assertAlmostEqual(a.reservoir(), b.reservoir())
self.assertAlmostEqual(a.forest(), b.forest())
self.assertAlmostEqual(a.unspecified(), b.unspecified())
self.assertAlmostEqual(a.snow_storage(),
1.0 - a.lake() - a.reservoir())
try:
b.set_fractions(glacier=0.9, forest=0.2, lake=0.0, reservoir=0.0)
self.fail("expected exception, nothing raised")
except:
self.assertTrue(True, "If we reach here all is ok")
def _create_std_geo_cell_data(self):
geo_point = api.GeoPoint(1, 2, 3)
ltf = api.LandTypeFractions()
ltf.set_fractions(0.2, 0.2, 0.1, 0.3)
geo_cell_data = api.GeoCellData(geo_point, 1000.0**2, 0, 0.7, ltf)
geo_cell_data.radiation_slope_factor = 0.7
return geo_cell_data
def test_create(self):
p = api.GeoPoint(100, 200, 300)
ltf = api.LandTypeFractions()
ltf.set_fractions(glacier=0.1, lake=0.1, reservoir=0.1, forest=0.1)
self.assertAlmostEqual(ltf.unspecified(), 0.6)
gcd = api.GeoCellData(p, 1000000.0, 1, 0.9, ltf)
self.assertAlmostEqual(gcd.area(), 1000000)
self.assertAlmostEqual(gcd.catchment_id(), 1)
def test_geo_cell_data_vector(self):
gcdv = api.GeoCellDataVector()
for i in range(5):
p = api.GeoPoint(100, 200, 300)
ltf = api.LandTypeFractions()
ltf.set_fractions(glacier=0.1, lake=0.1, reservoir=0.1, forest=0.1)
gcd = api.GeoCellData(p, 1000000.0, i, 0.9, ltf)
gcdv.append(gcd)
self.assertEqual(len(gcdv), 5)
for i in range(5):
self.assertEqual(gcdv[i].catchment_id(), i)
def build_model(model_t, parameter_t, model_size, num_catchments=1):
cell_area = 1000 * 1000
region_parameter = parameter_t()
gcds = api.GeoCellDataVector() # creating models from geo_cell-data is easier and more flexible
for i in range(model_size):
gp = api.GeoPoint(500+ 1000.0*i,500.0, 500.0*i/model_size)
cid = 0
if num_catchments > 1:
cid = random.randint(1, num_catchments + 1)
geo_cell_data = api.GeoCellData(gp, cell_area, cid, 0.9, api.LandTypeFractions(0.01, 0.05, 0.19, 0.3, 0.45))
geo_cell_data.land_type_fractions_info().set_fractions(glacier=0.01, lake=0.05, reservoir=0.19, forest=0.3)
gcds.append(geo_cell_data)
return model_t(gcds, region_parameter)
def test_create(self):
p = api.GeoPoint(100, 200, 300)
ltf = api.LandTypeFractions()
ltf.set_fractions(glacier=0.1, lake=0.1, reservoir=0.1, forest=0.1)
self.assertAlmostEqual(ltf.unspecified(), 0.6)
routing_info = api.RoutingInfo(2, 12000.0)
gcd = api.GeoCellData(p, 1000000.0, 1, 0.9, ltf, routing_info)
self.assertAlmostEqual(gcd.area(), 1000000)
self.assertAlmostEqual(gcd.catchment_id(), 1)
self.assertAlmostEqual(gcd.routing_info.distance, 12000.0)
self.assertAlmostEqual(gcd.routing_info.id, 2)
gcd.routing_info.distance = 13000.0
gcd.routing_info.id = 3
self.assertAlmostEqual(gcd.routing_info.distance, 13000.0)
self.assertAlmostEqual(gcd.routing_info.id, 3)
def build_model(model_t, parameter_t, model_size, num_catchments=1):
cells = model_t.cell_t.vector_t()
cell_area = 1000 * 1000
region_parameter = parameter_t()
for i in range(model_size):
loc = (10000 * random.random(2)).tolist() + (500 * random.random(1)).tolist()
gp = api.GeoPoint(*loc)
cid = 0
if num_catchments>1:
cid = random.randint(1,num_catchments)
geo_cell_data = api.GeoCellData(gp, cell_area,cid,0.9,api.LandTypeFractions(0.01, 0.05, 0.19, 0.3, 0.45))
# geo_cell_data.land_type_fractions_info().set_fractions(glacier=0.01, lake=0.05, reservoir=0.19, forest=0.3)
cell = model_t.cell_t()
cell.geo = geo_cell_data
cells.append(cell)
return model_t(cells, region_parameter)
def test_geo_cell_data_vector(self):
gcdv = api.GeoCellDataVector()
for i in range(5):
p = api.GeoPoint(100, 200, 300)
ltf = api.LandTypeFractions()
ltf.set_fractions(glacier=0.1, lake=0.1, reservoir=0.1, forest=0.1)
gcd = api.GeoCellData(p, 1000000.0, i, 0.9, ltf)
gcdv.append(gcd)
self.assertEqual(len(gcdv), 5)
for i in range(5):
self.assertEqual(gcdv[i].catchment_id(), i)
g2 = api.GeoCellDataVector(gcdv) # copy construct a new
self.assertTrue(g2 == gcdv)
g2[0].set_catchment_id(10)
self.assertTrue(g2 != gcdv)
# serialize
gcdv_s= gcdv.serialize()
self.assertGreater(len(gcdv_s),2)
gcdv_deserialized = api.GeoCellDataVector.deserialize(gcdv_s)
self.assertIsNotNone(gcdv_deserialized)
self.assertTrue(gcdv_deserialized == gcdv)
# get dimensions from netcdf file
num_cells = cell_data.dimensions['cell'].size
for i in range(num_cells):
gp = api.GeoPoint(x[i], y[i], z[i])
cid = cell_data.variables['catchment_id'][i]
cell_area = cell_data.variables['area'][i]
glac = cell_data.variables['glacier-fraction'][i]
lake = cell_data.variables['lake-fraction'][i]
rsvr = cell_data.variables['reservoir-fraction'][i]
frst = cell_data.variables['forest-fraction'][i]
unsp = 1 - (glac + lake + rsvr + frst)
land_cover_frac = api.LandTypeFractions(float(glac), float(lake),
float(rsvr), float(frst),
float(unsp))
rad_fx = 0.9
# note, for now we need to make sure we cast some types to pure python, not numpy
geo_cell_data = api.GeoCellData(gp, float(cell_area), int(cid), rad_fx,
land_cover_frac)
cell_data_vector.append(geo_cell_data)
# put it all together to initialize a model, we'll use PTGSK
params = api.pt_gs_k.PTGSKParameter()
model = api.pt_gs_k.PTGSKModel(cell_data_vector, params)
re = api.ARegionEnvironment()
def get_region_model(self, region_id, catchments=None):
"""
Return a fully specified shyft api region_model for region_id, based on data found
in netcdf dataset.
Parameters
-----------
region_id: string
unique identifier of region in data
catchments: list of unique integers
catchment indices when extracting a region consisting of a subset
of the catchments has attribs to construct params and cells etc.
Returns
-------
region_model: shyft.api type
"""
with Dataset(self._data_file) as dset:
Vars = dset.variables
c_ids = Vars["catchment_id"][:]
xcoord = Vars['x'][:]
ycoord = Vars['y'][:]
m_catch = np.ones(len(c_ids), dtype=bool)
if self._catch_ids is not None:
m_catch = np.in1d(c_ids, self._catch_ids)
xcoord_m = xcoord[m_catch]
ycoord_m = ycoord[m_catch]
dataset_epsg = None
if 'crs' in Vars.keys():
dataset_epsg = Vars['crs'].epsg_code.split(':')[1]
if not dataset_epsg:
raise interfaces.InterfaceError(
"netcdf: epsg attr not found in group elevation")
#if dataset_epsg != self._epsg:
target_cs = "+proj=utm +zone={} +ellps={} +datum={} +units=m +no_defs".format(
int(self._epsg) - 32600, "WGS84", "WGS84")
source_cs = "+proj=utm +zone={} +ellps={} +datum={} +units=m +no_defs".format(
int(dataset_epsg) - 32600, "WGS84", "WGS84")
# Construct bounding region
box_fields = set(("lower_left_x", "lower_left_y", "step_x",
"step_y", "nx", "ny", "EPSG"))
if box_fields.issubset(self._rconf.domain()):
tmp = self._rconf.domain()
epsg = tmp["EPSG"]
x_min = tmp["lower_left_x"]
x_max = x_min + tmp["nx"] * tmp["step_x"]
y_min = tmp["lower_left_y"]
y_max = y_min + tmp["ny"] * tmp["step_y"]
bounding_region = BoundingBoxRegion(np.array([x_min, x_max]),
np.array([y_min, y_max]),
epsg, self._epsg)
else:
bounding_region = BoundingBoxRegion(xcoord_m, ycoord_m,
dataset_epsg, self._epsg)
self.bounding_box = bounding_region.bounding_box(self._epsg)
x, y, m_xy, _ = self._limit(xcoord, ycoord, source_cs, target_cs)
mask = ((m_xy) & (m_catch))
areas = Vars['area'][mask]
elevation = Vars["z"][mask]
coordinates = np.dstack(
(x[mask], y[mask], elevation)).reshape(-1, 3)
c_ids = Vars["catchment_id"][mask]
c_ids_unique = list(np.unique(c_ids))
c_indx = np.array([c_ids_unique.index(cid) for cid in c_ids])
ff = Vars["forest-fraction"][mask]
lf = Vars["lake-fraction"][mask]
rf = Vars["reservoir-fraction"][mask]
gf = Vars["glacier-fraction"][mask]
# Construct region parameter:
name_map = {
"gamma_snow": "gs",
"priestley_taylor": "pt",
"kirchner": "kirchner",
"actual_evapotranspiration": "ae",
"skaugen": "skaugen",
"hbv_snow": "snow"
}
region_parameter = self._region_model.parameter_t()
for p_type_name, value_ in iteritems(self._mconf.model_parameters()):
if p_type_name in name_map:
if hasattr(region_parameter, name_map[p_type_name]):
sub_param = getattr(region_parameter,
name_map[p_type_name])
for p, v in iteritems(value_):
if hasattr(sub_param, p):
setattr(sub_param, p, v)
else:
raise RegionConfigError(
"Invalid parameter '{}' for parameter set '{}'"
.format(p, p_type_name))
else:
raise RegionConfigError(
"Invalid parameter set '{}' for selected model '{}'".
format(p_type_name, self._region_model.__name__))
elif p_type_name == "p_corr_scale_factor":
region_parameter.p_corr.scale_factor = value_
else:
raise RegionConfigError(
"Unknown parameter set '{}'".format(p_type_name))
# TODO: Move into yaml file similar to p_corr_scale_factor
radiation_slope_factor = 0.9
# Construct cells
cell_vector = self._region_model.cell_t.vector_t()
for pt, a, c_id, ff, lf, rf, gf in zip(coordinates, areas, c_indx, ff,
lf, rf, gf):
cell = self._region_model.cell_t()
cell.geo = api.GeoCellData(
api.GeoPoint(*pt), a, int(c_id), radiation_slope_factor,
api.LandTypeFractions(gf, lf, rf, ff, 0.0))
cell_vector.append(cell)
# Construct catchment overrides
catchment_parameters = self._region_model.parameter_t.map_t()
for k, v in iteritems(self._rconf.parameter_overrides()):
if k in c_ids_unique:
param = self._region_model.parameter_t(region_parameter)
for p_type_name, value_ in iteritems(v):
if p_type_name in name_map:
sub_param = getattr(param, name_map[p_type_name])
for p, pv in iteritems(value_):
setattr(sub_param, p, pv)
elif p_type_name == "p_corr_scale_factor":
param.p_corr.scale_factor = value_
else:
# Avoid unknown params to go unadvertised
raise RegionConfigError(
"parameter {} is not in the set of allowed ones".
format(p_type_name))
catchment_parameters[c_ids_unique.index(k)] = param
region_model = self._region_model(cell_vector, region_parameter,
catchment_parameters)
region_model.bounding_region = bounding_region
region_model.catchment_id_map = c_ids_unique
return region_model
def get_region_model(self, region_id, catchments=None):
"""
Return a fully specified shyft api region_model for region_id, based on data found
in netcdf dataset.
Parameters
-----------
region_id: string
unique identifier of region in data
catchments: list of unique integers
catchment indices when extracting a region consisting of a subset
of the catchments has attribs to construct params and cells etc.
Returns
-------
region_model: shyft.api type
"""
with Dataset(self._data_file) as dset:
grp = dset.groups["elevation"]
xcoord = grp.variables["xcoord"][:]
ycoord = grp.variables["ycoord"][:]
dataset_epsg = None
if hasattr(grp, "epsg"):
dataset_epsg = grp.epsg
if hasattr(grp, "EPSG"):
dataset_epsg = grp.EPSG
if not dataset_epsg:
raise interfaces.InterfaceError(
"netcdf: epsg attr not found in group elevation")
if dataset_epsg != self._epsg:
source_cs = "+proj=utm +zone={} +ellps={} +datum={} +units=m +no_defs".format(
int(self._epsg) - 32600, "WGS84", "WGS84")
target_cs = "+proj=utm +zone={} +ellps={} +datum={} +units=m +no_defs".format(
int(dataset_epsg) - 32600, "WGS84", "WGS84")
source_proj = Proj(source_cs)
target_proj = Proj(target_cs)
mesh2d = np.dstack(
transform(source_proj, target_proj,
*np.meshgrid(xcoord, ycoord))).reshape(-1, 2)
dx = xcoord[1] - xcoord[0]
dy = ycoord[1] - ycoord[0]
x_corners = np.empty(len(xcoord) + 1, dtype=xcoord.dtype)
y_corners = np.empty(len(ycoord) + 1, dtype=ycoord.dtype)
x_corners[1:] = xcoord + dx / 2.0
x_corners[0] = xcoord[0] - dx / 2.0
y_corners[1:] = ycoord + dy / 2.0
y_corners[0] = ycoord[0] - dy / 2.0
xc, yc = transform(source_proj, target_proj,
*np.meshgrid(x_corners, y_corners))
areas = np.empty((len(xcoord), len(ycoord)),
dtype=xcoord.dtype)
for i in range(len(xcoord)):
for j in range(len(ycoord)):
pts = [(xc[j, i], yc[j, i]),
(xc[j, i + 1], yc[j, i + 1]),
(xc[j + 1, i + 1], yc[j + 1, i + 1]),
(xc[j + 1, i], yc[j + 1, i])]
areas[i, j] = Polygon(pts).area
areas = areas.flatten()[self.mask]
else:
mesh2d = np.dstack(np.meshgrid(xcoord, ycoord)).reshape(-1, 2)
areas = np.ones(len(xcoord) * len(ycoord), dtype=xcoord.dtype)[
self.mask] * (xcoord[1] - xcoord[0]) * (ycoord[1] -
ycoord[0])
elevation = grp.variables["elevation"][:]
coordinates = np.hstack((mesh2d, elevation.reshape(-1,
1)))[self.mask]
catchments = dset.groups["catchments"].variables[
"catchments"][:].reshape(-1)[self.mask]
c_ids = dset.groups["catchments"].variables["catchment_indices"][:]
def frac_extract(name):
g = dset.groups # Alias for readability
return g[name].variables[name][:].reshape(-1)[self.mask]
ff = frac_extract("forest-fraction")
lf = frac_extract("lake-fraction")
rf = frac_extract("reservoir-fraction")
gf = frac_extract("glacier-fraction")
# Construct bounding region
box_fields = set(("upper_left_x", "upper_left_y", "step_x", "step_y",
"nx", "ny", "EPSG"))
if box_fields.issubset(self._rconf.domain()):
tmp = self._rconf.domain()
epsg = tmp["EPSG"]
x_min = tmp["upper_left_x"]
x_max = x_min + tmp["nx"] * tmp["step_x"]
y_max = tmp["upper_left_x"]
y_min = y_max - tmp["ny"] * tmp["step_y"]
bounding_region = BoundingBoxRegion(np.array([x_min, x_max]),
np.array([y_min, y_max]), epsg,
self._epsg)
else:
bounding_region = BoundingBoxRegion(xcoord, ycoord, dataset_epsg,
self._epsg)
# Construct region parameter:
name_map = {
"gamma_snow": "gs",
"priestley_taylor": "pt",
"kirchner": "kirchner",
"actual_evapotranspiration": "ae",
"skaugen": "skaugen",
"hbv_snow": "snow"
}
region_parameter = self._region_model.parameter_t()
for p_type_name, value_ in iteritems(self._mconf.model_parameters()):
if p_type_name in name_map:
if hasattr(region_parameter, name_map[p_type_name]):
sub_param = getattr(region_parameter,
name_map[p_type_name])
for p, v in iteritems(value_):
setattr(sub_param, p, v)
elif p_type_name == "p_corr_scale_factor":
region_parameter.p_corr.scale_factor = value_
# TODO: Move into yaml file similar to p_corr_scale_factor
radiation_slope_factor = 0.9
# Construct cells
cell_vector = self._region_model.cell_t.vector_t()
for pt, a, c_id, ff, lf, rf, gf in zip(coordinates, areas, catchments,
ff, lf, rf, gf):
cell = self._region_model.cell_t()
cell.geo = api.GeoCellData(
api.GeoPoint(*pt), a, int(c_id), radiation_slope_factor,
api.LandTypeFractions(gf, lf, rf, ff, 0.0))
cell_vector.append(cell)
# Construct catchment overrides
catchment_parameters = self._region_model.parameter_t.map_t()
for k, v in iteritems(self._rconf.parameter_overrides()):
if k in c_ids:
param = self._region_model.parameter_t(region_parameter)
for p_type_name, value_ in iteritems(v):
if p_type_name in name_map:
sub_param = getattr(param, name_map[p_type_name])
for p, pv in iteritems(value_):
setattr(sub_param, p, pv)
elif p_type_name == "p_corr_scale_factor":
param.p_corr.scale_factor = value_
else:
# Avoid unknown params to go unadvertised
raise RegionConfigError(
"parameter {} is not in the set of allowed ones".
format(p_type_name))
catchment_parameters[k] = param
region_model = self._region_model(cell_vector, region_parameter,
catchment_parameters)
def do_clone(x):
clone = x.__class__(x)
clone.bounding_region = bounding_region
return clone
region_model.bounding_region = bounding_region
region_model.clone = do_clone
return region_model
def get_region_model(self, region_id, catchments=None):
"""
Return a fully specified shyft api region_model for region_id.
Parameters
-----------
region_id: string
unique identifier of region in data
catchments: list of unique integers
catchment id_list when extracting a region consisting of a subset
of the catchments
has attribs to construct params and cells etc.
Returns
-------
region_model: shyft.api type with
.bounding_region = grid_specification - as fetched from the rm.config
.catchment_id_map = array, where i'th item is the external catchment'id
.gis_info = result from CellDataFetcher - used to fetch the grid spec (to help plot etc)
{
'cell_data': {catchment_id:[{'cell':shapely-shapes,'elevation':moh,'glacier':area,'lake':area,'reservoir':area,'forest':area}]}
'catchment_land_types':{catchment_id:{'glacier':[shapelys..],'forest':[shapelys..],'lake':[shapelys..],'reservoir':[shapelys..]}}
'elevation_raster': np.array(dtype.float64)
}
"""
rm = self._get_cell_data_info(
region_id,
catchments) # fetch region model info needed to fetch efficiently
cell_info_service = CellDataFetcher(rm.catchment_regulated_type,
rm.service_id_field_name,
rm.grid_specification, rm.id_list)
result = cell_info_service.fetch(
) # clumsy result, we can adjust this..
cell_info = result['cell_data'] # this is the part we need here
cell_vector = rm.region_model_type.cell_t.vector_t()
radiation_slope_factor = 0.9 # todo: get it from service layer
catchment_id_map = [
] # needed to build up the external c-id to shyft core internal 0-based c-ids
for c_id, c_info_list in cell_info.items():
if not c_id == 0: # only cells with c_id different from 0
if not c_id in catchment_id_map:
catchment_id_map.append(c_id)
c_id_0 = len(catchment_id_map) - 1
else:
c_id_0 = catchment_id_map.index(c_id)
for c_info in c_info_list:
shape = c_info[
'cell'] # todo fetcher should return geopoint,area, ltf..
z = c_info['elevation']
geopoint = api.GeoPoint(shape.centroid.x, shape.centroid.y,
z)
area = shape.area
ltf = api.LandTypeFractions()
ltf.set_fractions(c_info.get('glacier', 0.0),
c_info.get('lake', 0.0),
c_info.get('reservoir', 0.0),
c_info.get('forest', 0.0))
cell = rm.region_model_type.cell_t()
cell.geo = api.GeoCellData(geopoint, area, c_id_0,
radiation_slope_factor, ltf)
cell_vector.append(cell)
catchment_parameter_map = rm.region_model_type.parameter_t.map_t()
# todo add catchment level parameters to map
region_model = rm.region_model_type(cell_vector, rm.region_parameters,
catchment_parameter_map)
region_model.bounding_region = rm.grid_specification # mandatory for orchestration
region_model.catchment_id_map = catchment_id_map # needed to map from externa c_id to 0-based c_id used internally in
region_model.gis_info = result # opt:needed for internal statkraft use/presentation
def do_clone(x):
clone = x.__class__(x)
clone.bounding_region = x.bounding_region
clone.catchment_id_map = catchment_id_map
clone.gis_info = result
return clone
region_model.clone = do_clone
return region_model
