def plot_region_model(self, catchment_type, identifier, x0, y0, dx, dy, nx, ny,
catch_indicies, station_ids, epsg_id):
grid_spec = GridSpecification(epsg_id, x0, y0, dx, dy, nx, ny)
cf = CellDataFetcher(catchment_type, identifier, grid_spec, id_list=catch_indicies)
print("Start fetching data")
cf.fetch()
print("Done, now preparing plot")
# Plot the extracted data
fig, ax = plt.subplots(1)
color_map = {"forest": 'g', "lake": 'b', "glacier": 'r', "cell": "0.75", "reservoir": "purple"}
extent = grid_spec.geometry[0], grid_spec.geometry[2], grid_spec.geometry[1], grid_spec.geometry[3]
ax.imshow(cf.elevation_raster, origin='upper', extent=extent, cmap=cm.gray)
for catchment_cells in iter(cf.cell_data.values()):
self.add_plot_polygons(ax, [cell["cell"] for cell in catchment_cells], color=color_map["cell"])
for catchment_land_type in iter(cf.catchment_land_types.values()):
for k, v in iter(catchment_land_type.items()):
self.add_plot_polygons(ax, v, color=color_map[k])
geometry = grid_spec.geometry
if station_ids is not None:
glr = GisLocationService()
stations = glr.get_locations(station_ids, epsg_id)
if stations:
points = stations.values()
ax.scatter([pt[0] for pt in points], [pt[1] for pt in points], alpha=0.5)
station_min_x = min([v[0] for v in points]) - 1000
station_max_x = max([v[0] for v in points]) + 1000
station_min_y = min([v[1] for v in points]) - 1000
station_max_y = max([v[1] for v in points]) + 1000
geometry[0] = min(station_min_x, geometry[0])
geometry[1] = min(station_min_y, geometry[1])
geometry[2] = max(station_max_x, geometry[2])
geometry[3] = max(station_max_y, geometry[3])
base_dir = path.join(shyftdata_dir, "repository", "arome_data_repository")
EPSG = grid_spec.epsg()
bbox = grid_spec.bounding_box(EPSG)
arome4 = AromeDataRepository(EPSG, base_dir, filename="arome_metcoop_default2_5km_*.nc",
bounding_box=bbox, allow_subset=True)
# data_names = ("temperature", "wind_speed", "precipitation", "relative_humidity","radiation")
# utc_period = api.UtcPeriod(
# api.Calendar().time(api.YMDhms(2015, 10, 1, 0, 0, 0)),
# api.Calendar().time(api.YMDhms(2015, 10, 2, 0, 0, 0))
# )
# arome_ts=arome4.get_timeseries(["temperature"],utc_period)
# arome_points=[gts.mid_point() for gts in arome_ts['temperature']]
# ax.scatter( [pt.x for pt in arome_points ],[pt.y for pt in arome_points],c=[pt.z for pt in arome_points],alpha=0.5,cmap='gray',s=100)#, facecolors=('r'))
ax.set_xlim(geometry[0], geometry[2])
ax.set_ylim(geometry[1], geometry[3])
plt.show()
def plot_region_model(self, catchment_type, identifier ,x0, y0, dx, dy, nx, ny,
catch_indicies, station_ids,epsg_id):
grid_spec = GridSpecification(epsg_id, x0, y0, dx, dy, nx, ny)
cf = CellDataFetcher(catchment_type, identifier, grid_spec, id_list=catch_indicies)
print( "Start fetching data")
cf.fetch()
print ("Done, now preparing plot")
# Plot the extracted data
fig, ax = plt.subplots(1)
color_map = {"forest": 'g', "lake": 'b', "glacier": 'r', "cell": "0.75", "reservoir": "purple"}
extent = grid_spec.geometry[0], grid_spec.geometry[2], grid_spec.geometry[1], grid_spec.geometry[3]
ax.imshow(cf.elevation_raster, origin='upper', extent=extent, cmap=cm.gray)
for catchment_cells in iter(cf.cell_data.values()):
self.add_plot_polygons(ax, [cell["cell"] for cell in catchment_cells], color=color_map["cell"])
for catchment_land_type in iter(cf.catchment_land_types.values()):
for k,v in iter(catchment_land_type.items()):
self.add_plot_polygons(ax, v, color=color_map[k])
geometry = grid_spec.geometry
if station_ids is not None:
glr = GisLocationService()
stations = glr.get_locations(station_ids, epsg_id)
if stations:
points = stations.values()
ax.scatter([pt[0] for pt in points], [pt[1] for pt in points], alpha=0.5)
station_min_x = min([v[0] for v in points]) - 1000
station_max_x = max([v[0] for v in points]) + 1000
station_min_y = min([v[1] for v in points]) - 1000
station_max_y = max([v[1] for v in points]) + 1000
geometry[0] = min(station_min_x, geometry[0])
geometry[1] = min(station_min_y, geometry[1])
geometry[2] = max(station_max_x, geometry[2])
geometry[3] = max(station_max_y, geometry[3])
base_dir = path.join(shyftdata_dir, "repository", "arome_data_repository")
EPSG = grid_spec.epsg_id
bbox = grid_spec.bounding_box(EPSG)
arome4 = AromeDataRepository(EPSG, base_dir, filename="arome_metcoop_default2_5km_*.nc",
bounding_box=bbox, allow_subset=True)
#data_names = ("temperature", "wind_speed", "precipitation", "relative_humidity","radiation")
utc_period = api.UtcPeriod(
api.Calendar().time(api.YMDhms(2015,10,1,0,0,0)),
api.Calendar().time(api.YMDhms(2015,10,2,0,0,0))
)
#arome_ts=arome4.get_timeseries(["temperature"],utc_period)
#arome_points=[gts.mid_point() for gts in arome_ts['temperature']]
#ax.scatter( [pt.x for pt in arome_points ],[pt.y for pt in arome_points],c=[pt.z for pt in arome_points],alpha=0.5,cmap='gray',s=100)#, facecolors=('r'))
ax.set_xlim(geometry[0], geometry[2])
ax.set_ylim(geometry[1], geometry[3])
plt.show()
def test_reservoir_fetcher(self):
gs = GridSpecification(32632,
x0=557600,
y0=7040000,
dx=1000,
dy=1000,
nx=122,
ny=90)
rf = ReservoirFetcher(epsg_id=gs.epsg(), geometry=gs.geometry)
rpts = rf.fetch()
self.assertIsNotNone(rpts)
self.assertEqual(
24, len(rpts)
) # well, seems that this can change when doing maintenance in db ?
def test_region_model_nea_nidelv(self):
nea_nidelv_grid_spec = GridSpecification(epsg_id=32633,
x0=270000.0,
y0=7035000.0,
dx=1000,
dy=1000,
nx=105,
ny=75)
catch_ids = [
1228, 1308, 1394, 1443, 1726, 1867, 1996, 2041, 2129, 2195,
2198, 2277, 2402, 2446, 2465, 2545, 2640, 2718, 3002, 3536,
3630
] # , 1000010, 1000011]
ptgsk_params = self.std_ptgsk_parameters
cfg_list = [
RegionModelConfig("nea-nidelv-ptgsk", PTGSKModel, ptgsk_params,
nea_nidelv_grid_spec, "regulated",
"CATCH_ID", catch_ids)
]
rm_cfg_dict = {x.name: x for x in cfg_list}
rmr = GisRegionModelRepository(rm_cfg_dict)
nea_nidelv = rmr.get_region_model("nea-nidelv-ptgsk")
self.assertIsNotNone(nea_nidelv)
self.assertEqual(len(nea_nidelv.catchment_id_map), len(catch_ids))
print("nea-nidelv:{0}".format(nea_nidelv.catchment_id_map))
def grid_spec(self):
return GridSpecification(epsg_id=32633,
x0=35000.0,
y0=6788000.0,
dx=1000,
dy=1000,
nx=16,
ny=17)
def test_region_model_repository(self):
id_list = [1225]
epsg_id = 32632
# parameters can be loaded from yaml_config Model parameters..
pt_params = api.PriestleyTaylorParameter(
) # *params["priestley_taylor"])
gs_params = api.GammaSnowParameter() # *params["gamma_snow"])
ss_params = api.SkaugenParameter()
ae_params = api.ActualEvapotranspirationParameter(
) # *params["act_evap"])
k_params = api.KirchnerParameter() # *params["kirchner"])
p_params = api.PrecipitationCorrectionParameter(
) # TODO; default 1.0, is it used ??
ptgsk_rm_params = api.pt_gs_k.PTGSKParameter(
pt_params, gs_params, ae_params, k_params, p_params)
ptssk_rm_params = api.pt_ss_k.PTSSKParameter(
pt_params, ss_params, ae_params, k_params, p_params)
# create the description for 4 models of tistel,ptgsk, ptssk, full and optimized
tistel_grid_spec = GridSpecification(epsg_id=epsg_id,
x0=362000.0,
y0=6765000.0,
dx=1000,
dy=1000,
nx=8,
ny=8)
cfg_list = [
RegionModelConfig("tistel-ptgsk", PTGSKModel, ptgsk_rm_params,
tistel_grid_spec, "unregulated", "FELTNR",
id_list),
RegionModelConfig("tistel-ptgsk-opt", PTGSKOptModel,
ptgsk_rm_params, tistel_grid_spec,
"unregulated", "FELTNR", id_list),
RegionModelConfig("tistel-ptssk", PTSSKModel, ptssk_rm_params,
tistel_grid_spec, "unregulated", "FELTNR",
id_list)
]
rm_cfg_dict = {x.name: x for x in cfg_list}
rmr = GisRegionModelRepository(
rm_cfg_dict
) # ok, now we have a Gis RegionModelRepository that can handle all named entities we pass.
cm1 = rmr.get_region_model(
"tistel-ptgsk") # pull out a PTGSKModel for tistel
cm2 = rmr.get_region_model("tistel-ptgsk-opt")
# Does not work, fail on ct. model:
cm3 = rmr.get_region_model(
"tistel-ptssk") # pull out a PTGSKModel for tistel
# cm4= rmr.get_region_model("tistel-ptssk",PTSSKOptModel)
self.assertIsNotNone(cm3)
self.assertIsNotNone(cm1)
self.assertIsNotNone(cm2)
self.assertIsNotNone(
cm2.catchment_id_map
) # This one is needed in order to properly map catchment-id to internal id
self.assertEqual(cm2.catchment_id_map[0], id_list[0])
def test_land_type_fetcher(self):
gs = GridSpecification(32632,
x0=557600,
y0=7040000,
dx=1000,
dy=1000,
nx=10,
ny=10)
ltf = LandTypeFetcher(geometry=gs.geometry, epsg_id=32632)
for lt_name in ltf.en_field_names:
lt = ltf.fetch(name=lt_name)
self.assertIsNotNone(lt)
def statkraft_region_model_repo_constructor(region_config, model_config,
region_model_id):
from shyft.repository.service.gis_region_model_repository import GisRegionModelRepository
from shyft.repository.service.gis_region_model_repository import RegionModelConfig
from shyft.repository.service.gis_region_model_repository import GridSpecification
from six import iteritems # This replaces dictionary.iteritems() on Python 2 and dictionary.items() on Python 3
repo_params = region_config.repository()['params']
d = region_config.domain()
grid_specification = GridSpecification(d['EPSG'], d['lower_left_x'],
d['lower_left_y'], d['step_x'],
d['step_y'], d['nx'], d['ny'])
region_model_type = model_config.model_type()
# Construct region parameter:
name_map = {
"gamma_snow": "gs",
"priestley_taylor": "pt",
"kirchner": "kirchner",
"actual_evapotranspiration": "ae",
"skaugen": "skaugen",
"hbv_snow": "snow"
}
region_parameter = region_model_type.parameter_t()
for p_type_name, value_ in iteritems(model_config.model_parameters()):
if p_type_name in name_map:
if hasattr(region_parameter, name_map[p_type_name]):
sub_param = getattr(region_parameter, name_map[p_type_name])
for p, v in iteritems(value_):
if hasattr(sub_param, p):
setattr(sub_param, p, v)
else:
raise ConfigError(
"Invalid parameter '{}' for parameter set '{}'".
format(p, p_type_name))
else:
raise ConfigError(
"Invalid parameter set '{}' for selected model '{}'".
format(p_type_name, region_model_type.__name__))
elif p_type_name == "p_corr_scale_factor":
region_parameter.p_corr.scale_factor = value_
else:
raise ConfigError("Unknown parameter set '{}'".format(p_type_name))
cfg_list = [
RegionModelConfig(region_model_id, region_model_type, region_parameter,
grid_specification,
repo_params['catchment_regulated_type'],
repo_params['service_id_field_name'],
region_config.catchments()),
]
rm_cfg_dict = {x.name: x for x in cfg_list}
return GisRegionModelRepository(rm_cfg_dict)
def test_dtm_fetcher(self):
gs = GridSpecification(32632,
x0=557600,
y0=7040000,
dx=1000,
dy=1000,
nx=122,
ny=90)
dtmf = DTMFetcher(gs)
r = dtmf.fetch()
self.assertIsNotNone(r)
shape = r.shape
self.assertEqual(shape[0], gs.ny)
self.assertEqual(shape[1], gs.nx)
def test_cell_data_fetcher_ranalangvatn(self):
gs = GridSpecification(32632,
x0=704000,
y0=7431000,
dx=1000,
dy=1000,
nx=98,
ny=105)
pwrplants = [236]
cdf = CellDataFetcher(catchment_type="regulated",
identifier="POWER_PLANT_ID",
grid_specification=gs,
id_list=pwrplants)
cd = cdf.fetch()
self.assertIsNotNone(cd)
self.assertIsNotNone(cd['cell_data'])
self.assertIsNotNone(cd['cell_data'][pwrplants[0]])
self.assertIsNotNone(cd['catchment_land_types'])
self.assertIsNotNone(cd['elevation_raster'])
def region_model_repo_constructor(cls, region_config, model_config,
region_model_id):
if cls_path(
cls
) == 'shyft.repository.service.gis_region_model_repository.GisRegionModelRepository':
#from shyft.repository.service.gis_region_model_repository import GisRegionModelRepository
from shyft.repository.service.gis_region_model_repository import get_grid_spec_from_catch_poly
from shyft.repository.service.gis_region_model_repository import RegionModelConfig
from shyft.repository.service.gis_region_model_repository import GridSpecification
from six import iteritems # This replaces dictionary.iteritems() on Python 2 and dictionary.items() on Python 3
repo_params = region_config.repository()['params']
server_name = repo_params.get('server_name')
server_name_preprod = repo_params.get('server_name_preprod')
use_cache = repo_params.get('use_cache', False)
cache_folder = repo_params.get('cache_folder', None)
cache_folder = cache_folder.replace('${SHYFTDATA}',
os.getenv('SHYFTDATA', '.'))
cache_file_type = repo_params.get('cache_file_type', None)
calc_forest_frac = repo_params.get('calc_forest_frac', False)
c_ids = region_config.catchments()
d = region_config.domain()
get_bbox_from_catchment_boundary = d.get(
'get_bbox_from_catchment_boundary', False)
pad = d.get('buffer', 5)
epsg_id = d['EPSG']
dx, dy = [d['step_x'], d['step_y']]
if use_cache or get_bbox_from_catchment_boundary:
if dx != dy:
raise ConfigError(
"step_x({}) and step_y({}) should be the same "
"if 'use_cache' or 'get_bbox_from_catchment_boundary' is enabled"
.format(dx, dy))
if get_bbox_from_catchment_boundary:
grid_specification = get_grid_spec_from_catch_poly(
c_ids,
repo_params['catchment_regulated_type'],
repo_params['service_id_field_name'],
epsg_id,
dx,
pad,
server_name=server_name,
server_name_preprod=server_name_preprod)
else:
grid_specification = GridSpecification(epsg_id, d['lower_left_x'],
d['lower_left_y'], dx, dy,
d['nx'], d['ny'])
region_model_type = model_config.model_type()
# Construct region parameter:
name_map = {
"priestley_taylor": "pt",
"kirchner": "kirchner",
"precipitation_correction": "p_corr",
"actual_evapotranspiration": "ae",
"gamma_snow": "gs",
"skaugen_snow": "ss",
"hbv_snow": "hs",
"glacier_melt": "gm",
"hbv_actual_evapotranspiration": "ae",
"hbv_soil": "soil",
"hbv_tank": "tank"
}
region_parameter = region_model_type.parameter_t()
for p_type_name, value_ in iteritems(model_config.model_parameters()):
if p_type_name in name_map:
if hasattr(region_parameter, name_map[p_type_name]):
sub_param = getattr(region_parameter,
name_map[p_type_name])
for p, v in iteritems(value_):
if hasattr(sub_param, p):
setattr(sub_param, p, v)
else:
raise ConfigError(
"Invalid parameter '{}' for parameter set '{}'"
.format(p, p_type_name))
else:
raise ConfigError(
"Invalid parameter set '{}' for selected model '{}'".
format(p_type_name, region_model_type.__name__))
else:
raise ConfigError(
"Unknown parameter set '{}'".format(p_type_name))
# Construct catchment overrides
catchment_parameters = {}
for c_id, catch_param in iteritems(
region_config.parameter_overrides()):
if c_id in c_ids:
param = region_model_type.parameter_t(region_parameter)
for p_type_name, value_ in iteritems(catch_param):
if p_type_name in name_map:
if hasattr(param, name_map[p_type_name]):
sub_param = getattr(param, name_map[p_type_name])
for p, v in iteritems(value_):
if hasattr(sub_param, p):
setattr(sub_param, p, v)
else:
raise ConfigError(
"Invalid parameter '{}' for catchment parameter set '{}'"
.format(p, p_type_name))
else:
raise ConfigError(
"Invalid catchment parameter set '{}' for selected model '{}'"
.format(p_type_name,
region_model_type.__name__))
else:
raise ConfigError(
"Unknown catchment parameter set '{}'".format(
p_type_name))
catchment_parameters[c_id] = param
cfg_list = [
RegionModelConfig(region_model_id,
region_model_type,
region_parameter,
grid_specification,
repo_params['catchment_regulated_type'],
repo_params['service_id_field_name'],
region_config.catchments(),
catchment_parameters=catchment_parameters,
calc_forest_frac=calc_forest_frac),
]
rm_cfg_dict = {x.name: x for x in cfg_list}
# return GisRegionModelRepository(rm_cfg_dict)
if server_name is not None:
cls.server_name = repo_params.get('server_name')
if server_name_preprod is not None:
cls.server_name_preprod = repo_params.get('server_name_preprod')
return cls(rm_cfg_dict,
use_cache=use_cache,
cache_folder=cache_folder,
cache_file_type=cache_file_type)
else:
return cls(region_config, model_config)
def test_reservoir_fetcher(self):
gs = GridSpecification(32632, x0=557600, y0=7040000, dx=1000, dy=1000, nx=122, ny=90)
rf = ReservoirFetcher(epsg_id=gs.epsg(), geometry=gs.geometry)
rpts = rf.fetch()
self.assertIsNotNone(rpts)
self.assertEqual(24, len(rpts)) # well, seems that this can change when doing maintenance in db ?