def region_model_repo_constructor(cls, region_config, model_config,
region_model_id):
if cls_path(
cls
) == 'shyft.repository.service.gis_region_model_repository.GisRegionModelRepository':
#from shyft.repository.service.gis_region_model_repository import GisRegionModelRepository
from shyft.repository.service.gis_region_model_repository import get_grid_spec_from_catch_poly
from shyft.repository.service.gis_region_model_repository import RegionModelConfig
from shyft.repository.service.gis_region_model_repository import GridSpecification
from six import iteritems # This replaces dictionary.iteritems() on Python 2 and dictionary.items() on Python 3
repo_params = region_config.repository()['params']
server_name = repo_params.get('server_name')
server_name_preprod = repo_params.get('server_name_preprod')
use_cache = repo_params.get('use_cache', False)
cache_folder = repo_params.get('cache_folder', None)
cache_folder = cache_folder.replace('${SHYFTDATA}',
os.getenv('SHYFTDATA', '.'))
cache_file_type = repo_params.get('cache_file_type', None)
calc_forest_frac = repo_params.get('calc_forest_frac', False)
c_ids = region_config.catchments()
d = region_config.domain()
get_bbox_from_catchment_boundary = d.get(
'get_bbox_from_catchment_boundary', False)
pad = d.get('buffer', 5)
epsg_id = d['EPSG']
dx, dy = [d['step_x'], d['step_y']]
if use_cache or get_bbox_from_catchment_boundary:
if dx != dy:
raise ConfigError(
"step_x({}) and step_y({}) should be the same "
"if 'use_cache' or 'get_bbox_from_catchment_boundary' is enabled"
.format(dx, dy))
if get_bbox_from_catchment_boundary:
grid_specification = get_grid_spec_from_catch_poly(
c_ids,
repo_params['catchment_regulated_type'],
repo_params['service_id_field_name'],
epsg_id,
dx,
pad,
server_name=server_name,
server_name_preprod=server_name_preprod)
else:
grid_specification = GridSpecification(epsg_id, d['lower_left_x'],
d['lower_left_y'], dx, dy,
d['nx'], d['ny'])
region_model_type = model_config.model_type()
# Construct region parameter:
name_map = {
"priestley_taylor": "pt",
"kirchner": "kirchner",
"precipitation_correction": "p_corr",
"actual_evapotranspiration": "ae",
"gamma_snow": "gs",
"skaugen_snow": "ss",
"hbv_snow": "hs",
"glacier_melt": "gm",
"hbv_actual_evapotranspiration": "ae",
"hbv_soil": "soil",
"hbv_tank": "tank"
}
region_parameter = region_model_type.parameter_t()
for p_type_name, value_ in iteritems(model_config.model_parameters()):
if p_type_name in name_map:
if hasattr(region_parameter, name_map[p_type_name]):
sub_param = getattr(region_parameter,
name_map[p_type_name])
for p, v in iteritems(value_):
if hasattr(sub_param, p):
setattr(sub_param, p, v)
else:
raise ConfigError(
"Invalid parameter '{}' for parameter set '{}'"
.format(p, p_type_name))
else:
raise ConfigError(
"Invalid parameter set '{}' for selected model '{}'".
format(p_type_name, region_model_type.__name__))
else:
raise ConfigError(
"Unknown parameter set '{}'".format(p_type_name))
# Construct catchment overrides
catchment_parameters = {}
for c_id, catch_param in iteritems(
region_config.parameter_overrides()):
if c_id in c_ids:
param = region_model_type.parameter_t(region_parameter)
for p_type_name, value_ in iteritems(catch_param):
if p_type_name in name_map:
if hasattr(param, name_map[p_type_name]):
sub_param = getattr(param, name_map[p_type_name])
for p, v in iteritems(value_):
if hasattr(sub_param, p):
setattr(sub_param, p, v)
else:
raise ConfigError(
"Invalid parameter '{}' for catchment parameter set '{}'"
.format(p, p_type_name))
else:
raise ConfigError(
"Invalid catchment parameter set '{}' for selected model '{}'"
.format(p_type_name,
region_model_type.__name__))
else:
raise ConfigError(
"Unknown catchment parameter set '{}'".format(
p_type_name))
catchment_parameters[c_id] = param
cfg_list = [
RegionModelConfig(region_model_id,
region_model_type,
region_parameter,
grid_specification,
repo_params['catchment_regulated_type'],
repo_params['service_id_field_name'],
region_config.catchments(),
catchment_parameters=catchment_parameters,
calc_forest_frac=calc_forest_frac),
]
rm_cfg_dict = {x.name: x for x in cfg_list}
# return GisRegionModelRepository(rm_cfg_dict)
if server_name is not None:
cls.server_name = repo_params.get('server_name')
if server_name_preprod is not None:
cls.server_name_preprod = repo_params.get('server_name_preprod')
return cls(rm_cfg_dict,
use_cache=use_cache,
cache_folder=cache_folder,
cache_file_type=cache_file_type)
else:
return cls(region_config, model_config)
def test_serialize_and_deserialize(self):
rm_type = pt_gs_k.PTGSKModel
reg_param = rm_type.parameter_t()
epsg_id = 32633
catchment_type = 'regulated'
identifier = 'SUBCATCH_ID'
dxy = 1000.
pad = 5
# LTM5-Tya
id_list = [172, 174, 177, 185, 187, 190]
grid_specification = get_grid_spec_from_catch_poly(
id_list, catchment_type, identifier, epsg_id, dxy, pad)
# LTM5-Nea
id_list_1 = [180, 188, 191, 196]
grid_specification_1 = get_grid_spec_from_catch_poly(
id_list_1, catchment_type, identifier, epsg_id, dxy, pad)
cfg_list = [
RegionModelConfig('Tya', rm_type, reg_param,
grid_specification, catchment_type,
identifier, id_list),
RegionModelConfig('Nea', rm_type, reg_param,
grid_specification_1, catchment_type,
identifier, id_list_1),
]
rm_cfg_dict = {x.name: x for x in cfg_list}
rm_repo = GisRegionModelRepository(rm_cfg_dict, use_cache=True)
# Start by removing existing cache file
GisRegionModelRepository.remove_cache(identifier,
grid_specification)
# Update cache with Nea cell_data
GisRegionModelRepository.update_cache(catchment_type, identifier,
grid_specification_1,
id_list_1)
# Construct a region_model for Tya - this process should automatically append Tya cell_data to cache
rm = rm_repo.get_region_model('Tya')
cells_from_rm = rm.get_cells()
# Get Tya cell_data from the region_model
cell_data_from_region_model = cells_from_rm.geo_cell_data_vector(
cells_from_rm).to_numpy().reshape(-1, 11)
# Get Tya cell_data from auto generated cache
cell_data_from_auto_cache = GisRegionModelRepository.get_cell_data_from_cache(
identifier, grid_specification, id_list)
# Remove the cache file
GisRegionModelRepository.remove_cache(identifier,
grid_specification)
# Update cache with Tya cell_data
GisRegionModelRepository.update_cache(catchment_type, identifier,
grid_specification, id_list)
# Update cache with Nea cell_data
GisRegionModelRepository.update_cache(catchment_type, identifier,
grid_specification_1,
id_list_1)
# Get Tya cell_data from updated cache
cell_data_from_updated_cache = GisRegionModelRepository.get_cell_data_from_cache(
identifier, grid_specification, id_list)
# Get Nea cell_data from updated cache
cell_data_from_updated_cache_1 = GisRegionModelRepository.get_cell_data_from_cache(
identifier, grid_specification_1, id_list_1)
# Remove the cache file
GisRegionModelRepository.remove_cache(identifier,
grid_specification)
# Get Nea cell_data from GIS
cell_data_from_gis_1 = GisRegionModelRepository.get_cell_data_from_gis(
catchment_type, identifier, grid_specification_1, id_list_1)
# Get Tya cell_data from GIS
cell_data_from_gis = GisRegionModelRepository.get_cell_data_from_gis(
catchment_type, identifier, grid_specification, id_list)
# Update cache with Tya cell_data
GisRegionModelRepository.update_cache(catchment_type, identifier,
grid_specification, id_list)
# Construct a region_model for Nea - this process should automatically append Nea cell_data to cache')
rm_1 = rm_repo.get_region_model('Nea')
cells_from_rm_1 = rm_1.get_cells()
# Get Nea cell_data from the region_model
cell_data_from_region_model_1 = cells_from_rm_1.geo_cell_data_vector(
cells_from_rm_1).to_numpy().reshape(-1, 11)
# Get Nea cell_data from auto generated cache
cell_data_from_auto_cache_1 = GisRegionModelRepository.get_cell_data_from_cache(
identifier, grid_specification_1, id_list_1)
# Remove the cache file
GisRegionModelRepository.remove_cache(identifier,
grid_specification)
# Comparing results...
# Arrays to compare
# Tya
# cell_data_from_region_model, cell_data_from_auto_cache, cell_data_from_updated_cache, cell_data_from_gis
atol = 1e-08
self.assertTrue(
np.allclose(cell_data_from_region_model,
cell_data_from_gis['geo_data'],
atol=atol))
self.assertTrue(
np.allclose(cell_data_from_auto_cache['geo_data'],
cell_data_from_gis['geo_data'],
atol=atol))
self.assertTrue(
np.allclose(cell_data_from_updated_cache['geo_data'],
cell_data_from_gis['geo_data'],
atol=atol))
self.assertTrue(
np.allclose(rm.catchment_id_map,
cell_data_from_gis['cid_map'],
atol=atol))
self.assertTrue(
np.allclose(cell_data_from_auto_cache['cid_map'],
cell_data_from_gis['cid_map'],
atol=atol))
self.assertTrue(
np.allclose(cell_data_from_updated_cache['cid_map'],
cell_data_from_gis['cid_map'],
atol=atol))
# Nea
# cell_data_from_region_model_1, cell_data_from_auto_cache_1, cell_data_from_updated_cache_1, cell_data_from_gis_1
self.assertTrue(
np.allclose(cell_data_from_region_model_1,
cell_data_from_gis_1['geo_data'],
atol=atol))
self.assertTrue(
np.allclose(cell_data_from_auto_cache_1['geo_data'],
cell_data_from_gis_1['geo_data'],
atol=atol))
self.assertTrue(
np.allclose(cell_data_from_updated_cache_1['geo_data'],
cell_data_from_gis_1['geo_data'],
atol=atol))
self.assertTrue(
np.allclose(rm_1.catchment_id_map,
cell_data_from_gis_1['cid_map'],
atol=atol))
self.assertTrue(
np.allclose(cell_data_from_auto_cache_1['cid_map'],
cell_data_from_gis_1['cid_map'],
atol=atol))
self.assertTrue(
np.allclose(cell_data_from_updated_cache_1['cid_map'],
cell_data_from_gis_1['cid_map'],
atol=atol))
def region_model_repo_constructor(cls,region_config, model_config, region_model_id):
if cls_path(cls) == 'shyft.repository.service.gis_region_model_repository.GisRegionModelRepository':
#from shyft.repository.service.gis_region_model_repository import GisRegionModelRepository
from shyft.repository.service.gis_region_model_repository import get_grid_spec_from_catch_poly
from shyft.repository.service.gis_region_model_repository import RegionModelConfig
from shyft.repository.service.gis_region_model_repository import GridSpecification
from six import iteritems # This replaces dictionary.iteritems() on Python 2 and dictionary.items() on Python 3
repo_params = region_config.repository()['params']
server_name = repo_params.get('server_name')
server_name_preprod = repo_params.get('server_name_preprod')
use_cache = repo_params.get('use_cache', False)
cache_folder = repo_params.get('cache_folder', None)
cache_folder = cache_folder.replace('${SHYFTDATA}', os.getenv('SHYFTDATA', '.'))
cache_file_type = repo_params.get('cache_file_type', None)
calc_forest_frac = repo_params.get('calc_forest_frac', False)
c_ids = region_config.catchments()
d = region_config.domain()
get_bbox_from_catchment_boundary = d.get('get_bbox_from_catchment_boundary', False)
pad = d.get('buffer', 5)
epsg_id = d['EPSG']
dx, dy = [d['step_x'], d['step_y']]
if use_cache or get_bbox_from_catchment_boundary:
if dx != dy:
raise ConfigError("step_x({}) and step_y({}) should be the same "
"if 'use_cache' or 'get_bbox_from_catchment_boundary' is enabled".format(dx, dy))
if get_bbox_from_catchment_boundary:
grid_specification = get_grid_spec_from_catch_poly(c_ids, repo_params['catchment_regulated_type'],
repo_params['service_id_field_name'], epsg_id, dx, pad,
server_name=server_name, server_name_preprod=server_name_preprod)
else:
grid_specification = GridSpecification(epsg_id, d['lower_left_x'], d['lower_left_y'],
dx, dy, d['nx'], d['ny'])
region_model_type = model_config.model_type()
# Construct region parameter:
name_map = {"priestley_taylor": "pt", "kirchner": "kirchner",
"precipitation_correction": "p_corr", "actual_evapotranspiration": "ae",
"gamma_snow": "gs", "skaugen_snow": "ss", "hbv_snow": "hs", "glacier_melt": "gm",
"hbv_actual_evapotranspiration":"ae", "hbv_soil": "soil", "hbv_tank": "tank"}
region_parameter = region_model_type.parameter_t()
for p_type_name, value_ in iteritems(model_config.model_parameters()):
if p_type_name in name_map:
if hasattr(region_parameter, name_map[p_type_name]):
sub_param = getattr(region_parameter, name_map[p_type_name])
for p, v in iteritems(value_):
if hasattr(sub_param, p):
setattr(sub_param, p, v)
else:
raise ConfigError("Invalid parameter '{}' for parameter set '{}'".format(p, p_type_name))
else:
raise ConfigError("Invalid parameter set '{}' for selected model '{}'".format(p_type_name, region_model_type.__name__))
else:
raise ConfigError("Unknown parameter set '{}'".format(p_type_name))
# Construct catchment overrides
catchment_parameters = {}
for c_id, catch_param in iteritems(region_config.parameter_overrides()):
if c_id in c_ids:
param = region_model_type.parameter_t(region_parameter)
for p_type_name, value_ in iteritems(catch_param):
if p_type_name in name_map:
if hasattr(param, name_map[p_type_name]):
sub_param = getattr(param, name_map[p_type_name])
for p, v in iteritems(value_):
if hasattr(sub_param, p):
setattr(sub_param, p, v)
else:
raise ConfigError("Invalid parameter '{}' for catchment parameter set '{}'".format(p, p_type_name))
else:
raise ConfigError("Invalid catchment parameter set '{}' for selected model '{}'".format(p_type_name, region_model_type.__name__))
else:
raise ConfigError("Unknown catchment parameter set '{}'".format(p_type_name))
catchment_parameters[c_id] = param
cfg_list=[
RegionModelConfig(region_model_id, region_model_type, region_parameter, grid_specification,
repo_params['catchment_regulated_type'], repo_params['service_id_field_name'],
region_config.catchments(), catchment_parameters=catchment_parameters,
calc_forest_frac=calc_forest_frac),
]
rm_cfg_dict = {x.name: x for x in cfg_list}
# return GisRegionModelRepository(rm_cfg_dict)
if server_name is not None:
cls.server_name = repo_params.get('server_name')
if server_name_preprod is not None:
cls.server_name_preprod = repo_params.get('server_name_preprod')
return cls(rm_cfg_dict, use_cache=use_cache, cache_folder=cache_folder, cache_file_type=cache_file_type)
else:
return cls(region_config, model_config)
def test_serialize_and_deserialize(self):
rm_type = pt_gs_k.PTGSKModel
reg_param = rm_type.parameter_t()
epsg_id = 32633
catchment_type = 'regulated'
identifier = 'SUBCATCH_ID'
dxy = 1000.
pad = 5
# LTM5-Tya
id_list = [172, 174, 177, 185, 187, 190]
grid_specification = get_grid_spec_from_catch_poly(id_list, catchment_type, identifier, epsg_id, dxy, pad)
# LTM5-Nea
id_list_1 = [180, 188, 191, 196]
grid_specification_1 = get_grid_spec_from_catch_poly(id_list_1, catchment_type, identifier, epsg_id, dxy, pad)
cfg_list = [
RegionModelConfig('Tya', rm_type, reg_param, grid_specification, catchment_type, identifier, id_list),
RegionModelConfig('Nea', rm_type, reg_param, grid_specification_1, catchment_type, identifier, id_list_1),
]
rm_cfg_dict = {x.name: x for x in cfg_list}
rm_repo = GisRegionModelRepository(rm_cfg_dict, use_cache=True)
# Start by removing existing cache file
GisRegionModelRepository.remove_cache(identifier, grid_specification)
# Update cache with Nea cell_data
GisRegionModelRepository.update_cache(catchment_type, identifier, grid_specification_1, id_list_1)
# Construct a region_model for Tya - this process should automatically append Tya cell_data to cache
rm = rm_repo.get_region_model('Tya')
cells_from_rm = rm.get_cells()
# Get Tya cell_data from the region_model
cell_data_from_region_model = cells_from_rm.geo_cell_data_vector(cells_from_rm).to_numpy().reshape(-1, 11)
# Get Tya cell_data from auto generated cache
cell_data_from_auto_cache = GisRegionModelRepository.get_cell_data_from_cache(
identifier, grid_specification, id_list)
# Remove the cache file
GisRegionModelRepository.remove_cache(identifier, grid_specification)
# Update cache with Tya cell_data
GisRegionModelRepository.update_cache(catchment_type, identifier, grid_specification, id_list)
# Update cache with Nea cell_data
GisRegionModelRepository.update_cache(catchment_type, identifier, grid_specification_1, id_list_1)
# Get Tya cell_data from updated cache
cell_data_from_updated_cache = GisRegionModelRepository.get_cell_data_from_cache(
identifier, grid_specification, id_list)
# Get Nea cell_data from updated cache
cell_data_from_updated_cache_1 = GisRegionModelRepository.get_cell_data_from_cache(
identifier, grid_specification_1, id_list_1)
# Remove the cache file
GisRegionModelRepository.remove_cache(identifier, grid_specification)
# Get Nea cell_data from GIS
cell_data_from_gis_1 = GisRegionModelRepository.get_cell_data_from_gis(
catchment_type, identifier, grid_specification_1, id_list_1)
# Get Tya cell_data from GIS
cell_data_from_gis = GisRegionModelRepository.get_cell_data_from_gis(
catchment_type, identifier, grid_specification, id_list)
# Update cache with Tya cell_data
GisRegionModelRepository.update_cache(catchment_type, identifier, grid_specification, id_list)
# Construct a region_model for Nea - this process should automatically append Nea cell_data to cache')
rm_1 = rm_repo.get_region_model('Nea')
cells_from_rm_1 = rm_1.get_cells()
# Get Nea cell_data from the region_model
cell_data_from_region_model_1 = cells_from_rm_1.geo_cell_data_vector(cells_from_rm_1).to_numpy().reshape(-1, 11)
# Get Nea cell_data from auto generated cache
cell_data_from_auto_cache_1 = GisRegionModelRepository.get_cell_data_from_cache(
identifier, grid_specification_1, id_list_1)
# Remove the cache file
GisRegionModelRepository.remove_cache(identifier, grid_specification)
# Comparing results...
# Arrays to compare
# Tya
# cell_data_from_region_model, cell_data_from_auto_cache, cell_data_from_updated_cache, cell_data_from_gis
atol = 1e-08
self.assertTrue(np.allclose(cell_data_from_region_model, cell_data_from_gis['geo_data'], atol=atol))
self.assertTrue(np.allclose(cell_data_from_auto_cache['geo_data'], cell_data_from_gis['geo_data'], atol=atol))
self.assertTrue(np.allclose(cell_data_from_updated_cache['geo_data'], cell_data_from_gis['geo_data'], atol=atol))
self.assertTrue(np.allclose(rm.catchment_id_map, cell_data_from_gis['cid_map'], atol=atol))
self.assertTrue(np.allclose(cell_data_from_auto_cache['cid_map'], cell_data_from_gis['cid_map'], atol=atol))
self.assertTrue(np.allclose(cell_data_from_updated_cache['cid_map'], cell_data_from_gis['cid_map'], atol=atol))
# Nea
# cell_data_from_region_model_1, cell_data_from_auto_cache_1, cell_data_from_updated_cache_1, cell_data_from_gis_1
self.assertTrue(np.allclose(cell_data_from_region_model_1, cell_data_from_gis_1['geo_data'], atol=atol))
self.assertTrue(np.allclose(cell_data_from_auto_cache_1['geo_data'], cell_data_from_gis_1['geo_data'], atol=atol))
self.assertTrue(np.allclose(cell_data_from_updated_cache_1['geo_data'], cell_data_from_gis_1['geo_data'], atol=atol))
self.assertTrue(np.allclose(rm_1.catchment_id_map, cell_data_from_gis_1['cid_map'], atol=atol))
self.assertTrue(np.allclose(cell_data_from_auto_cache_1['cid_map'], cell_data_from_gis_1['cid_map'], atol=atol))
self.assertTrue(np.allclose(cell_data_from_updated_cache_1['cid_map'], cell_data_from_gis_1['cid_map'], atol=atol))
