def region_model_repo_constructor(cls, region_config, model_config, region_model_id): if cls_path( cls ) == 'shyft.repository.service.gis_region_model_repository.GisRegionModelRepository': #from shyft.repository.service.gis_region_model_repository import GisRegionModelRepository from shyft.repository.service.gis_region_model_repository import get_grid_spec_from_catch_poly from shyft.repository.service.gis_region_model_repository import RegionModelConfig from shyft.repository.service.gis_region_model_repository import GridSpecification from six import iteritems # This replaces dictionary.iteritems() on Python 2 and dictionary.items() on Python 3 repo_params = region_config.repository()['params'] server_name = repo_params.get('server_name') server_name_preprod = repo_params.get('server_name_preprod') use_cache = repo_params.get('use_cache', False) cache_folder = repo_params.get('cache_folder', None) cache_folder = cache_folder.replace('${SHYFTDATA}', os.getenv('SHYFTDATA', '.')) cache_file_type = repo_params.get('cache_file_type', None) calc_forest_frac = repo_params.get('calc_forest_frac', False) c_ids = region_config.catchments() d = region_config.domain() get_bbox_from_catchment_boundary = d.get( 'get_bbox_from_catchment_boundary', False) pad = d.get('buffer', 5) epsg_id = d['EPSG'] dx, dy = [d['step_x'], d['step_y']] if use_cache or get_bbox_from_catchment_boundary: if dx != dy: raise ConfigError( "step_x({}) and step_y({}) should be the same " "if 'use_cache' or 'get_bbox_from_catchment_boundary' is enabled" .format(dx, dy)) if get_bbox_from_catchment_boundary: grid_specification = get_grid_spec_from_catch_poly( c_ids, repo_params['catchment_regulated_type'], repo_params['service_id_field_name'], epsg_id, dx, pad, server_name=server_name, server_name_preprod=server_name_preprod) else: grid_specification = GridSpecification(epsg_id, d['lower_left_x'], d['lower_left_y'], dx, dy, d['nx'], d['ny']) region_model_type = model_config.model_type() # Construct region parameter: name_map = { "priestley_taylor": "pt", "kirchner": "kirchner", "precipitation_correction": "p_corr", "actual_evapotranspiration": "ae", "gamma_snow": "gs", "skaugen_snow": "ss", "hbv_snow": "hs", "glacier_melt": "gm", "hbv_actual_evapotranspiration": "ae", "hbv_soil": "soil", "hbv_tank": "tank" } region_parameter = region_model_type.parameter_t() for p_type_name, value_ in iteritems(model_config.model_parameters()): if p_type_name in name_map: if hasattr(region_parameter, name_map[p_type_name]): sub_param = getattr(region_parameter, name_map[p_type_name]) for p, v in iteritems(value_): if hasattr(sub_param, p): setattr(sub_param, p, v) else: raise ConfigError( "Invalid parameter '{}' for parameter set '{}'" .format(p, p_type_name)) else: raise ConfigError( "Invalid parameter set '{}' for selected model '{}'". format(p_type_name, region_model_type.__name__)) else: raise ConfigError( "Unknown parameter set '{}'".format(p_type_name)) # Construct catchment overrides catchment_parameters = {} for c_id, catch_param in iteritems( region_config.parameter_overrides()): if c_id in c_ids: param = region_model_type.parameter_t(region_parameter) for p_type_name, value_ in iteritems(catch_param): if p_type_name in name_map: if hasattr(param, name_map[p_type_name]): sub_param = getattr(param, name_map[p_type_name]) for p, v in iteritems(value_): if hasattr(sub_param, p): setattr(sub_param, p, v) else: raise ConfigError( "Invalid parameter '{}' for catchment parameter set '{}'" .format(p, p_type_name)) else: raise ConfigError( "Invalid catchment parameter set '{}' for selected model '{}'" .format(p_type_name, region_model_type.__name__)) else: raise ConfigError( "Unknown catchment parameter set '{}'".format( p_type_name)) catchment_parameters[c_id] = param cfg_list = [ RegionModelConfig(region_model_id, region_model_type, region_parameter, grid_specification, repo_params['catchment_regulated_type'], repo_params['service_id_field_name'], region_config.catchments(), catchment_parameters=catchment_parameters, calc_forest_frac=calc_forest_frac), ] rm_cfg_dict = {x.name: x for x in cfg_list} # return GisRegionModelRepository(rm_cfg_dict) if server_name is not None: cls.server_name = repo_params.get('server_name') if server_name_preprod is not None: cls.server_name_preprod = repo_params.get('server_name_preprod') return cls(rm_cfg_dict, use_cache=use_cache, cache_folder=cache_folder, cache_file_type=cache_file_type) else: return cls(region_config, model_config)
def test_serialize_and_deserialize(self): rm_type = pt_gs_k.PTGSKModel reg_param = rm_type.parameter_t() epsg_id = 32633 catchment_type = 'regulated' identifier = 'SUBCATCH_ID' dxy = 1000. pad = 5 # LTM5-Tya id_list = [172, 174, 177, 185, 187, 190] grid_specification = get_grid_spec_from_catch_poly( id_list, catchment_type, identifier, epsg_id, dxy, pad) # LTM5-Nea id_list_1 = [180, 188, 191, 196] grid_specification_1 = get_grid_spec_from_catch_poly( id_list_1, catchment_type, identifier, epsg_id, dxy, pad) cfg_list = [ RegionModelConfig('Tya', rm_type, reg_param, grid_specification, catchment_type, identifier, id_list), RegionModelConfig('Nea', rm_type, reg_param, grid_specification_1, catchment_type, identifier, id_list_1), ] rm_cfg_dict = {x.name: x for x in cfg_list} rm_repo = GisRegionModelRepository(rm_cfg_dict, use_cache=True) # Start by removing existing cache file GisRegionModelRepository.remove_cache(identifier, grid_specification) # Update cache with Nea cell_data GisRegionModelRepository.update_cache(catchment_type, identifier, grid_specification_1, id_list_1) # Construct a region_model for Tya - this process should automatically append Tya cell_data to cache rm = rm_repo.get_region_model('Tya') cells_from_rm = rm.get_cells() # Get Tya cell_data from the region_model cell_data_from_region_model = cells_from_rm.geo_cell_data_vector( cells_from_rm).to_numpy().reshape(-1, 11) # Get Tya cell_data from auto generated cache cell_data_from_auto_cache = GisRegionModelRepository.get_cell_data_from_cache( identifier, grid_specification, id_list) # Remove the cache file GisRegionModelRepository.remove_cache(identifier, grid_specification) # Update cache with Tya cell_data GisRegionModelRepository.update_cache(catchment_type, identifier, grid_specification, id_list) # Update cache with Nea cell_data GisRegionModelRepository.update_cache(catchment_type, identifier, grid_specification_1, id_list_1) # Get Tya cell_data from updated cache cell_data_from_updated_cache = GisRegionModelRepository.get_cell_data_from_cache( identifier, grid_specification, id_list) # Get Nea cell_data from updated cache cell_data_from_updated_cache_1 = GisRegionModelRepository.get_cell_data_from_cache( identifier, grid_specification_1, id_list_1) # Remove the cache file GisRegionModelRepository.remove_cache(identifier, grid_specification) # Get Nea cell_data from GIS cell_data_from_gis_1 = GisRegionModelRepository.get_cell_data_from_gis( catchment_type, identifier, grid_specification_1, id_list_1) # Get Tya cell_data from GIS cell_data_from_gis = GisRegionModelRepository.get_cell_data_from_gis( catchment_type, identifier, grid_specification, id_list) # Update cache with Tya cell_data GisRegionModelRepository.update_cache(catchment_type, identifier, grid_specification, id_list) # Construct a region_model for Nea - this process should automatically append Nea cell_data to cache') rm_1 = rm_repo.get_region_model('Nea') cells_from_rm_1 = rm_1.get_cells() # Get Nea cell_data from the region_model cell_data_from_region_model_1 = cells_from_rm_1.geo_cell_data_vector( cells_from_rm_1).to_numpy().reshape(-1, 11) # Get Nea cell_data from auto generated cache cell_data_from_auto_cache_1 = GisRegionModelRepository.get_cell_data_from_cache( identifier, grid_specification_1, id_list_1) # Remove the cache file GisRegionModelRepository.remove_cache(identifier, grid_specification) # Comparing results... # Arrays to compare # Tya # cell_data_from_region_model, cell_data_from_auto_cache, cell_data_from_updated_cache, cell_data_from_gis atol = 1e-08 self.assertTrue( np.allclose(cell_data_from_region_model, cell_data_from_gis['geo_data'], atol=atol)) self.assertTrue( np.allclose(cell_data_from_auto_cache['geo_data'], cell_data_from_gis['geo_data'], atol=atol)) self.assertTrue( np.allclose(cell_data_from_updated_cache['geo_data'], cell_data_from_gis['geo_data'], atol=atol)) self.assertTrue( np.allclose(rm.catchment_id_map, cell_data_from_gis['cid_map'], atol=atol)) self.assertTrue( np.allclose(cell_data_from_auto_cache['cid_map'], cell_data_from_gis['cid_map'], atol=atol)) self.assertTrue( np.allclose(cell_data_from_updated_cache['cid_map'], cell_data_from_gis['cid_map'], atol=atol)) # Nea # cell_data_from_region_model_1, cell_data_from_auto_cache_1, cell_data_from_updated_cache_1, cell_data_from_gis_1 self.assertTrue( np.allclose(cell_data_from_region_model_1, cell_data_from_gis_1['geo_data'], atol=atol)) self.assertTrue( np.allclose(cell_data_from_auto_cache_1['geo_data'], cell_data_from_gis_1['geo_data'], atol=atol)) self.assertTrue( np.allclose(cell_data_from_updated_cache_1['geo_data'], cell_data_from_gis_1['geo_data'], atol=atol)) self.assertTrue( np.allclose(rm_1.catchment_id_map, cell_data_from_gis_1['cid_map'], atol=atol)) self.assertTrue( np.allclose(cell_data_from_auto_cache_1['cid_map'], cell_data_from_gis_1['cid_map'], atol=atol)) self.assertTrue( np.allclose(cell_data_from_updated_cache_1['cid_map'], cell_data_from_gis_1['cid_map'], atol=atol))
def region_model_repo_constructor(cls,region_config, model_config, region_model_id): if cls_path(cls) == 'shyft.repository.service.gis_region_model_repository.GisRegionModelRepository': #from shyft.repository.service.gis_region_model_repository import GisRegionModelRepository from shyft.repository.service.gis_region_model_repository import get_grid_spec_from_catch_poly from shyft.repository.service.gis_region_model_repository import RegionModelConfig from shyft.repository.service.gis_region_model_repository import GridSpecification from six import iteritems # This replaces dictionary.iteritems() on Python 2 and dictionary.items() on Python 3 repo_params = region_config.repository()['params'] server_name = repo_params.get('server_name') server_name_preprod = repo_params.get('server_name_preprod') use_cache = repo_params.get('use_cache', False) cache_folder = repo_params.get('cache_folder', None) cache_folder = cache_folder.replace('${SHYFTDATA}', os.getenv('SHYFTDATA', '.')) cache_file_type = repo_params.get('cache_file_type', None) calc_forest_frac = repo_params.get('calc_forest_frac', False) c_ids = region_config.catchments() d = region_config.domain() get_bbox_from_catchment_boundary = d.get('get_bbox_from_catchment_boundary', False) pad = d.get('buffer', 5) epsg_id = d['EPSG'] dx, dy = [d['step_x'], d['step_y']] if use_cache or get_bbox_from_catchment_boundary: if dx != dy: raise ConfigError("step_x({}) and step_y({}) should be the same " "if 'use_cache' or 'get_bbox_from_catchment_boundary' is enabled".format(dx, dy)) if get_bbox_from_catchment_boundary: grid_specification = get_grid_spec_from_catch_poly(c_ids, repo_params['catchment_regulated_type'], repo_params['service_id_field_name'], epsg_id, dx, pad, server_name=server_name, server_name_preprod=server_name_preprod) else: grid_specification = GridSpecification(epsg_id, d['lower_left_x'], d['lower_left_y'], dx, dy, d['nx'], d['ny']) region_model_type = model_config.model_type() # Construct region parameter: name_map = {"priestley_taylor": "pt", "kirchner": "kirchner", "precipitation_correction": "p_corr", "actual_evapotranspiration": "ae", "gamma_snow": "gs", "skaugen_snow": "ss", "hbv_snow": "hs", "glacier_melt": "gm", "hbv_actual_evapotranspiration":"ae", "hbv_soil": "soil", "hbv_tank": "tank"} region_parameter = region_model_type.parameter_t() for p_type_name, value_ in iteritems(model_config.model_parameters()): if p_type_name in name_map: if hasattr(region_parameter, name_map[p_type_name]): sub_param = getattr(region_parameter, name_map[p_type_name]) for p, v in iteritems(value_): if hasattr(sub_param, p): setattr(sub_param, p, v) else: raise ConfigError("Invalid parameter '{}' for parameter set '{}'".format(p, p_type_name)) else: raise ConfigError("Invalid parameter set '{}' for selected model '{}'".format(p_type_name, region_model_type.__name__)) else: raise ConfigError("Unknown parameter set '{}'".format(p_type_name)) # Construct catchment overrides catchment_parameters = {} for c_id, catch_param in iteritems(region_config.parameter_overrides()): if c_id in c_ids: param = region_model_type.parameter_t(region_parameter) for p_type_name, value_ in iteritems(catch_param): if p_type_name in name_map: if hasattr(param, name_map[p_type_name]): sub_param = getattr(param, name_map[p_type_name]) for p, v in iteritems(value_): if hasattr(sub_param, p): setattr(sub_param, p, v) else: raise ConfigError("Invalid parameter '{}' for catchment parameter set '{}'".format(p, p_type_name)) else: raise ConfigError("Invalid catchment parameter set '{}' for selected model '{}'".format(p_type_name, region_model_type.__name__)) else: raise ConfigError("Unknown catchment parameter set '{}'".format(p_type_name)) catchment_parameters[c_id] = param cfg_list=[ RegionModelConfig(region_model_id, region_model_type, region_parameter, grid_specification, repo_params['catchment_regulated_type'], repo_params['service_id_field_name'], region_config.catchments(), catchment_parameters=catchment_parameters, calc_forest_frac=calc_forest_frac), ] rm_cfg_dict = {x.name: x for x in cfg_list} # return GisRegionModelRepository(rm_cfg_dict) if server_name is not None: cls.server_name = repo_params.get('server_name') if server_name_preprod is not None: cls.server_name_preprod = repo_params.get('server_name_preprod') return cls(rm_cfg_dict, use_cache=use_cache, cache_folder=cache_folder, cache_file_type=cache_file_type) else: return cls(region_config, model_config)
def test_serialize_and_deserialize(self): rm_type = pt_gs_k.PTGSKModel reg_param = rm_type.parameter_t() epsg_id = 32633 catchment_type = 'regulated' identifier = 'SUBCATCH_ID' dxy = 1000. pad = 5 # LTM5-Tya id_list = [172, 174, 177, 185, 187, 190] grid_specification = get_grid_spec_from_catch_poly(id_list, catchment_type, identifier, epsg_id, dxy, pad) # LTM5-Nea id_list_1 = [180, 188, 191, 196] grid_specification_1 = get_grid_spec_from_catch_poly(id_list_1, catchment_type, identifier, epsg_id, dxy, pad) cfg_list = [ RegionModelConfig('Tya', rm_type, reg_param, grid_specification, catchment_type, identifier, id_list), RegionModelConfig('Nea', rm_type, reg_param, grid_specification_1, catchment_type, identifier, id_list_1), ] rm_cfg_dict = {x.name: x for x in cfg_list} rm_repo = GisRegionModelRepository(rm_cfg_dict, use_cache=True) # Start by removing existing cache file GisRegionModelRepository.remove_cache(identifier, grid_specification) # Update cache with Nea cell_data GisRegionModelRepository.update_cache(catchment_type, identifier, grid_specification_1, id_list_1) # Construct a region_model for Tya - this process should automatically append Tya cell_data to cache rm = rm_repo.get_region_model('Tya') cells_from_rm = rm.get_cells() # Get Tya cell_data from the region_model cell_data_from_region_model = cells_from_rm.geo_cell_data_vector(cells_from_rm).to_numpy().reshape(-1, 11) # Get Tya cell_data from auto generated cache cell_data_from_auto_cache = GisRegionModelRepository.get_cell_data_from_cache( identifier, grid_specification, id_list) # Remove the cache file GisRegionModelRepository.remove_cache(identifier, grid_specification) # Update cache with Tya cell_data GisRegionModelRepository.update_cache(catchment_type, identifier, grid_specification, id_list) # Update cache with Nea cell_data GisRegionModelRepository.update_cache(catchment_type, identifier, grid_specification_1, id_list_1) # Get Tya cell_data from updated cache cell_data_from_updated_cache = GisRegionModelRepository.get_cell_data_from_cache( identifier, grid_specification, id_list) # Get Nea cell_data from updated cache cell_data_from_updated_cache_1 = GisRegionModelRepository.get_cell_data_from_cache( identifier, grid_specification_1, id_list_1) # Remove the cache file GisRegionModelRepository.remove_cache(identifier, grid_specification) # Get Nea cell_data from GIS cell_data_from_gis_1 = GisRegionModelRepository.get_cell_data_from_gis( catchment_type, identifier, grid_specification_1, id_list_1) # Get Tya cell_data from GIS cell_data_from_gis = GisRegionModelRepository.get_cell_data_from_gis( catchment_type, identifier, grid_specification, id_list) # Update cache with Tya cell_data GisRegionModelRepository.update_cache(catchment_type, identifier, grid_specification, id_list) # Construct a region_model for Nea - this process should automatically append Nea cell_data to cache') rm_1 = rm_repo.get_region_model('Nea') cells_from_rm_1 = rm_1.get_cells() # Get Nea cell_data from the region_model cell_data_from_region_model_1 = cells_from_rm_1.geo_cell_data_vector(cells_from_rm_1).to_numpy().reshape(-1, 11) # Get Nea cell_data from auto generated cache cell_data_from_auto_cache_1 = GisRegionModelRepository.get_cell_data_from_cache( identifier, grid_specification_1, id_list_1) # Remove the cache file GisRegionModelRepository.remove_cache(identifier, grid_specification) # Comparing results... # Arrays to compare # Tya # cell_data_from_region_model, cell_data_from_auto_cache, cell_data_from_updated_cache, cell_data_from_gis atol = 1e-08 self.assertTrue(np.allclose(cell_data_from_region_model, cell_data_from_gis['geo_data'], atol=atol)) self.assertTrue(np.allclose(cell_data_from_auto_cache['geo_data'], cell_data_from_gis['geo_data'], atol=atol)) self.assertTrue(np.allclose(cell_data_from_updated_cache['geo_data'], cell_data_from_gis['geo_data'], atol=atol)) self.assertTrue(np.allclose(rm.catchment_id_map, cell_data_from_gis['cid_map'], atol=atol)) self.assertTrue(np.allclose(cell_data_from_auto_cache['cid_map'], cell_data_from_gis['cid_map'], atol=atol)) self.assertTrue(np.allclose(cell_data_from_updated_cache['cid_map'], cell_data_from_gis['cid_map'], atol=atol)) # Nea # cell_data_from_region_model_1, cell_data_from_auto_cache_1, cell_data_from_updated_cache_1, cell_data_from_gis_1 self.assertTrue(np.allclose(cell_data_from_region_model_1, cell_data_from_gis_1['geo_data'], atol=atol)) self.assertTrue(np.allclose(cell_data_from_auto_cache_1['geo_data'], cell_data_from_gis_1['geo_data'], atol=atol)) self.assertTrue(np.allclose(cell_data_from_updated_cache_1['geo_data'], cell_data_from_gis_1['geo_data'], atol=atol)) self.assertTrue(np.allclose(rm_1.catchment_id_map, cell_data_from_gis_1['cid_map'], atol=atol)) self.assertTrue(np.allclose(cell_data_from_auto_cache_1['cid_map'], cell_data_from_gis_1['cid_map'], atol=atol)) self.assertTrue(np.allclose(cell_data_from_updated_cache_1['cid_map'], cell_data_from_gis_1['cid_map'], atol=atol))