def test_read_template_pass(self): """Wrong exposures definition""" df = pd.read_excel(ENT_TEMPLATE_XLS) exp_df = Exposures(df) # set metadata exp_df.ref_year = 2020 exp_df.tag = Tag(ENT_TEMPLATE_XLS, 'ENT_TEMPLATE_XLS') exp_df.value_unit = 'XSD' exp_df.check()
def check(self): """ Check which variables are present """ # check metadata for var in self._metadata: if var[0] == '_': continue try: if getattr(self, var) is None and var == 'crs': self.crs = DEF_CRS LOGGER.info('%s set to default value: %s', var, self.__dict__[var]) except AttributeError: if var == 'tag': self.tag = Tag() elif var == 'ref_year': self.ref_year = DEF_REF_YEAR elif var == 'value_unit': self.value_unit = DEF_VALUE_UNIT LOGGER.info('%s metadata set to default value: %s', var, self.__dict__[var]) for var in self.vars_oblig: if not var in self.columns: LOGGER.error("%s missing.", var) raise ValueError for var in self.vars_def: if var == INDICATOR_IF: found = np.array([var in var_col for var_col in self.columns]).any() if INDICATOR_IF in self.columns: LOGGER.info("Hazard type not set in %s", var) else: found = var in self.columns if not found and var == INDICATOR_IF: LOGGER.info("Setting %s to default impact functions ids 1.", var) self[INDICATOR_IF] = np.ones(self.shape[0], dtype=int) elif not found: LOGGER.info("%s not set.", var) for var in self.vars_opt: if var == INDICATOR_CENTR: found = np.array([var in var_col for var_col in self.columns]).any() if INDICATOR_CENTR in self.columns: LOGGER.info("Hazard type not set in %s", var) else: found = var in self.columns if not found: LOGGER.info("%s not set.", var) elif var == 'geometry' and \ (self.geometry.values[0].x != self.longitude.values[0] or \ self.geometry.values[0].y != self.latitude.values[0]): LOGGER.error('Geometry values do not correspond to latitude ' +\ 'and longitude. Use set_geometry_points() or set_lat_lon().') raise ValueError
def set_countries(self, countries=[], reg=[], ref_year=2000, path=None): """Model countries using values at reference year. If GDP or income group not available for that year, consider the value of the closest available year. Parameters: countries (list): list of country names ISO3 ref_year (int, optional): reference year. Default: 2016 path (string): path to exposure dataset (ISIMIP) """ gdp2a_list = [] tag = Tag() if path is None: raise NameError('No path for exposure data set') if not Path(path).is_file(): raise NameError('Invalid path %s' % path) try: if not countries: if reg: natISO = u_coord.region2isos(reg) countries = np.array(natISO) else: raise ValueError('set_countries requires countries or reg') for cntr_ind in range(len(countries)): gdp2a_list.append( self._set_one_country(countries[cntr_ind], ref_year, path)) tag.description += ("{} GDP2Asset \n").\ format(countries[cntr_ind]) except KeyError as err: raise KeyError( f'Exposure countries: {countries} or reg {reg} could not be set, ' f'check ISO3 or reference year {ref_year}') from err tag.description += 'GDP2Asset ' + str(self.ref_year) Exposures.__init__(self, data=Exposures.concat(gdp2a_list).gdf, ref_year=ref_year, tag=tag, value_unit='USD') # set meta res = 0.0416666 rows, cols, ras_trans = u_coord.pts_to_raster_meta( (self.gdf.longitude.min(), self.gdf.latitude.min(), self.gdf.longitude.max(), self.gdf.latitude.max()), res) self.meta = { 'width': cols, 'height': rows, 'crs': self.crs, 'transform': ras_trans }
def test_io_hdf5_pass(self): """write and read hdf5""" exp_df = Exposures(pd.read_excel(ENT_TEMPLATE_XLS), crs="epsg:32632") exp_df.set_geometry_points() exp_df.check() # set metadata exp_df.ref_year = 2020 exp_df.tag = Tag(ENT_TEMPLATE_XLS, 'ENT_TEMPLATE_XLS') exp_df.value_unit = 'XSD' file_name = DATA_DIR.joinpath('test_hdf5_exp.h5') # pd.errors.PerformanceWarning should be suppressed. Therefore, make sure that # PerformanceWarning would result in test failure here import warnings with warnings.catch_warnings(): warnings.simplefilter("error", category=pd.errors.PerformanceWarning) exp_df.write_hdf5(file_name) exp_read = Exposures.from_hdf5(file_name) self.assertEqual(exp_df.ref_year, exp_read.ref_year) self.assertEqual(exp_df.value_unit, exp_read.value_unit) self.assertDictEqual(exp_df.meta, exp_read.meta) self.assertTrue(u_coord.equal_crs(exp_df.crs, exp_read.crs)) self.assertTrue(u_coord.equal_crs(exp_df.gdf.crs, exp_read.gdf.crs)) self.assertEqual(exp_df.tag.file_name, exp_read.tag.file_name) self.assertEqual(exp_df.tag.description, exp_read.tag.description) np.testing.assert_array_equal(exp_df.gdf.latitude.values, exp_read.gdf.latitude.values) np.testing.assert_array_equal(exp_df.gdf.longitude.values, exp_read.gdf.longitude.values) np.testing.assert_array_equal(exp_df.gdf.value.values, exp_read.gdf.value.values) np.testing.assert_array_equal(exp_df.gdf.deductible.values, exp_read.gdf.deductible.values) np.testing.assert_array_equal(exp_df.gdf.cover.values, exp_read.gdf.cover.values) np.testing.assert_array_equal(exp_df.gdf.region_id.values, exp_read.gdf.region_id.values) np.testing.assert_array_equal(exp_df.gdf.category_id.values, exp_read.gdf.category_id.values) np.testing.assert_array_equal(exp_df.gdf.impf_TC.values, exp_read.gdf.impf_TC.values) np.testing.assert_array_equal(exp_df.gdf.centr_TC.values, exp_read.gdf.centr_TC.values) np.testing.assert_array_equal(exp_df.gdf.impf_FL.values, exp_read.gdf.impf_FL.values) np.testing.assert_array_equal(exp_df.gdf.centr_FL.values, exp_read.gdf.centr_FL.values) for point_df, point_read in zip(exp_df.gdf.geometry.values, exp_read.gdf.geometry.values): self.assertEqual(point_df.x, point_read.x) self.assertEqual(point_df.y, point_read.y)
def test_io_hdf5_pass(self): """write and read hdf5""" exp_df = Exposures(pd.read_excel(ENT_TEMPLATE_XLS)) exp_df.set_geometry_points() exp_df.check() # set metadata exp_df.ref_year = 2020 exp_df.tag = Tag(ENT_TEMPLATE_XLS, 'ENT_TEMPLATE_XLS') exp_df.value_unit = 'XSD' file_name = DATA_DIR.joinpath('test_hdf5_exp.h5') exp_df.write_hdf5(file_name) exp_read = Exposures() exp_read.read_hdf5(file_name) self.assertEqual(exp_df.ref_year, exp_read.ref_year) self.assertEqual(exp_df.value_unit, exp_read.value_unit) self.assertEqual(exp_df.crs, exp_read.crs) self.assertEqual(exp_df.tag.file_name, exp_read.tag.file_name) self.assertEqual(exp_df.tag.description, exp_read.tag.description) self.assertTrue( np.array_equal(exp_df.gdf.latitude.values, exp_read.gdf.latitude.values)) self.assertTrue( np.array_equal(exp_df.gdf.longitude.values, exp_read.gdf.longitude.values)) self.assertTrue( np.array_equal(exp_df.gdf.value.values, exp_read.gdf.value.values)) self.assertTrue( np.array_equal(exp_df.gdf.deductible.values, exp_read.gdf.deductible.values)) self.assertTrue( np.array_equal(exp_df.gdf.cover.values, exp_read.gdf.cover.values)) self.assertTrue( np.array_equal(exp_df.gdf.region_id.values, exp_read.gdf.region_id.values)) self.assertTrue( np.array_equal(exp_df.gdf.category_id.values, exp_read.gdf.category_id.values)) self.assertTrue( np.array_equal(exp_df.gdf.if_TC.values, exp_read.gdf.if_TC.values)) self.assertTrue( np.array_equal(exp_df.gdf.centr_TC.values, exp_read.gdf.centr_TC.values)) self.assertTrue( np.array_equal(exp_df.gdf.if_FL.values, exp_read.gdf.if_FL.values)) self.assertTrue( np.array_equal(exp_df.gdf.centr_FL.values, exp_read.gdf.centr_FL.values)) for point_df, point_read in zip(exp_df.gdf.geometry.values, exp_read.gdf.geometry.values): self.assertEqual(point_df.x, point_read.x) self.assertEqual(point_df.y, point_read.y)
def from_raster(cls, file_name, band=1, src_crs=None, window=False, geometry=False, dst_crs=False, transform=None, width=None, height=None, resampling=Resampling.nearest): """Read raster data and set latitude, longitude, value and meta Parameters ---------- file_name : str file name containing values band : int, optional bands to read (starting at 1) src_crs : crs, optional source CRS. Provide it if error without it. window : rasterio.windows.Windows, optional window where data is extracted geometry : shapely.geometry, optional consider pixels only in shape dst_crs : crs, optional reproject to given crs transform : rasterio.Affine affine transformation to apply wdith : float number of lons for transform height : float number of lats for transform resampling : rasterio.warp,.Resampling optional resampling function used for reprojection to dst_crs returns -------- Exposures """ exp = cls() if 'geometry' in exp.gdf: raise ValueError("there is already a geometry column defined in the GeoDataFrame") exp.tag = Tag() exp.tag.file_name = str(file_name) meta, value = u_coord.read_raster(file_name, [band], src_crs, window, geometry, dst_crs, transform, width, height, resampling) ulx, xres, _, uly, _, yres = meta['transform'].to_gdal() lrx = ulx + meta['width'] * xres lry = uly + meta['height'] * yres x_grid, y_grid = np.meshgrid(np.arange(ulx + xres / 2, lrx, xres), np.arange(uly + yres / 2, lry, yres)) if exp.crs is None: exp.set_crs() exp.gdf['longitude'] = x_grid.flatten() exp.gdf['latitude'] = y_grid.flatten() exp.gdf['value'] = value.reshape(-1) exp.meta = meta return exp
def _build_exp(self): eai_exp = Exposures() eai_exp['value'] = self.eai_exp eai_exp['latitude'] = self.coord_exp[:, 0] eai_exp['longitude'] = self.coord_exp[:, 1] eai_exp.crs = self.crs eai_exp.value_unit = self.unit eai_exp.ref_year = 0 eai_exp.tag = Tag() eai_exp.meta = None return eai_exp
def set_countries(self, countries, ref_year=2016, res_km=None, from_hr=None, **kwargs): """ Model countries using values at reference year. If GDP or income group not available for that year, consider the value of the closest available year. Parameters: countries (list or dict): list of country names (admin0) or dict with key = admin0 name and value = [admin1 names] ref_year (int, optional): reference year. Default: 2016 res_km (float, optional): approx resolution in km. Default: nightlights resolution. from_hr (bool, optional): force to use higher resolution image, independently of its year of acquisition. kwargs (optional): 'gdp' and 'inc_grp' dictionaries with keys the country ISO_alpha3 code. 'poly_val' polynomial transformation [1,x,x^2,...] to apply to nightlight (DEF_POLY_VAL used if not provided). If provided, these are used. """ shp_file = shapereader.natural_earth(resolution='10m', category='cultural', name='admin_0_countries') shp_file = shapereader.Reader(shp_file) cntry_info, cntry_admin1 = country_iso_geom(countries, shp_file) fill_econ_indicators(ref_year, cntry_info, shp_file, **kwargs) nightlight, coord_nl, fn_nl, res_fact, res_km = get_nightlight(\ ref_year, cntry_info, res_km, from_hr) tag = Tag() bkmrbl_list = [] for cntry_iso, cntry_val in cntry_info.items(): bkmrbl_list.append(self._set_one_country(cntry_val, nightlight, \ coord_nl, res_fact, res_km, cntry_admin1[cntry_iso], **kwargs)) tag.description += ("{} {:d} GDP: {:.3e} income group: {:d} \n").\ format(cntry_val[1], cntry_val[3], cntry_val[4], cntry_val[5]) Exposures.__init__( self, gpd.GeoDataFrame(pd.concat(bkmrbl_list, ignore_index=True))) # set metadata self.ref_year = ref_year self.tag = tag self.tag.file_name = fn_nl self.value_unit = 'USD' self.crs = {'init': 'epsg:4326'}
def test_write_read_exp_test(self): ''' Test result against reference value''' # Create impact object num_ev = 5 num_exp = 10 imp_write = Impact() imp_write.tag = {'exp': Tag('file_exp.p', 'descr exp'), 'haz': TagHaz('TC', 'file_haz.p', 'descr haz'), 'if_set': Tag()} imp_write.event_id = np.arange(num_ev) imp_write.event_name = ['event_'+str(num) for num in imp_write.event_id] imp_write.date = np.ones(num_ev) imp_write.coord_exp = np.zeros((num_exp, 2)) imp_write.coord_exp[:, 0] = 1.5 imp_write.coord_exp[:, 1] = 2.5 imp_write.eai_exp = np.arange(num_exp) * 100 imp_write.at_event = np.arange(num_ev) * 50 imp_write.frequency = np.ones(num_ev) * 0.1 imp_write.tot_value = 1000 imp_write.aai_agg = 1001 imp_write.unit = 'USD' file_name = os.path.join(DATA_FOLDER, 'test.csv') imp_write.write_csv(file_name) imp_read = Impact() imp_read.read_csv(file_name) self.assertTrue(np.array_equal(imp_write.event_id, imp_read.event_id)) self.assertTrue(np.array_equal(imp_write.date, imp_read.date)) self.assertTrue(np.array_equal(imp_write.coord_exp, imp_read.coord_exp)) self.assertTrue(np.array_equal(imp_write.eai_exp, imp_read.eai_exp)) self.assertTrue(np.array_equal(imp_write.at_event, imp_read.at_event)) self.assertTrue(np.array_equal(imp_write.frequency, imp_read.frequency)) self.assertEqual(imp_write.tot_value, imp_read.tot_value) self.assertEqual(imp_write.aai_agg, imp_read.aai_agg) self.assertEqual(imp_write.unit, imp_read.unit) self.assertEqual(0, len([i for i, j in zip(imp_write.event_name, imp_read.event_name) if i != j])) self.assertIsInstance(imp_read.crs, dict)
def read_excel(self, file_name): """Read excel file containing impact data generated by write_excel. Parameters: file_name (str): absolute path of the file """ LOGGER.info('Reading %s', file_name) dfr = pd.read_excel(file_name) self.__init__() self.tag['haz'] = TagHaz() self.tag['haz'].haz_type = dfr['tag_hazard'][0] self.tag['haz'].file_name = dfr['tag_hazard'][1] self.tag['haz'].description = dfr['tag_hazard'][2] self.tag['exp'] = Tag() self.tag['exp'].file_name = dfr['tag_exposure'][0] self.tag['exp'].description = dfr['tag_exposure'][1] self.tag['if_set'] = Tag() self.tag['if_set'].file_name = dfr['tag_impact_func'][0] self.tag['if_set'].description = dfr['tag_impact_func'][1] self.unit = dfr.unit[0] self.tot_value = dfr.tot_value[0] self.aai_agg = dfr.aai_agg[0] self.event_id = dfr.event_id[~np.isnan(dfr.event_id.values)].values self.event_name = dfr.event_name[:self.event_id.size].values self.date = dfr.event_date[:self.event_id.size].values self.frequency = dfr.event_frequency[:self.event_id.size].values self.at_event = dfr.at_event[:self.event_id.size].values self.eai_exp = dfr.eai_exp[~np.isnan(dfr.eai_exp.values)].values self.coord_exp = np.zeros((self.eai_exp.size, 2)) self.coord_exp[:, 0] = dfr.exp_lat.values[:self.eai_exp.size] self.coord_exp[:, 1] = dfr.exp_lon.values[:self.eai_exp.size] try: self.crs = ast.literal_eval(dfr.exp_crs.values[0]) except AttributeError: self.crs = DEF_CRS
def _read_mat_metadata(exposures, data, file_name, var_names): """ Fille metadata in DataFrame object """ try: exposures.ref_year = int(np.squeeze(data[var_names['var_name']['ref']])) except KeyError: exposures.ref_year = DEF_REF_YEAR try: exposures.value_unit = hdf5.get_str_from_ref(file_name, \ data[var_names['var_name']['uni']][0][0]) except KeyError: exposures.value_unit = DEF_VALUE_UNIT exposures.tag = Tag(file_name)
def set_countries(self, countries=[], reg=[], ref_year=2000, path=None): """ Model countries using values at reference year. If GDP or income group not available for that year, consider the value of the closest available year. Parameters: countries (list): list of country names ISO3 ref_year (int, optional): reference year. Default: 2016 path (string): path to exposure dataset """ gdp2a_list = [] tag = Tag() if path is None: LOGGER.error('No path for exposure data set') raise NameError if not os.path.exists(path): LOGGER.error('Invalid path ' + path) raise NameError try: if not countries: if reg: natID_info = pd.read_csv(NAT_REG_ID) natISO = natID_info["ISO"][np.isin(natID_info["Reg_name"], reg)] countries = np.array(natISO) else: LOGGER.error('set_countries requires countries or reg') raise ValueError for cntr_ind in range(len(countries)): gdp2a_list.append(self._set_one_country(countries[cntr_ind], ref_year, path)) tag.description += ("{} GDP2Asset \n").\ format(countries[cntr_ind]) Exposures.__init__(self, gpd.GeoDataFrame( pd.concat(gdp2a_list, ignore_index=True))) except KeyError: LOGGER.error('Exposure countries: ' + str(countries) + ' or reg ' + str(reg) + ' could not be set, check ISO3 or' + ' reference year ' + str(ref_year)) raise KeyError self.ref_year = ref_year self.value_unit = 'USD' self.tag = tag self.crs = DEF_CRS
def _build_exp_event(self, event_id): """Write impact of an event as Exposures Parameters: event_id(int): id of the event """ impact_csr_exp = Exposures() impact_csr_exp['value'] = self.imp_mat.toarray()[event_id - 1, :] impact_csr_exp['latitude'] = self.coord_exp[:, 0] impact_csr_exp['longitude'] = self.coord_exp[:, 1] impact_csr_exp.crs = self.crs impact_csr_exp.value_unit = self.unit impact_csr_exp.ref_year = 0 impact_csr_exp.tag = Tag() impact_csr_exp.meta = None return impact_csr_exp
def set_from_raster(self, file_name, band=1, src_crs=None, window=False, geometry=False, dst_crs=False, transform=None, width=None, height=None, resampling=Resampling.nearest): """ Read raster data and set latitude, longitude, value and meta Parameters: file_name (str): file name containing values band (int, optional): bands to read (starting at 1) src_crs (crs, optional): source CRS. Provide it if error without it. window (rasterio.windows.Windows, optional): window where data is extracted geometry (shapely.geometry, optional): consider pixels only in shape dst_crs (crs, optional): reproject to given crs transform (rasterio.Affine): affine transformation to apply wdith (float): number of lons for transform height (float): number of lats for transform resampling (rasterio.warp,.Resampling optional): resampling function used for reprojection to dst_crs """ self.__init__() self.tag = Tag() self.tag.file_name = file_name meta, value = co.read_raster(file_name, [band], src_crs, window, geometry, dst_crs, transform, width, height, resampling) ulx, xres, _, uly, _, yres = meta['transform'].to_gdal() lrx = ulx + meta['width'] * xres lry = uly + meta['height'] * yres x_grid, y_grid = np.meshgrid(np.arange(ulx + xres / 2, lrx, xres), np.arange(uly + yres / 2, lry, yres)) try: self.crs = meta['crs'].to_dict() except AttributeError: self.crs = meta['crs'] self['longitude'] = x_grid.flatten() self['latitude'] = y_grid.flatten() self['value'] = value.reshape(-1) self.meta = meta
def __init__(self, years=None, rates=None, tag=None): """ Fill discount rates with values and check consistency data Parameters ---------- years : numpy.ndarray(int) Array of years. Default is numpy.array([]). rates : numpy.ndarray(float) Discount rates for each year in years. Default is numpy.array([]). Note: rates given in float, e.g., to set 1% rate use 0.01 tag : climate.entity.tag Metadata. Default is None. """ years = np.array([]) if years is None else years rates = np.array([]) if rates is None else rates self.years = years self.rates = rates tag = Tag() if tag is None else tag self.tag = tag
def from_mat(cls, file_name, description='', var_names=None): """ Read MATLAB file generated with previous MATLAB CLIMADA version. Parameters ---------- file_name: str filename including path and extension description: str, optional description of the data. The default is '' var_names: dict, optional name of the variables in the file. The Default is DEF_VAR_MAT = {'sup_field_name': 'entity', 'field_name': 'discount', 'var_name': {'year': 'year', 'disc': 'discount_rate'}} Returns ------- climada.entity.DiscRates : The disc rates from matlab """ if var_names is None: var_names = DEF_VAR_MAT disc = u_hdf5.read(file_name) tag = Tag(file_name=str(file_name), description=description) try: disc = disc[var_names['sup_field_name']] except KeyError: pass try: disc = disc[var_names['field_name']] years = np.squeeze(disc[var_names['var_name']['year']]). \ astype(int, copy=False) rates = np.squeeze(disc[var_names['var_name']['disc']]) except KeyError as err: raise KeyError("Not existing variable: %s" % str(err)) from err return cls(years=years, rates=rates, tag=tag)
def clear(self): """Reinitialize attributes.""" self.tag = Tag() self._data = dict() # {hazard_type : {name: Measure()}}
def clear(self): """Reinitialize attributes.""" self.tag = Tag() self._data = dict() # {hazard_type : {id:ImpactFunc}}
class ImpactFuncSet(): """Contains impact functions of type ImpactFunc. Loads from files with format defined in FILE_EXT. Attributes: tag (Tag): information about the source data _data (dict): contains ImpactFunc classes. It's not suppossed to be directly accessed. Use the class methods instead. """ def __init__(self): """Empty initialization. Examples: Fill impact functions with values and check consistency data: >>> fun_1 = ImpactFunc() >>> fun_1.haz_type = 'TC' >>> fun_1.id = 3 >>> fun_1.intensity = np.array([0, 20]) >>> fun_1.paa = np.array([0, 1]) >>> fun_1.mdd = np.array([0, 0.5]) >>> imp_fun = ImpactFuncSet() >>> imp_fun.append(fun_1) >>> imp_fun.check() Read impact functions from file and checks consistency data. >>> imp_fun = ImpactFuncSet() >>> imp_fun.read(ENT_TEMPLATE_XLS) """ self.clear() def clear(self): """Reinitialize attributes.""" self.tag = Tag() self._data = dict() # {hazard_type : {id:ImpactFunc}} def append(self, func): """Append a ImpactFunc. Overwrite existing if same id and haz_type. Parameters: func (ImpactFunc): ImpactFunc instance Raises: ValueError """ if not isinstance(func, ImpactFunc): LOGGER.error("Input value is not of type ImpactFunc.") raise ValueError if not func.haz_type: LOGGER.warning("Input ImpactFunc's hazard type not set.") if not func.id: LOGGER.warning("Input ImpactFunc's id not set.") if func.haz_type not in self._data: self._data[func.haz_type] = dict() self._data[func.haz_type][func.id] = func def remove_func(self, haz_type=None, fun_id=None): """Remove impact function(s) with provided hazard type and/or id. If no input provided, all impact functions are removed. Parameters: haz_type (str, optional): all impact functions with this hazard fun_id (int, optional): all impact functions with this id """ if (haz_type is not None) and (fun_id is not None): try: del self._data[haz_type][fun_id] except KeyError: LOGGER.warning("No ImpactFunc with hazard %s and id %s.", haz_type, fun_id) elif haz_type is not None: try: del self._data[haz_type] except KeyError: LOGGER.warning("No ImpactFunc with hazard %s.", haz_type) elif fun_id is not None: haz_remove = self.get_hazard_types(fun_id) if not haz_remove: LOGGER.warning("No ImpactFunc with id %s.", fun_id) for vul_haz in haz_remove: del self._data[vul_haz][fun_id] else: self._data = dict() def get_func(self, haz_type=None, fun_id=None): """Get ImpactFunc(s) of input hazard type and/or id. If no input provided, all impact functions are returned. Parameters: haz_type (str, optional): hazard type fun_id (int, optional): ImpactFunc id Returns: ImpactFunc (if haz_type and fun_id), list(ImpactFunc) (if haz_type or fun_id), {ImpactFunc.haz_type: {ImpactFunc.id : ImpactFunc}} (if None) """ if (haz_type is not None) and (fun_id is not None): try: return self._data[haz_type][fun_id] except KeyError: return list() elif haz_type is not None: try: return list(self._data[haz_type].values()) except KeyError: return list() elif fun_id is not None: haz_return = self.get_hazard_types(fun_id) vul_return = [] for vul_haz in haz_return: vul_return.append(self._data[vul_haz][fun_id]) return vul_return else: return self._data def get_hazard_types(self, fun_id=None): """Get impact functions hazard types contained for the id provided. Return all hazard types if no input id. Parameters: fun_id (int, optional): id of an impact function Returns: list(str) """ if fun_id is None: return list(self._data.keys()) haz_types = [] for vul_haz, vul_dict in self._data.items(): if fun_id in vul_dict: haz_types.append(vul_haz) return haz_types def get_ids(self, haz_type=None): """Get impact functions ids contained for the hazard type provided. Return all ids for each hazard type if no input hazard type. Parameters: haz_type (str, optional): hazard type from which to obtain the ids Returns: list(ImpactFunc.id) (if haz_type provided), {ImpactFunc.haz_type : list(ImpactFunc.id)} (if no haz_type) """ if haz_type is None: out_dict = dict() for vul_haz, vul_dict in self._data.items(): out_dict[vul_haz] = list(vul_dict.keys()) return out_dict try: return list(self._data[haz_type].keys()) except KeyError: return list() def size(self, haz_type=None, fun_id=None): """Get number of impact functions contained with input hazard type and /or id. If no input provided, get total number of impact functions. Parameters: haz_type (str, optional): hazard type fun_id (int, optional): ImpactFunc id Returns: int """ if (haz_type is not None) and (fun_id is not None) and \ (isinstance(self.get_func(haz_type, fun_id), ImpactFunc)): return 1 if (haz_type is not None) or (fun_id is not None): return len(self.get_func(haz_type, fun_id)) return sum(len(vul_list) for vul_list in self.get_ids().values()) def check(self): """Check instance attributes. Raises: ValueError """ for key_haz, vul_dict in self._data.items(): for fun_id, vul in vul_dict.items(): if (fun_id != vul.id) | (fun_id == ''): LOGGER.error("Wrong ImpactFunc.id: %s != %s.", fun_id, vul.id) raise ValueError if (key_haz != vul.haz_type) | (key_haz == ''): LOGGER.error("Wrong ImpactFunc.haz_type: %s != %s.", key_haz, vul.haz_type) raise ValueError vul.check() def extend(self, impact_funcs): """Append impact functions of input ImpactFuncSet to current ImpactFuncSet. Overwrite ImpactFunc if same id and haz_type. Parameters: impact_funcs (ImpactFuncSet): ImpactFuncSet instance to extend Raises: ValueError """ impact_funcs.check() if self.size() == 0: self.__dict__ = copy.deepcopy(impact_funcs.__dict__) return self.tag.append(impact_funcs.tag) new_func = impact_funcs.get_func() for _, vul_dict in new_func.items(): for _, vul in vul_dict.items(): self.append(vul) def plot(self, haz_type=None, fun_id=None, axis=None, **kwargs): """Plot impact functions of selected hazard (all if not provided) and selected function id (all if not provided). Parameters: haz_type (str, optional): hazard type fun_id (int, optional): id of the function Returns: matplotlib.axes._subplots.AxesSubplot """ num_plts = self.size(haz_type, fun_id) num_row, num_col = u_plot._get_row_col_size(num_plts) # Select all hazard types to plot if haz_type is not None: hazards = [haz_type] else: hazards = self._data.keys() if not axis: _, axis = plt.subplots(num_row, num_col) if num_plts > 1: axes = axis.flatten() else: axes = [axis] i_axis = 0 for sel_haz in hazards: if fun_id is not None: self._data[sel_haz][fun_id].plot(axis=axes[i_axis], **kwargs) i_axis += 1 else: for sel_id in self._data[sel_haz].keys(): self._data[sel_haz][sel_id].plot(axis=axes[i_axis], **kwargs) i_axis += 1 return axis def read_excel(self, file_name, description='', var_names=DEF_VAR_EXCEL): """Read excel file following template and store variables. Parameters: file_name (str): absolute file name description (str, optional): description of the data var_names (dict, optional): name of the variables in the file """ dfr = pd.read_excel(file_name, var_names['sheet_name']) self.clear() self.tag.file_name = str(file_name) self.tag.description = description self._fill_dfr(dfr, var_names) def read_mat(self, file_name, description='', var_names=DEF_VAR_MAT): """Read MATLAB file generated with previous MATLAB CLIMADA version. Parameters: file_name (str): absolute file name description (str, optional): description of the data var_names (dict, optional): name of the variables in the file """ def _get_hdf5_funcs(imp, file_name, var_names): """Get rows that fill every impact function and its name.""" func_pos = dict() for row, (fun_id, fun_type) in enumerate( zip(imp[var_names['var_name']['fun_id']].squeeze(), imp[var_names['var_name']['peril']].squeeze())): type_str = u_hdf5.get_str_from_ref(file_name, fun_type) key = (type_str, int(fun_id)) if key not in func_pos: func_pos[key] = list() func_pos[key].append(row) return func_pos def _get_hdf5_str(imp, idxs, file_name, var_name): """Get rows with same string in var_name.""" prev_str = "" for row in idxs: cur_str = u_hdf5.get_str_from_ref(file_name, imp[var_name][row][0]) if prev_str == "": prev_str = cur_str elif prev_str != cur_str: LOGGER.error("Impact function with two different %s.", var_name) raise ValueError return prev_str imp = u_hdf5.read(file_name) self.clear() self.tag.file_name = str(file_name) self.tag.description = description try: imp = imp[var_names['sup_field_name']] except KeyError: pass try: imp = imp[var_names['field_name']] funcs_idx = _get_hdf5_funcs(imp, file_name, var_names) for imp_key, imp_rows in funcs_idx.items(): func = ImpactFunc() func.haz_type = imp_key[0] func.id = imp_key[1] # check that this function only has one intensity unit, if provided try: func.intensity_unit = _get_hdf5_str(imp, imp_rows, file_name, var_names['var_name']['unit']) except KeyError: pass # check that this function only has one name try: func.name = _get_hdf5_str(imp, imp_rows, file_name, var_names['var_name']['name']) except KeyError: func.name = str(func.id) func.intensity = np.take(imp[var_names['var_name']['inten']], imp_rows) func.mdd = np.take(imp[var_names['var_name']['mdd']], imp_rows) func.paa = np.take(imp[var_names['var_name']['paa']], imp_rows) self.append(func) except KeyError as err: LOGGER.error("Not existing variable: %s", str(err)) raise err def write_excel(self, file_name, var_names=DEF_VAR_EXCEL): """Write excel file following template. Parameters: file_name (str): absolute file name to write var_names (dict, optional): name of the variables in the file """ def write_if(row_ini, imp_ws, xls_data): """Write one impact function""" for icol, col_dat in enumerate(xls_data): for irow, data in enumerate(col_dat, row_ini): imp_ws.write(irow, icol, data) imp_wb = xlsxwriter.Workbook(file_name) imp_ws = imp_wb.add_worksheet(var_names['sheet_name']) header = [var_names['col_name']['func_id'], var_names['col_name']['inten'], var_names['col_name']['mdd'], var_names['col_name']['paa'], var_names['col_name']['peril'], var_names['col_name']['unit'], var_names['col_name']['name']] for icol, head_dat in enumerate(header): imp_ws.write(0, icol, head_dat) row_ini = 1 for fun_haz_id, fun_haz in self._data.items(): for fun_id, fun in fun_haz.items(): n_inten = fun.intensity.size xls_data = [repeat(fun_id, n_inten), fun.intensity, fun.mdd, fun.paa, repeat(fun_haz_id, n_inten), repeat(fun.intensity_unit, n_inten), repeat(fun.name, n_inten)] write_if(row_ini, imp_ws, xls_data) row_ini += n_inten imp_wb.close() def _fill_dfr(self, dfr, var_names): def _get_xls_funcs(dfr, var_names): """Parse individual impact functions.""" dist_func = [] for (haz_type, imp_id) in zip(dfr[var_names['col_name']['peril']], dfr[var_names['col_name']['func_id']]): if (haz_type, imp_id) not in dist_func: dist_func.append((haz_type, imp_id)) return dist_func try: dist_func = _get_xls_funcs(dfr, var_names) for haz_type, imp_id in dist_func: df_func = dfr[dfr[var_names['col_name']['peril']] == haz_type] df_func = df_func[df_func[var_names['col_name']['func_id']] == imp_id] func = ImpactFunc() func.haz_type = haz_type func.id = imp_id # check that the unit of the intensity is the same try: if len(df_func[var_names['col_name']['name']].unique()) != 1: raise ValueError('Impact function with two different names.') func.name = df_func[var_names['col_name']['name']].values[0] except KeyError: func.name = str(func.id) # check that the unit of the intensity is the same, if provided try: if len(df_func[var_names['col_name']['unit']].unique()) != 1: raise ValueError('Impact function with two different \ intensity units.') func.intensity_unit = \ df_func[var_names['col_name']['unit']].values[0] except KeyError: pass func.intensity = df_func[var_names['col_name']['inten']].values func.mdd = df_func[var_names['col_name']['mdd']].values func.paa = df_func[var_names['col_name']['paa']].values self.append(func) except KeyError as err: LOGGER.error("Not existing variable: %s", str(err)) raise err
def set_from_isimip_netcdf(self, input_dir=None, filename=None, hist_mean=None, bbox=None, yearrange=None, cl_model=None, scenario=None, crop=None, irr=None, isimip_version=None, unit=None, fn_str_var=None): """Wrapper to fill exposure from NetCDF file from ISIMIP. Requires historical mean relative cropyield module as additional input. Optional Parameters: input_dir (string): path to input data directory filename (string): name of the landuse data file to use, e.g. "histsoc_landuse-15crops_annual_1861_2005.nc"" hist_mean (str or array): historic mean crop yield per centroid (or path) bbox (list of four floats): bounding box: [lon min, lat min, lon max, lat max] yearrange (int tuple): year range for exposure set e.g., (1990, 2010) scenario (string): climate change and socio economic scenario e.g., '1860soc', 'histsoc', '2005soc', 'rcp26soc','rcp60soc','2100rcp26soc' cl_model (string): abbrev. climate model (only for future projections of lu data) e.g., 'gfdl-esm2m', 'hadgem2-es', 'ipsl-cm5a-lr','miroc5' crop (string): crop type e.g., 'mai', 'ric', 'whe', 'soy' irr (string): irrigation type, default: 'combined' f.i 'firr' (full irrigation), 'noirr' (no irrigation) or 'combined'= firr+noirr isimip_version(str): 'ISIMIP2' (default) or 'ISIMIP3' unit (string): unit of the exposure (per year) f.i 't/y' (default), 'USD/y', or 'kcal/y' fn_str_var (string): FileName STRing depending on VARiable and ISIMIP simuation round Returns: Exposure """ # parameters not provided in method call are set to default values: if irr is None: irr = 'combined' if not bbox: bbox = BBOX if not input_dir: input_dir = INPUT_DIR if hist_mean is None: hist_mean = HIST_MEAN_PATH if not fn_str_var: fn_str_var = FN_STR_VAR if (not isimip_version) or (isimip_version in ('ISIMIP2a', 'ISIMIP2b')): isimip_version = 'ISIMIP2' elif isimip_version in ('ISIMIP3a', 'ISIMIP3b'): isimip_version = 'ISIMIP3' if (not scenario) or (scenario in ('historical', 'hist')): scenario = 'histsoc' if yearrange is None: yearrange = YEARCHUNKS[isimip_version][scenario]['yearrange'] if not unit: unit = 't/y' #if not soc: soc='' # The filename is set or other variables (cl_model, scenario) are extracted of the # specified filename if filename is None: yearchunk = YEARCHUNKS[isimip_version][scenario] # if scenario == 'histsoc' or scenario == '1860soc': if scenario in ('histsoc', '1860soc'): string = '{}_{}_{}_{}.nc' filepath = Path(input_dir, string.format(scenario, fn_str_var, yearchunk['startyear'], yearchunk['endyear'])) else: string = '{}_{}_{}_{}_{}.nc' filepath = Path(input_dir, string.format(scenario, cl_model, fn_str_var, yearchunk['startyear'], yearchunk['endyear'])) elif scenario == 'flexible': _, _, _, _, _, _, startyear, endyearnc = filename.split('_') endyear = endyearnc.split('.')[0] yearchunk = dict() yearchunk = {'yearrange': (int(startyear), int(endyear)), 'startyear': int(startyear), 'endyear': int(endyear)} filepath = Path(input_dir, filename) else: scenario, *_ = filename.split('_') yearchunk = YEARCHUNKS[isimip_version][scenario] filepath = Path(input_dir, filename) # Dataset is opened and data within the bbox extends is extracted data_set = xr.open_dataset(filepath, decode_times=False) [lonmin, latmin, lonmax, latmax] = bbox data = data_set.sel(lon=slice(lonmin, lonmax), lat=slice(latmax, latmin)) # The latitude and longitude are set; the region_id is determined lon, lat = np.meshgrid(data.lon.values, data.lat.values) self.gdf['latitude'] = lat.flatten() self.gdf['longitude'] = lon.flatten() self.gdf['region_id'] = u_coord.get_country_code(self.gdf.latitude, self.gdf.longitude) # The indeces of the yearrange to be extracted are determined time_idx = (int(yearrange[0] - yearchunk['startyear']), int(yearrange[1] - yearchunk['startyear'])) # The area covered by a grid cell is calculated depending on the latitude # 1 degree = 111.12km (at the equator); resolution data: 0.5 degree; # longitudal distance in km = 111.12*0.5*cos(lat); # latitudal distance in km = 111.12*0.5; # area = longitudal distance * latitudal distance; # 1km2 = 100ha area = (111.12 * 0.5)**2 * np.cos(np.deg2rad(lat)) * 100 # The area covered by a crop is calculated as the product of the fraction and # the grid cell size if irr == 'combined': irr_types = ['firr', 'noirr'] else: irr_types = [irr] area_crop = dict() for irr_var in irr_types: area_crop[irr_var] = ( getattr( data, (CROP_NAME[crop])['input']+'_'+ (IRR_NAME[irr_var])['name'] )[time_idx[0]:time_idx[1], :, :].mean(dim='time')*area ).values area_crop[irr_var] = np.nan_to_num(area_crop[irr_var]).flatten() # set historic mean, its latitude, and longitude: hist_mean_dict = dict() # if hist_mean is given as np.ndarray or dict, # code assumes it contains hist_mean as returned by relative_cropyield # however structured in dictionary as hist_mean_dict, with same # bbox extensions as the exposure: if isinstance(hist_mean, dict): if not ('firr' in hist_mean.keys() or 'noirr' in hist_mean.keys()): # as a dict hist_mean, needs to contain key 'firr' or 'noirr'; # if irr=='combined', both 'firr' and 'noirr' are required. LOGGER.error('Invalid hist_mean provided: %s', hist_mean) raise ValueError('invalid hist_mean.') hist_mean_dict = hist_mean lat_mean = self.gdf.latitude.values elif isinstance(hist_mean, np.ndarray) or isinstance(hist_mean, list): hist_mean_dict[irr_types[0]] = np.array(hist_mean) lat_mean = self.gdf.latitude.values elif Path(hist_mean).is_dir(): # else if hist_mean is given as path to directory # The adequate file from the directory (depending on crop and irrigation) is extracted # and the variables hist_mean, lat_mean and lon_mean are set accordingly for irr_var in irr_types: filename = str(Path(hist_mean, 'hist_mean_%s-%s_%i-%i.hdf5' %( crop, irr_var, yearrange[0], yearrange[1]))) hist_mean_dict[irr_var] = (h5py.File(filename, 'r'))['mean'][()] lat_mean = (h5py.File(filename, 'r'))['lat'][()] lon_mean = (h5py.File(filename, 'r'))['lon'][()] elif Path(input_dir, hist_mean).is_file(): # file path # Hist_mean, lat_mean and lon_mean are extracted from the given file if len(irr_types) > 1: LOGGER.error("For irr=='combined', hist_mean can not be single file. Aborting.") raise ValueError('Wrong combination of parameters irr and hist_mean.') hist_mean = h5py.File(str(Path(input_dir, hist_mean)), 'r') hist_mean_dict[irr_types[0]] = hist_mean['mean'][()] lat_mean = hist_mean['lat'][()] lon_mean = hist_mean['lon'][()] else: LOGGER.error('Invalid hist_mean provided: %s', hist_mean) raise ValueError('invalid hist_mean.') # The bbox is cut out of the hist_mean data file if needed if len(lat_mean) != len(self.gdf.latitude.values): idx_mean = np.zeros(len(self.gdf.latitude.values), dtype=int) for i in range(len(self.gdf.latitude.values)): idx_mean[i] = np.where( (lat_mean == self.gdf.latitude.values[i]) & (lon_mean == self.gdf.longitude.values[i]) )[0][0] else: idx_mean = np.arange(0, len(lat_mean)) # The exposure [t/y] is computed per grid cell as the product of the area covered # by a crop [ha] and its yield [t/ha/y] self.gdf['value'] = np.squeeze(area_crop[irr_types[0]] * \ hist_mean_dict[irr_types[0]][idx_mean]) self.gdf['value'] = np.nan_to_num(self.gdf.value) # replace NaN by 0.0 for irr_val in irr_types[1:]: # add other irrigation types if irr=='combined' value_tmp = np.squeeze(area_crop[irr_val]*hist_mean_dict[irr_val][idx_mean]) value_tmp = np.nan_to_num(value_tmp) # replace NaN by 0.0 self.gdf['value'] += value_tmp self.tag = Tag() self.tag.description = ("Crop production exposure from ISIMIP " + (CROP_NAME[crop])['print'] + ' ' + irr + ' ' + str(yearrange[0]) + '-' + str(yearrange[-1])) self.value_unit = 't/y' # input unit, will be reset below if required by user self.crop = crop self.ref_year = yearrange try: rows, cols, ras_trans = pts_to_raster_meta( (self.gdf.longitude.min(), self.gdf.latitude.min(), self.gdf.longitude.max(), self.gdf.latitude.max()), get_resolution(self.gdf.longitude, self.gdf.latitude)) self.meta = { 'width': cols, 'height': rows, 'crs': self.crs, 'transform': ras_trans, } except ValueError: LOGGER.warning('Could not write attribute meta, because exposure' ' has only 1 data point') self.meta = {} if 'USD' in unit: # set_value_to_usd() is called to compute the exposure in USD/y (country specific) self.set_value_to_usd(input_dir=input_dir) elif 'kcal' in unit: # set_value_to_kcal() is called to compute the exposure in kcal/y self.set_value_to_kcal() self.check() return self
def clear(self): """Reinitialize attributes.""" self.tag = Tag() # Following values are given for each defined year self.years = np.array([], int) self.rates = np.array([], float)
class DiscRates(): """Defines discount rates and basic methods. Loads from files with format defined in FILE_EXT. Attributes: tag (Tag): information about the source data years (np.array): years rates (np.array): discount rates for each year (between 0 and 1) """ def __init__(self): """Empty initialization. Examples: Fill discount rates with values and check consistency data: >>> disc_rates = DiscRates() >>> disc_rates.years = np.array([2000, 2001]) >>> disc_rates.rates = np.array([0.02, 0.02]) >>> disc_rates.check() Read discount rates from year_2050.mat and checks consistency data. >>> disc_rates = DiscRates(ENT_TEMPLATE_XLS) """ self.clear() def clear(self): """Reinitialize attributes.""" self.tag = Tag() # Following values are given for each defined year self.years = np.array([], int) self.rates = np.array([], float) def check(self): """Check attributes consistency. Raises: ValueError """ check.size(len(self.years), self.rates, 'DiscRates.rates') def select(self, year_range): """Select discount rates in given years. Parameters: year_range (np.array): continuous sequence of selected years. Returns: DiscRates """ pos_year = np.isin(year_range, self.years) if not np.all(pos_year): LOGGER.info('No discount rates for given years.') return None pos_year = np.isin(self.years, year_range) sel_disc = self.__class__() sel_disc.tag = self.tag sel_disc.years = self.years[pos_year] sel_disc.rates = self.rates[pos_year] return sel_disc def append(self, disc_rates): """Check and append discount rates to current DiscRates. Overwrite discount rate if same year. Parameters: disc_rates (DiscRates): DiscRates instance to append Raises: ValueError """ disc_rates.check() if self.years.size == 0: self.__dict__ = copy.deepcopy(disc_rates.__dict__) return self.tag.append(disc_rates.tag) new_year = array('l') new_rate = array('d') for year, rate in zip(disc_rates.years, disc_rates.rates): found = np.where(year == self.years)[0] if found.size > 0: self.rates[found[0]] = rate else: new_year.append(year) new_rate.append(rate) self.years = np.append(self.years, new_year).astype(int, copy=False) self.rates = np.append(self.rates, new_rate) def net_present_value(self, ini_year, end_year, val_years): """Compute net present value between present year and future year. Parameters: ini_year (float): initial year end_year (float): end year val_years (np.array): cash flow at each year btw ini_year and end_year (both included) Returns: float """ year_range = np.arange(ini_year, end_year + 1) if year_range.size != val_years.size: LOGGER.error('Wrong size of yearly values.') raise ValueError sel_disc = self.select(year_range) if sel_disc is None: LOGGER.error('No information of discount rates for provided years:'\ ' %s - %s', ini_year, end_year) raise ValueError return u_fin.net_present_value(sel_disc.years, sel_disc.rates, val_years) def plot(self, axis=None, **kwargs): """Plot discount rates per year. Parameters: axis (matplotlib.axes._subplots.AxesSubplot, optional): axis to use kwargs (optional): arguments for plot matplotlib function, e.g. marker='x' Returns: matplotlib.axes._subplots.AxesSubplot """ if not axis: _, axis = plt.subplots(1, 1) axis.set_title('Discount rates') axis.set_xlabel('Year') axis.set_ylabel('discount rate (%)') axis.plot(self.years, self.rates * 100, **kwargs) axis.set_xlim((self.years.min(), self.years.max())) return axis def read_mat(self, file_name, description='', var_names=DEF_VAR_MAT): """Read MATLAB file generated with previous MATLAB CLIMADA version. Parameters: file_name (str): absolute file name description (str, optional): description of the data var_names (dict, optional): name of the variables in the file """ disc = hdf5.read(file_name) self.clear() self.tag.file_name = file_name self.tag.description = description try: disc = disc[var_names['sup_field_name']] except KeyError: pass try: disc = disc[var_names['field_name']] self.years = np.squeeze(disc[var_names['var_name']['year']]). \ astype(int, copy=False) self.rates = np.squeeze(disc[var_names['var_name']['disc']]) except KeyError as err: LOGGER.error("Not existing variable: %s", str(err)) raise err def read_excel(self, file_name, description='', var_names=DEF_VAR_EXCEL): """Read excel file following template and store variables. Parameters: file_name (str): absolute file name description (str, optional): description of the data var_names (dict, optional): name of the variables in the file """ dfr = pd.read_excel(file_name, var_names['sheet_name']) self.clear() self.tag.file_name = file_name self.tag.description = description try: self.years = dfr[var_names['col_name']['year']].values. \ astype(int, copy=False) self.rates = dfr[var_names['col_name']['disc']].values except KeyError as err: LOGGER.error("Not existing variable: %s", str(err)) raise err def write_excel(self, file_name, var_names=DEF_VAR_EXCEL): """ Write excel file following template. Parameters: file_name (str): absolute file name to write var_names (dict, optional): name of the variables in the file """ disc_wb = xlsxwriter.Workbook(file_name) disc_ws = disc_wb.add_worksheet(var_names['sheet_name']) header = [var_names['col_name']['year'], var_names['col_name']['disc']] for icol, head_dat in enumerate(header): disc_ws.write(0, icol, head_dat) for i_yr, (disc_yr, disc_rt) in enumerate(zip(self.years, self.rates), 1): disc_ws.write(i_yr, 0, disc_yr) disc_ws.write(i_yr, 1, disc_rt) disc_wb.close()
def init_spam_agrar(self, **parameters): """initiates agriculture exposure from SPAM data: https://dataverse.harvard.edu/ dataset.xhtml?persistentId=doi:10.7910/DVN/DHXBJX Optional parameters: data_path (str): absolute path where files are stored. Default: SYSTEM_DIR country (str): Three letter country code of country to be cut out. No default (global) name_adm1 (str): Name of admin1 (e.g. Federal State) to be cut out. No default name_adm2 (str): Name of admin2 to be cut out. No default spam_variable (str): select one agricultural variable: 'A' physical area 'H' harvested area 'P' production 'Y' yield 'V_agg' value of production, aggregated to all crops, food and non-food (default) Warning: for A, H, P and Y, currently all crops are summed up spam_technology (str): select one agricultural technology type: 'TA' all technologies together, ie complete crop (default) 'TI' irrigated portion of crop 'TH' rainfed high inputs portion of crop 'TL' rainfed low inputs portion of crop 'TS' rainfed subsistence portion of crop 'TR' rainfed portion of crop (= TA - TI, or TH + TL + TS) ! different impact_ids are assigned to each technology (1-6) save_name_adm1 (Boolean): Determines how many aditional data are saved: False: only basics (lat, lon, total value), region_id per country True: like 1 + name of admin1 haz_type (str): hazard type abbreviation, e.g. 'DR' for Drought or 'CP' for CropPotential Returns: """ data_p = parameters.get('data_path', SYSTEM_DIR) spam_t = parameters.get('spam_technology', 'TA') spam_v = parameters.get('spam_variable', 'V_agg') adm0 = parameters.get('country') adm1 = parameters.get('name_adm1') adm2 = parameters.get('name_adm2') save_adm1 = parameters.get('save_name_adm1', False) haz_type = parameters.get('haz_type', DEF_HAZ_TYPE) # Test if parameters make sense: if spam_v not in ['A', 'H', 'P', 'Y', 'V_agg'] or \ spam_t not in ['TA', 'TI', 'TH', 'TL', 'TS', 'TR']: LOGGER.error('Invalid input parameter(s).') raise ValueError('Invalid input parameter(s).') # read data from CSV: data = self._read_spam_file(data_path=data_p, spam_technology=spam_t, spam_variable=spam_v, result_mode=1) # extract country or admin level (if provided) data, region = self._spam_set_country(data, country=adm0, name_adm1=adm1, name_adm2=adm2) # sort by alloc_key to make extraction of lat / lon easier: data = data.sort_values(by=['alloc_key']) lat, lon = self._spam_get_coordinates(data.loc[:, 'alloc_key'], data_path=data_p) if save_adm1: self.name_adm1 = data.loc[:, 'name_adm1'].values if spam_v == 'V_agg': # total only (column 7) i_1 = 7 i_2 = 8 else: i_1 = 7 # get sum over all crops (columns 7 to 48) i_2 = 49 self.gdf['value'] = data.iloc[:, i_1:i_2].sum(axis=1).values self.gdf['latitude'] = lat.values self.gdf['longitude'] = lon.values LOGGER.info('Lat. range: {:+.3f} to {:+.3f}.'.format( np.min(self.gdf.latitude), np.max(self.gdf.latitude))) LOGGER.info('Lon. range: {:+.3f} to {:+.3f}.'.format( np.min(self.gdf.longitude), np.max(self.gdf.longitude))) # set region_id (numeric ISO3): country_id = data.loc[:, 'iso3'] if country_id.unique().size == 1: region_id = np.ones(self.gdf.value.size, int)\ * int(iso_cntry.get(country_id.iloc[0]).numeric) else: region_id = np.zeros(self.gdf.value.size, int) for i in range(0, self.gdf.value.size): region_id[i] = int(iso_cntry.get(country_id.iloc[i]).numeric) self.gdf['region_id'] = region_id self.ref_year = 2005 self.tag = Tag() self.tag.description = ("SPAM agrar exposure for variable " + spam_v + " and technology " + spam_t) # if impact id variation iiv = 1, assign different damage function ID # per technology type. self._set_if(spam_t, haz_type) self.tag.file_name = (FILENAME_SPAM + '_' + spam_v + '_' + spam_t + '.csv') # self.tag.shape = cntry_info[2] #self.tag.country = cntry_info[1] if spam_v in ('A', 'H'): self.value_unit = 'Ha' elif spam_v == 'Y': self.value_unit = 'kg/Ha' elif spam_v == 'P': self.value_unit = 'mt' else: self.value_unit = 'USD' LOGGER.info('Total {} {} {}: {:.1f} {}.'.format( spam_v, spam_t, region, self.gdf.value.sum(), self.value_unit)) self.check()
def set_countries(self, countries, ref_year=2016, res_km=None, from_hr=None, admin_file='admin_0_countries', **kwargs): """ Model countries using values at reference year. If GDP or income group not available for that year, consider the value of the closest available year. Parameters: countries (list or dict): list of country names (admin0 or subunits) or dict with key = admin0 name and value = [admin1 names] ref_year (int, optional): reference year. Default: 2016 res_km (float, optional): approx resolution in km. Default: nightlights resolution. from_hr (bool, optional): force to use higher resolution image, independently of its year of acquisition. admin_file (str): file name, admin_0_countries or admin_0_map_subunits kwargs (optional): 'gdp' and 'inc_grp' dictionaries with keys the country ISO_alpha3 code. 'poly_val' list of polynomial coefficients [1,x,x^2,...] to apply to nightlight (DEF_POLY_VAL used if not provided). If provided, these are used. """ admin_key_dict = { 'admin_0_countries': ['ADMIN', 'ADM0_A3'], 'admin_0_map_subunits': ['SUBUNIT', 'SU_A3'] } shp_file = shapereader.natural_earth(resolution='10m', category='cultural', name=admin_file) shp_file = shapereader.Reader(shp_file) cntry_info, cntry_admin1 = country_iso_geom(countries, shp_file, admin_key_dict[admin_file]) fill_econ_indicators(ref_year, cntry_info, shp_file, **kwargs) nightlight, coord_nl, fn_nl, res_fact, res_km = get_nightlight( ref_year, cntry_info, res_km, from_hr) tag = Tag(file_name=fn_nl) bkmrbl_list = [] for cntry_iso, cntry_val in cntry_info.items(): bkmrbl_list.append( self._set_one_country(cntry_val, nightlight, coord_nl, res_fact, res_km, cntry_admin1[cntry_iso], **kwargs).gdf) tag.description += ("{} {:d} GDP: {:.3e} income group: {:d} \n").\ format(cntry_val[1], cntry_val[3], cntry_val[4], cntry_val[5]) Exposures.__init__(self, data=Exposures.concat(bkmrbl_list).gdf, crs=DEF_CRS, ref_year=ref_year, tag=tag, value_unit='USD') rows, cols, ras_trans = pts_to_raster_meta( (self.gdf.longitude.min(), self.gdf.latitude.min(), self.gdf.longitude.max(), self.gdf.latitude.max()), (coord_nl[0, 1], -coord_nl[0, 1])) self.meta = { 'width': cols, 'height': rows, 'crs': self.crs, 'transform': ras_trans }
def emdat_to_impact(emdat_file_csv, hazard_type_climada, year_range=None, countries=None, hazard_type_emdat=None, reference_year=None, imp_str="Total Damages"): """function to load EM-DAT data return impact per event Parameters: emdat_file_csv (str): Full path to EMDAT-file (CSV), i.e.: emdat_file_csv = SYSTEM_DIR.joinpath('emdat_201810.csv') hazard_type_climada (str): Hazard type CLIMADA abbreviation, i.e. 'TC' for tropical cyclone Optional parameters: hazard_type_emdat (list or str): List of Disaster (sub-)type accordung EMDAT terminology, e.g.: Animal accident, Drought, Earthquake, Epidemic, Extreme temperature, Flood, Fog, Impact, Insect infestation, Landslide, Mass movement (dry), Storm, Volcanic activity, Wildfire; Coastal Flooding, Convective Storm, Riverine Flood, Tropical cyclone, Tsunami, etc.; OR CLIMADA hazard type abbreviations, e.g. TC, BF, etc. If not given, it is deducted from hazard_type_climada year_range (list with 2 integers): start and end year e.g. [1980, 2017] default: None --> take year range from EM-DAT file countries (list of str): country ISO3-codes or names, e.g. ['JAM']. Set to None or ['all'] for all countries (default) reference_year (int): reference year of exposures. Impact is scaled proportional to GDP to the value of the reference year. No scaling for reference_year=0 (default) imp_str (str): Column name of impact metric in EMDAT CSV, default = "Total Damages ('000 US$)" Returns: impact_instance (instance of climada.engine.Impact): impact object of same format as output from CLIMADA impact computation. Values scaled with GDP to reference_year if reference_year is given. i.e. current US$ for imp_str="Total Damages ('000 US$) scaled" (factor 1000 is applied) impact_instance.eai_exp holds expected annual impact for each country. impact_instance.coord_exp holds rough central coordinates for each country. countries (list): ISO3-codes of countries in same order as in impact_instance.eai_exp """ if "Total Damages" in imp_str: imp_str = "Total Damages ('000 US$)" elif "Insured Damages" in imp_str: imp_str = "Insured Damages ('000 US$)" elif "Reconstruction Costs" in imp_str: imp_str = "Reconstruction Costs ('000 US$)" imp_str = VARNAMES_EMDAT[max(VARNAMES_EMDAT.keys())][imp_str] if not hazard_type_emdat: hazard_type_emdat = [hazard_type_climada] if reference_year == 0: reference_year = None # Inititate Impact-instance: impact_instance = Impact() impact_instance.tag = dict() impact_instance.tag['haz'] = TagHaz( haz_type=hazard_type_climada, file_name=emdat_file_csv, description='EM-DAT impact, direct import') impact_instance.tag['exp'] = Tag( file_name=emdat_file_csv, description='EM-DAT impact, direct import') impact_instance.tag['if_set'] = Tag(file_name=None, description=None) # Load EM-DAT impact data by event: em_data = emdat_impact_event(emdat_file_csv, countries=countries, hazard=hazard_type_emdat, year_range=year_range, reference_year=reference_year, imp_str=imp_str, version=max(VARNAMES_EMDAT.keys())) if isinstance(countries, str): countries = [countries] elif not countries: countries = emdat_countries_by_hazard(emdat_file_csv, year_range=year_range, hazard=hazard_type_emdat)[0] if em_data.empty: return impact_instance, countries impact_instance.event_id = np.array(em_data.index, int) impact_instance.event_name = list(em_data[VARNAMES_EMDAT[max( VARNAMES_EMDAT.keys())]['Dis No']]) date_list = list() for year in list(em_data['Year']): date_list.append(datetime.toordinal(datetime.strptime(str(year), '%Y'))) if 'Start Year' in em_data.columns and 'Start Month' in em_data.columns \ and 'Start Day' in em_data.columns: idx = 0 for year, month, day in zip(em_data['Start Year'], em_data['Start Month'], em_data['Start Day']): if np.isnan(year): idx += 1 continue if np.isnan(month): month = 1 if np.isnan(day): day = 1 date_list[idx] = datetime.toordinal( datetime.strptime('%02i/%02i/%04i' % (day, month, year), '%d/%m/%Y')) idx += 1 impact_instance.date = np.array(date_list, int) impact_instance.crs = DEF_CRS if not reference_year: impact_instance.at_event = np.array(em_data["impact"]) else: impact_instance.at_event = np.array(em_data["impact_scaled"]) impact_instance.at_event[np.isnan(impact_instance.at_event)] = 0 if not year_range: year_range = [em_data['Year'].min(), em_data['Year'].max()] impact_instance.frequency = np.ones( em_data.shape[0]) / (1 + np.diff(year_range)) impact_instance.tot_value = 0 impact_instance.aai_agg = np.nansum(impact_instance.at_event * impact_instance.frequency) impact_instance.unit = 'USD' impact_instance.imp_mat = [] # init rough exposure with central point per country shp = shapereader.natural_earth(resolution='110m', category='cultural', name='admin_0_countries') shp = shapefile.Reader(shp) countries_reg_id = list() countries_lat = list() countries_lon = list() impact_instance.eai_exp = np.zeros( len(countries)) # empty: damage at exposure for idx, cntry in enumerate(countries): try: cntry = iso_cntry.get(cntry).alpha3 except KeyError: print(cntry) LOGGER.error('Country not found in iso_country: %s', cntry) cntry_boolean = False for rec_i, rec in enumerate(shp.records()): if rec[9].casefold() == cntry.casefold(): bbox = shp.shapes()[rec_i].bbox cntry_boolean = True break if cntry_boolean: countries_lat.append(np.mean([bbox[1], bbox[3]])) countries_lon.append(np.mean([bbox[0], bbox[2]])) else: countries_lat.append(np.nan) countries_lon.append(np.nan) try: countries_reg_id.append(int(iso_cntry.get(cntry).numeric)) except KeyError: countries_reg_id.append(0) df_tmp = em_data[em_data[VARNAMES_EMDAT[max( VARNAMES_EMDAT.keys())]['ISO']].str.contains(cntry)] if not reference_year: impact_instance.eai_exp[idx] = sum( np.array(df_tmp["impact"]) * impact_instance.frequency[0]) else: impact_instance.eai_exp[idx] = sum( np.array(df_tmp["impact_scaled"]) * impact_instance.frequency[0]) impact_instance.coord_exp = np.stack([countries_lat, countries_lon], axis=1) return impact_instance, countries
def emdat_to_impact(emdat_file_csv, year_range=None, countries=None,\ hazard_type_emdat=None, hazard_type_climada=None, \ reference_year=0, imp_str="Total damage ('000 US$)"): """function to load EM-DAT data return impact per event Parameters: emdat_file_csv (str): Full path to EMDAT-file (CSV), i.e.: emdat_file_csv = os.path.join(SYSTEM_DIR, 'emdat_201810.csv') hazard_type_emdat (str): Hazard (sub-)type according to EMDAT terminology, i.e. 'Tropical cyclone' for tropical cyclone OR hazard_type_climada (str): Hazard type CLIMADA abbreviation, i.e. 'TC' for tropical cyclone Optional parameters: year_range (list with 2 integers): start and end year i.e. [1980, 2017] default: None --> take year range from EM-DAT file countries (list of str): country ISO3-codes or names, i.e. ['JAM']. Set to None or ['all'] for all countries (default) reference_year (int): reference year of exposures. Impact is scaled proportional to GDP to the value of the reference year. No scaling for reference_year=0 (default) imp_str (str): Column name of impact metric in EMDAT CSV, default = "Total damage ('000 US$)" Returns: impact_instance (instance of climada.engine.Impact): impact object of same format as output from CLIMADA impact computation scaled with GDP to reference_year if reference_year noit equal 0 i.e. 1000 current US$ for imp_str="Total damage ('000 US$) scaled". impact_instance.eai_exp holds expected annual impact for each country. impact_instance.coord_exp holds rough central coordinates for each country. countries (list): ISO3-codes of countries imn same order as in impact_instance.eai_exp """ # Mapping of hazard type between EM-DAT and CLIMADA: if not hazard_type_climada: if not hazard_type_emdat: LOGGER.error( 'Either hazard_type_climada or hazard_type_emdat need to be defined.' ) return None if hazard_type_emdat == 'Tropical cyclone': hazard_type_climada = 'TC' elif hazard_type_emdat == 'Drought': hazard_type_climada = 'DR' elif hazard_type_emdat == 'Landslide': hazard_type_climada = 'LS' elif hazard_type_emdat == 'Riverine flood': hazard_type_climada = 'RF' elif hazard_type_emdat in [ 'Wildfire', 'Forest Fire', 'Land fire (Brush, Bush, Pasture)' ]: hazard_type_climada = 'BF' elif hazard_type_emdat == 'Extra-tropical storm': hazard_type_climada = 'WS' elif not hazard_type_emdat: if hazard_type_climada == 'TC': hazard_type_emdat = 'Tropical cyclone' elif hazard_type_climada == 'DR': hazard_type_emdat = 'Drought' elif hazard_type_climada == 'LS': hazard_type_emdat = 'Landslide' elif hazard_type_climada == 'RF': hazard_type_emdat = 'Riverine flood' elif hazard_type_climada == 'BF': hazard_type_emdat = 'Wildfire' elif hazard_type_climada == 'WS': hazard_type_emdat = 'Extra-tropical storm' # Inititate Impact-instance: impact_instance = Impact() impact_instance.tag = dict() impact_instance.tag['haz'] = TagHaz(haz_type=hazard_type_climada, \ file_name=emdat_file_csv, description='EM-DAT impact, direct import') impact_instance.tag['exp'] = Tag(file_name=emdat_file_csv, \ description='EM-DAT impact, direct import') impact_instance.tag['if_set'] = Tag(file_name=None, description=None) if not countries or countries == ['all']: countries = emdat_countries_by_hazard(hazard_type_emdat, emdat_file_csv, \ ignore_missing=True, verbose=True)[0] else: if isinstance(countries, str): countries = [countries] # Load EM-DAT impact data by event: em_data = emdat_impact_event(countries, hazard_type_emdat, emdat_file_csv, \ year_range, reference_year=reference_year) if em_data.empty: return impact_instance, countries impact_instance.event_id = np.array(em_data.index, int) impact_instance.event_name = list(em_data['Disaster No.']) date_list = list() for year in list(em_data['year']): date_list.append(datetime.toordinal(datetime.strptime(str(year), '%Y'))) boolean_warning = True for idx, datestr in enumerate(list(em_data['Start date'])): try: date_list[idx] = datetime.toordinal( datetime.strptime(datestr[-7:], '%m/%Y')) except ValueError: if boolean_warning: LOGGER.warning('EM_DAT CSV contains invalid time formats') boolean_warning = False try: date_list[idx] = datetime.toordinal( datetime.strptime(datestr, '%d/%m/%Y')) except ValueError: if boolean_warning: LOGGER.warning('EM_DAT CSV contains invalid time formats') boolean_warning = False impact_instance.date = np.array(date_list, int) impact_instance.crs = DEF_CRS if reference_year == 0: impact_instance.at_event = np.array(em_data[imp_str]) else: impact_instance.at_event = np.array(em_data[imp_str + " scaled"]) if not year_range: year_range = [em_data['year'].min(), em_data['year'].max()] impact_instance.frequency = np.ones( em_data.shape[0]) / (1 + np.diff(year_range)) impact_instance.tot_value = 0 impact_instance.aai_agg = sum(impact_instance.at_event * impact_instance.frequency) impact_instance.unit = 'USD' impact_instance.imp_mat = [] # init rough exposure with central point per country shp = shapereader.natural_earth(resolution='110m', category='cultural', name='admin_0_countries') shp = shapefile.Reader(shp) countries_reg_id = list() countries_lat = list() countries_lon = list() impact_instance.eai_exp = np.zeros( len(countries)) # empty: damage at exposure for idx, cntry in enumerate(countries): try: cntry = iso_cntry.get(cntry).alpha3 except KeyError: LOGGER.error('Country not found in iso_country: ' + cntry) cntry_boolean = False for rec_i, rec in enumerate(shp.records()): if rec[9].casefold() == cntry.casefold(): bbox = shp.shapes()[rec_i].bbox cntry_boolean = True break if cntry_boolean: countries_lat.append(np.mean([bbox[1], bbox[3]])) countries_lon.append(np.mean([bbox[0], bbox[2]])) else: countries_lat.append(np.nan) countries_lon.append(np.nan) try: countries_reg_id.append(int(iso_cntry.get(cntry).numeric)) except KeyError: countries_reg_id.append(0) df_tmp = em_data[em_data['ISO'].str.contains(cntry)] if reference_year == 0: impact_instance.eai_exp[idx] = sum(np.array(df_tmp[imp_str])*\ impact_instance.frequency[0]) else: impact_instance.eai_exp[idx] = sum(np.array(df_tmp[imp_str + " scaled"])*\ impact_instance.frequency[0]) impact_instance.coord_exp = np.stack([countries_lat, countries_lon], axis=1) #impact_instance.plot_raster_eai_exposure() return impact_instance, countries
class MeasureSet(): """Contains measures of type Measure. Loads from files with format defined in FILE_EXT. Attributes: tag (Tag): information about the source data _data (dict): cotains Measure classes. It's not suppossed to be directly accessed. Use the class methods instead. """ def __init__(self): """Empty initialization. Examples: Fill MeasureSet with values and check consistency data: >>> act_1 = Measure() >>> act_1.name = 'Seawall' >>> act_1.color_rgb = np.array([0.1529, 0.2510, 0.5451]) >>> act_1.hazard_intensity = (1, 0) >>> act_1.mdd_impact = (1, 0) >>> act_1.paa_impact = (1, 0) >>> meas = MeasureSet() >>> meas.append(act_1) >>> meas.tag.description = "my dummy MeasureSet." >>> meas.check() Read measures from file and checks consistency data: >>> meas = MeasureSet() >>> meas.read_excel(ENT_TEMPLATE_XLS) """ self.clear() def clear(self): """Reinitialize attributes.""" self.tag = Tag() self._data = dict() # {hazard_type : {name: Measure()}} def append(self, meas): """Append an Measure. Override if same name and haz_type. Parameters: meas (Measure): Measure instance Raises: ValueError """ if not isinstance(meas, Measure): LOGGER.error("Input value is not of type Measure.") raise ValueError if not meas.haz_type: LOGGER.warning("Input Measure's hazard type not set.") if not meas.name: LOGGER.warning("Input Measure's name not set.") if meas.haz_type not in self._data: self._data[meas.haz_type] = dict() self._data[meas.haz_type][meas.name] = meas def remove_measure(self, haz_type=None, name=None): """Remove impact function(s) with provided hazard type and/or id. If no input provided, all impact functions are removed. Parameters: haz_type (str, optional): all impact functions with this hazard name (str, optional): measure name """ if (haz_type is not None) and (name is not None): try: del self._data[haz_type][name] except KeyError: LOGGER.info("No Measure with hazard %s and id %s.", haz_type, name) elif haz_type is not None: try: del self._data[haz_type] except KeyError: LOGGER.info("No Measure with hazard %s.", haz_type) elif name is not None: haz_remove = self.get_hazard_types(name) if not haz_remove: LOGGER.info("No Measure with name %s.", name) for haz in haz_remove: del self._data[haz][name] else: self._data = dict() def get_measure(self, haz_type=None, name=None): """Get ImpactFunc(s) of input hazard type and/or id. If no input provided, all impact functions are returned. Parameters: haz_type (str, optional): hazard type name (str, optional): measure name Returns: Measure (if haz_type and name), list(Measure) (if haz_type or name), {Measure.haz_type: {Measure.name : Measure}} (if None) """ if (haz_type is not None) and (name is not None): try: return self._data[haz_type][name] except KeyError: LOGGER.info("No Measure with hazard %s and id %s.", haz_type, name) return list() elif haz_type is not None: try: return list(self._data[haz_type].values()) except KeyError: LOGGER.info("No Measure with hazard %s.", haz_type) return list() elif name is not None: haz_return = self.get_hazard_types(name) if not haz_return: LOGGER.info("No Measure with name %s.", name) meas_return = [] for haz in haz_return: meas_return.append(self._data[haz][name]) return meas_return else: return self._data def get_hazard_types(self, meas=None): """Get measures hazard types contained for the name provided. Return all hazard types if no input name. Parameters: name (str, optional): measure name Returns: list(str) """ if meas is None: return list(self._data.keys()) haz_return = [] for haz, haz_dict in self._data.items(): if meas in haz_dict: haz_return.append(haz) return haz_return def get_names(self, haz_type=None): """Get measures names contained for the hazard type provided. Return all names for each hazard type if no input hazard type. Parameters: haz_type (str, optional): hazard type from which to obtain the names Returns: list(Measure.name) (if haz_type provided), {Measure.haz_type : list(Measure.name)} (if no haz_type) """ if haz_type is None: out_dict = dict() for haz, haz_dict in self._data.items(): out_dict[haz] = list(haz_dict.keys()) return out_dict try: return list(self._data[haz_type].keys()) except KeyError: LOGGER.info("No Measure with hazard %s.", haz_type) return list() def size(self, haz_type=None, name=None): """Get number of measures contained with input hazard type and /or id. If no input provided, get total number of impact functions. Parameters: haz_type (str, optional): hazard type name (str, optional): measure name Returns: int """ if (haz_type is not None) and (name is not None) and \ (isinstance(self.get_measure(haz_type, name), Measure)): return 1 if (haz_type is not None) or (name is not None): return len(self.get_measure(haz_type, name)) return sum(len(meas_list) for meas_list in self.get_names().values()) def check(self): """Check instance attributes. Raises: ValueError """ for key_haz, meas_dict in self._data.items(): def_color = plt.cm.get_cmap('Greys', len(meas_dict)) for i_meas, (name, meas) in enumerate(meas_dict.items()): if (name != meas.name) | (name == ''): LOGGER.error("Wrong Measure.name: %s != %s.", name, meas.name) raise ValueError if key_haz != meas.haz_type: LOGGER.error("Wrong Measure.haz_type: %s != %s.", key_haz, meas.haz_type) raise ValueError # set default color if not set if np.array_equal(meas.color_rgb, np.zeros(3)): meas.color_rgb = def_color(i_meas) meas.check() def extend(self, meas_set): """Extend measures of input MeasureSet to current MeasureSet. Overwrite Measure if same name and haz_type. Parameters: impact_funcs (MeasureSet): ImpactFuncSet instance to extend Raises: ValueError """ meas_set.check() if self.size() == 0: self.__dict__ = copy.deepcopy(meas_set.__dict__) return self.tag.append(meas_set.tag) new_func = meas_set.get_measure() for _, meas_dict in new_func.items(): for _, meas in meas_dict.items(): self.append(meas) def read_mat(self, file_name, description='', var_names=DEF_VAR_MAT): """Read MATLAB file generated with previous MATLAB CLIMADA version. Parameters: file_name (str): absolute file name description (str, optional): description of the data var_names (dict, optional): name of the variables in the file """ def read_att_mat(measures, data, file_name, var_names): """Read MATLAB measures attributes""" num_mes = len(data[var_names['var_name']['name']]) for idx in range(0, num_mes): meas = Measure() meas.name = hdf5.get_str_from_ref( file_name, data[var_names['var_name']['name']][idx][0]) color_str = hdf5.get_str_from_ref( file_name, data[var_names['var_name']['color']][idx][0]) meas.color_rgb = np.fromstring(color_str, dtype=float, sep=' ') meas.cost = data[var_names['var_name']['cost']][idx][0] meas.haz_type = hdf5.get_str_from_ref( file_name, data[var_names['var_name']['haz']][idx][0]) meas.hazard_freq_cutoff = data[var_names['var_name'] ['haz_frq']][idx][0] meas.hazard_set = hdf5.get_str_from_ref( file_name, data[var_names['var_name']['haz_set']][idx][0]) try: meas.hazard_inten_imp = ( data[var_names['var_name']['haz_int_a']][idx][0], data[var_names['var_name']['haz_int_b']][0][idx]) except KeyError: meas.hazard_inten_imp = ( data[var_names['var_name']['haz_int_a'][:-2]][idx][0], 0) # different convention of signes followed in MATLAB! meas.mdd_impact = ( data[var_names['var_name']['mdd_a']][idx][0], data[var_names['var_name']['mdd_b']][idx][0]) meas.paa_impact = ( data[var_names['var_name']['paa_a']][idx][0], data[var_names['var_name']['paa_b']][idx][0]) meas.imp_fun_map = hdf5.get_str_from_ref( file_name, data[var_names['var_name']['fun_map']][idx][0]) meas.exposures_set = hdf5.get_str_from_ref( file_name, data[var_names['var_name']['exp_set']][idx][0]) exp_region_id = data[var_names['var_name']['exp_reg']][idx][0] if exp_region_id: meas.exp_region_id = [exp_region_id] meas.risk_transf_attach = data[var_names['var_name'] ['risk_att']][idx][0] meas.risk_transf_cover = data[var_names['var_name'] ['risk_cov']][idx][0] measures.append(meas) data = hdf5.read(file_name) self.clear() self.tag.file_name = file_name self.tag.description = description try: data = data[var_names['sup_field_name']] except KeyError: pass try: data = data[var_names['field_name']] read_att_mat(self, data, file_name, var_names) except KeyError as var_err: LOGGER.error("Not existing variable %s", str(var_err)) raise var_err def read_excel(self, file_name, description='', var_names=DEF_VAR_EXCEL): """Read excel file following template and store variables. Parameters: file_name (str): absolute file name description (str, optional): description of the data var_names (dict, optional): name of the variables in the file """ def read_att_excel(measures, dfr, var_names): """Read Excel measures attributes""" num_mes = len(dfr.index) for idx in range(0, num_mes): meas = Measure() meas.name = dfr[var_names['col_name']['name']][idx] try: meas.haz_type = dfr[var_names['col_name']['haz']][idx] except KeyError: pass meas.color_rgb = np.fromstring( dfr[var_names['col_name']['color']][idx], dtype=float, sep=' ') meas.cost = dfr[var_names['col_name']['cost']][idx] meas.hazard_freq_cutoff = dfr[var_names['col_name'] ['haz_frq']][idx] meas.hazard_set = dfr[var_names['col_name']['haz_set']][idx] # Search for (a, b) values, put a = 1 otherwise try: meas.hazard_inten_imp = ( dfr[var_names['col_name']['haz_int_a']][idx], dfr[var_names['col_name']['haz_int_b']][idx]) except KeyError: meas.hazard_inten_imp = ( 1, dfr['hazard intensity impact'][idx]) try: meas.exposures_set = dfr[var_names['col_name'] ['exp_set']][idx] meas.exp_region_id = ast.literal_eval( dfr[var_names['col_name']['exp_reg']][idx]) except KeyError: pass except ValueError: meas.exp_region_id = dfr[var_names['col_name'] ['exp_reg']][idx] meas.mdd_impact = (dfr[var_names['col_name']['mdd_a']][idx], dfr[var_names['col_name']['mdd_b']][idx]) meas.paa_impact = (dfr[var_names['col_name']['paa_a']][idx], dfr[var_names['col_name']['paa_b']][idx]) meas.imp_fun_map = dfr[var_names['col_name']['fun_map']][idx] meas.risk_transf_attach = dfr[var_names['col_name'] ['risk_att']][idx] meas.risk_transf_cover = dfr[var_names['col_name'] ['risk_cov']][idx] try: meas.risk_transf_cost_factor = dfr[var_names['col_name'] ['risk_fact']][idx] except KeyError: pass measures.append(meas) dfr = pd.read_excel(file_name, var_names['sheet_name']) dfr = dfr.fillna('') self.clear() self.tag.file_name = file_name self.tag.description = description try: read_att_excel(self, dfr, var_names) except KeyError as var_err: LOGGER.error("Not existing variable: %s", str(var_err)) raise var_err def write_excel(self, file_name, var_names=DEF_VAR_EXCEL): """Write excel file following template. Parameters: file_name (str): absolute file name to write var_names (dict, optional): name of the variables in the file """ def write_meas(row_ini, imp_ws, xls_data): """Write one measure""" for icol, col_dat in enumerate(xls_data): imp_ws.write(row_ini, icol, col_dat) meas_wb = xlsxwriter.Workbook(file_name) mead_ws = meas_wb.add_worksheet(var_names['sheet_name']) header = [ var_names['col_name']['name'], var_names['col_name']['color'], var_names['col_name']['cost'], var_names['col_name']['haz_int_a'], var_names['col_name']['haz_int_b'], var_names['col_name']['haz_frq'], var_names['col_name']['haz_set'], var_names['col_name']['mdd_a'], var_names['col_name']['mdd_b'], var_names['col_name']['paa_a'], var_names['col_name']['paa_b'], var_names['col_name']['fun_map'], var_names['col_name']['exp_set'], var_names['col_name']['exp_reg'], var_names['col_name']['risk_att'], var_names['col_name']['risk_cov'], var_names['col_name']['haz'] ] for icol, head_dat in enumerate(header): mead_ws.write(0, icol, head_dat) for row_ini, (_, haz_dict) in enumerate(self._data.items(), 1): for meas_name, meas in haz_dict.items(): xls_data = [ meas_name, ' '.join(list(map(str, meas.color_rgb))), meas.cost, meas.hazard_inten_imp[0], meas.hazard_inten_imp[1], meas.hazard_freq_cutoff, meas.hazard_set, meas.mdd_impact[0], meas.mdd_impact[1], meas.paa_impact[0], meas.paa_impact[1], meas.imp_fun_map, meas.exposures_set, str(meas.exp_region_id), meas.risk_transf_attach, meas.risk_transf_cover, meas.haz_type ] write_meas(row_ini, mead_ws, xls_data) meas_wb.close()
class DiscRates(): """ Defines discount rates and basic methods. Loads from files with format defined in FILE_EXT. Attributes --------- tag: Tag information about the source data years: np.array list of years rates: np.array list of discount rates for each year (between 0 and 1) """ def __init__(self, years=None, rates=None, tag=None): """ Fill discount rates with values and check consistency data Parameters ---------- years : numpy.ndarray(int) Array of years. Default is numpy.array([]). rates : numpy.ndarray(float) Discount rates for each year in years. Default is numpy.array([]). Note: rates given in float, e.g., to set 1% rate use 0.01 tag : climate.entity.tag Metadata. Default is None. """ years = np.array([]) if years is None else years rates = np.array([]) if rates is None else rates self.years = years self.rates = rates tag = Tag() if tag is None else tag self.tag = tag def clear(self): """Reinitialize attributes.""" self.tag = Tag() # Following values are given for each defined year self.years = np.array([], int) self.rates = np.array([], float) def check(self): """ Check attributes consistency. Raises ------ ValueError """ u_check.size(len(self.years), self.rates, 'DiscRates.rates') def select(self, year_range): """ Select discount rates in given years. Parameters ---------- year_range: np.array(int) continuous sequence of selected years. Returns: climada.entity.DiscRates The selected discrates in the year_range """ pos_year = np.isin(year_range, self.years) if not np.all(pos_year): LOGGER.info('No discount rates for given years.') return None pos_year = np.isin(self.years, year_range) return DiscRates(years=self.years[pos_year], rates=self.rates[pos_year], tag=self.tag) def append(self, disc_rates): """ Check and append discount rates to current DiscRates. Overwrite discount rate if same year. Parameters ---------- disc_rates: climada.entity.DiscRates DiscRates instance to append Raises ------ ValueError """ disc_rates.check() if self.years.size == 0: self.__dict__ = copy.deepcopy(disc_rates.__dict__) return self.tag.append(disc_rates.tag) new_year = array('l') new_rate = array('d') for year, rate in zip(disc_rates.years, disc_rates.rates): found = np.where(year == self.years)[0] if found.size > 0: self.rates[found[0]] = rate else: new_year.append(year) new_rate.append(rate) self.years = np.append(self.years, new_year).astype(int, copy=False) self.rates = np.append(self.rates, new_rate) def net_present_value(self, ini_year, end_year, val_years): """ Compute net present value between present year and future year. Parameters ---------- ini_year: float initial year end_year: float end year val_years: np.array cash flow at each year btw ini_year and end_year (both included) Returns ------- net_present_value: float net present value between present year and future year. """ year_range = np.arange(ini_year, end_year + 1) if year_range.size != val_years.size: raise ValueError('Wrong size of yearly values.') sel_disc = self.select(year_range) if sel_disc is None: raise ValueError( 'No information of discount rates for provided years:' f' {ini_year} - {end_year}') return u_fin.net_present_value(sel_disc.years, sel_disc.rates, val_years) def plot(self, axis=None, figsize=(6, 8), **kwargs): """ Plot discount rates per year. Parameters ---------- axis: matplotlib.axes._subplots.AxesSubplot, optional axis to use figsize: tuple(int, int), optional size of the figure. The default is (6,8) kwargs: optional keyword arguments passed to plotting function axis.plot Returns ------- axis: matplotlib.axes._subplots.AxesSubplot axis handles of the plot """ if not axis: _, axis = plt.subplots(1, 1, figsize=figsize) axis.set_title('Discount rates') axis.set_xlabel('Year') axis.set_ylabel('discount rate (%)') axis.plot(self.years, self.rates * 100, **kwargs) axis.set_xlim((self.years.min(), self.years.max())) return axis @classmethod def from_mat(cls, file_name, description='', var_names=None): """ Read MATLAB file generated with previous MATLAB CLIMADA version. Parameters ---------- file_name: str filename including path and extension description: str, optional description of the data. The default is '' var_names: dict, optional name of the variables in the file. The Default is DEF_VAR_MAT = {'sup_field_name': 'entity', 'field_name': 'discount', 'var_name': {'year': 'year', 'disc': 'discount_rate'}} Returns ------- climada.entity.DiscRates : The disc rates from matlab """ if var_names is None: var_names = DEF_VAR_MAT disc = u_hdf5.read(file_name) tag = Tag(file_name=str(file_name), description=description) try: disc = disc[var_names['sup_field_name']] except KeyError: pass try: disc = disc[var_names['field_name']] years = np.squeeze(disc[var_names['var_name']['year']]). \ astype(int, copy=False) rates = np.squeeze(disc[var_names['var_name']['disc']]) except KeyError as err: raise KeyError("Not existing variable: %s" % str(err)) from err return cls(years=years, rates=rates, tag=tag) def read_mat(self, *args, **kwargs): """This function is deprecated, use DiscRates.from_mats instead.""" LOGGER.warning("The use of DiscRates.read_mats is deprecated." "Use DiscRates.from_mats instead.") self.__dict__ = DiscRates.from_mat(*args, **kwargs).__dict__ @classmethod def from_excel(cls, file_name, description='', var_names=None): """ Read excel file following template and store variables. Parameters ---------- file_name: str filename including path and extension description: str, optional description of the data. The default is '' var_names: dict, optional name of the variables in the file. The Default is DEF_VAR_EXCEL = {'sheet_name': 'discount', 'col_name': {'year': 'year', 'disc': 'discount_rate'}} Returns ------- climada.entity.DiscRates : The disc rates from excel """ if var_names is None: var_names = DEF_VAR_EXCEL dfr = pd.read_excel(file_name, var_names['sheet_name']) tag = Tag(file_name=str(file_name), description=description) try: years = dfr[var_names['col_name']['year']].values. \ astype(int, copy=False) rates = dfr[var_names['col_name']['disc']].values except KeyError as err: raise KeyError("Not existing variable: %s" % str(err)) from err return cls(years=years, rates=rates, tag=tag) def read_excel(self, *args, **kwargs): """This function is deprecated, use DiscRates.from_excel instead.""" LOGGER.warning("The use of DiscRates.read_excel is deprecated." "Use DiscRates.from_excel instead.") self.__dict__ = DiscRates.from_mat(*args, **kwargs).__dict__ def write_excel(self, file_name, var_names=None): """ Write excel file following template. Parameters ---------- file_name: str filename including path and extension var_names: dict, optional name of the variables in the file. The Default is DEF_VAR_EXCEL = {'sheet_name': 'discount', 'col_name': {'year': 'year', 'disc': 'discount_rate'}} """ if var_names is None: var_names = DEF_VAR_EXCEL disc_wb = xlsxwriter.Workbook(file_name) disc_ws = disc_wb.add_worksheet(var_names['sheet_name']) header = [var_names['col_name']['year'], var_names['col_name']['disc']] for icol, head_dat in enumerate(header): disc_ws.write(0, icol, head_dat) for i_yr, (disc_yr, disc_rt) in enumerate(zip(self.years, self.rates), 1): disc_ws.write(i_yr, 0, disc_yr) disc_ws.write(i_yr, 1, disc_rt) disc_wb.close()
def __init__(self, *args, **kwargs): """Creates an Exposures object from a GeoDataFrame Parameters ---------- *args : Arguments of the GeoDataFrame constructor **kwargs : Named arguments of the GeoDataFrame constructor, additionally tag : climada.entity.exposures.tag.Tag Exopusres tag ref_year : int Reference Year value_unit : str Unit of the exposed value meta : dict Metadata dictionary """ # meta data try: self.meta = kwargs.pop('meta') if self.meta is None: self.meta = {} if not isinstance(self.meta, dict): raise ValueError("meta must be a dictionary") except KeyError: self.meta = {} LOGGER.info('meta set to default value %s', self.meta) # tag try: self.tag = kwargs.pop('tag') except KeyError: self.tag = self.meta.get('tag', Tag()) if 'tag' not in self.meta: LOGGER.info('tag set to default value %s', self.tag) # reference year try: self.ref_year = kwargs.pop('ref_year') except KeyError: self.ref_year = self.meta.get('ref_year', DEF_REF_YEAR) if 'ref_year' not in self.meta: LOGGER.info('ref_year set to default value %s', self.ref_year) # value unit try: self.value_unit = kwargs.pop('value_unit') except KeyError: self.value_unit = self.meta.get('ref_year', DEF_VALUE_UNIT) if 'value_unit' not in self.meta: LOGGER.info('value_unit set to default value %s', self.value_unit) # remaining generic attributes for mda in type(self)._metadata: if mda not in Exposures._metadata: if mda in kwargs: setattr(self, mda, kwargs.pop(mda)) elif mda in self.meta: setattr(self, mda, self.meta[mda]) else: setattr(self, mda, None) # make the data frame self.gdf = GeoDataFrame(*args, **kwargs) # align crs from gdf and meta data if self.gdf.crs: crs = self.gdf.crs # With geopandas 3.1, the crs attribute is not conserved by the constructor # without a geometry column. Therefore the conservation is done 'manually': elif len(args) > 0: try: crs = args[0].crs except AttributeError: crs = None elif 'data' in kwargs: try: crs = kwargs['data'].crs except AttributeError: crs = None else: crs = None # store the crs in the meta dictionary if crs: if self.meta.get('crs') and not u_coord.equal_crs( self.meta.get('crs'), crs): LOGGER.info( 'crs from `meta` argument ignored and overwritten by GeoDataFrame' ' crs: %s', self.gdf.crs) self.meta['crs'] = crs if not self.gdf.crs: self.gdf.crs = crs else: if 'crs' not in self.meta: LOGGER.info('crs set to default value: %s', DEF_CRS) self.meta['crs'] = DEF_CRS self.gdf.crs = self.meta['crs']
def __init__(self, *args, meta=None, tag=None, ref_year=DEF_REF_YEAR, value_unit=DEF_VALUE_UNIT, crs=None, **kwargs): """Creates an Exposures object from a GeoDataFrame Parameters ---------- args : Arguments of the GeoDataFrame constructor kwargs : Named arguments of the GeoDataFrame constructor, additionally meta : dict, optional Metadata dictionary. Default: {} (empty dictionary) tag : climada.entity.exposures.tag.Tag, optional Exposures tag. Defaults to the entry of the same name in `meta` or an empty Tag object. ref_year : int, optional Reference Year. Defaults to the entry of the same name in `meta` or 2018. value_unit : str, optional Unit of the exposed value. Defaults to the entry of the same name in `meta` or 'USD'. crs : object, anything accepted by pyproj.CRS.from_user_input Coordinate reference system. Defaults to the entry of the same name in `meta`, or to the CRS of the GeoDataFrame (if provided) or to 'epsg:4326'. """ # meta data self.meta = {} if meta is None else meta if not isinstance(self.meta, dict): raise ValueError("meta must be a dictionary") self.tag = self.meta.get('tag', Tag()) if tag is None else tag self.ref_year = self.meta.get( 'ref_year', DEF_REF_YEAR) if ref_year is None else ref_year self.value_unit = (self.meta.get('value_unit', DEF_VALUE_UNIT) if value_unit is None else value_unit) # remaining generic attributes from derived classes for mda in type(self)._metadata: if mda not in Exposures._metadata: if mda in kwargs: setattr(self, mda, kwargs.pop(mda)) elif mda in self.meta: setattr(self, mda, self.meta[mda]) else: setattr(self, mda, None) # crs (property) and geometry data = args[0] if args else kwargs.get('data', {}) try: data_crs = data.geometry.crs except AttributeError: data_crs = None if data_crs and data.crs and not u_coord.equal_crs(data_crs, data.crs): raise ValueError( "Inconsistent crs definition in data and data.geometry") crs = (crs if crs is not None else self.meta['crs'] if 'crs' in self.meta else data_crs if data_crs else None) if 'crs' in self.meta and not u_coord.equal_crs(self.meta['crs'], crs): raise ValueError( "Inconsistent CRS definition, crs and meta arguments don't match" ) if data_crs and not u_coord.equal_crs(data_crs, crs): raise ValueError( "Inconsistent CRS definition, data doesn't match meta or crs argument" ) if not crs: crs = DEF_CRS geometry = kwargs.get('geometry') if geometry and isinstance(geometry, str): raise ValueError( "Exposures is not able to handle customized 'geometry' column names." ) # make the data frame self.set_gdf(GeoDataFrame(*args, **kwargs), crs=crs)