def test_read_template_pass(self):
"""Wrong exposures definition"""
df = pd.read_excel(ENT_TEMPLATE_XLS)
exp_df = Exposures(df)
# set metadata
exp_df.ref_year = 2020
exp_df.tag = Tag(ENT_TEMPLATE_XLS, 'ENT_TEMPLATE_XLS')
exp_df.value_unit = 'XSD'
exp_df.check()
def check(self):
""" Check which variables are present """
# check metadata
for var in self._metadata:
if var[0] == '_':
continue
try:
if getattr(self, var) is None and var == 'crs':
self.crs = DEF_CRS
LOGGER.info('%s set to default value: %s', var,
self.__dict__[var])
except AttributeError:
if var == 'tag':
self.tag = Tag()
elif var == 'ref_year':
self.ref_year = DEF_REF_YEAR
elif var == 'value_unit':
self.value_unit = DEF_VALUE_UNIT
LOGGER.info('%s metadata set to default value: %s', var,
self.__dict__[var])
for var in self.vars_oblig:
if not var in self.columns:
LOGGER.error("%s missing.", var)
raise ValueError
for var in self.vars_def:
if var == INDICATOR_IF:
found = np.array([var in var_col
for var_col in self.columns]).any()
if INDICATOR_IF in self.columns:
LOGGER.info("Hazard type not set in %s", var)
else:
found = var in self.columns
if not found and var == INDICATOR_IF:
LOGGER.info("Setting %s to default impact functions ids 1.",
var)
self[INDICATOR_IF] = np.ones(self.shape[0], dtype=int)
elif not found:
LOGGER.info("%s not set.", var)
for var in self.vars_opt:
if var == INDICATOR_CENTR:
found = np.array([var in var_col
for var_col in self.columns]).any()
if INDICATOR_CENTR in self.columns:
LOGGER.info("Hazard type not set in %s", var)
else:
found = var in self.columns
if not found:
LOGGER.info("%s not set.", var)
elif var == 'geometry' and \
(self.geometry.values[0].x != self.longitude.values[0] or \
self.geometry.values[0].y != self.latitude.values[0]):
LOGGER.error('Geometry values do not correspond to latitude ' +\
'and longitude. Use set_geometry_points() or set_lat_lon().')
raise ValueError
def set_countries(self, countries=[], reg=[], ref_year=2000, path=None):
"""Model countries using values at reference year. If GDP or income
group not available for that year, consider the value of the closest
available year.
Parameters:
countries (list): list of country names ISO3
ref_year (int, optional): reference year. Default: 2016
path (string): path to exposure dataset (ISIMIP)
"""
gdp2a_list = []
tag = Tag()
if path is None:
raise NameError('No path for exposure data set')
if not Path(path).is_file():
raise NameError('Invalid path %s' % path)
try:
if not countries:
if reg:
natISO = u_coord.region2isos(reg)
countries = np.array(natISO)
else:
raise ValueError('set_countries requires countries or reg')
for cntr_ind in range(len(countries)):
gdp2a_list.append(
self._set_one_country(countries[cntr_ind], ref_year, path))
tag.description += ("{} GDP2Asset \n").\
format(countries[cntr_ind])
except KeyError as err:
raise KeyError(
f'Exposure countries: {countries} or reg {reg} could not be set, '
f'check ISO3 or reference year {ref_year}') from err
tag.description += 'GDP2Asset ' + str(self.ref_year)
Exposures.__init__(self,
data=Exposures.concat(gdp2a_list).gdf,
ref_year=ref_year,
tag=tag,
value_unit='USD')
# set meta
res = 0.0416666
rows, cols, ras_trans = u_coord.pts_to_raster_meta(
(self.gdf.longitude.min(), self.gdf.latitude.min(),
self.gdf.longitude.max(), self.gdf.latitude.max()), res)
self.meta = {
'width': cols,
'height': rows,
'crs': self.crs,
'transform': ras_trans
}
def test_io_hdf5_pass(self):
"""write and read hdf5"""
exp_df = Exposures(pd.read_excel(ENT_TEMPLATE_XLS), crs="epsg:32632")
exp_df.set_geometry_points()
exp_df.check()
# set metadata
exp_df.ref_year = 2020
exp_df.tag = Tag(ENT_TEMPLATE_XLS, 'ENT_TEMPLATE_XLS')
exp_df.value_unit = 'XSD'
file_name = DATA_DIR.joinpath('test_hdf5_exp.h5')
# pd.errors.PerformanceWarning should be suppressed. Therefore, make sure that
# PerformanceWarning would result in test failure here
import warnings
with warnings.catch_warnings():
warnings.simplefilter("error",
category=pd.errors.PerformanceWarning)
exp_df.write_hdf5(file_name)
exp_read = Exposures.from_hdf5(file_name)
self.assertEqual(exp_df.ref_year, exp_read.ref_year)
self.assertEqual(exp_df.value_unit, exp_read.value_unit)
self.assertDictEqual(exp_df.meta, exp_read.meta)
self.assertTrue(u_coord.equal_crs(exp_df.crs, exp_read.crs))
self.assertTrue(u_coord.equal_crs(exp_df.gdf.crs, exp_read.gdf.crs))
self.assertEqual(exp_df.tag.file_name, exp_read.tag.file_name)
self.assertEqual(exp_df.tag.description, exp_read.tag.description)
np.testing.assert_array_equal(exp_df.gdf.latitude.values,
exp_read.gdf.latitude.values)
np.testing.assert_array_equal(exp_df.gdf.longitude.values,
exp_read.gdf.longitude.values)
np.testing.assert_array_equal(exp_df.gdf.value.values,
exp_read.gdf.value.values)
np.testing.assert_array_equal(exp_df.gdf.deductible.values,
exp_read.gdf.deductible.values)
np.testing.assert_array_equal(exp_df.gdf.cover.values,
exp_read.gdf.cover.values)
np.testing.assert_array_equal(exp_df.gdf.region_id.values,
exp_read.gdf.region_id.values)
np.testing.assert_array_equal(exp_df.gdf.category_id.values,
exp_read.gdf.category_id.values)
np.testing.assert_array_equal(exp_df.gdf.impf_TC.values,
exp_read.gdf.impf_TC.values)
np.testing.assert_array_equal(exp_df.gdf.centr_TC.values,
exp_read.gdf.centr_TC.values)
np.testing.assert_array_equal(exp_df.gdf.impf_FL.values,
exp_read.gdf.impf_FL.values)
np.testing.assert_array_equal(exp_df.gdf.centr_FL.values,
exp_read.gdf.centr_FL.values)
for point_df, point_read in zip(exp_df.gdf.geometry.values,
exp_read.gdf.geometry.values):
self.assertEqual(point_df.x, point_read.x)
self.assertEqual(point_df.y, point_read.y)
def test_io_hdf5_pass(self):
"""write and read hdf5"""
exp_df = Exposures(pd.read_excel(ENT_TEMPLATE_XLS))
exp_df.set_geometry_points()
exp_df.check()
# set metadata
exp_df.ref_year = 2020
exp_df.tag = Tag(ENT_TEMPLATE_XLS, 'ENT_TEMPLATE_XLS')
exp_df.value_unit = 'XSD'
file_name = DATA_DIR.joinpath('test_hdf5_exp.h5')
exp_df.write_hdf5(file_name)
exp_read = Exposures()
exp_read.read_hdf5(file_name)
self.assertEqual(exp_df.ref_year, exp_read.ref_year)
self.assertEqual(exp_df.value_unit, exp_read.value_unit)
self.assertEqual(exp_df.crs, exp_read.crs)
self.assertEqual(exp_df.tag.file_name, exp_read.tag.file_name)
self.assertEqual(exp_df.tag.description, exp_read.tag.description)
self.assertTrue(
np.array_equal(exp_df.gdf.latitude.values,
exp_read.gdf.latitude.values))
self.assertTrue(
np.array_equal(exp_df.gdf.longitude.values,
exp_read.gdf.longitude.values))
self.assertTrue(
np.array_equal(exp_df.gdf.value.values, exp_read.gdf.value.values))
self.assertTrue(
np.array_equal(exp_df.gdf.deductible.values,
exp_read.gdf.deductible.values))
self.assertTrue(
np.array_equal(exp_df.gdf.cover.values, exp_read.gdf.cover.values))
self.assertTrue(
np.array_equal(exp_df.gdf.region_id.values,
exp_read.gdf.region_id.values))
self.assertTrue(
np.array_equal(exp_df.gdf.category_id.values,
exp_read.gdf.category_id.values))
self.assertTrue(
np.array_equal(exp_df.gdf.if_TC.values, exp_read.gdf.if_TC.values))
self.assertTrue(
np.array_equal(exp_df.gdf.centr_TC.values,
exp_read.gdf.centr_TC.values))
self.assertTrue(
np.array_equal(exp_df.gdf.if_FL.values, exp_read.gdf.if_FL.values))
self.assertTrue(
np.array_equal(exp_df.gdf.centr_FL.values,
exp_read.gdf.centr_FL.values))
for point_df, point_read in zip(exp_df.gdf.geometry.values,
exp_read.gdf.geometry.values):
self.assertEqual(point_df.x, point_read.x)
self.assertEqual(point_df.y, point_read.y)
def from_raster(cls, file_name, band=1, src_crs=None, window=False,
geometry=False, dst_crs=False, transform=None,
width=None, height=None, resampling=Resampling.nearest):
"""Read raster data and set latitude, longitude, value and meta
Parameters
----------
file_name : str
file name containing values
band : int, optional
bands to read (starting at 1)
src_crs : crs, optional
source CRS. Provide it if error without it.
window : rasterio.windows.Windows, optional
window where data is
extracted
geometry : shapely.geometry, optional
consider pixels only in shape
dst_crs : crs, optional
reproject to given crs
transform : rasterio.Affine
affine transformation to apply
wdith : float
number of lons for transform
height : float
number of lats for transform
resampling : rasterio.warp,.Resampling optional
resampling
function used for reprojection to dst_crs
returns
--------
Exposures
"""
exp = cls()
if 'geometry' in exp.gdf:
raise ValueError("there is already a geometry column defined in the GeoDataFrame")
exp.tag = Tag()
exp.tag.file_name = str(file_name)
meta, value = u_coord.read_raster(file_name, [band], src_crs, window,
geometry, dst_crs, transform, width,
height, resampling)
ulx, xres, _, uly, _, yres = meta['transform'].to_gdal()
lrx = ulx + meta['width'] * xres
lry = uly + meta['height'] * yres
x_grid, y_grid = np.meshgrid(np.arange(ulx + xres / 2, lrx, xres),
np.arange(uly + yres / 2, lry, yres))
if exp.crs is None:
exp.set_crs()
exp.gdf['longitude'] = x_grid.flatten()
exp.gdf['latitude'] = y_grid.flatten()
exp.gdf['value'] = value.reshape(-1)
exp.meta = meta
return exp
def _build_exp(self):
eai_exp = Exposures()
eai_exp['value'] = self.eai_exp
eai_exp['latitude'] = self.coord_exp[:, 0]
eai_exp['longitude'] = self.coord_exp[:, 1]
eai_exp.crs = self.crs
eai_exp.value_unit = self.unit
eai_exp.ref_year = 0
eai_exp.tag = Tag()
eai_exp.meta = None
return eai_exp
def set_countries(self,
countries,
ref_year=2016,
res_km=None,
from_hr=None,
**kwargs):
""" Model countries using values at reference year. If GDP or income
group not available for that year, consider the value of the closest
available year.
Parameters:
countries (list or dict): list of country names (admin0) or dict
with key = admin0 name and value = [admin1 names]
ref_year (int, optional): reference year. Default: 2016
res_km (float, optional): approx resolution in km. Default:
nightlights resolution.
from_hr (bool, optional): force to use higher resolution image,
independently of its year of acquisition.
kwargs (optional): 'gdp' and 'inc_grp' dictionaries with keys the
country ISO_alpha3 code. 'poly_val' polynomial transformation
[1,x,x^2,...] to apply to nightlight (DEF_POLY_VAL used if not
provided). If provided, these are used.
"""
shp_file = shapereader.natural_earth(resolution='10m',
category='cultural',
name='admin_0_countries')
shp_file = shapereader.Reader(shp_file)
cntry_info, cntry_admin1 = country_iso_geom(countries, shp_file)
fill_econ_indicators(ref_year, cntry_info, shp_file, **kwargs)
nightlight, coord_nl, fn_nl, res_fact, res_km = get_nightlight(\
ref_year, cntry_info, res_km, from_hr)
tag = Tag()
bkmrbl_list = []
for cntry_iso, cntry_val in cntry_info.items():
bkmrbl_list.append(self._set_one_country(cntry_val, nightlight, \
coord_nl, res_fact, res_km, cntry_admin1[cntry_iso], **kwargs))
tag.description += ("{} {:d} GDP: {:.3e} income group: {:d} \n").\
format(cntry_val[1], cntry_val[3], cntry_val[4], cntry_val[5])
Exposures.__init__(
self, gpd.GeoDataFrame(pd.concat(bkmrbl_list, ignore_index=True)))
# set metadata
self.ref_year = ref_year
self.tag = tag
self.tag.file_name = fn_nl
self.value_unit = 'USD'
self.crs = {'init': 'epsg:4326'}
def test_write_read_exp_test(self):
''' Test result against reference value'''
# Create impact object
num_ev = 5
num_exp = 10
imp_write = Impact()
imp_write.tag = {'exp': Tag('file_exp.p', 'descr exp'),
'haz': TagHaz('TC', 'file_haz.p', 'descr haz'),
'if_set': Tag()}
imp_write.event_id = np.arange(num_ev)
imp_write.event_name = ['event_'+str(num) for num in imp_write.event_id]
imp_write.date = np.ones(num_ev)
imp_write.coord_exp = np.zeros((num_exp, 2))
imp_write.coord_exp[:, 0] = 1.5
imp_write.coord_exp[:, 1] = 2.5
imp_write.eai_exp = np.arange(num_exp) * 100
imp_write.at_event = np.arange(num_ev) * 50
imp_write.frequency = np.ones(num_ev) * 0.1
imp_write.tot_value = 1000
imp_write.aai_agg = 1001
imp_write.unit = 'USD'
file_name = os.path.join(DATA_FOLDER, 'test.csv')
imp_write.write_csv(file_name)
imp_read = Impact()
imp_read.read_csv(file_name)
self.assertTrue(np.array_equal(imp_write.event_id, imp_read.event_id))
self.assertTrue(np.array_equal(imp_write.date, imp_read.date))
self.assertTrue(np.array_equal(imp_write.coord_exp, imp_read.coord_exp))
self.assertTrue(np.array_equal(imp_write.eai_exp, imp_read.eai_exp))
self.assertTrue(np.array_equal(imp_write.at_event, imp_read.at_event))
self.assertTrue(np.array_equal(imp_write.frequency, imp_read.frequency))
self.assertEqual(imp_write.tot_value, imp_read.tot_value)
self.assertEqual(imp_write.aai_agg, imp_read.aai_agg)
self.assertEqual(imp_write.unit, imp_read.unit)
self.assertEqual(0, len([i for i, j in
zip(imp_write.event_name, imp_read.event_name) if i != j]))
self.assertIsInstance(imp_read.crs, dict)
def read_excel(self, file_name):
"""Read excel file containing impact data generated by write_excel.
Parameters:
file_name (str): absolute path of the file
"""
LOGGER.info('Reading %s', file_name)
dfr = pd.read_excel(file_name)
self.__init__()
self.tag['haz'] = TagHaz()
self.tag['haz'].haz_type = dfr['tag_hazard'][0]
self.tag['haz'].file_name = dfr['tag_hazard'][1]
self.tag['haz'].description = dfr['tag_hazard'][2]
self.tag['exp'] = Tag()
self.tag['exp'].file_name = dfr['tag_exposure'][0]
self.tag['exp'].description = dfr['tag_exposure'][1]
self.tag['if_set'] = Tag()
self.tag['if_set'].file_name = dfr['tag_impact_func'][0]
self.tag['if_set'].description = dfr['tag_impact_func'][1]
self.unit = dfr.unit[0]
self.tot_value = dfr.tot_value[0]
self.aai_agg = dfr.aai_agg[0]
self.event_id = dfr.event_id[~np.isnan(dfr.event_id.values)].values
self.event_name = dfr.event_name[:self.event_id.size].values
self.date = dfr.event_date[:self.event_id.size].values
self.frequency = dfr.event_frequency[:self.event_id.size].values
self.at_event = dfr.at_event[:self.event_id.size].values
self.eai_exp = dfr.eai_exp[~np.isnan(dfr.eai_exp.values)].values
self.coord_exp = np.zeros((self.eai_exp.size, 2))
self.coord_exp[:, 0] = dfr.exp_lat.values[:self.eai_exp.size]
self.coord_exp[:, 1] = dfr.exp_lon.values[:self.eai_exp.size]
try:
self.crs = ast.literal_eval(dfr.exp_crs.values[0])
except AttributeError:
self.crs = DEF_CRS
def _read_mat_metadata(exposures, data, file_name, var_names):
""" Fille metadata in DataFrame object """
try:
exposures.ref_year = int(np.squeeze(data[var_names['var_name']['ref']]))
except KeyError:
exposures.ref_year = DEF_REF_YEAR
try:
exposures.value_unit = hdf5.get_str_from_ref(file_name, \
data[var_names['var_name']['uni']][0][0])
except KeyError:
exposures.value_unit = DEF_VALUE_UNIT
exposures.tag = Tag(file_name)
def set_countries(self, countries=[], reg=[], ref_year=2000,
path=None):
""" Model countries using values at reference year. If GDP or income
group not available for that year, consider the value of the closest
available year.
Parameters:
countries (list): list of country names ISO3
ref_year (int, optional): reference year. Default: 2016
path (string): path to exposure dataset
"""
gdp2a_list = []
tag = Tag()
if path is None:
LOGGER.error('No path for exposure data set')
raise NameError
if not os.path.exists(path):
LOGGER.error('Invalid path ' + path)
raise NameError
try:
if not countries:
if reg:
natID_info = pd.read_csv(NAT_REG_ID)
natISO = natID_info["ISO"][np.isin(natID_info["Reg_name"],
reg)]
countries = np.array(natISO)
else:
LOGGER.error('set_countries requires countries or reg')
raise ValueError
for cntr_ind in range(len(countries)):
gdp2a_list.append(self._set_one_country(countries[cntr_ind],
ref_year, path))
tag.description += ("{} GDP2Asset \n").\
format(countries[cntr_ind])
Exposures.__init__(self, gpd.GeoDataFrame(
pd.concat(gdp2a_list, ignore_index=True)))
except KeyError:
LOGGER.error('Exposure countries: ' + str(countries) + ' or reg ' +
str(reg) + ' could not be set, check ISO3 or' +
' reference year ' + str(ref_year))
raise KeyError
self.ref_year = ref_year
self.value_unit = 'USD'
self.tag = tag
self.crs = DEF_CRS
def _build_exp_event(self, event_id):
"""Write impact of an event as Exposures
Parameters:
event_id(int): id of the event
"""
impact_csr_exp = Exposures()
impact_csr_exp['value'] = self.imp_mat.toarray()[event_id - 1, :]
impact_csr_exp['latitude'] = self.coord_exp[:, 0]
impact_csr_exp['longitude'] = self.coord_exp[:, 1]
impact_csr_exp.crs = self.crs
impact_csr_exp.value_unit = self.unit
impact_csr_exp.ref_year = 0
impact_csr_exp.tag = Tag()
impact_csr_exp.meta = None
return impact_csr_exp
def set_from_raster(self,
file_name,
band=1,
src_crs=None,
window=False,
geometry=False,
dst_crs=False,
transform=None,
width=None,
height=None,
resampling=Resampling.nearest):
""" Read raster data and set latitude, longitude, value and meta
Parameters:
file_name (str): file name containing values
band (int, optional): bands to read (starting at 1)
src_crs (crs, optional): source CRS. Provide it if error without it.
window (rasterio.windows.Windows, optional): window where data is
extracted
geometry (shapely.geometry, optional): consider pixels only in shape
dst_crs (crs, optional): reproject to given crs
transform (rasterio.Affine): affine transformation to apply
wdith (float): number of lons for transform
height (float): number of lats for transform
resampling (rasterio.warp,.Resampling optional): resampling
function used for reprojection to dst_crs
"""
self.__init__()
self.tag = Tag()
self.tag.file_name = file_name
meta, value = co.read_raster(file_name, [band], src_crs, window,
geometry, dst_crs, transform, width,
height, resampling)
ulx, xres, _, uly, _, yres = meta['transform'].to_gdal()
lrx = ulx + meta['width'] * xres
lry = uly + meta['height'] * yres
x_grid, y_grid = np.meshgrid(np.arange(ulx + xres / 2, lrx, xres),
np.arange(uly + yres / 2, lry, yres))
try:
self.crs = meta['crs'].to_dict()
except AttributeError:
self.crs = meta['crs']
self['longitude'] = x_grid.flatten()
self['latitude'] = y_grid.flatten()
self['value'] = value.reshape(-1)
self.meta = meta
def __init__(self, years=None, rates=None, tag=None):
"""
Fill discount rates with values and check consistency data
Parameters
----------
years : numpy.ndarray(int)
Array of years. Default is numpy.array([]).
rates : numpy.ndarray(float)
Discount rates for each year in years.
Default is numpy.array([]).
Note: rates given in float, e.g., to set 1% rate use 0.01
tag : climate.entity.tag
Metadata. Default is None.
"""
years = np.array([]) if years is None else years
rates = np.array([]) if rates is None else rates
self.years = years
self.rates = rates
tag = Tag() if tag is None else tag
self.tag = tag
def from_mat(cls, file_name, description='', var_names=None):
"""
Read MATLAB file generated with previous MATLAB CLIMADA version.
Parameters
----------
file_name: str
filename including path and extension
description: str, optional
description of the data. The default is ''
var_names: dict, optional
name of the variables in the file. The Default is
DEF_VAR_MAT = {'sup_field_name': 'entity', 'field_name': 'discount',
'var_name': {'year': 'year', 'disc': 'discount_rate'}}
Returns
-------
climada.entity.DiscRates :
The disc rates from matlab
"""
if var_names is None:
var_names = DEF_VAR_MAT
disc = u_hdf5.read(file_name)
tag = Tag(file_name=str(file_name), description=description)
try:
disc = disc[var_names['sup_field_name']]
except KeyError:
pass
try:
disc = disc[var_names['field_name']]
years = np.squeeze(disc[var_names['var_name']['year']]). \
astype(int, copy=False)
rates = np.squeeze(disc[var_names['var_name']['disc']])
except KeyError as err:
raise KeyError("Not existing variable: %s" % str(err)) from err
return cls(years=years, rates=rates, tag=tag)
def clear(self):
"""Reinitialize attributes."""
self.tag = Tag()
self._data = dict() # {hazard_type : {name: Measure()}}
def clear(self):
"""Reinitialize attributes."""
self.tag = Tag()
self._data = dict() # {hazard_type : {id:ImpactFunc}}
class ImpactFuncSet():
"""Contains impact functions of type ImpactFunc. Loads from
files with format defined in FILE_EXT.
Attributes:
tag (Tag): information about the source data
_data (dict): contains ImpactFunc classes. It's not suppossed to be
directly accessed. Use the class methods instead.
"""
def __init__(self):
"""Empty initialization.
Examples:
Fill impact functions with values and check consistency data:
>>> fun_1 = ImpactFunc()
>>> fun_1.haz_type = 'TC'
>>> fun_1.id = 3
>>> fun_1.intensity = np.array([0, 20])
>>> fun_1.paa = np.array([0, 1])
>>> fun_1.mdd = np.array([0, 0.5])
>>> imp_fun = ImpactFuncSet()
>>> imp_fun.append(fun_1)
>>> imp_fun.check()
Read impact functions from file and checks consistency data.
>>> imp_fun = ImpactFuncSet()
>>> imp_fun.read(ENT_TEMPLATE_XLS)
"""
self.clear()
def clear(self):
"""Reinitialize attributes."""
self.tag = Tag()
self._data = dict() # {hazard_type : {id:ImpactFunc}}
def append(self, func):
"""Append a ImpactFunc. Overwrite existing if same id and haz_type.
Parameters:
func (ImpactFunc): ImpactFunc instance
Raises:
ValueError
"""
if not isinstance(func, ImpactFunc):
LOGGER.error("Input value is not of type ImpactFunc.")
raise ValueError
if not func.haz_type:
LOGGER.warning("Input ImpactFunc's hazard type not set.")
if not func.id:
LOGGER.warning("Input ImpactFunc's id not set.")
if func.haz_type not in self._data:
self._data[func.haz_type] = dict()
self._data[func.haz_type][func.id] = func
def remove_func(self, haz_type=None, fun_id=None):
"""Remove impact function(s) with provided hazard type and/or id.
If no input provided, all impact functions are removed.
Parameters:
haz_type (str, optional): all impact functions with this hazard
fun_id (int, optional): all impact functions with this id
"""
if (haz_type is not None) and (fun_id is not None):
try:
del self._data[haz_type][fun_id]
except KeyError:
LOGGER.warning("No ImpactFunc with hazard %s and id %s.",
haz_type, fun_id)
elif haz_type is not None:
try:
del self._data[haz_type]
except KeyError:
LOGGER.warning("No ImpactFunc with hazard %s.", haz_type)
elif fun_id is not None:
haz_remove = self.get_hazard_types(fun_id)
if not haz_remove:
LOGGER.warning("No ImpactFunc with id %s.", fun_id)
for vul_haz in haz_remove:
del self._data[vul_haz][fun_id]
else:
self._data = dict()
def get_func(self, haz_type=None, fun_id=None):
"""Get ImpactFunc(s) of input hazard type and/or id.
If no input provided, all impact functions are returned.
Parameters:
haz_type (str, optional): hazard type
fun_id (int, optional): ImpactFunc id
Returns:
ImpactFunc (if haz_type and fun_id),
list(ImpactFunc) (if haz_type or fun_id),
{ImpactFunc.haz_type: {ImpactFunc.id : ImpactFunc}} (if None)
"""
if (haz_type is not None) and (fun_id is not None):
try:
return self._data[haz_type][fun_id]
except KeyError:
return list()
elif haz_type is not None:
try:
return list(self._data[haz_type].values())
except KeyError:
return list()
elif fun_id is not None:
haz_return = self.get_hazard_types(fun_id)
vul_return = []
for vul_haz in haz_return:
vul_return.append(self._data[vul_haz][fun_id])
return vul_return
else:
return self._data
def get_hazard_types(self, fun_id=None):
"""Get impact functions hazard types contained for the id provided.
Return all hazard types if no input id.
Parameters:
fun_id (int, optional): id of an impact function
Returns:
list(str)
"""
if fun_id is None:
return list(self._data.keys())
haz_types = []
for vul_haz, vul_dict in self._data.items():
if fun_id in vul_dict:
haz_types.append(vul_haz)
return haz_types
def get_ids(self, haz_type=None):
"""Get impact functions ids contained for the hazard type provided.
Return all ids for each hazard type if no input hazard type.
Parameters:
haz_type (str, optional): hazard type from which to obtain the ids
Returns:
list(ImpactFunc.id) (if haz_type provided),
{ImpactFunc.haz_type : list(ImpactFunc.id)} (if no haz_type)
"""
if haz_type is None:
out_dict = dict()
for vul_haz, vul_dict in self._data.items():
out_dict[vul_haz] = list(vul_dict.keys())
return out_dict
try:
return list(self._data[haz_type].keys())
except KeyError:
return list()
def size(self, haz_type=None, fun_id=None):
"""Get number of impact functions contained with input hazard type and
/or id. If no input provided, get total number of impact functions.
Parameters:
haz_type (str, optional): hazard type
fun_id (int, optional): ImpactFunc id
Returns:
int
"""
if (haz_type is not None) and (fun_id is not None) and \
(isinstance(self.get_func(haz_type, fun_id), ImpactFunc)):
return 1
if (haz_type is not None) or (fun_id is not None):
return len(self.get_func(haz_type, fun_id))
return sum(len(vul_list) for vul_list in self.get_ids().values())
def check(self):
"""Check instance attributes.
Raises:
ValueError
"""
for key_haz, vul_dict in self._data.items():
for fun_id, vul in vul_dict.items():
if (fun_id != vul.id) | (fun_id == ''):
LOGGER.error("Wrong ImpactFunc.id: %s != %s.", fun_id,
vul.id)
raise ValueError
if (key_haz != vul.haz_type) | (key_haz == ''):
LOGGER.error("Wrong ImpactFunc.haz_type: %s != %s.",
key_haz, vul.haz_type)
raise ValueError
vul.check()
def extend(self, impact_funcs):
"""Append impact functions of input ImpactFuncSet to current
ImpactFuncSet. Overwrite ImpactFunc if same id and haz_type.
Parameters:
impact_funcs (ImpactFuncSet): ImpactFuncSet instance to extend
Raises:
ValueError
"""
impact_funcs.check()
if self.size() == 0:
self.__dict__ = copy.deepcopy(impact_funcs.__dict__)
return
self.tag.append(impact_funcs.tag)
new_func = impact_funcs.get_func()
for _, vul_dict in new_func.items():
for _, vul in vul_dict.items():
self.append(vul)
def plot(self, haz_type=None, fun_id=None, axis=None, **kwargs):
"""Plot impact functions of selected hazard (all if not provided) and
selected function id (all if not provided).
Parameters:
haz_type (str, optional): hazard type
fun_id (int, optional): id of the function
Returns:
matplotlib.axes._subplots.AxesSubplot
"""
num_plts = self.size(haz_type, fun_id)
num_row, num_col = u_plot._get_row_col_size(num_plts)
# Select all hazard types to plot
if haz_type is not None:
hazards = [haz_type]
else:
hazards = self._data.keys()
if not axis:
_, axis = plt.subplots(num_row, num_col)
if num_plts > 1:
axes = axis.flatten()
else:
axes = [axis]
i_axis = 0
for sel_haz in hazards:
if fun_id is not None:
self._data[sel_haz][fun_id].plot(axis=axes[i_axis], **kwargs)
i_axis += 1
else:
for sel_id in self._data[sel_haz].keys():
self._data[sel_haz][sel_id].plot(axis=axes[i_axis], **kwargs)
i_axis += 1
return axis
def read_excel(self, file_name, description='', var_names=DEF_VAR_EXCEL):
"""Read excel file following template and store variables.
Parameters:
file_name (str): absolute file name
description (str, optional): description of the data
var_names (dict, optional): name of the variables in the file
"""
dfr = pd.read_excel(file_name, var_names['sheet_name'])
self.clear()
self.tag.file_name = str(file_name)
self.tag.description = description
self._fill_dfr(dfr, var_names)
def read_mat(self, file_name, description='', var_names=DEF_VAR_MAT):
"""Read MATLAB file generated with previous MATLAB CLIMADA version.
Parameters:
file_name (str): absolute file name
description (str, optional): description of the data
var_names (dict, optional): name of the variables in the file
"""
def _get_hdf5_funcs(imp, file_name, var_names):
"""Get rows that fill every impact function and its name."""
func_pos = dict()
for row, (fun_id, fun_type) in enumerate(
zip(imp[var_names['var_name']['fun_id']].squeeze(),
imp[var_names['var_name']['peril']].squeeze())):
type_str = u_hdf5.get_str_from_ref(file_name, fun_type)
key = (type_str, int(fun_id))
if key not in func_pos:
func_pos[key] = list()
func_pos[key].append(row)
return func_pos
def _get_hdf5_str(imp, idxs, file_name, var_name):
"""Get rows with same string in var_name."""
prev_str = ""
for row in idxs:
cur_str = u_hdf5.get_str_from_ref(file_name, imp[var_name][row][0])
if prev_str == "":
prev_str = cur_str
elif prev_str != cur_str:
LOGGER.error("Impact function with two different %s.", var_name)
raise ValueError
return prev_str
imp = u_hdf5.read(file_name)
self.clear()
self.tag.file_name = str(file_name)
self.tag.description = description
try:
imp = imp[var_names['sup_field_name']]
except KeyError:
pass
try:
imp = imp[var_names['field_name']]
funcs_idx = _get_hdf5_funcs(imp, file_name, var_names)
for imp_key, imp_rows in funcs_idx.items():
func = ImpactFunc()
func.haz_type = imp_key[0]
func.id = imp_key[1]
# check that this function only has one intensity unit, if provided
try:
func.intensity_unit = _get_hdf5_str(imp, imp_rows,
file_name,
var_names['var_name']['unit'])
except KeyError:
pass
# check that this function only has one name
try:
func.name = _get_hdf5_str(imp, imp_rows, file_name,
var_names['var_name']['name'])
except KeyError:
func.name = str(func.id)
func.intensity = np.take(imp[var_names['var_name']['inten']], imp_rows)
func.mdd = np.take(imp[var_names['var_name']['mdd']], imp_rows)
func.paa = np.take(imp[var_names['var_name']['paa']], imp_rows)
self.append(func)
except KeyError as err:
LOGGER.error("Not existing variable: %s", str(err))
raise err
def write_excel(self, file_name, var_names=DEF_VAR_EXCEL):
"""Write excel file following template.
Parameters:
file_name (str): absolute file name to write
var_names (dict, optional): name of the variables in the file
"""
def write_if(row_ini, imp_ws, xls_data):
"""Write one impact function"""
for icol, col_dat in enumerate(xls_data):
for irow, data in enumerate(col_dat, row_ini):
imp_ws.write(irow, icol, data)
imp_wb = xlsxwriter.Workbook(file_name)
imp_ws = imp_wb.add_worksheet(var_names['sheet_name'])
header = [var_names['col_name']['func_id'], var_names['col_name']['inten'],
var_names['col_name']['mdd'], var_names['col_name']['paa'],
var_names['col_name']['peril'], var_names['col_name']['unit'],
var_names['col_name']['name']]
for icol, head_dat in enumerate(header):
imp_ws.write(0, icol, head_dat)
row_ini = 1
for fun_haz_id, fun_haz in self._data.items():
for fun_id, fun in fun_haz.items():
n_inten = fun.intensity.size
xls_data = [repeat(fun_id, n_inten), fun.intensity, fun.mdd,
fun.paa, repeat(fun_haz_id, n_inten),
repeat(fun.intensity_unit, n_inten),
repeat(fun.name, n_inten)]
write_if(row_ini, imp_ws, xls_data)
row_ini += n_inten
imp_wb.close()
def _fill_dfr(self, dfr, var_names):
def _get_xls_funcs(dfr, var_names):
"""Parse individual impact functions."""
dist_func = []
for (haz_type, imp_id) in zip(dfr[var_names['col_name']['peril']],
dfr[var_names['col_name']['func_id']]):
if (haz_type, imp_id) not in dist_func:
dist_func.append((haz_type, imp_id))
return dist_func
try:
dist_func = _get_xls_funcs(dfr, var_names)
for haz_type, imp_id in dist_func:
df_func = dfr[dfr[var_names['col_name']['peril']] == haz_type]
df_func = df_func[df_func[var_names['col_name']['func_id']]
== imp_id]
func = ImpactFunc()
func.haz_type = haz_type
func.id = imp_id
# check that the unit of the intensity is the same
try:
if len(df_func[var_names['col_name']['name']].unique()) != 1:
raise ValueError('Impact function with two different names.')
func.name = df_func[var_names['col_name']['name']].values[0]
except KeyError:
func.name = str(func.id)
# check that the unit of the intensity is the same, if provided
try:
if len(df_func[var_names['col_name']['unit']].unique()) != 1:
raise ValueError('Impact function with two different \
intensity units.')
func.intensity_unit = \
df_func[var_names['col_name']['unit']].values[0]
except KeyError:
pass
func.intensity = df_func[var_names['col_name']['inten']].values
func.mdd = df_func[var_names['col_name']['mdd']].values
func.paa = df_func[var_names['col_name']['paa']].values
self.append(func)
except KeyError as err:
LOGGER.error("Not existing variable: %s", str(err))
raise err
def set_from_isimip_netcdf(self, input_dir=None, filename=None, hist_mean=None,
bbox=None, yearrange=None, cl_model=None, scenario=None,
crop=None, irr=None, isimip_version=None,
unit=None, fn_str_var=None):
"""Wrapper to fill exposure from NetCDF file from ISIMIP. Requires historical
mean relative cropyield module as additional input.
Optional Parameters:
input_dir (string): path to input data directory
filename (string): name of the landuse data file to use,
e.g. "histsoc_landuse-15crops_annual_1861_2005.nc""
hist_mean (str or array): historic mean crop yield per centroid (or path)
bbox (list of four floats): bounding box:
[lon min, lat min, lon max, lat max]
yearrange (int tuple): year range for exposure set
e.g., (1990, 2010)
scenario (string): climate change and socio economic scenario
e.g., '1860soc', 'histsoc', '2005soc', 'rcp26soc','rcp60soc','2100rcp26soc'
cl_model (string): abbrev. climate model (only for future projections of lu data)
e.g., 'gfdl-esm2m', 'hadgem2-es', 'ipsl-cm5a-lr','miroc5'
crop (string): crop type
e.g., 'mai', 'ric', 'whe', 'soy'
irr (string): irrigation type, default: 'combined'
f.i 'firr' (full irrigation), 'noirr' (no irrigation) or 'combined'= firr+noirr
isimip_version(str): 'ISIMIP2' (default) or 'ISIMIP3'
unit (string): unit of the exposure (per year)
f.i 't/y' (default), 'USD/y', or 'kcal/y'
fn_str_var (string): FileName STRing depending on VARiable and
ISIMIP simuation round
Returns:
Exposure
"""
# parameters not provided in method call are set to default values:
if irr is None:
irr = 'combined'
if not bbox:
bbox = BBOX
if not input_dir:
input_dir = INPUT_DIR
if hist_mean is None:
hist_mean = HIST_MEAN_PATH
if not fn_str_var:
fn_str_var = FN_STR_VAR
if (not isimip_version) or (isimip_version in ('ISIMIP2a', 'ISIMIP2b')):
isimip_version = 'ISIMIP2'
elif isimip_version in ('ISIMIP3a', 'ISIMIP3b'):
isimip_version = 'ISIMIP3'
if (not scenario) or (scenario in ('historical', 'hist')):
scenario = 'histsoc'
if yearrange is None:
yearrange = YEARCHUNKS[isimip_version][scenario]['yearrange']
if not unit:
unit = 't/y'
#if not soc: soc=''
# The filename is set or other variables (cl_model, scenario) are extracted of the
# specified filename
if filename is None:
yearchunk = YEARCHUNKS[isimip_version][scenario]
# if scenario == 'histsoc' or scenario == '1860soc':
if scenario in ('histsoc', '1860soc'):
string = '{}_{}_{}_{}.nc'
filepath = Path(input_dir, string.format(scenario, fn_str_var,
yearchunk['startyear'],
yearchunk['endyear']))
else:
string = '{}_{}_{}_{}_{}.nc'
filepath = Path(input_dir, string.format(scenario, cl_model, fn_str_var,
yearchunk['startyear'],
yearchunk['endyear']))
elif scenario == 'flexible':
_, _, _, _, _, _, startyear, endyearnc = filename.split('_')
endyear = endyearnc.split('.')[0]
yearchunk = dict()
yearchunk = {'yearrange': (int(startyear), int(endyear)),
'startyear': int(startyear), 'endyear': int(endyear)}
filepath = Path(input_dir, filename)
else:
scenario, *_ = filename.split('_')
yearchunk = YEARCHUNKS[isimip_version][scenario]
filepath = Path(input_dir, filename)
# Dataset is opened and data within the bbox extends is extracted
data_set = xr.open_dataset(filepath, decode_times=False)
[lonmin, latmin, lonmax, latmax] = bbox
data = data_set.sel(lon=slice(lonmin, lonmax), lat=slice(latmax, latmin))
# The latitude and longitude are set; the region_id is determined
lon, lat = np.meshgrid(data.lon.values, data.lat.values)
self.gdf['latitude'] = lat.flatten()
self.gdf['longitude'] = lon.flatten()
self.gdf['region_id'] = u_coord.get_country_code(self.gdf.latitude, self.gdf.longitude)
# The indeces of the yearrange to be extracted are determined
time_idx = (int(yearrange[0] - yearchunk['startyear']),
int(yearrange[1] - yearchunk['startyear']))
# The area covered by a grid cell is calculated depending on the latitude
# 1 degree = 111.12km (at the equator); resolution data: 0.5 degree;
# longitudal distance in km = 111.12*0.5*cos(lat);
# latitudal distance in km = 111.12*0.5;
# area = longitudal distance * latitudal distance;
# 1km2 = 100ha
area = (111.12 * 0.5)**2 * np.cos(np.deg2rad(lat)) * 100
# The area covered by a crop is calculated as the product of the fraction and
# the grid cell size
if irr == 'combined':
irr_types = ['firr', 'noirr']
else:
irr_types = [irr]
area_crop = dict()
for irr_var in irr_types:
area_crop[irr_var] = (
getattr(
data, (CROP_NAME[crop])['input']+'_'+ (IRR_NAME[irr_var])['name']
)[time_idx[0]:time_idx[1], :, :].mean(dim='time')*area
).values
area_crop[irr_var] = np.nan_to_num(area_crop[irr_var]).flatten()
# set historic mean, its latitude, and longitude:
hist_mean_dict = dict()
# if hist_mean is given as np.ndarray or dict,
# code assumes it contains hist_mean as returned by relative_cropyield
# however structured in dictionary as hist_mean_dict, with same
# bbox extensions as the exposure:
if isinstance(hist_mean, dict):
if not ('firr' in hist_mean.keys() or 'noirr' in hist_mean.keys()):
# as a dict hist_mean, needs to contain key 'firr' or 'noirr';
# if irr=='combined', both 'firr' and 'noirr' are required.
LOGGER.error('Invalid hist_mean provided: %s', hist_mean)
raise ValueError('invalid hist_mean.')
hist_mean_dict = hist_mean
lat_mean = self.gdf.latitude.values
elif isinstance(hist_mean, np.ndarray) or isinstance(hist_mean, list):
hist_mean_dict[irr_types[0]] = np.array(hist_mean)
lat_mean = self.gdf.latitude.values
elif Path(hist_mean).is_dir(): # else if hist_mean is given as path to directory
# The adequate file from the directory (depending on crop and irrigation) is extracted
# and the variables hist_mean, lat_mean and lon_mean are set accordingly
for irr_var in irr_types:
filename = str(Path(hist_mean, 'hist_mean_%s-%s_%i-%i.hdf5' %(
crop, irr_var, yearrange[0], yearrange[1])))
hist_mean_dict[irr_var] = (h5py.File(filename, 'r'))['mean'][()]
lat_mean = (h5py.File(filename, 'r'))['lat'][()]
lon_mean = (h5py.File(filename, 'r'))['lon'][()]
elif Path(input_dir, hist_mean).is_file(): # file path
# Hist_mean, lat_mean and lon_mean are extracted from the given file
if len(irr_types) > 1:
LOGGER.error("For irr=='combined', hist_mean can not be single file. Aborting.")
raise ValueError('Wrong combination of parameters irr and hist_mean.')
hist_mean = h5py.File(str(Path(input_dir, hist_mean)), 'r')
hist_mean_dict[irr_types[0]] = hist_mean['mean'][()]
lat_mean = hist_mean['lat'][()]
lon_mean = hist_mean['lon'][()]
else:
LOGGER.error('Invalid hist_mean provided: %s', hist_mean)
raise ValueError('invalid hist_mean.')
# The bbox is cut out of the hist_mean data file if needed
if len(lat_mean) != len(self.gdf.latitude.values):
idx_mean = np.zeros(len(self.gdf.latitude.values), dtype=int)
for i in range(len(self.gdf.latitude.values)):
idx_mean[i] = np.where(
(lat_mean == self.gdf.latitude.values[i])
& (lon_mean == self.gdf.longitude.values[i])
)[0][0]
else:
idx_mean = np.arange(0, len(lat_mean))
# The exposure [t/y] is computed per grid cell as the product of the area covered
# by a crop [ha] and its yield [t/ha/y]
self.gdf['value'] = np.squeeze(area_crop[irr_types[0]] * \
hist_mean_dict[irr_types[0]][idx_mean])
self.gdf['value'] = np.nan_to_num(self.gdf.value) # replace NaN by 0.0
for irr_val in irr_types[1:]: # add other irrigation types if irr=='combined'
value_tmp = np.squeeze(area_crop[irr_val]*hist_mean_dict[irr_val][idx_mean])
value_tmp = np.nan_to_num(value_tmp) # replace NaN by 0.0
self.gdf['value'] += value_tmp
self.tag = Tag()
self.tag.description = ("Crop production exposure from ISIMIP " +
(CROP_NAME[crop])['print'] + ' ' +
irr + ' ' + str(yearrange[0]) + '-' + str(yearrange[-1]))
self.value_unit = 't/y' # input unit, will be reset below if required by user
self.crop = crop
self.ref_year = yearrange
try:
rows, cols, ras_trans = pts_to_raster_meta(
(self.gdf.longitude.min(), self.gdf.latitude.min(),
self.gdf.longitude.max(), self.gdf.latitude.max()),
get_resolution(self.gdf.longitude, self.gdf.latitude))
self.meta = {
'width': cols,
'height': rows,
'crs': self.crs,
'transform': ras_trans,
}
except ValueError:
LOGGER.warning('Could not write attribute meta, because exposure'
' has only 1 data point')
self.meta = {}
if 'USD' in unit:
# set_value_to_usd() is called to compute the exposure in USD/y (country specific)
self.set_value_to_usd(input_dir=input_dir)
elif 'kcal' in unit:
# set_value_to_kcal() is called to compute the exposure in kcal/y
self.set_value_to_kcal()
self.check()
return self
def clear(self):
"""Reinitialize attributes."""
self.tag = Tag()
# Following values are given for each defined year
self.years = np.array([], int)
self.rates = np.array([], float)
class DiscRates():
"""Defines discount rates and basic methods. Loads from
files with format defined in FILE_EXT.
Attributes:
tag (Tag): information about the source data
years (np.array): years
rates (np.array): discount rates for each year (between 0 and 1)
"""
def __init__(self):
"""Empty initialization.
Examples:
Fill discount rates with values and check consistency data:
>>> disc_rates = DiscRates()
>>> disc_rates.years = np.array([2000, 2001])
>>> disc_rates.rates = np.array([0.02, 0.02])
>>> disc_rates.check()
Read discount rates from year_2050.mat and checks consistency data.
>>> disc_rates = DiscRates(ENT_TEMPLATE_XLS)
"""
self.clear()
def clear(self):
"""Reinitialize attributes."""
self.tag = Tag()
# Following values are given for each defined year
self.years = np.array([], int)
self.rates = np.array([], float)
def check(self):
"""Check attributes consistency.
Raises:
ValueError
"""
check.size(len(self.years), self.rates, 'DiscRates.rates')
def select(self, year_range):
"""Select discount rates in given years.
Parameters:
year_range (np.array): continuous sequence of selected years.
Returns:
DiscRates
"""
pos_year = np.isin(year_range, self.years)
if not np.all(pos_year):
LOGGER.info('No discount rates for given years.')
return None
pos_year = np.isin(self.years, year_range)
sel_disc = self.__class__()
sel_disc.tag = self.tag
sel_disc.years = self.years[pos_year]
sel_disc.rates = self.rates[pos_year]
return sel_disc
def append(self, disc_rates):
"""Check and append discount rates to current DiscRates. Overwrite
discount rate if same year.
Parameters:
disc_rates (DiscRates): DiscRates instance to append
Raises:
ValueError
"""
disc_rates.check()
if self.years.size == 0:
self.__dict__ = copy.deepcopy(disc_rates.__dict__)
return
self.tag.append(disc_rates.tag)
new_year = array('l')
new_rate = array('d')
for year, rate in zip(disc_rates.years, disc_rates.rates):
found = np.where(year == self.years)[0]
if found.size > 0:
self.rates[found[0]] = rate
else:
new_year.append(year)
new_rate.append(rate)
self.years = np.append(self.years, new_year).astype(int, copy=False)
self.rates = np.append(self.rates, new_rate)
def net_present_value(self, ini_year, end_year, val_years):
"""Compute net present value between present year and future year.
Parameters:
ini_year (float): initial year
end_year (float): end year
val_years (np.array): cash flow at each year btw ini_year and
end_year (both included)
Returns:
float
"""
year_range = np.arange(ini_year, end_year + 1)
if year_range.size != val_years.size:
LOGGER.error('Wrong size of yearly values.')
raise ValueError
sel_disc = self.select(year_range)
if sel_disc is None:
LOGGER.error('No information of discount rates for provided years:'\
' %s - %s', ini_year, end_year)
raise ValueError
return u_fin.net_present_value(sel_disc.years, sel_disc.rates,
val_years)
def plot(self, axis=None, **kwargs):
"""Plot discount rates per year.
Parameters:
axis (matplotlib.axes._subplots.AxesSubplot, optional): axis to use
kwargs (optional): arguments for plot matplotlib function, e.g. marker='x'
Returns:
matplotlib.axes._subplots.AxesSubplot
"""
if not axis:
_, axis = plt.subplots(1, 1)
axis.set_title('Discount rates')
axis.set_xlabel('Year')
axis.set_ylabel('discount rate (%)')
axis.plot(self.years, self.rates * 100, **kwargs)
axis.set_xlim((self.years.min(), self.years.max()))
return axis
def read_mat(self, file_name, description='', var_names=DEF_VAR_MAT):
"""Read MATLAB file generated with previous MATLAB CLIMADA version.
Parameters:
file_name (str): absolute file name
description (str, optional): description of the data
var_names (dict, optional): name of the variables in the file
"""
disc = hdf5.read(file_name)
self.clear()
self.tag.file_name = file_name
self.tag.description = description
try:
disc = disc[var_names['sup_field_name']]
except KeyError:
pass
try:
disc = disc[var_names['field_name']]
self.years = np.squeeze(disc[var_names['var_name']['year']]). \
astype(int, copy=False)
self.rates = np.squeeze(disc[var_names['var_name']['disc']])
except KeyError as err:
LOGGER.error("Not existing variable: %s", str(err))
raise err
def read_excel(self, file_name, description='', var_names=DEF_VAR_EXCEL):
"""Read excel file following template and store variables.
Parameters:
file_name (str): absolute file name
description (str, optional): description of the data
var_names (dict, optional): name of the variables in the file
"""
dfr = pd.read_excel(file_name, var_names['sheet_name'])
self.clear()
self.tag.file_name = file_name
self.tag.description = description
try:
self.years = dfr[var_names['col_name']['year']].values. \
astype(int, copy=False)
self.rates = dfr[var_names['col_name']['disc']].values
except KeyError as err:
LOGGER.error("Not existing variable: %s", str(err))
raise err
def write_excel(self, file_name, var_names=DEF_VAR_EXCEL):
""" Write excel file following template.
Parameters:
file_name (str): absolute file name to write
var_names (dict, optional): name of the variables in the file
"""
disc_wb = xlsxwriter.Workbook(file_name)
disc_ws = disc_wb.add_worksheet(var_names['sheet_name'])
header = [var_names['col_name']['year'], var_names['col_name']['disc']]
for icol, head_dat in enumerate(header):
disc_ws.write(0, icol, head_dat)
for i_yr, (disc_yr, disc_rt) in enumerate(zip(self.years, self.rates),
1):
disc_ws.write(i_yr, 0, disc_yr)
disc_ws.write(i_yr, 1, disc_rt)
disc_wb.close()
def init_spam_agrar(self, **parameters):
"""initiates agriculture exposure from SPAM data:
https://dataverse.harvard.edu/
dataset.xhtml?persistentId=doi:10.7910/DVN/DHXBJX
Optional parameters:
data_path (str): absolute path where files are stored.
Default: SYSTEM_DIR
country (str): Three letter country code of country to be cut out.
No default (global)
name_adm1 (str): Name of admin1 (e.g. Federal State) to be cut out.
No default
name_adm2 (str): Name of admin2 to be cut out.
No default
spam_variable (str): select one agricultural variable:
'A' physical area
'H' harvested area
'P' production
'Y' yield
'V_agg' value of production, aggregated to all crops,
food and non-food (default)
Warning: for A, H, P and Y, currently all crops are summed up
spam_technology (str): select one agricultural technology type:
'TA' all technologies together, ie complete crop (default)
'TI' irrigated portion of crop
'TH' rainfed high inputs portion of crop
'TL' rainfed low inputs portion of crop
'TS' rainfed subsistence portion of crop
'TR' rainfed portion of crop (= TA - TI, or TH + TL + TS)
! different impact_ids are assigned to each technology (1-6)
save_name_adm1 (Boolean): Determines how many aditional data are saved:
False: only basics (lat, lon, total value), region_id per country
True: like 1 + name of admin1
haz_type (str): hazard type abbreviation, e.g.
'DR' for Drought or
'CP' for CropPotential
Returns:
"""
data_p = parameters.get('data_path', SYSTEM_DIR)
spam_t = parameters.get('spam_technology', 'TA')
spam_v = parameters.get('spam_variable', 'V_agg')
adm0 = parameters.get('country')
adm1 = parameters.get('name_adm1')
adm2 = parameters.get('name_adm2')
save_adm1 = parameters.get('save_name_adm1', False)
haz_type = parameters.get('haz_type', DEF_HAZ_TYPE)
# Test if parameters make sense:
if spam_v not in ['A', 'H', 'P', 'Y', 'V_agg'] or \
spam_t not in ['TA', 'TI', 'TH', 'TL', 'TS', 'TR']:
LOGGER.error('Invalid input parameter(s).')
raise ValueError('Invalid input parameter(s).')
# read data from CSV:
data = self._read_spam_file(data_path=data_p, spam_technology=spam_t,
spam_variable=spam_v, result_mode=1)
# extract country or admin level (if provided)
data, region = self._spam_set_country(data, country=adm0,
name_adm1=adm1, name_adm2=adm2)
# sort by alloc_key to make extraction of lat / lon easier:
data = data.sort_values(by=['alloc_key'])
lat, lon = self._spam_get_coordinates(data.loc[:, 'alloc_key'],
data_path=data_p)
if save_adm1:
self.name_adm1 = data.loc[:, 'name_adm1'].values
if spam_v == 'V_agg': # total only (column 7)
i_1 = 7
i_2 = 8
else:
i_1 = 7 # get sum over all crops (columns 7 to 48)
i_2 = 49
self.gdf['value'] = data.iloc[:, i_1:i_2].sum(axis=1).values
self.gdf['latitude'] = lat.values
self.gdf['longitude'] = lon.values
LOGGER.info('Lat. range: {:+.3f} to {:+.3f}.'.format(
np.min(self.gdf.latitude), np.max(self.gdf.latitude)))
LOGGER.info('Lon. range: {:+.3f} to {:+.3f}.'.format(
np.min(self.gdf.longitude), np.max(self.gdf.longitude)))
# set region_id (numeric ISO3):
country_id = data.loc[:, 'iso3']
if country_id.unique().size == 1:
region_id = np.ones(self.gdf.value.size, int)\
* int(iso_cntry.get(country_id.iloc[0]).numeric)
else:
region_id = np.zeros(self.gdf.value.size, int)
for i in range(0, self.gdf.value.size):
region_id[i] = int(iso_cntry.get(country_id.iloc[i]).numeric)
self.gdf['region_id'] = region_id
self.ref_year = 2005
self.tag = Tag()
self.tag.description = ("SPAM agrar exposure for variable "
+ spam_v + " and technology " + spam_t)
# if impact id variation iiv = 1, assign different damage function ID
# per technology type.
self._set_if(spam_t, haz_type)
self.tag.file_name = (FILENAME_SPAM + '_' + spam_v + '_' + spam_t + '.csv')
# self.tag.shape = cntry_info[2]
#self.tag.country = cntry_info[1]
if spam_v in ('A', 'H'):
self.value_unit = 'Ha'
elif spam_v == 'Y':
self.value_unit = 'kg/Ha'
elif spam_v == 'P':
self.value_unit = 'mt'
else:
self.value_unit = 'USD'
LOGGER.info('Total {} {} {}: {:.1f} {}.'.format(
spam_v, spam_t, region, self.gdf.value.sum(), self.value_unit))
self.check()
def set_countries(self,
countries,
ref_year=2016,
res_km=None,
from_hr=None,
admin_file='admin_0_countries',
**kwargs):
""" Model countries using values at reference year. If GDP or income
group not available for that year, consider the value of the closest
available year.
Parameters:
countries (list or dict): list of country names (admin0 or subunits)
or dict with key = admin0 name and value = [admin1 names]
ref_year (int, optional): reference year. Default: 2016
res_km (float, optional): approx resolution in km. Default:
nightlights resolution.
from_hr (bool, optional): force to use higher resolution image,
independently of its year of acquisition.
admin_file (str): file name, admin_0_countries or admin_0_map_subunits
kwargs (optional): 'gdp' and 'inc_grp' dictionaries with keys the
country ISO_alpha3 code. 'poly_val' list of polynomial coefficients
[1,x,x^2,...] to apply to nightlight (DEF_POLY_VAL used if not
provided). If provided, these are used.
"""
admin_key_dict = {
'admin_0_countries': ['ADMIN', 'ADM0_A3'],
'admin_0_map_subunits': ['SUBUNIT', 'SU_A3']
}
shp_file = shapereader.natural_earth(resolution='10m',
category='cultural',
name=admin_file)
shp_file = shapereader.Reader(shp_file)
cntry_info, cntry_admin1 = country_iso_geom(countries, shp_file,
admin_key_dict[admin_file])
fill_econ_indicators(ref_year, cntry_info, shp_file, **kwargs)
nightlight, coord_nl, fn_nl, res_fact, res_km = get_nightlight(
ref_year, cntry_info, res_km, from_hr)
tag = Tag(file_name=fn_nl)
bkmrbl_list = []
for cntry_iso, cntry_val in cntry_info.items():
bkmrbl_list.append(
self._set_one_country(cntry_val, nightlight, coord_nl,
res_fact, res_km,
cntry_admin1[cntry_iso], **kwargs).gdf)
tag.description += ("{} {:d} GDP: {:.3e} income group: {:d} \n").\
format(cntry_val[1], cntry_val[3], cntry_val[4], cntry_val[5])
Exposures.__init__(self,
data=Exposures.concat(bkmrbl_list).gdf,
crs=DEF_CRS,
ref_year=ref_year,
tag=tag,
value_unit='USD')
rows, cols, ras_trans = pts_to_raster_meta(
(self.gdf.longitude.min(), self.gdf.latitude.min(),
self.gdf.longitude.max(), self.gdf.latitude.max()),
(coord_nl[0, 1], -coord_nl[0, 1]))
self.meta = {
'width': cols,
'height': rows,
'crs': self.crs,
'transform': ras_trans
}
def emdat_to_impact(emdat_file_csv,
hazard_type_climada,
year_range=None,
countries=None,
hazard_type_emdat=None,
reference_year=None,
imp_str="Total Damages"):
"""function to load EM-DAT data return impact per event
Parameters:
emdat_file_csv (str): Full path to EMDAT-file (CSV), i.e.:
emdat_file_csv = SYSTEM_DIR.joinpath('emdat_201810.csv')
hazard_type_climada (str): Hazard type CLIMADA abbreviation,
i.e. 'TC' for tropical cyclone
Optional parameters:
hazard_type_emdat (list or str): List of Disaster (sub-)type accordung
EMDAT terminology, e.g.:
Animal accident, Drought, Earthquake, Epidemic, Extreme temperature,
Flood, Fog, Impact, Insect infestation, Landslide, Mass movement (dry),
Storm, Volcanic activity, Wildfire;
Coastal Flooding, Convective Storm, Riverine Flood, Tropical cyclone,
Tsunami, etc.;
OR CLIMADA hazard type abbreviations, e.g. TC, BF, etc.
If not given, it is deducted from hazard_type_climada
year_range (list with 2 integers): start and end year e.g. [1980, 2017]
default: None --> take year range from EM-DAT file
countries (list of str): country ISO3-codes or names, e.g. ['JAM'].
Set to None or ['all'] for all countries (default)
reference_year (int): reference year of exposures. Impact is scaled
proportional to GDP to the value of the reference year. No scaling
for reference_year=0 (default)
imp_str (str): Column name of impact metric in EMDAT CSV,
default = "Total Damages ('000 US$)"
Returns:
impact_instance (instance of climada.engine.Impact):
impact object of same format as output from CLIMADA
impact computation.
Values scaled with GDP to reference_year if reference_year is given.
i.e. current US$ for imp_str="Total Damages ('000 US$) scaled" (factor 1000 is applied)
impact_instance.eai_exp holds expected annual impact for each country.
impact_instance.coord_exp holds rough central coordinates for each country.
countries (list): ISO3-codes of countries in same order as in impact_instance.eai_exp
"""
if "Total Damages" in imp_str:
imp_str = "Total Damages ('000 US$)"
elif "Insured Damages" in imp_str:
imp_str = "Insured Damages ('000 US$)"
elif "Reconstruction Costs" in imp_str:
imp_str = "Reconstruction Costs ('000 US$)"
imp_str = VARNAMES_EMDAT[max(VARNAMES_EMDAT.keys())][imp_str]
if not hazard_type_emdat:
hazard_type_emdat = [hazard_type_climada]
if reference_year == 0:
reference_year = None
# Inititate Impact-instance:
impact_instance = Impact()
impact_instance.tag = dict()
impact_instance.tag['haz'] = TagHaz(
haz_type=hazard_type_climada,
file_name=emdat_file_csv,
description='EM-DAT impact, direct import')
impact_instance.tag['exp'] = Tag(
file_name=emdat_file_csv, description='EM-DAT impact, direct import')
impact_instance.tag['if_set'] = Tag(file_name=None, description=None)
# Load EM-DAT impact data by event:
em_data = emdat_impact_event(emdat_file_csv,
countries=countries,
hazard=hazard_type_emdat,
year_range=year_range,
reference_year=reference_year,
imp_str=imp_str,
version=max(VARNAMES_EMDAT.keys()))
if isinstance(countries, str):
countries = [countries]
elif not countries:
countries = emdat_countries_by_hazard(emdat_file_csv,
year_range=year_range,
hazard=hazard_type_emdat)[0]
if em_data.empty:
return impact_instance, countries
impact_instance.event_id = np.array(em_data.index, int)
impact_instance.event_name = list(em_data[VARNAMES_EMDAT[max(
VARNAMES_EMDAT.keys())]['Dis No']])
date_list = list()
for year in list(em_data['Year']):
date_list.append(datetime.toordinal(datetime.strptime(str(year),
'%Y')))
if 'Start Year' in em_data.columns and 'Start Month' in em_data.columns \
and 'Start Day' in em_data.columns:
idx = 0
for year, month, day in zip(em_data['Start Year'],
em_data['Start Month'],
em_data['Start Day']):
if np.isnan(year):
idx += 1
continue
if np.isnan(month):
month = 1
if np.isnan(day):
day = 1
date_list[idx] = datetime.toordinal(
datetime.strptime('%02i/%02i/%04i' % (day, month, year),
'%d/%m/%Y'))
idx += 1
impact_instance.date = np.array(date_list, int)
impact_instance.crs = DEF_CRS
if not reference_year:
impact_instance.at_event = np.array(em_data["impact"])
else:
impact_instance.at_event = np.array(em_data["impact_scaled"])
impact_instance.at_event[np.isnan(impact_instance.at_event)] = 0
if not year_range:
year_range = [em_data['Year'].min(), em_data['Year'].max()]
impact_instance.frequency = np.ones(
em_data.shape[0]) / (1 + np.diff(year_range))
impact_instance.tot_value = 0
impact_instance.aai_agg = np.nansum(impact_instance.at_event *
impact_instance.frequency)
impact_instance.unit = 'USD'
impact_instance.imp_mat = []
# init rough exposure with central point per country
shp = shapereader.natural_earth(resolution='110m',
category='cultural',
name='admin_0_countries')
shp = shapefile.Reader(shp)
countries_reg_id = list()
countries_lat = list()
countries_lon = list()
impact_instance.eai_exp = np.zeros(
len(countries)) # empty: damage at exposure
for idx, cntry in enumerate(countries):
try:
cntry = iso_cntry.get(cntry).alpha3
except KeyError:
print(cntry)
LOGGER.error('Country not found in iso_country: %s', cntry)
cntry_boolean = False
for rec_i, rec in enumerate(shp.records()):
if rec[9].casefold() == cntry.casefold():
bbox = shp.shapes()[rec_i].bbox
cntry_boolean = True
break
if cntry_boolean:
countries_lat.append(np.mean([bbox[1], bbox[3]]))
countries_lon.append(np.mean([bbox[0], bbox[2]]))
else:
countries_lat.append(np.nan)
countries_lon.append(np.nan)
try:
countries_reg_id.append(int(iso_cntry.get(cntry).numeric))
except KeyError:
countries_reg_id.append(0)
df_tmp = em_data[em_data[VARNAMES_EMDAT[max(
VARNAMES_EMDAT.keys())]['ISO']].str.contains(cntry)]
if not reference_year:
impact_instance.eai_exp[idx] = sum(
np.array(df_tmp["impact"]) * impact_instance.frequency[0])
else:
impact_instance.eai_exp[idx] = sum(
np.array(df_tmp["impact_scaled"]) *
impact_instance.frequency[0])
impact_instance.coord_exp = np.stack([countries_lat, countries_lon],
axis=1)
return impact_instance, countries
def emdat_to_impact(emdat_file_csv, year_range=None, countries=None,\
hazard_type_emdat=None, hazard_type_climada=None, \
reference_year=0, imp_str="Total damage ('000 US$)"):
"""function to load EM-DAT data return impact per event
Parameters:
emdat_file_csv (str): Full path to EMDAT-file (CSV), i.e.:
emdat_file_csv = os.path.join(SYSTEM_DIR, 'emdat_201810.csv')
hazard_type_emdat (str): Hazard (sub-)type according to EMDAT terminology,
i.e. 'Tropical cyclone' for tropical cyclone
OR
hazard_type_climada (str): Hazard type CLIMADA abbreviation,
i.e. 'TC' for tropical cyclone
Optional parameters:
year_range (list with 2 integers): start and end year i.e. [1980, 2017]
default: None --> take year range from EM-DAT file
countries (list of str): country ISO3-codes or names, i.e. ['JAM'].
Set to None or ['all'] for all countries (default)
reference_year (int): reference year of exposures. Impact is scaled
proportional to GDP to the value of the reference year. No scaling
for reference_year=0 (default)
imp_str (str): Column name of impact metric in EMDAT CSV,
default = "Total damage ('000 US$)"
Returns:
impact_instance (instance of climada.engine.Impact):
impact object of same format as output from CLIMADA
impact computation
scaled with GDP to reference_year if reference_year noit equal 0
i.e. 1000 current US$ for imp_str="Total damage ('000 US$) scaled".
impact_instance.eai_exp holds expected annual impact for each country.
impact_instance.coord_exp holds rough central coordinates for each country.
countries (list): ISO3-codes of countries imn same order as in impact_instance.eai_exp
"""
# Mapping of hazard type between EM-DAT and CLIMADA:
if not hazard_type_climada:
if not hazard_type_emdat:
LOGGER.error(
'Either hazard_type_climada or hazard_type_emdat need to be defined.'
)
return None
if hazard_type_emdat == 'Tropical cyclone':
hazard_type_climada = 'TC'
elif hazard_type_emdat == 'Drought':
hazard_type_climada = 'DR'
elif hazard_type_emdat == 'Landslide':
hazard_type_climada = 'LS'
elif hazard_type_emdat == 'Riverine flood':
hazard_type_climada = 'RF'
elif hazard_type_emdat in [
'Wildfire', 'Forest Fire', 'Land fire (Brush, Bush, Pasture)'
]:
hazard_type_climada = 'BF'
elif hazard_type_emdat == 'Extra-tropical storm':
hazard_type_climada = 'WS'
elif not hazard_type_emdat:
if hazard_type_climada == 'TC':
hazard_type_emdat = 'Tropical cyclone'
elif hazard_type_climada == 'DR':
hazard_type_emdat = 'Drought'
elif hazard_type_climada == 'LS':
hazard_type_emdat = 'Landslide'
elif hazard_type_climada == 'RF':
hazard_type_emdat = 'Riverine flood'
elif hazard_type_climada == 'BF':
hazard_type_emdat = 'Wildfire'
elif hazard_type_climada == 'WS':
hazard_type_emdat = 'Extra-tropical storm'
# Inititate Impact-instance:
impact_instance = Impact()
impact_instance.tag = dict()
impact_instance.tag['haz'] = TagHaz(haz_type=hazard_type_climada, \
file_name=emdat_file_csv, description='EM-DAT impact, direct import')
impact_instance.tag['exp'] = Tag(file_name=emdat_file_csv, \
description='EM-DAT impact, direct import')
impact_instance.tag['if_set'] = Tag(file_name=None, description=None)
if not countries or countries == ['all']:
countries = emdat_countries_by_hazard(hazard_type_emdat, emdat_file_csv, \
ignore_missing=True, verbose=True)[0]
else:
if isinstance(countries, str):
countries = [countries]
# Load EM-DAT impact data by event:
em_data = emdat_impact_event(countries, hazard_type_emdat, emdat_file_csv, \
year_range, reference_year=reference_year)
if em_data.empty:
return impact_instance, countries
impact_instance.event_id = np.array(em_data.index, int)
impact_instance.event_name = list(em_data['Disaster No.'])
date_list = list()
for year in list(em_data['year']):
date_list.append(datetime.toordinal(datetime.strptime(str(year),
'%Y')))
boolean_warning = True
for idx, datestr in enumerate(list(em_data['Start date'])):
try:
date_list[idx] = datetime.toordinal(
datetime.strptime(datestr[-7:], '%m/%Y'))
except ValueError:
if boolean_warning:
LOGGER.warning('EM_DAT CSV contains invalid time formats')
boolean_warning = False
try:
date_list[idx] = datetime.toordinal(
datetime.strptime(datestr, '%d/%m/%Y'))
except ValueError:
if boolean_warning:
LOGGER.warning('EM_DAT CSV contains invalid time formats')
boolean_warning = False
impact_instance.date = np.array(date_list, int)
impact_instance.crs = DEF_CRS
if reference_year == 0:
impact_instance.at_event = np.array(em_data[imp_str])
else:
impact_instance.at_event = np.array(em_data[imp_str + " scaled"])
if not year_range:
year_range = [em_data['year'].min(), em_data['year'].max()]
impact_instance.frequency = np.ones(
em_data.shape[0]) / (1 + np.diff(year_range))
impact_instance.tot_value = 0
impact_instance.aai_agg = sum(impact_instance.at_event *
impact_instance.frequency)
impact_instance.unit = 'USD'
impact_instance.imp_mat = []
# init rough exposure with central point per country
shp = shapereader.natural_earth(resolution='110m',
category='cultural',
name='admin_0_countries')
shp = shapefile.Reader(shp)
countries_reg_id = list()
countries_lat = list()
countries_lon = list()
impact_instance.eai_exp = np.zeros(
len(countries)) # empty: damage at exposure
for idx, cntry in enumerate(countries):
try:
cntry = iso_cntry.get(cntry).alpha3
except KeyError:
LOGGER.error('Country not found in iso_country: ' + cntry)
cntry_boolean = False
for rec_i, rec in enumerate(shp.records()):
if rec[9].casefold() == cntry.casefold():
bbox = shp.shapes()[rec_i].bbox
cntry_boolean = True
break
if cntry_boolean:
countries_lat.append(np.mean([bbox[1], bbox[3]]))
countries_lon.append(np.mean([bbox[0], bbox[2]]))
else:
countries_lat.append(np.nan)
countries_lon.append(np.nan)
try:
countries_reg_id.append(int(iso_cntry.get(cntry).numeric))
except KeyError:
countries_reg_id.append(0)
df_tmp = em_data[em_data['ISO'].str.contains(cntry)]
if reference_year == 0:
impact_instance.eai_exp[idx] = sum(np.array(df_tmp[imp_str])*\
impact_instance.frequency[0])
else:
impact_instance.eai_exp[idx] = sum(np.array(df_tmp[imp_str + " scaled"])*\
impact_instance.frequency[0])
impact_instance.coord_exp = np.stack([countries_lat, countries_lon],
axis=1)
#impact_instance.plot_raster_eai_exposure()
return impact_instance, countries
class MeasureSet():
"""Contains measures of type Measure. Loads from
files with format defined in FILE_EXT.
Attributes:
tag (Tag): information about the source data
_data (dict): cotains Measure classes. It's not suppossed to be
directly accessed. Use the class methods instead.
"""
def __init__(self):
"""Empty initialization.
Examples:
Fill MeasureSet with values and check consistency data:
>>> act_1 = Measure()
>>> act_1.name = 'Seawall'
>>> act_1.color_rgb = np.array([0.1529, 0.2510, 0.5451])
>>> act_1.hazard_intensity = (1, 0)
>>> act_1.mdd_impact = (1, 0)
>>> act_1.paa_impact = (1, 0)
>>> meas = MeasureSet()
>>> meas.append(act_1)
>>> meas.tag.description = "my dummy MeasureSet."
>>> meas.check()
Read measures from file and checks consistency data:
>>> meas = MeasureSet()
>>> meas.read_excel(ENT_TEMPLATE_XLS)
"""
self.clear()
def clear(self):
"""Reinitialize attributes."""
self.tag = Tag()
self._data = dict() # {hazard_type : {name: Measure()}}
def append(self, meas):
"""Append an Measure. Override if same name and haz_type.
Parameters:
meas (Measure): Measure instance
Raises:
ValueError
"""
if not isinstance(meas, Measure):
LOGGER.error("Input value is not of type Measure.")
raise ValueError
if not meas.haz_type:
LOGGER.warning("Input Measure's hazard type not set.")
if not meas.name:
LOGGER.warning("Input Measure's name not set.")
if meas.haz_type not in self._data:
self._data[meas.haz_type] = dict()
self._data[meas.haz_type][meas.name] = meas
def remove_measure(self, haz_type=None, name=None):
"""Remove impact function(s) with provided hazard type and/or id.
If no input provided, all impact functions are removed.
Parameters:
haz_type (str, optional): all impact functions with this hazard
name (str, optional): measure name
"""
if (haz_type is not None) and (name is not None):
try:
del self._data[haz_type][name]
except KeyError:
LOGGER.info("No Measure with hazard %s and id %s.", haz_type,
name)
elif haz_type is not None:
try:
del self._data[haz_type]
except KeyError:
LOGGER.info("No Measure with hazard %s.", haz_type)
elif name is not None:
haz_remove = self.get_hazard_types(name)
if not haz_remove:
LOGGER.info("No Measure with name %s.", name)
for haz in haz_remove:
del self._data[haz][name]
else:
self._data = dict()
def get_measure(self, haz_type=None, name=None):
"""Get ImpactFunc(s) of input hazard type and/or id.
If no input provided, all impact functions are returned.
Parameters:
haz_type (str, optional): hazard type
name (str, optional): measure name
Returns:
Measure (if haz_type and name),
list(Measure) (if haz_type or name),
{Measure.haz_type: {Measure.name : Measure}} (if None)
"""
if (haz_type is not None) and (name is not None):
try:
return self._data[haz_type][name]
except KeyError:
LOGGER.info("No Measure with hazard %s and id %s.", haz_type,
name)
return list()
elif haz_type is not None:
try:
return list(self._data[haz_type].values())
except KeyError:
LOGGER.info("No Measure with hazard %s.", haz_type)
return list()
elif name is not None:
haz_return = self.get_hazard_types(name)
if not haz_return:
LOGGER.info("No Measure with name %s.", name)
meas_return = []
for haz in haz_return:
meas_return.append(self._data[haz][name])
return meas_return
else:
return self._data
def get_hazard_types(self, meas=None):
"""Get measures hazard types contained for the name provided.
Return all hazard types if no input name.
Parameters:
name (str, optional): measure name
Returns:
list(str)
"""
if meas is None:
return list(self._data.keys())
haz_return = []
for haz, haz_dict in self._data.items():
if meas in haz_dict:
haz_return.append(haz)
return haz_return
def get_names(self, haz_type=None):
"""Get measures names contained for the hazard type provided.
Return all names for each hazard type if no input hazard type.
Parameters:
haz_type (str, optional): hazard type from which to obtain the names
Returns:
list(Measure.name) (if haz_type provided),
{Measure.haz_type : list(Measure.name)} (if no haz_type)
"""
if haz_type is None:
out_dict = dict()
for haz, haz_dict in self._data.items():
out_dict[haz] = list(haz_dict.keys())
return out_dict
try:
return list(self._data[haz_type].keys())
except KeyError:
LOGGER.info("No Measure with hazard %s.", haz_type)
return list()
def size(self, haz_type=None, name=None):
"""Get number of measures contained with input hazard type and
/or id. If no input provided, get total number of impact functions.
Parameters:
haz_type (str, optional): hazard type
name (str, optional): measure name
Returns:
int
"""
if (haz_type is not None) and (name is not None) and \
(isinstance(self.get_measure(haz_type, name), Measure)):
return 1
if (haz_type is not None) or (name is not None):
return len(self.get_measure(haz_type, name))
return sum(len(meas_list) for meas_list in self.get_names().values())
def check(self):
"""Check instance attributes.
Raises:
ValueError
"""
for key_haz, meas_dict in self._data.items():
def_color = plt.cm.get_cmap('Greys', len(meas_dict))
for i_meas, (name, meas) in enumerate(meas_dict.items()):
if (name != meas.name) | (name == ''):
LOGGER.error("Wrong Measure.name: %s != %s.", name,
meas.name)
raise ValueError
if key_haz != meas.haz_type:
LOGGER.error("Wrong Measure.haz_type: %s != %s.", key_haz,
meas.haz_type)
raise ValueError
# set default color if not set
if np.array_equal(meas.color_rgb, np.zeros(3)):
meas.color_rgb = def_color(i_meas)
meas.check()
def extend(self, meas_set):
"""Extend measures of input MeasureSet to current
MeasureSet. Overwrite Measure if same name and haz_type.
Parameters:
impact_funcs (MeasureSet): ImpactFuncSet instance to extend
Raises:
ValueError
"""
meas_set.check()
if self.size() == 0:
self.__dict__ = copy.deepcopy(meas_set.__dict__)
return
self.tag.append(meas_set.tag)
new_func = meas_set.get_measure()
for _, meas_dict in new_func.items():
for _, meas in meas_dict.items():
self.append(meas)
def read_mat(self, file_name, description='', var_names=DEF_VAR_MAT):
"""Read MATLAB file generated with previous MATLAB CLIMADA version.
Parameters:
file_name (str): absolute file name
description (str, optional): description of the data
var_names (dict, optional): name of the variables in the file
"""
def read_att_mat(measures, data, file_name, var_names):
"""Read MATLAB measures attributes"""
num_mes = len(data[var_names['var_name']['name']])
for idx in range(0, num_mes):
meas = Measure()
meas.name = hdf5.get_str_from_ref(
file_name, data[var_names['var_name']['name']][idx][0])
color_str = hdf5.get_str_from_ref(
file_name, data[var_names['var_name']['color']][idx][0])
meas.color_rgb = np.fromstring(color_str, dtype=float, sep=' ')
meas.cost = data[var_names['var_name']['cost']][idx][0]
meas.haz_type = hdf5.get_str_from_ref(
file_name, data[var_names['var_name']['haz']][idx][0])
meas.hazard_freq_cutoff = data[var_names['var_name']
['haz_frq']][idx][0]
meas.hazard_set = hdf5.get_str_from_ref(
file_name, data[var_names['var_name']['haz_set']][idx][0])
try:
meas.hazard_inten_imp = (
data[var_names['var_name']['haz_int_a']][idx][0],
data[var_names['var_name']['haz_int_b']][0][idx])
except KeyError:
meas.hazard_inten_imp = (
data[var_names['var_name']['haz_int_a'][:-2]][idx][0],
0)
# different convention of signes followed in MATLAB!
meas.mdd_impact = (
data[var_names['var_name']['mdd_a']][idx][0],
data[var_names['var_name']['mdd_b']][idx][0])
meas.paa_impact = (
data[var_names['var_name']['paa_a']][idx][0],
data[var_names['var_name']['paa_b']][idx][0])
meas.imp_fun_map = hdf5.get_str_from_ref(
file_name, data[var_names['var_name']['fun_map']][idx][0])
meas.exposures_set = hdf5.get_str_from_ref(
file_name, data[var_names['var_name']['exp_set']][idx][0])
exp_region_id = data[var_names['var_name']['exp_reg']][idx][0]
if exp_region_id:
meas.exp_region_id = [exp_region_id]
meas.risk_transf_attach = data[var_names['var_name']
['risk_att']][idx][0]
meas.risk_transf_cover = data[var_names['var_name']
['risk_cov']][idx][0]
measures.append(meas)
data = hdf5.read(file_name)
self.clear()
self.tag.file_name = file_name
self.tag.description = description
try:
data = data[var_names['sup_field_name']]
except KeyError:
pass
try:
data = data[var_names['field_name']]
read_att_mat(self, data, file_name, var_names)
except KeyError as var_err:
LOGGER.error("Not existing variable %s", str(var_err))
raise var_err
def read_excel(self, file_name, description='', var_names=DEF_VAR_EXCEL):
"""Read excel file following template and store variables.
Parameters:
file_name (str): absolute file name
description (str, optional): description of the data
var_names (dict, optional): name of the variables in the file
"""
def read_att_excel(measures, dfr, var_names):
"""Read Excel measures attributes"""
num_mes = len(dfr.index)
for idx in range(0, num_mes):
meas = Measure()
meas.name = dfr[var_names['col_name']['name']][idx]
try:
meas.haz_type = dfr[var_names['col_name']['haz']][idx]
except KeyError:
pass
meas.color_rgb = np.fromstring(
dfr[var_names['col_name']['color']][idx],
dtype=float,
sep=' ')
meas.cost = dfr[var_names['col_name']['cost']][idx]
meas.hazard_freq_cutoff = dfr[var_names['col_name']
['haz_frq']][idx]
meas.hazard_set = dfr[var_names['col_name']['haz_set']][idx]
# Search for (a, b) values, put a = 1 otherwise
try:
meas.hazard_inten_imp = (
dfr[var_names['col_name']['haz_int_a']][idx],
dfr[var_names['col_name']['haz_int_b']][idx])
except KeyError:
meas.hazard_inten_imp = (
1, dfr['hazard intensity impact'][idx])
try:
meas.exposures_set = dfr[var_names['col_name']
['exp_set']][idx]
meas.exp_region_id = ast.literal_eval(
dfr[var_names['col_name']['exp_reg']][idx])
except KeyError:
pass
except ValueError:
meas.exp_region_id = dfr[var_names['col_name']
['exp_reg']][idx]
meas.mdd_impact = (dfr[var_names['col_name']['mdd_a']][idx],
dfr[var_names['col_name']['mdd_b']][idx])
meas.paa_impact = (dfr[var_names['col_name']['paa_a']][idx],
dfr[var_names['col_name']['paa_b']][idx])
meas.imp_fun_map = dfr[var_names['col_name']['fun_map']][idx]
meas.risk_transf_attach = dfr[var_names['col_name']
['risk_att']][idx]
meas.risk_transf_cover = dfr[var_names['col_name']
['risk_cov']][idx]
try:
meas.risk_transf_cost_factor = dfr[var_names['col_name']
['risk_fact']][idx]
except KeyError:
pass
measures.append(meas)
dfr = pd.read_excel(file_name, var_names['sheet_name'])
dfr = dfr.fillna('')
self.clear()
self.tag.file_name = file_name
self.tag.description = description
try:
read_att_excel(self, dfr, var_names)
except KeyError as var_err:
LOGGER.error("Not existing variable: %s", str(var_err))
raise var_err
def write_excel(self, file_name, var_names=DEF_VAR_EXCEL):
"""Write excel file following template.
Parameters:
file_name (str): absolute file name to write
var_names (dict, optional): name of the variables in the file
"""
def write_meas(row_ini, imp_ws, xls_data):
"""Write one measure"""
for icol, col_dat in enumerate(xls_data):
imp_ws.write(row_ini, icol, col_dat)
meas_wb = xlsxwriter.Workbook(file_name)
mead_ws = meas_wb.add_worksheet(var_names['sheet_name'])
header = [
var_names['col_name']['name'], var_names['col_name']['color'],
var_names['col_name']['cost'], var_names['col_name']['haz_int_a'],
var_names['col_name']['haz_int_b'],
var_names['col_name']['haz_frq'], var_names['col_name']['haz_set'],
var_names['col_name']['mdd_a'], var_names['col_name']['mdd_b'],
var_names['col_name']['paa_a'], var_names['col_name']['paa_b'],
var_names['col_name']['fun_map'], var_names['col_name']['exp_set'],
var_names['col_name']['exp_reg'],
var_names['col_name']['risk_att'],
var_names['col_name']['risk_cov'], var_names['col_name']['haz']
]
for icol, head_dat in enumerate(header):
mead_ws.write(0, icol, head_dat)
for row_ini, (_, haz_dict) in enumerate(self._data.items(), 1):
for meas_name, meas in haz_dict.items():
xls_data = [
meas_name, ' '.join(list(map(str,
meas.color_rgb))), meas.cost,
meas.hazard_inten_imp[0], meas.hazard_inten_imp[1],
meas.hazard_freq_cutoff, meas.hazard_set,
meas.mdd_impact[0], meas.mdd_impact[1], meas.paa_impact[0],
meas.paa_impact[1], meas.imp_fun_map, meas.exposures_set,
str(meas.exp_region_id), meas.risk_transf_attach,
meas.risk_transf_cover, meas.haz_type
]
write_meas(row_ini, mead_ws, xls_data)
meas_wb.close()
class DiscRates():
"""
Defines discount rates and basic methods. Loads from
files with format defined in FILE_EXT.
Attributes
---------
tag: Tag
information about the source data
years: np.array
list of years
rates: np.array
list of discount rates for each year (between 0 and 1)
"""
def __init__(self, years=None, rates=None, tag=None):
"""
Fill discount rates with values and check consistency data
Parameters
----------
years : numpy.ndarray(int)
Array of years. Default is numpy.array([]).
rates : numpy.ndarray(float)
Discount rates for each year in years.
Default is numpy.array([]).
Note: rates given in float, e.g., to set 1% rate use 0.01
tag : climate.entity.tag
Metadata. Default is None.
"""
years = np.array([]) if years is None else years
rates = np.array([]) if rates is None else rates
self.years = years
self.rates = rates
tag = Tag() if tag is None else tag
self.tag = tag
def clear(self):
"""Reinitialize attributes."""
self.tag = Tag()
# Following values are given for each defined year
self.years = np.array([], int)
self.rates = np.array([], float)
def check(self):
"""
Check attributes consistency.
Raises
------
ValueError
"""
u_check.size(len(self.years), self.rates, 'DiscRates.rates')
def select(self, year_range):
"""
Select discount rates in given years.
Parameters
----------
year_range: np.array(int)
continuous sequence of selected years.
Returns: climada.entity.DiscRates
The selected discrates in the year_range
"""
pos_year = np.isin(year_range, self.years)
if not np.all(pos_year):
LOGGER.info('No discount rates for given years.')
return None
pos_year = np.isin(self.years, year_range)
return DiscRates(years=self.years[pos_year],
rates=self.rates[pos_year],
tag=self.tag)
def append(self, disc_rates):
"""
Check and append discount rates to current DiscRates. Overwrite
discount rate if same year.
Parameters
----------
disc_rates: climada.entity.DiscRates
DiscRates instance to append
Raises
------
ValueError
"""
disc_rates.check()
if self.years.size == 0:
self.__dict__ = copy.deepcopy(disc_rates.__dict__)
return
self.tag.append(disc_rates.tag)
new_year = array('l')
new_rate = array('d')
for year, rate in zip(disc_rates.years, disc_rates.rates):
found = np.where(year == self.years)[0]
if found.size > 0:
self.rates[found[0]] = rate
else:
new_year.append(year)
new_rate.append(rate)
self.years = np.append(self.years, new_year).astype(int, copy=False)
self.rates = np.append(self.rates, new_rate)
def net_present_value(self, ini_year, end_year, val_years):
"""
Compute net present value between present year and future year.
Parameters
----------
ini_year: float
initial year
end_year: float
end year
val_years: np.array
cash flow at each year btw ini_year and end_year (both included)
Returns
-------
net_present_value: float
net present value between present year and future year.
"""
year_range = np.arange(ini_year, end_year + 1)
if year_range.size != val_years.size:
raise ValueError('Wrong size of yearly values.')
sel_disc = self.select(year_range)
if sel_disc is None:
raise ValueError(
'No information of discount rates for provided years:'
f' {ini_year} - {end_year}')
return u_fin.net_present_value(sel_disc.years, sel_disc.rates,
val_years)
def plot(self, axis=None, figsize=(6, 8), **kwargs):
"""
Plot discount rates per year.
Parameters
----------
axis: matplotlib.axes._subplots.AxesSubplot, optional
axis to use
figsize: tuple(int, int), optional
size of the figure. The default is (6,8)
kwargs: optional
keyword arguments passed to plotting function axis.plot
Returns
-------
axis: matplotlib.axes._subplots.AxesSubplot
axis handles of the plot
"""
if not axis:
_, axis = plt.subplots(1, 1, figsize=figsize)
axis.set_title('Discount rates')
axis.set_xlabel('Year')
axis.set_ylabel('discount rate (%)')
axis.plot(self.years, self.rates * 100, **kwargs)
axis.set_xlim((self.years.min(), self.years.max()))
return axis
@classmethod
def from_mat(cls, file_name, description='', var_names=None):
"""
Read MATLAB file generated with previous MATLAB CLIMADA version.
Parameters
----------
file_name: str
filename including path and extension
description: str, optional
description of the data. The default is ''
var_names: dict, optional
name of the variables in the file. The Default is
DEF_VAR_MAT = {'sup_field_name': 'entity', 'field_name': 'discount',
'var_name': {'year': 'year', 'disc': 'discount_rate'}}
Returns
-------
climada.entity.DiscRates :
The disc rates from matlab
"""
if var_names is None:
var_names = DEF_VAR_MAT
disc = u_hdf5.read(file_name)
tag = Tag(file_name=str(file_name), description=description)
try:
disc = disc[var_names['sup_field_name']]
except KeyError:
pass
try:
disc = disc[var_names['field_name']]
years = np.squeeze(disc[var_names['var_name']['year']]). \
astype(int, copy=False)
rates = np.squeeze(disc[var_names['var_name']['disc']])
except KeyError as err:
raise KeyError("Not existing variable: %s" % str(err)) from err
return cls(years=years, rates=rates, tag=tag)
def read_mat(self, *args, **kwargs):
"""This function is deprecated, use DiscRates.from_mats instead."""
LOGGER.warning("The use of DiscRates.read_mats is deprecated."
"Use DiscRates.from_mats instead.")
self.__dict__ = DiscRates.from_mat(*args, **kwargs).__dict__
@classmethod
def from_excel(cls, file_name, description='', var_names=None):
"""
Read excel file following template and store variables.
Parameters
----------
file_name: str
filename including path and extension
description: str, optional
description of the data. The default is ''
var_names: dict, optional
name of the variables in the file. The Default is
DEF_VAR_EXCEL = {'sheet_name': 'discount',
'col_name': {'year': 'year', 'disc': 'discount_rate'}}
Returns
-------
climada.entity.DiscRates :
The disc rates from excel
"""
if var_names is None:
var_names = DEF_VAR_EXCEL
dfr = pd.read_excel(file_name, var_names['sheet_name'])
tag = Tag(file_name=str(file_name), description=description)
try:
years = dfr[var_names['col_name']['year']].values. \
astype(int, copy=False)
rates = dfr[var_names['col_name']['disc']].values
except KeyError as err:
raise KeyError("Not existing variable: %s" % str(err)) from err
return cls(years=years, rates=rates, tag=tag)
def read_excel(self, *args, **kwargs):
"""This function is deprecated, use DiscRates.from_excel instead."""
LOGGER.warning("The use of DiscRates.read_excel is deprecated."
"Use DiscRates.from_excel instead.")
self.__dict__ = DiscRates.from_mat(*args, **kwargs).__dict__
def write_excel(self, file_name, var_names=None):
"""
Write excel file following template.
Parameters
----------
file_name: str
filename including path and extension
var_names: dict, optional
name of the variables in the file. The Default is
DEF_VAR_EXCEL = {'sheet_name': 'discount',
'col_name': {'year': 'year', 'disc': 'discount_rate'}}
"""
if var_names is None:
var_names = DEF_VAR_EXCEL
disc_wb = xlsxwriter.Workbook(file_name)
disc_ws = disc_wb.add_worksheet(var_names['sheet_name'])
header = [var_names['col_name']['year'], var_names['col_name']['disc']]
for icol, head_dat in enumerate(header):
disc_ws.write(0, icol, head_dat)
for i_yr, (disc_yr, disc_rt) in enumerate(zip(self.years, self.rates),
1):
disc_ws.write(i_yr, 0, disc_yr)
disc_ws.write(i_yr, 1, disc_rt)
disc_wb.close()
def __init__(self, *args, **kwargs):
"""Creates an Exposures object from a GeoDataFrame
Parameters
----------
*args :
Arguments of the GeoDataFrame constructor
**kwargs :
Named arguments of the GeoDataFrame constructor, additionally
tag : climada.entity.exposures.tag.Tag
Exopusres tag
ref_year : int
Reference Year
value_unit : str
Unit of the exposed value
meta : dict
Metadata dictionary
"""
# meta data
try:
self.meta = kwargs.pop('meta')
if self.meta is None:
self.meta = {}
if not isinstance(self.meta, dict):
raise ValueError("meta must be a dictionary")
except KeyError:
self.meta = {}
LOGGER.info('meta set to default value %s', self.meta)
# tag
try:
self.tag = kwargs.pop('tag')
except KeyError:
self.tag = self.meta.get('tag', Tag())
if 'tag' not in self.meta:
LOGGER.info('tag set to default value %s', self.tag)
# reference year
try:
self.ref_year = kwargs.pop('ref_year')
except KeyError:
self.ref_year = self.meta.get('ref_year', DEF_REF_YEAR)
if 'ref_year' not in self.meta:
LOGGER.info('ref_year set to default value %s', self.ref_year)
# value unit
try:
self.value_unit = kwargs.pop('value_unit')
except KeyError:
self.value_unit = self.meta.get('ref_year', DEF_VALUE_UNIT)
if 'value_unit' not in self.meta:
LOGGER.info('value_unit set to default value %s',
self.value_unit)
# remaining generic attributes
for mda in type(self)._metadata:
if mda not in Exposures._metadata:
if mda in kwargs:
setattr(self, mda, kwargs.pop(mda))
elif mda in self.meta:
setattr(self, mda, self.meta[mda])
else:
setattr(self, mda, None)
# make the data frame
self.gdf = GeoDataFrame(*args, **kwargs)
# align crs from gdf and meta data
if self.gdf.crs:
crs = self.gdf.crs
# With geopandas 3.1, the crs attribute is not conserved by the constructor
# without a geometry column. Therefore the conservation is done 'manually':
elif len(args) > 0:
try:
crs = args[0].crs
except AttributeError:
crs = None
elif 'data' in kwargs:
try:
crs = kwargs['data'].crs
except AttributeError:
crs = None
else:
crs = None
# store the crs in the meta dictionary
if crs:
if self.meta.get('crs') and not u_coord.equal_crs(
self.meta.get('crs'), crs):
LOGGER.info(
'crs from `meta` argument ignored and overwritten by GeoDataFrame'
' crs: %s', self.gdf.crs)
self.meta['crs'] = crs
if not self.gdf.crs:
self.gdf.crs = crs
else:
if 'crs' not in self.meta:
LOGGER.info('crs set to default value: %s', DEF_CRS)
self.meta['crs'] = DEF_CRS
self.gdf.crs = self.meta['crs']
def __init__(self,
*args,
meta=None,
tag=None,
ref_year=DEF_REF_YEAR,
value_unit=DEF_VALUE_UNIT,
crs=None,
**kwargs):
"""Creates an Exposures object from a GeoDataFrame
Parameters
----------
args :
Arguments of the GeoDataFrame constructor
kwargs :
Named arguments of the GeoDataFrame constructor, additionally
meta : dict, optional
Metadata dictionary. Default: {} (empty dictionary)
tag : climada.entity.exposures.tag.Tag, optional
Exposures tag. Defaults to the entry of the same name in `meta` or an empty Tag object.
ref_year : int, optional
Reference Year. Defaults to the entry of the same name in `meta` or 2018.
value_unit : str, optional
Unit of the exposed value. Defaults to the entry of the same name in `meta` or 'USD'.
crs : object, anything accepted by pyproj.CRS.from_user_input
Coordinate reference system. Defaults to the entry of the same name in `meta`, or to
the CRS of the GeoDataFrame (if provided) or to 'epsg:4326'.
"""
# meta data
self.meta = {} if meta is None else meta
if not isinstance(self.meta, dict):
raise ValueError("meta must be a dictionary")
self.tag = self.meta.get('tag', Tag()) if tag is None else tag
self.ref_year = self.meta.get(
'ref_year', DEF_REF_YEAR) if ref_year is None else ref_year
self.value_unit = (self.meta.get('value_unit', DEF_VALUE_UNIT)
if value_unit is None else value_unit)
# remaining generic attributes from derived classes
for mda in type(self)._metadata:
if mda not in Exposures._metadata:
if mda in kwargs:
setattr(self, mda, kwargs.pop(mda))
elif mda in self.meta:
setattr(self, mda, self.meta[mda])
else:
setattr(self, mda, None)
# crs (property) and geometry
data = args[0] if args else kwargs.get('data', {})
try:
data_crs = data.geometry.crs
except AttributeError:
data_crs = None
if data_crs and data.crs and not u_coord.equal_crs(data_crs, data.crs):
raise ValueError(
"Inconsistent crs definition in data and data.geometry")
crs = (crs if crs is not None else self.meta['crs']
if 'crs' in self.meta else data_crs if data_crs else None)
if 'crs' in self.meta and not u_coord.equal_crs(self.meta['crs'], crs):
raise ValueError(
"Inconsistent CRS definition, crs and meta arguments don't match"
)
if data_crs and not u_coord.equal_crs(data_crs, crs):
raise ValueError(
"Inconsistent CRS definition, data doesn't match meta or crs argument"
)
if not crs:
crs = DEF_CRS
geometry = kwargs.get('geometry')
if geometry and isinstance(geometry, str):
raise ValueError(
"Exposures is not able to handle customized 'geometry' column names."
)
# make the data frame
self.set_gdf(GeoDataFrame(*args, **kwargs), crs=crs)
