def test_calib_instance(self):
""" Test save calib instance """
# Read default entity values
ent = Entity()
ent.read_excel(ENT_DEMO_TODAY)
ent.check()
# Read default hazard file
hazard = Hazard('TC')
hazard.read_mat(HAZ_TEST_MAT)
# get impact function from set
imp_func = ent.impact_funcs.get_func(hazard.tag.haz_type,
ent.exposures.if_TC.median())
# Assign centroids to exposures
ent.exposures.assign_centroids(hazard)
# create input frame
df_in = pd.DataFrame.from_dict({
'v_threshold': [25.7],
'other_param': [2],
'hazard': [HAZ_TEST_MAT]
})
df_in_yearly = pd.DataFrame.from_dict({
'v_threshold': [25.7],
'other_param': [2],
'hazard': [HAZ_TEST_MAT]
})
# Compute the impact over the whole exposures
df_out = calib_instance(hazard, ent.exposures, imp_func, df_in)
df_out_yearly = calib_instance(hazard,
ent.exposures,
imp_func,
df_in_yearly,
yearly_impact=True)
# calc Impact as comparison
impact = Impact()
impact.calc(ent.exposures, ent.impact_funcs, hazard)
IYS = impact.calc_impact_year_set(all_years=True)
# do the tests
self.assertTrue(isinstance(df_out, pd.DataFrame))
self.assertTrue(isinstance(df_out_yearly, pd.DataFrame))
self.assertEqual(df_out.shape[0], hazard.event_id.size)
self.assertEqual(df_out_yearly.shape[0], 161)
self.assertTrue(all(df_out['event_id'] == hazard.event_id))
self.assertTrue(
all(df_out[df_in.columns[0]].isin(df_in[df_in.columns[0]])))
self.assertTrue(
all(df_out_yearly[df_in.columns[1]].isin(df_in[df_in.columns[1]])))
self.assertTrue(
all(df_out_yearly[df_in.columns[2]].isin(df_in[df_in.columns[2]])))
self.assertTrue(
all(df_out['impact_CLIMADA'].values == impact.at_event))
self.assertTrue(
all(df_out_yearly['impact_CLIMADA'].values == [*IYS.values()]))
def calib_instance(hazard,
exposure,
impact_func,
df_out=pd.DataFrame(),
yearly_impact=False):
""" calculate one impact instance for the calibration algorithm and write
to given DataFrame
Parameters:
hazard: hazard set instance
exposure: exposure set instance
impact_func: impact function instance
Optional Parameters:
df_out: Output DataFrame with headers of columns defined and optionally with
first row (index=0) defined with values. If columns "impact",
"event_id", or "year" are not included, they are created here.
Data like reported impacts or impact function parameters can be
given here; values are preserved.
yearly_impact (boolean): if set True, impact is returned per year,
not per event
Returns:
df_out: DataFrame with modelled impact written to rows for each year
or event.
"""
IFS = ImpactFuncSet()
IFS.append(impact_func)
impacts = Impact()
impacts.calc(exposure, IFS, hazard)
if yearly_impact: # impact per year
IYS = impacts.calc_impact_year_set(all_years=True)
# Loop over whole year range:
for cnt_, year in enumerate(np.sort(list((IYS.keys())))):
if cnt_ > 0:
df_out.loc[cnt_] = df_out.loc[0] # copy info from first row
if year in IYS:
df_out.loc[cnt_, 'impact'] = IYS[year]
else:
df_out.loc[cnt_, 'impact'] = 0
df_out.loc[cnt_, 'year'] = year
else: # impact per event
for cnt_, impact in enumerate(impacts.at_event):
if cnt_ > 0:
df_out.loc[cnt_] = df_out.loc[0] # copy info from first row
df_out.loc[cnt_, 'impact'] = impact
df_out.loc[cnt_, 'event_id'] = int(impacts.event_id[cnt_])
df_out.loc[cnt_, 'event_name'] = impacts.event_name[cnt_]
df_out.loc[cnt_, 'year'] = \
dt.datetime.fromordinal(impacts.date[cnt_]).year
return df_out
def make_Y(parameter, *args):
"""
Score function for Optimization process.
Multiple scoring options are present (spearman, pearson, RMSE, RMSF)
Parameters
----------
parameter : np.ndarray
array containing parameter that are optimized.
*args :
imp_fun_parameter: dict
Contains ind and Parameter for Impact function
exp: climada.entity.exposures.base.Exposures
CLIMADA Expusure.
haz: climada.hazard.base.Hazard
CLIMADA hazard
haz_type: str
Type of Hazard ("HL")
num_fct: int
number of Impact functions ([1,3])
Returns
-------
score: float
Variable that is minimizes by optimization. Mulitple variables possible.
"""
# *args = imp_fun_parameter, exp, agr, haz_type
# a = time.perf_counter()
parameter_optimize, exp, haz, haz_type, num_fct, score_type, type_imp_fun = args
ifset_hail = ImpactFuncSet()
if type_imp_fun == "sig":
if num_fct ==1:
parameter_optimize[0]["L"] = parameter[0]
parameter_optimize[0]["x_0"] = parameter[1]
parameter_optimize[0]["k"] = parameter[2]
else:
parameter_optimize[0]["L"] = parameter[0]
parameter_optimize[0]["x_0"] = parameter[1]
parameter_optimize[0]["k"] = parameter[2]
parameter_optimize[1]["L"] = parameter[3]
parameter_optimize[1]["x_0"] = parameter[4]
parameter_optimize[1]["k"] = parameter[5]
parameter_optimize[2]["L"] = parameter[6]
parameter_optimize[2]["x_0"] = parameter[7]
parameter_optimize[2]["k"] = parameter[8]
# b = time.perf_counter()
# print("time to write parameter_optimize: ", b-a)
for imp_fun_dict in parameter_optimize:
imp_fun = create_impact_func(haz_type,
imp_fun_dict["imp_id"],
imp_fun_dict["L"],
imp_fun_dict["x_0"],
imp_fun_dict["k"])
ifset_hail.append(imp_fun)
elif type_imp_fun == "lin":
parameter_optimize[0]["m"] = parameter[0]
imp_fun = create_impact_func_lin(haz_type,
parameter_optimize[0]["imp_id"],
m = parameter[0])
ifset_hail.append(imp_fun)
c = time.perf_counter()
# print("time to make imp_fun: ", c-b)
imp = Impact()
# imp.calc(self = imp, exposures = exp, impact_funcs = ifset_hail, hazard = haz, save_mat = True)
imp.calc(exp, ifset_hail, haz, save_mat = False)
d = time.perf_counter()
print("time to calc impact: ", d-c)
Y = list(imp.calc_impact_year_set(year_range = [2002, 2019]).values())
all_eq = 0
#very stupid bugfix. Ther where problem when all y values where 0,
#so this test if this is the case and changes the last value if so
for count, y in enumerate(Y):
if y==0:
all_eq += 1
Y[count] = 0.1
if all_eq == len(Y):
Y[-1] = 0.2
Y_norm = np.divide(Y, min(Y))
Observ = [27.48, 46.14, 80.67, 76.80, 32.66, 62.47, 26.30, 110.60, 13.01,
34.53, 21.50, 71.77, 22.80, 19.84, 17.50, 35.80, 24.40, 33.30]
O_norm = np.divide(Observ, min(Observ))
# res = mean_squared_error(Y_norm, O_norm)**0.5
rmsf = RMSF(Y_norm, O_norm)
rmse = mean_squared_error(O_norm, Y_norm)
print("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")
print("Params {}".format(parameter_optimize))
print("The sum of the new Impact is: {}".format(sum(Y)))
spear_coef, spear_p_value = spearmanr(O_norm, Y_norm)
print("spearman for agr (score, p_value) = ({}, {})".format(spear_coef, spear_p_value))
pears_coef, pears_p_value = stats.pearsonr(O_norm, Y_norm)
print("pearson for agr (score, p_value) = ({}, {})".format(pears_coef, pears_p_value))
print("RMSF: ", rmsf)
print("RMSE", rmse)
print("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")
# e= time.perf_counter()
# print("time to get result: ", e-d)
if score_type == "pearson":
score = pears_coef * -1
elif score_type == "spearman":
score = spear_coef * -1
elif score_type == "RMSF":
score = rmsf
elif score_type == "RMSE":
score = rmse
return score
# for ev_name in ev_list:
# imp_infr.plot_hexbin_impact_exposure(event_id = haz_real.get_event_id(event_name=ev_list[1])[0], ignore_zero = False)
# imp_agr.plot_hexbin_impact_exposure(event_id = haz_real.get_event_id(event_name=ev_name)[0])
print(
"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")
print("I'm done with the script")
#Secret test chamber pssst
if True:
print(
"dmg infr {} Mio CHF, dmg agr_meshs {} Mio CHF, dmg agr_dur {} Mio CHF"
.format(imp_infr.aai_agg / 1e6, imp_agr.aai_agg / 1e6,
imp_agr_dur.aai_agg / 1e6))
agr_meshs_yearly_imp = list(
imp_agr.calc_impact_year_set(year_range=[2002, 2019]).values())
agr_dur_yearly_imp = list(
imp_agr_dur.calc_impact_year_set(year_range=[2002, 2019]).values())
# plt.figure()
# plt.bar(years, agr_dur_yearly_imp)
# plt.show()
# plt.figure()
# plt.bar(years, agr_meshs_yearly_imp)
dmg_from_sturmarchiv = [
29.12, 48.61, 84.34, 79.35, 33.37, 63.40, 26.05, 110.06, 12.87, 34.05,
21.35, 71.41, 22.71, 19.98, 17.69, 35.39, 24.30, 33.07
]
dmg_from_sturmarchiv = [int(i * 1e6) for i in dmg_from_sturmarchiv]
dmg_from_vkg = [
164.27, 32.35, 101.18, 169.90, 26.60, 31.00, 17.66, 311.96, 14.96,
237.43, 76.12, 188.37, 8.98, 25.60, 17.15, 60.80, 29.50, 16.55
def calc_sector_direct_impact(self, hazard, exposure, imp_fun_set,
selected_subsec="service"):
"""Calculate direct impacts.
Parameters:
----------
hazard : Hazard
Hazard object for impact calculation.
exposure : Exposures
Exposures object for impact calculation. For WIOD tables, exposure.region_id
must be country names following ISO3 codes.
imp_fun_set : ImpactFuncSet
Set of impact functions.
selected_subsec : str or list
Positions of the selected sectors. These positions can be either
defined by the user by passing a list of values, or by using built-in
sectors' aggregations for the WIOD data passing a string with possible
values being "service", "manufacturing", "agriculture" or "mining".
Default is "service".
"""
if isinstance(selected_subsec, str):
built_in_subsec_pos = {'service': range(26, 56),
'manufacturing': range(4, 23),
'agriculture': range(0, 1),
'mining': range(3, 4)}
selected_subsec = built_in_subsec_pos[selected_subsec]
dates = [
dt.datetime.strptime(date, "%Y-%m-%d")
for date in hazard.get_event_date()
]
self.years = np.unique([date.year for date in dates])
unique_exp_regid = exposure.gdf.region_id.unique()
self.direct_impact = np.zeros(shape=(len(self.years),
len(self.mriot_reg_names)*len(self.sectors)))
self.reg_dir_imp = []
for exp_regid in unique_exp_regid:
reg_exp = Exposures(exposure.gdf[exposure.gdf.region_id == exp_regid])
reg_exp.check()
# Normalize exposure
total_reg_value = reg_exp.gdf['value'].sum()
reg_exp.gdf['value'] /= total_reg_value
# Calc impact for country
imp = Impact()
imp.calc(reg_exp, imp_fun_set, hazard)
imp_year_set = np.array(list(imp.calc_impact_year_set(imp).values()))
mriot_reg_name = self._map_exp_to_mriot(exp_regid, self.mriot_type)
self.reg_dir_imp.append(mriot_reg_name)
subsec_reg_pos = np.array(selected_subsec) + self.reg_pos[mriot_reg_name][0]
subsec_reg_prod = self.mriot_data[subsec_reg_pos].sum(axis=1)
imp_year_set = np.repeat(imp_year_set, len(selected_subsec)
).reshape(len(self.years),
len(selected_subsec))
direct_impact_reg = np.multiply(imp_year_set, subsec_reg_prod)
# Sum needed below in case of many ROWs, which are aggregated into
# one country as per WIOD table.
self.direct_impact[:, subsec_reg_pos] += direct_impact_reg.astype(np.float32)
# average impact across years
self.direct_aai_agg = self.direct_impact.mean(axis=0)
def calib_instance(hazard,
exposure,
impact_func,
df_out=pd.DataFrame(),
yearly_impact=False,
return_cost='False'):
"""calculate one impact instance for the calibration algorithm and write
to given DataFrame
Parameters
----------
hazard : Hazard
exposure : Exposure
impact_func : ImpactFunc
df_out : Dataframe, optional
Output DataFrame with headers of columns defined and optionally with
first row (index=0) defined with values. If columns "impact",
"event_id", or "year" are not included, they are created here.
Data like reported impacts or impact function parameters can be
given here; values are preserved.
yearly_impact : boolean, optional
if set True, impact is returned per year, not per event
return_cost : str, optional
if not 'False' but any of 'R2', 'logR2',
cost is returned instead of df_out
Returns
-------
df_out: DataFrame
DataFrame with modelled impact written to rows for each year
or event.
"""
IFS = ImpactFuncSet()
IFS.append(impact_func)
impacts = Impact()
impacts.calc(exposure, IFS, hazard)
if yearly_impact: # impact per year
IYS = impacts.calc_impact_year_set(all_years=True)
# Loop over whole year range:
if df_out.empty | df_out.index.shape[0] == 1:
for cnt_, year in enumerate(np.sort(list((IYS.keys())))):
if cnt_ > 0:
df_out.loc[cnt_] = df_out.loc[
0] # copy info from first row
if year in IYS:
df_out.loc[cnt_, 'impact_CLIMADA'] = IYS[year]
else:
df_out.loc[cnt_, 'impact_CLIMADA'] = 0.0
df_out.loc[cnt_, 'year'] = year
else:
years_in_common = df_out.loc[
df_out['year'].isin(np.sort(list((IYS.keys())))), 'year']
for cnt_, year in years_in_common.iteritems():
df_out.loc[df_out['year'] == year,
'impact_CLIMADA'] = IYS[year]
else: # impact per event
if df_out.empty | df_out.index.shape[0] == 1:
for cnt_, impact in enumerate(impacts.at_event):
if cnt_ > 0:
df_out.loc[cnt_] = df_out.loc[
0] # copy info from first row
df_out.loc[cnt_, 'impact_CLIMADA'] = impact
df_out.loc[cnt_, 'event_id'] = int(impacts.event_id[cnt_])
df_out.loc[cnt_, 'event_name'] = impacts.event_name[cnt_]
df_out.loc[cnt_, 'year'] = \
dt.datetime.fromordinal(impacts.date[cnt_]).year
df_out.loc[cnt_, 'date'] = impacts.date[cnt_]
elif df_out.index.shape[0] == impacts.at_event.shape[0]:
for cnt_, (impact,
ind) in enumerate(zip(impacts.at_event, df_out.index)):
df_out.loc[ind, 'impact_CLIMADA'] = impact
df_out.loc[ind, 'event_id'] = int(impacts.event_id[cnt_])
df_out.loc[ind, 'event_name'] = impacts.event_name[cnt_]
df_out.loc[ind, 'year'] = \
dt.datetime.fromordinal(impacts.date[cnt_]).year
df_out.loc[ind, 'date'] = impacts.date[cnt_]
else:
raise ValueError('adding simulated impacts to reported impacts not'
' yet implemented. use yearly_impact=True or run'
' without init_impact_data.')
if not return_cost == 'False':
df_out = calib_cost_calc(df_out, return_cost)
return df_out
plt.show()
# for ev_name in ev_list:
# imp_infr.plot_basemap_impact_exposure(event_id = haz_real.get_event_id(event_name=ev_name)[0])
# imp_agr.plot_basemap_impact_exposure(event_id = haz_real.get_event_id(event_name=ev_name)[0])
# for ev_name in ev_list:
# imp_infr.plot_hexbin_impact_exposure(event_id = haz_real.get_event_id(event_name=ev_list[1])[0], ignore_zero = False)
# imp_agr.plot_hexbin_impact_exposure(event_id = haz_real.get_event_id(event_name=ev_name)[0])
print("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")
print("I'm done with the script")
#Secret test chamber pssst
if True:
print("dmg infr {} Mio CHF, dmg agr_meshs {} Mio CHF, dmg agr_dur {} Mio CHF".format(imp_infr.aai_agg/1e6, imp_agr.aai_agg/1e6, imp_agr_dur.aai_agg/1e6))
agr_meshs_yearly_imp = list(imp_agr.calc_impact_year_set(year_range = [2002, 2019]).values())
agr_dur_yearly_imp = list(imp_agr_dur.calc_impact_year_set(year_range = [2002, 2019]).values())
# plt.figure()
# plt.bar(years, agr_dur_yearly_imp)
# plt.show()
# plt.figure()
# plt.bar(years, agr_meshs_yearly_imp)
dmg_from_sturmarchiv = [27.48, 46.14, 80.67, 76.80, 32.66, 62.47, 26.30, 110.60, 13.01, 34.53, 21.50, 71.77, 22.80, 19.84, 17.50, 35.80, 24.40, 33.30]
dmg_from_sturmarchiv = [i*1e6 for i in dmg_from_sturmarchiv]
norm_agr_meshs_yearly_imp = np.divide(agr_meshs_yearly_imp, min(agr_meshs_yearly_imp))
norm_agr_dur_yearly_imp = np.divide(agr_dur_yearly_imp, min(agr_dur_yearly_imp))
norm_dmg_from_sturmarchiv = np.divide(dmg_from_sturmarchiv, min(dmg_from_sturmarchiv)) #[i / min(dmg_from_sturmarchiv) for i in dmg_from_sturmarchiv]
#plot
plt.figure()
plt.plot(years, agr_meshs_yearly_imp)
imp_agr_dur.plot_scatter_eai_exposure()
imp_agr_dur.plot_raster_eai_exposure(raster_res=0.001)
print("dmg agr_meshs {} Mio CHF, dmg agr_dur {} Mio CHF".format(
imp_agr_meshs.aai_agg / 1e6, imp_agr_dur.aai_agg / 1e6))
print(
"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")
print("I'm done with the script")
#Secret test chamber pssst
if False:
print("dmg agr_meshs {} Mio CHF, dmg agr_dur {} Mio CHF".format(
imp_agr_meshs.aai_agg / 1e6, imp_agr_dur.aai_agg / 1e6))
agr_meshs_yearly_imp = list(
imp_agr_meshs.calc_impact_year_set(year_range=[2002, 2019]).values())
agr_dur_yearly_imp = list(
imp_agr_dur.calc_impact_year_set(year_range=[2002, 2019]).values())
# plt.figure()
# plt.bar(years, agr_dur_yearly_imp)
# plt.show()
# plt.figure()
# plt.bar(years, agr_meshs_yearly_imp)
dmg_from_sturmarchiv = [
27.48, 46.14, 80.67, 76.80, 32.66, 62.47, 26.30, 110.60, 13.01, 34.53,
21.50, 71.77, 22.80, 19.84, 17.50, 35.80, 24.40, 33.30
]
dmg_from_sturmarchiv = [i * 1e6 for i in dmg_from_sturmarchiv]
norm_agr_meshs_yearly_imp = np.divide(agr_meshs_yearly_imp,
min(agr_meshs_yearly_imp))
norm_agr_dur_yearly_imp = np.divide(agr_dur_yearly_imp,
