def read_excel(self, file_name): """ Read excel file containing impact data generated by write_excel. Parameters: file_name (str): absolute path of the file """ dfr = pd.read_excel(file_name) self.__init__() self.tag['haz'] = TagHaz() self.tag['haz'].haz_type = dfr['tag_hazard'][0] self.tag['haz'].file_name = dfr['tag_hazard'][1] self.tag['haz'].description = dfr['tag_hazard'][2] self.tag['exp'] = Tag() self.tag['exp'].file_name = dfr['tag_exposure'][0] self.tag['exp'].description = dfr['tag_exposure'][1] self.tag['if_set'] = Tag() self.tag['if_set'].file_name = dfr['tag_impact_func'][0] self.tag['if_set'].description = dfr['tag_impact_func'][1] self.unit = dfr.unit[0] self.tot_value = dfr.tot_value[0] self.aai_agg = dfr.aai_agg[0] self.event_id = dfr.event_id[~np.isnan(dfr.event_id.values)].values self.event_name = dfr.event_name[:self.event_id.size].values self.date = dfr.event_date[:self.event_id.size].values self.frequency = dfr.event_frequency[:self.event_id.size].values self.at_event = dfr.at_event[:self.event_id.size].values self.eai_exp = dfr.eai_exp[~np.isnan(dfr.eai_exp.values)].values self.coord_exp = np.zeros((self.eai_exp.size, 2)) self.coord_exp[:, 0] = dfr.exp_lat.values[:self.eai_exp.size] self.coord_exp[:, 1] = dfr.exp_lon.values[:self.eai_exp.size] self.crs = ast.literal_eval(dfr.exp_crs.values[0])
def read_csv(self, file_name): """ Read csv file containing impact data generated by write_csv. Parameters: file_name (str): absolute path of the file """ imp_df = pd.read_csv(file_name) self.__init__() self.unit = imp_df.unit[0] self.tot_value = imp_df.tot_value[0] self.aai_agg = imp_df.aai_agg[0] self.event_id = imp_df.event_id[~np.isnan(imp_df.event_id)] num_ev = self.event_id.size self.event_name = imp_df.event_name[:num_ev] self.date = imp_df.event_date[:num_ev] self.at_event = imp_df.at_event[:num_ev] self.frequency = imp_df.event_frequency[:num_ev] self.eai_exp = imp_df.eai_exp[~np.isnan(imp_df.eai_exp)] num_exp = self.eai_exp.size self.coord_exp = np.zeros((num_exp, 2)) self.coord_exp[:, 0] = imp_df.exp_lat[:num_exp] self.coord_exp[:, 1] = imp_df.exp_lon[:num_exp] self.crs = ast.literal_eval(imp_df.exp_crs.values[0]) self.tag['haz'] = TagHaz(str(imp_df.tag_hazard[0]), str(imp_df.tag_hazard[1]), str(imp_df.tag_hazard[2])) self.tag['exp'] = Tag(str(imp_df.tag_exposure[0]), str(imp_df.tag_exposure[1])) self.tag['if_set'] = Tag(str(imp_df.tag_impact_func[0]), str(imp_df.tag_impact_func[1]))
def test_write_read_exp_test(self): """Test result against reference value""" # Create impact object num_ev = 5 num_exp = 10 imp_write = Impact() imp_write.tag = { 'exp': Tag('file_exp.p', 'descr exp'), 'haz': TagHaz('TC', 'file_haz.p', 'descr haz'), 'if_set': Tag() } imp_write.event_id = np.arange(num_ev) imp_write.event_name = [ 'event_' + str(num) for num in imp_write.event_id ] imp_write.date = np.ones(num_ev) imp_write.coord_exp = np.zeros((num_exp, 2)) imp_write.coord_exp[:, 0] = 1.5 imp_write.coord_exp[:, 1] = 2.5 imp_write.eai_exp = np.arange(num_exp) * 100 imp_write.at_event = np.arange(num_ev) * 50 imp_write.frequency = np.ones(num_ev) * 0.1 imp_write.tot_value = 1000 imp_write.aai_agg = 1001 imp_write.unit = 'USD' file_name = os.path.join(DATA_FOLDER, 'test.csv') imp_write.write_csv(file_name) imp_read = Impact() imp_read.read_csv(file_name) self.assertTrue(np.array_equal(imp_write.event_id, imp_read.event_id)) self.assertTrue(np.array_equal(imp_write.date, imp_read.date)) self.assertTrue(np.array_equal(imp_write.coord_exp, imp_read.coord_exp)) self.assertTrue(np.array_equal(imp_write.eai_exp, imp_read.eai_exp)) self.assertTrue(np.array_equal(imp_write.at_event, imp_read.at_event)) self.assertTrue(np.array_equal(imp_write.frequency, imp_read.frequency)) self.assertEqual(imp_write.tot_value, imp_read.tot_value) self.assertEqual(imp_write.aai_agg, imp_read.aai_agg) self.assertEqual(imp_write.unit, imp_read.unit) self.assertEqual( 0, len([ i for i, j in zip(imp_write.event_name, imp_read.event_name) if i != j ])) self.assertIsInstance(imp_read.crs, dict)
def emdat_to_impact(emdat_file_csv, year_range=None, countries=None,\ hazard_type_emdat=None, hazard_type_climada=None, \ reference_year=0, imp_str="Total damage ('000 US$)"): """function to load EM-DAT data return impact per event Parameters: emdat_file_csv (str): Full path to EMDAT-file (CSV), i.e.: emdat_file_csv = os.path.join(SYSTEM_DIR, 'emdat_201810.csv') hazard_type_emdat (str): Hazard (sub-)type according to EMDAT terminology, i.e. 'Tropical cyclone' for tropical cyclone OR hazard_type_climada (str): Hazard type CLIMADA abbreviation, i.e. 'TC' for tropical cyclone Optional parameters: year_range (list with 2 integers): start and end year i.e. [1980, 2017] default: None --> take year range from EM-DAT file countries (list of str): country ISO3-codes or names, i.e. ['JAM']. Set to None or ['all'] for all countries (default) reference_year (int): reference year of exposures. Impact is scaled proportional to GDP to the value of the reference year. No scaling for reference_year=0 (default) imp_str (str): Column name of impact metric in EMDAT CSV, default = "Total damage ('000 US$)" Returns: impact_instance (instance of climada.engine.Impact): impact object of same format as output from CLIMADA impact computation scaled with GDP to reference_year if reference_year noit equal 0 i.e. 1000 current US$ for imp_str="Total damage ('000 US$) scaled". impact_instance.eai_exp holds expected annual impact for each country. impact_instance.coord_exp holds rough central coordinates for each country. countries (list): ISO3-codes of countries imn same order as in impact_instance.eai_exp """ # Mapping of hazard type between EM-DAT and CLIMADA: if not hazard_type_climada: if not hazard_type_emdat: LOGGER.error( 'Either hazard_type_climada or hazard_type_emdat need to be defined.' ) return None if hazard_type_emdat == 'Tropical cyclone': hazard_type_climada = 'TC' elif hazard_type_emdat == 'Drought': hazard_type_climada = 'DR' elif hazard_type_emdat == 'Landslide': hazard_type_climada = 'LS' elif hazard_type_emdat == 'Riverine flood': hazard_type_climada = 'RF' elif hazard_type_emdat in [ 'Wildfire', 'Forest Fire', 'Land fire (Brush, Bush, Pasture)' ]: hazard_type_climada = 'BF' elif hazard_type_emdat == 'Extra-tropical storm': hazard_type_climada = 'WS' elif not hazard_type_emdat: if hazard_type_climada == 'TC': hazard_type_emdat = 'Tropical cyclone' elif hazard_type_climada == 'DR': hazard_type_emdat = 'Drought' elif hazard_type_climada == 'LS': hazard_type_emdat = 'Landslide' elif hazard_type_climada == 'RF': hazard_type_emdat = 'Riverine flood' elif hazard_type_climada == 'BF': hazard_type_emdat = 'Wildfire' elif hazard_type_climada == 'WS': hazard_type_emdat = 'Extra-tropical storm' # Inititate Impact-instance: impact_instance = Impact() impact_instance.tag = dict() impact_instance.tag['haz'] = TagHaz(haz_type=hazard_type_climada, \ file_name=emdat_file_csv, description='EM-DAT impact, direct import') impact_instance.tag['exp'] = Tag(file_name=emdat_file_csv, \ description='EM-DAT impact, direct import') impact_instance.tag['if_set'] = Tag(file_name=None, description=None) if not countries or countries == ['all']: countries = emdat_countries_by_hazard(hazard_type_emdat, emdat_file_csv, \ ignore_missing=True, verbose=True)[0] else: if isinstance(countries, str): countries = [countries] # Load EM-DAT impact data by event: em_data = emdat_impact_event(countries, hazard_type_emdat, emdat_file_csv, \ year_range, reference_year=reference_year) if em_data.empty: return impact_instance, countries impact_instance.event_id = np.array(em_data.index, int) impact_instance.event_name = list(em_data['Disaster No.']) date_list = list() for year in list(em_data['year']): date_list.append(datetime.toordinal(datetime.strptime(str(year), '%Y'))) boolean_warning = True for idx, datestr in enumerate(list(em_data['Start date'])): try: date_list[idx] = datetime.toordinal( datetime.strptime(datestr[-7:], '%m/%Y')) except ValueError: if boolean_warning: LOGGER.warning('EM_DAT CSV contains invalid time formats') boolean_warning = False try: date_list[idx] = datetime.toordinal( datetime.strptime(datestr, '%d/%m/%Y')) except ValueError: if boolean_warning: LOGGER.warning('EM_DAT CSV contains invalid time formats') boolean_warning = False impact_instance.date = np.array(date_list, int) impact_instance.crs = DEF_CRS if reference_year == 0: impact_instance.at_event = np.array(em_data[imp_str]) else: impact_instance.at_event = np.array(em_data[imp_str + " scaled"]) if not year_range: year_range = [em_data['year'].min(), em_data['year'].max()] impact_instance.frequency = np.ones( em_data.shape[0]) / (1 + np.diff(year_range)) impact_instance.tot_value = 0 impact_instance.aai_agg = sum(impact_instance.at_event * impact_instance.frequency) impact_instance.unit = 'USD' impact_instance.imp_mat = [] # init rough exposure with central point per country shp = shapereader.natural_earth(resolution='110m', category='cultural', name='admin_0_countries') shp = shapefile.Reader(shp) countries_reg_id = list() countries_lat = list() countries_lon = list() impact_instance.eai_exp = np.zeros( len(countries)) # empty: damage at exposure for idx, cntry in enumerate(countries): try: cntry = iso_cntry.get(cntry).alpha3 except KeyError: LOGGER.error('Country not found in iso_country: ' + cntry) cntry_boolean = False for rec_i, rec in enumerate(shp.records()): if rec[9].casefold() == cntry.casefold(): bbox = shp.shapes()[rec_i].bbox cntry_boolean = True break if cntry_boolean: countries_lat.append(np.mean([bbox[1], bbox[3]])) countries_lon.append(np.mean([bbox[0], bbox[2]])) else: countries_lat.append(np.nan) countries_lon.append(np.nan) try: countries_reg_id.append(int(iso_cntry.get(cntry).numeric)) except KeyError: countries_reg_id.append(0) df_tmp = em_data[em_data['ISO'].str.contains(cntry)] if reference_year == 0: impact_instance.eai_exp[idx] = sum(np.array(df_tmp[imp_str])*\ impact_instance.frequency[0]) else: impact_instance.eai_exp[idx] = sum(np.array(df_tmp[imp_str + " scaled"])*\ impact_instance.frequency[0]) impact_instance.coord_exp = np.stack([countries_lat, countries_lon], axis=1) #impact_instance.plot_raster_eai_exposure() return impact_instance, countries
def emdat_to_impact(emdat_file_csv, hazard_type_climada, year_range=None, countries=None, hazard_type_emdat=None, reference_year=None, imp_str="Total Damages"): """function to load EM-DAT data return impact per event Parameters: emdat_file_csv (str): Full path to EMDAT-file (CSV), i.e.: emdat_file_csv = SYSTEM_DIR.joinpath('emdat_201810.csv') hazard_type_climada (str): Hazard type CLIMADA abbreviation, i.e. 'TC' for tropical cyclone Optional parameters: hazard_type_emdat (list or str): List of Disaster (sub-)type accordung EMDAT terminology, e.g.: Animal accident, Drought, Earthquake, Epidemic, Extreme temperature, Flood, Fog, Impact, Insect infestation, Landslide, Mass movement (dry), Storm, Volcanic activity, Wildfire; Coastal Flooding, Convective Storm, Riverine Flood, Tropical cyclone, Tsunami, etc.; OR CLIMADA hazard type abbreviations, e.g. TC, BF, etc. If not given, it is deducted from hazard_type_climada year_range (list with 2 integers): start and end year e.g. [1980, 2017] default: None --> take year range from EM-DAT file countries (list of str): country ISO3-codes or names, e.g. ['JAM']. Set to None or ['all'] for all countries (default) reference_year (int): reference year of exposures. Impact is scaled proportional to GDP to the value of the reference year. No scaling for reference_year=0 (default) imp_str (str): Column name of impact metric in EMDAT CSV, default = "Total Damages ('000 US$)" Returns: impact_instance (instance of climada.engine.Impact): impact object of same format as output from CLIMADA impact computation. Values scaled with GDP to reference_year if reference_year is given. i.e. current US$ for imp_str="Total Damages ('000 US$) scaled" (factor 1000 is applied) impact_instance.eai_exp holds expected annual impact for each country. impact_instance.coord_exp holds rough central coordinates for each country. countries (list): ISO3-codes of countries in same order as in impact_instance.eai_exp """ if "Total Damages" in imp_str: imp_str = "Total Damages ('000 US$)" elif "Insured Damages" in imp_str: imp_str = "Insured Damages ('000 US$)" elif "Reconstruction Costs" in imp_str: imp_str = "Reconstruction Costs ('000 US$)" imp_str = VARNAMES_EMDAT[max(VARNAMES_EMDAT.keys())][imp_str] if not hazard_type_emdat: hazard_type_emdat = [hazard_type_climada] if reference_year == 0: reference_year = None # Inititate Impact-instance: impact_instance = Impact() impact_instance.tag = dict() impact_instance.tag['haz'] = TagHaz( haz_type=hazard_type_climada, file_name=emdat_file_csv, description='EM-DAT impact, direct import') impact_instance.tag['exp'] = Tag( file_name=emdat_file_csv, description='EM-DAT impact, direct import') impact_instance.tag['if_set'] = Tag(file_name=None, description=None) # Load EM-DAT impact data by event: em_data = emdat_impact_event(emdat_file_csv, countries=countries, hazard=hazard_type_emdat, year_range=year_range, reference_year=reference_year, imp_str=imp_str, version=max(VARNAMES_EMDAT.keys())) if isinstance(countries, str): countries = [countries] elif not countries: countries = emdat_countries_by_hazard(emdat_file_csv, year_range=year_range, hazard=hazard_type_emdat)[0] if em_data.empty: return impact_instance, countries impact_instance.event_id = np.array(em_data.index, int) impact_instance.event_name = list(em_data[VARNAMES_EMDAT[max( VARNAMES_EMDAT.keys())]['Dis No']]) date_list = list() for year in list(em_data['Year']): date_list.append(datetime.toordinal(datetime.strptime(str(year), '%Y'))) if 'Start Year' in em_data.columns and 'Start Month' in em_data.columns \ and 'Start Day' in em_data.columns: idx = 0 for year, month, day in zip(em_data['Start Year'], em_data['Start Month'], em_data['Start Day']): if np.isnan(year): idx += 1 continue if np.isnan(month): month = 1 if np.isnan(day): day = 1 date_list[idx] = datetime.toordinal( datetime.strptime('%02i/%02i/%04i' % (day, month, year), '%d/%m/%Y')) idx += 1 impact_instance.date = np.array(date_list, int) impact_instance.crs = DEF_CRS if not reference_year: impact_instance.at_event = np.array(em_data["impact"]) else: impact_instance.at_event = np.array(em_data["impact_scaled"]) impact_instance.at_event[np.isnan(impact_instance.at_event)] = 0 if not year_range: year_range = [em_data['Year'].min(), em_data['Year'].max()] impact_instance.frequency = np.ones( em_data.shape[0]) / (1 + np.diff(year_range)) impact_instance.tot_value = 0 impact_instance.aai_agg = np.nansum(impact_instance.at_event * impact_instance.frequency) impact_instance.unit = 'USD' impact_instance.imp_mat = [] # init rough exposure with central point per country shp = shapereader.natural_earth(resolution='110m', category='cultural', name='admin_0_countries') shp = shapefile.Reader(shp) countries_reg_id = list() countries_lat = list() countries_lon = list() impact_instance.eai_exp = np.zeros( len(countries)) # empty: damage at exposure for idx, cntry in enumerate(countries): try: cntry = iso_cntry.get(cntry).alpha3 except KeyError: print(cntry) LOGGER.error('Country not found in iso_country: %s', cntry) cntry_boolean = False for rec_i, rec in enumerate(shp.records()): if rec[9].casefold() == cntry.casefold(): bbox = shp.shapes()[rec_i].bbox cntry_boolean = True break if cntry_boolean: countries_lat.append(np.mean([bbox[1], bbox[3]])) countries_lon.append(np.mean([bbox[0], bbox[2]])) else: countries_lat.append(np.nan) countries_lon.append(np.nan) try: countries_reg_id.append(int(iso_cntry.get(cntry).numeric)) except KeyError: countries_reg_id.append(0) df_tmp = em_data[em_data[VARNAMES_EMDAT[max( VARNAMES_EMDAT.keys())]['ISO']].str.contains(cntry)] if not reference_year: impact_instance.eai_exp[idx] = sum( np.array(df_tmp["impact"]) * impact_instance.frequency[0]) else: impact_instance.eai_exp[idx] = sum( np.array(df_tmp["impact_scaled"]) * impact_instance.frequency[0]) impact_instance.coord_exp = np.stack([countries_lat, countries_lon], axis=1) return impact_instance, countries