def test_download_file(self):
""""""
client = Client()
client.MAX_WAITING_PERIOD = 0.1
dataset = client.get_dataset_info(name='FAOSTAT_data_producer_prices',
status='test_dataset')
# test failure
def fail(x, y):
raise Download.Failed("on purpose")
self.assertRaises(Download.Failed,
client._download_file,
DATA_DIR,
dataset.files[0],
check=fail)
self.assertFalse(
DATA_DIR.joinpath(dataset.files[0].file_name).is_file())
# test success
download = client._download_file(DATA_DIR, dataset.files[0])
self.assertEqual(download, DATA_DIR / dataset.files[0].file_name)
self.assertTrue(download.is_file())
self.assertEqual(download.stat().st_size, dataset.files[0].file_size)
download.unlink()
self.assertFalse(download.is_file())
def test_get_litpop_default(self):
client = Client()
litpop = client.get_litpop_default(country='LUX', dump_dir=DATA_DIR)
self.assertEqual(len(litpop.gdf), 188)
self.assertEqual(np.unique(litpop.gdf.region_id), 442)
self.assertTrue('[1, 1]' in litpop.tag.description)
self.assertTrue('pc' in litpop.tag.description)
def get_haz_test_file(ds_name):
# As this module is part of the installation test suite, we want tom make sure it is running
# also in offline mode even when installing from pypi, where there is no test configuration.
# So we set cache_enabled explicitly to true
client = Client(cache_enabled=True)
test_ds = client.get_dataset_info(name=ds_name, status='test_dataset')
_, [haz_test_file] = client.download_dataset(test_ds)
return haz_test_file
def test_dataset(self):
""""""
client = Client()
dataset = client.get_dataset(name='FAOSTAT_data_producer_prices')
self.assertEqual(dataset.version, 'v1')
self.assertEqual(len(dataset.files), 1)
self.assertEqual(dataset.files[0].file_size, 26481)
self.assertEqual(dataset.data_type.data_type, 'crop_production')
dataset2 = client.get_dataset_by_uuid(dataset.uuid)
self.assertEqual(dataset, dataset2)
def test_get_hazard(self):
client = Client()
hazard = client.get_hazard(hazard_type='river_flood',
properties={
'country_name': 'Austria',
'year_range': '2010_2030',
'climate_scenario': 'rcp26'
},
dump_dir=DATA_DIR)
self.assertEqual(np.shape(hazard.intensity), (480, 5784))
self.assertEqual(np.unique(hazard.centroids.region_id), 40)
self.assertEqual(np.unique(hazard.date).size, 20)
self.assertEqual(hazard.tag.haz_type, 'RF')
def test_get_exposures(self):
client = Client()
exposures = client.get_exposures(exposures_type='litpop',
properties={
'country_iso3alpha': 'AUT',
'fin_mode': 'pop',
'exponents': '(0,1)'
},
dump_dir=DATA_DIR)
self.assertEqual(len(exposures.gdf), 5782)
self.assertEqual(np.unique(exposures.gdf.region_id), 40)
self.assertTrue('[0, 1]' in exposures.tag.description)
self.assertTrue('pop' in exposures.tag.description)
exposures
def test_dataset(self):
""""""
client = Client()
dataset = client.get_dataset_info(name='FAOSTAT_data_producer_prices',
status='test_dataset')
self.assertEqual(dataset.version, 'v1')
self.assertEqual(len(dataset.files), 1)
self.assertEqual(dataset.files[0].file_size, 26481)
self.assertEqual(dataset.data_type,
DataTypeShortInfo('crop_production', 'exposures'))
dataset2 = client.get_dataset_info_by_uuid(dataset.uuid)
self.assertEqual(dataset, dataset2)
def test_download_dataset(self):
""""""
client = Client()
client.MAX_WAITING_PERIOD = 0.1
dataset = client.get_dataset(name='test_write_raster')
downloads = client.download_dataset(dataset, target_dir=DATA_DIR)
self.assertEqual(len(downloads), 2)
for download, dsfile in zip(downloads, dataset.files):
self.assertTrue(download.is_file())
self.assertEqual(download.stat().st_size, dsfile.file_size)
self.assertEqual(download.name, dsfile.file_name)
self.assertEqual(download.parent.name, dataset.version)
self.assertEqual(download.parent.parent.name, dataset.name)
self.assertEqual(download.parent.parent.parent.name, dataset.data_type.data_type)
download.unlink()
rm_empty_dir(download.parent.parent.parent)
def test_data_type(self):
""""""
lpdt = Client().get_data_type_info("tropical_cyclone")
self.assertEqual(lpdt.data_type, 'tropical_cyclone')
self.assertEqual(lpdt.data_type_group, 'hazard')
self.assertTrue(
'res_arcsec' in
[p['property'] for p in lpdt.properties if p['mandatory']])
self.assertTrue(
'ref_year' in
[p['property'] for p in lpdt.properties if not p['mandatory']])
def test_icon_read(self):
"""test reading from icon grib"""
# for this test the forecast file is supposed to be already downloaded from the dwd
# another download would fail because the files are available for 24h only
# instead, we download it as a test dataset through the climada data api
apiclient = Client()
ds = apiclient.get_dataset_info(name='test_storm_europe_icon_2021012800', status='test_dataset')
dsdir, _ = apiclient.download_dataset(ds)
haz = StormEurope.from_icon_grib(
dt.datetime(2021, 1, 28),
dt.datetime(2021, 1, 28),
model_name='test',
grib_dir=dsdir,
delete_raw_data=False)
self.assertEqual(haz.tag.haz_type, 'WS')
self.assertEqual(haz.units, 'm/s')
self.assertEqual(haz.event_id.size, 40)
self.assertEqual(haz.date.size, 40)
self.assertEqual(dt.datetime.fromordinal(haz.date[0]).year, 2021)
self.assertEqual(dt.datetime.fromordinal(haz.date[0]).month, 1)
self.assertEqual(dt.datetime.fromordinal(haz.date[0]).day, 28)
self.assertEqual(haz.event_id[-1], 40)
self.assertEqual(haz.event_name[-1], '2021-01-28_ens40')
self.assertIsInstance(haz.intensity,
sparse.csr.csr_matrix)
self.assertIsInstance(haz.fraction,
sparse.csr.csr_matrix)
self.assertEqual(haz.intensity.shape, (40, 49))
self.assertAlmostEqual(haz.intensity.max(), 17.276321,places=3)
self.assertEqual(haz.fraction.shape, (40, 49))
with self.assertLogs('climada.hazard.storm_europe', level='WARNING') as cm:
with self.assertRaises(ValueError):
haz = StormEurope.from_icon_grib(
dt.datetime(2021, 1, 28, 6),
dt.datetime(2021, 1, 28),
model_name='test',
grib_dir=CONFIG.hazard.test_data.str(),
delete_raw_data=False)
self.assertEqual(len(cm.output), 1)
self.assertIn('event definition is inaccuratly implemented', cm.output[0])
def test_dataset_offline(self):
""""""
client = Client()
client.online = False
with self.assertLogs('climada.util.api_client', level='WARNING') as cm:
dataset = client.get_dataset_info(
name='FAOSTAT_data_producer_prices', status='test_dataset')
self.assertIn(
"there is no internet connection but the client has stored ",
cm.output[0])
self.assertEqual(dataset.version, 'v1')
self.assertEqual(len(dataset.files), 1)
self.assertEqual(dataset.files[0].file_size, 26481)
self.assertEqual(dataset.data_type,
DataTypeShortInfo('crop_production', 'exposures'))
with self.assertRaises(AssertionError) as ar:
with self.assertLogs('climada.util.api_client',
level='WARNING') as cm:
dataset2 = Client().get_dataset_info_by_uuid(dataset.uuid)
self.assertIn("no logs of level WARNING or higher triggered",
str(ar.exception))
self.assertEqual(dataset, dataset2)
with self.assertLogs('climada.util.api_client', level='WARNING') as cm:
dataset2 = client.get_dataset_info_by_uuid(dataset.uuid)
self.assertIn(
"there is no internet connection but the client has stored ",
cm.output[0])
self.assertEqual(dataset, dataset2)
def test_multi_filter(self):
client = Client()
testds = client.list_dataset_infos(data_type='storm_europe')
# assert no systemic loss in filtering
still = client._filter_datasets(testds, dict())
for o, r in zip(testds, still):
self.assertEqual(o, r)
# assert filter is effective
p = 'country_name'
a, b = 'Germany', 'Netherlands'
less = client._filter_datasets(testds, {p: [a, b]})
self.assertLess(len(less), len(testds))
only = client._filter_datasets(testds, {p: [a]})
self.assertLess(len(only), len(less))
self.assertLess(0, len(only))
def test_get_exposures_fails(self):
client = Client()
with self.assertRaises(ValueError) as cm:
client.get_exposures(exposures_type='river_flood',
properties={
'country_iso3alpha': 'AUT',
'fin_mode': 'pop',
'exponents': '(0,1)'
},
dump_dir=DATA_DIR)
self.assertIn(
'Valid exposures types are a subset of CLIMADA exposures types. Currently',
str(cm.exception))
with self.assertRaises(Client.AmbiguousResult) as cm:
client.get_exposures(exposures_type='litpop',
properties={'country_iso3alpha': 'AUT'},
dump_dir=DATA_DIR)
self.assertIn('there are several datasets meeting the requirements',
str(cm.exception))
def test_get_hazard_fails(self):
client = Client()
with self.assertRaises(ValueError) as cm:
client.get_hazard(hazard_type='litpop',
properties={
'country_name': 'Austria',
'year_range': '2010_2030',
'climate_scenario': 'rcp26'
},
dump_dir=DATA_DIR)
self.assertIn(
'Valid hazard types are a subset of CLIMADA hazard types. Currently',
str(cm.exception))
with self.assertRaises(Client.AmbiguousResult) as cm:
client.get_hazard(hazard_type='river_flood',
properties={
'country_name': ['Switzerland', 'Austria'],
'year_range': '2010_2030',
'climate_scenario': ['rcp26', 'rcp85']
},
dump_dir=DATA_DIR)
self.assertIn('there are several datasets meeting the requirements:',
str(cm.exception))
from climada.entity import Exposures
import climada.util.lines_polys_handler as u_lp
import climada.util.coordinates as u_coord
from climada.util.api_client import Client
from climada.engine.impact import Impact
from climada.hazard import Hazard
from climada.entity.impact_funcs import ImpactFuncSet
from climada.entity.impact_funcs.storm_europe import ImpfStormEurope
#TODO: add tests for the private methods
# Load gdfs and hazard and impact functions for tests
HAZ = Client().get_hazard('storm_europe',
name='test_haz_WS_nl',
status='test_dataset')
EXP_POLY = Client().get_exposures('base',
name='test_polygon_exp',
status='test_dataset')
GDF_POLY = EXP_POLY.gdf
EXP_LINE = Client().get_exposures('base',
name='test_line_exp',
status='test_dataset')
GDF_LINE = EXP_LINE.gdf
EXP_POINT = Client().get_exposures('base',
name='test_point_exp',
status='test_dataset')
def test_datasets(self):
""""""
datasets = Client().list_dataset_infos(
status=None, name='FAOSTAT_data_producer_prices')
self.assertEqual(len(datasets), 1)
def test_data_types(self):
""""""
exdts = Client().list_data_type_infos("exposures")
self.assertTrue('litpop' in [exdt.data_type for exdt in exdts])
def test_data_type(self):
""""""
lpdt = Client().get_data_type("litpop")
self.assertEqual(lpdt.data_type, 'litpop')
self.assertEqual(lpdt.data_type_group, 'exposures')
def test_multiplicity_split(self):
properties = {'country_name': ['x', 'y', 'z'], 'b': '1'}
# assert split matches expectations
straight, multi = Client._divide_straight_from_multi(properties)
self.assertEqual(straight, {'b': '1'})
self.assertEqual(multi, {'country_name': ['x', 'y', 'z']})
import scipy as sp
from pathos.pools import ProcessPool as Pool
from tables.exceptions import HDF5ExtError
from climada.entity import ImpactFunc, ImpactFuncSet
from climada.entity.entity_def import Entity
from climada.entity import Exposures
from climada.hazard import Hazard
from climada.engine.unsequa import InputVar, CalcImpact, UncOutput, CalcCostBenefit
from climada.util.constants import EXP_DEMO_H5, HAZ_DEMO_H5, ENT_DEMO_TODAY, ENT_DEMO_FUTURE
from climada.util.constants import TEST_UNC_OUTPUT_IMPACT, TEST_UNC_OUTPUT_COSTBEN
from climada.util.api_client import Client
apiclient = Client()
ds = apiclient.get_dataset_info(name=TEST_UNC_OUTPUT_IMPACT,
status='test_dataset')
_target_dir, [test_unc_output_impact] = apiclient.download_dataset(ds)
ds = apiclient.get_dataset_info(name=TEST_UNC_OUTPUT_COSTBEN,
status='test_dataset')
_target_dir, [test_unc_output_costben] = apiclient.download_dataset(ds)
def impf_dem(x_paa=1, x_mdd=1):
impf = ImpactFunc()
impf.haz_type = 'TC'
impf.id = 1
impf.intensity_unit = 'm/s'
impf.intensity = np.linspace(0, 150, num=100)
def get_haz_test_file(ds_name):
client = Client()
test_ds = client.get_dataset_info(name=ds_name, status='test_dataset')
_, [haz_test_file] = client.download_dataset(test_ds)
return haz_test_file