def test_dataset_offline(self):
""""""
client = Client()
client.online = False
with self.assertLogs('climada.util.api_client', level='WARNING') as cm:
dataset = client.get_dataset_info(
name='FAOSTAT_data_producer_prices', status='test_dataset')
self.assertIn(
"there is no internet connection but the client has stored ",
cm.output[0])
self.assertEqual(dataset.version, 'v1')
self.assertEqual(len(dataset.files), 1)
self.assertEqual(dataset.files[0].file_size, 26481)
self.assertEqual(dataset.data_type,
DataTypeShortInfo('crop_production', 'exposures'))
with self.assertRaises(AssertionError) as ar:
with self.assertLogs('climada.util.api_client',
level='WARNING') as cm:
dataset2 = Client().get_dataset_info_by_uuid(dataset.uuid)
self.assertIn("no logs of level WARNING or higher triggered",
str(ar.exception))
self.assertEqual(dataset, dataset2)
with self.assertLogs('climada.util.api_client', level='WARNING') as cm:
dataset2 = client.get_dataset_info_by_uuid(dataset.uuid)
self.assertIn(
"there is no internet connection but the client has stored ",
cm.output[0])
self.assertEqual(dataset, dataset2)
def test_download_file(self):
""""""
client = Client()
client.MAX_WAITING_PERIOD = 0.1
dataset = client.get_dataset_info(name='FAOSTAT_data_producer_prices',
status='test_dataset')
# test failure
def fail(x, y):
raise Download.Failed("on purpose")
self.assertRaises(Download.Failed,
client._download_file,
DATA_DIR,
dataset.files[0],
check=fail)
self.assertFalse(
DATA_DIR.joinpath(dataset.files[0].file_name).is_file())
# test success
download = client._download_file(DATA_DIR, dataset.files[0])
self.assertEqual(download, DATA_DIR / dataset.files[0].file_name)
self.assertTrue(download.is_file())
self.assertEqual(download.stat().st_size, dataset.files[0].file_size)
download.unlink()
self.assertFalse(download.is_file())
def get_haz_test_file(ds_name):
# As this module is part of the installation test suite, we want tom make sure it is running
# also in offline mode even when installing from pypi, where there is no test configuration.
# So we set cache_enabled explicitly to true
client = Client(cache_enabled=True)
test_ds = client.get_dataset_info(name=ds_name, status='test_dataset')
_, [haz_test_file] = client.download_dataset(test_ds)
return haz_test_file
def test_dataset(self):
""""""
client = Client()
dataset = client.get_dataset_info(name='FAOSTAT_data_producer_prices',
status='test_dataset')
self.assertEqual(dataset.version, 'v1')
self.assertEqual(len(dataset.files), 1)
self.assertEqual(dataset.files[0].file_size, 26481)
self.assertEqual(dataset.data_type,
DataTypeShortInfo('crop_production', 'exposures'))
dataset2 = client.get_dataset_info_by_uuid(dataset.uuid)
self.assertEqual(dataset, dataset2)
def test_download_dataset(self):
""""""
client = Client()
client.MAX_WAITING_PERIOD = 0.1
dataset = client.get_dataset_info(name='test_write_raster',
status='test_dataset')
download_dir, downloads = client.download_dataset(dataset,
target_dir=DATA_DIR)
self.assertEqual(download_dir.name, dataset.version)
self.assertEqual(download_dir.parent.name, dataset.name)
self.assertEqual(download_dir.parent.parent.name,
dataset.data_type.data_type)
self.assertEqual(len(downloads), 2)
for download, dsfile in zip(downloads, dataset.files):
self.assertTrue(download.is_file())
self.assertEqual(download.stat().st_size, dsfile.file_size)
self.assertEqual(download.name, dsfile.file_name)
self.assertEqual(download.parent, download_dir)
download.unlink()
rm_empty_dir(download.parent.parent.parent)
def test_icon_read(self):
"""test reading from icon grib"""
# for this test the forecast file is supposed to be already downloaded from the dwd
# another download would fail because the files are available for 24h only
# instead, we download it as a test dataset through the climada data api
apiclient = Client()
ds = apiclient.get_dataset_info(name='test_storm_europe_icon_2021012800', status='test_dataset')
dsdir, _ = apiclient.download_dataset(ds)
haz = StormEurope.from_icon_grib(
dt.datetime(2021, 1, 28),
dt.datetime(2021, 1, 28),
model_name='test',
grib_dir=dsdir,
delete_raw_data=False)
self.assertEqual(haz.tag.haz_type, 'WS')
self.assertEqual(haz.units, 'm/s')
self.assertEqual(haz.event_id.size, 40)
self.assertEqual(haz.date.size, 40)
self.assertEqual(dt.datetime.fromordinal(haz.date[0]).year, 2021)
self.assertEqual(dt.datetime.fromordinal(haz.date[0]).month, 1)
self.assertEqual(dt.datetime.fromordinal(haz.date[0]).day, 28)
self.assertEqual(haz.event_id[-1], 40)
self.assertEqual(haz.event_name[-1], '2021-01-28_ens40')
self.assertIsInstance(haz.intensity,
sparse.csr.csr_matrix)
self.assertIsInstance(haz.fraction,
sparse.csr.csr_matrix)
self.assertEqual(haz.intensity.shape, (40, 49))
self.assertAlmostEqual(haz.intensity.max(), 17.276321,places=3)
self.assertEqual(haz.fraction.shape, (40, 49))
with self.assertLogs('climada.hazard.storm_europe', level='WARNING') as cm:
with self.assertRaises(ValueError):
haz = StormEurope.from_icon_grib(
dt.datetime(2021, 1, 28, 6),
dt.datetime(2021, 1, 28),
model_name='test',
grib_dir=CONFIG.hazard.test_data.str(),
delete_raw_data=False)
self.assertEqual(len(cm.output), 1)
self.assertIn('event definition is inaccuratly implemented', cm.output[0])
from pathos.pools import ProcessPool as Pool
from tables.exceptions import HDF5ExtError
from climada.entity import ImpactFunc, ImpactFuncSet
from climada.entity.entity_def import Entity
from climada.entity import Exposures
from climada.hazard import Hazard
from climada.engine.unsequa import InputVar, CalcImpact, UncOutput, CalcCostBenefit
from climada.util.constants import EXP_DEMO_H5, HAZ_DEMO_H5, ENT_DEMO_TODAY, ENT_DEMO_FUTURE
from climada.util.constants import TEST_UNC_OUTPUT_IMPACT, TEST_UNC_OUTPUT_COSTBEN
from climada.util.api_client import Client
apiclient = Client()
ds = apiclient.get_dataset_info(name=TEST_UNC_OUTPUT_IMPACT,
status='test_dataset')
_target_dir, [test_unc_output_impact] = apiclient.download_dataset(ds)
ds = apiclient.get_dataset_info(name=TEST_UNC_OUTPUT_COSTBEN,
status='test_dataset')
_target_dir, [test_unc_output_costben] = apiclient.download_dataset(ds)
def impf_dem(x_paa=1, x_mdd=1):
impf = ImpactFunc()
impf.haz_type = 'TC'
impf.id = 1
impf.intensity_unit = 'm/s'
impf.intensity = np.linspace(0, 150, num=100)
impf.mdd = np.repeat(1, len(impf.intensity)) * x_mdd
impf.paa = np.arange(0, len(impf.intensity)) / len(impf.intensity) * x_paa
def get_haz_test_file(ds_name):
client = Client()
test_ds = client.get_dataset_info(name=ds_name, status='test_dataset')
_, [haz_test_file] = client.download_dataset(test_ds)
return haz_test_file
