def test_dataset_offline(self): """""" client = Client() client.online = False with self.assertLogs('climada.util.api_client', level='WARNING') as cm: dataset = client.get_dataset_info( name='FAOSTAT_data_producer_prices', status='test_dataset') self.assertIn( "there is no internet connection but the client has stored ", cm.output[0]) self.assertEqual(dataset.version, 'v1') self.assertEqual(len(dataset.files), 1) self.assertEqual(dataset.files[0].file_size, 26481) self.assertEqual(dataset.data_type, DataTypeShortInfo('crop_production', 'exposures')) with self.assertRaises(AssertionError) as ar: with self.assertLogs('climada.util.api_client', level='WARNING') as cm: dataset2 = Client().get_dataset_info_by_uuid(dataset.uuid) self.assertIn("no logs of level WARNING or higher triggered", str(ar.exception)) self.assertEqual(dataset, dataset2) with self.assertLogs('climada.util.api_client', level='WARNING') as cm: dataset2 = client.get_dataset_info_by_uuid(dataset.uuid) self.assertIn( "there is no internet connection but the client has stored ", cm.output[0]) self.assertEqual(dataset, dataset2)
def test_download_file(self): """""" client = Client() client.MAX_WAITING_PERIOD = 0.1 dataset = client.get_dataset_info(name='FAOSTAT_data_producer_prices', status='test_dataset') # test failure def fail(x, y): raise Download.Failed("on purpose") self.assertRaises(Download.Failed, client._download_file, DATA_DIR, dataset.files[0], check=fail) self.assertFalse( DATA_DIR.joinpath(dataset.files[0].file_name).is_file()) # test success download = client._download_file(DATA_DIR, dataset.files[0]) self.assertEqual(download, DATA_DIR / dataset.files[0].file_name) self.assertTrue(download.is_file()) self.assertEqual(download.stat().st_size, dataset.files[0].file_size) download.unlink() self.assertFalse(download.is_file())
def get_haz_test_file(ds_name): # As this module is part of the installation test suite, we want tom make sure it is running # also in offline mode even when installing from pypi, where there is no test configuration. # So we set cache_enabled explicitly to true client = Client(cache_enabled=True) test_ds = client.get_dataset_info(name=ds_name, status='test_dataset') _, [haz_test_file] = client.download_dataset(test_ds) return haz_test_file
def test_dataset(self): """""" client = Client() dataset = client.get_dataset_info(name='FAOSTAT_data_producer_prices', status='test_dataset') self.assertEqual(dataset.version, 'v1') self.assertEqual(len(dataset.files), 1) self.assertEqual(dataset.files[0].file_size, 26481) self.assertEqual(dataset.data_type, DataTypeShortInfo('crop_production', 'exposures')) dataset2 = client.get_dataset_info_by_uuid(dataset.uuid) self.assertEqual(dataset, dataset2)
def test_download_dataset(self): """""" client = Client() client.MAX_WAITING_PERIOD = 0.1 dataset = client.get_dataset_info(name='test_write_raster', status='test_dataset') download_dir, downloads = client.download_dataset(dataset, target_dir=DATA_DIR) self.assertEqual(download_dir.name, dataset.version) self.assertEqual(download_dir.parent.name, dataset.name) self.assertEqual(download_dir.parent.parent.name, dataset.data_type.data_type) self.assertEqual(len(downloads), 2) for download, dsfile in zip(downloads, dataset.files): self.assertTrue(download.is_file()) self.assertEqual(download.stat().st_size, dsfile.file_size) self.assertEqual(download.name, dsfile.file_name) self.assertEqual(download.parent, download_dir) download.unlink() rm_empty_dir(download.parent.parent.parent)
def test_icon_read(self): """test reading from icon grib""" # for this test the forecast file is supposed to be already downloaded from the dwd # another download would fail because the files are available for 24h only # instead, we download it as a test dataset through the climada data api apiclient = Client() ds = apiclient.get_dataset_info(name='test_storm_europe_icon_2021012800', status='test_dataset') dsdir, _ = apiclient.download_dataset(ds) haz = StormEurope.from_icon_grib( dt.datetime(2021, 1, 28), dt.datetime(2021, 1, 28), model_name='test', grib_dir=dsdir, delete_raw_data=False) self.assertEqual(haz.tag.haz_type, 'WS') self.assertEqual(haz.units, 'm/s') self.assertEqual(haz.event_id.size, 40) self.assertEqual(haz.date.size, 40) self.assertEqual(dt.datetime.fromordinal(haz.date[0]).year, 2021) self.assertEqual(dt.datetime.fromordinal(haz.date[0]).month, 1) self.assertEqual(dt.datetime.fromordinal(haz.date[0]).day, 28) self.assertEqual(haz.event_id[-1], 40) self.assertEqual(haz.event_name[-1], '2021-01-28_ens40') self.assertIsInstance(haz.intensity, sparse.csr.csr_matrix) self.assertIsInstance(haz.fraction, sparse.csr.csr_matrix) self.assertEqual(haz.intensity.shape, (40, 49)) self.assertAlmostEqual(haz.intensity.max(), 17.276321,places=3) self.assertEqual(haz.fraction.shape, (40, 49)) with self.assertLogs('climada.hazard.storm_europe', level='WARNING') as cm: with self.assertRaises(ValueError): haz = StormEurope.from_icon_grib( dt.datetime(2021, 1, 28, 6), dt.datetime(2021, 1, 28), model_name='test', grib_dir=CONFIG.hazard.test_data.str(), delete_raw_data=False) self.assertEqual(len(cm.output), 1) self.assertIn('event definition is inaccuratly implemented', cm.output[0])
from pathos.pools import ProcessPool as Pool from tables.exceptions import HDF5ExtError from climada.entity import ImpactFunc, ImpactFuncSet from climada.entity.entity_def import Entity from climada.entity import Exposures from climada.hazard import Hazard from climada.engine.unsequa import InputVar, CalcImpact, UncOutput, CalcCostBenefit from climada.util.constants import EXP_DEMO_H5, HAZ_DEMO_H5, ENT_DEMO_TODAY, ENT_DEMO_FUTURE from climada.util.constants import TEST_UNC_OUTPUT_IMPACT, TEST_UNC_OUTPUT_COSTBEN from climada.util.api_client import Client apiclient = Client() ds = apiclient.get_dataset_info(name=TEST_UNC_OUTPUT_IMPACT, status='test_dataset') _target_dir, [test_unc_output_impact] = apiclient.download_dataset(ds) ds = apiclient.get_dataset_info(name=TEST_UNC_OUTPUT_COSTBEN, status='test_dataset') _target_dir, [test_unc_output_costben] = apiclient.download_dataset(ds) def impf_dem(x_paa=1, x_mdd=1): impf = ImpactFunc() impf.haz_type = 'TC' impf.id = 1 impf.intensity_unit = 'm/s' impf.intensity = np.linspace(0, 150, num=100) impf.mdd = np.repeat(1, len(impf.intensity)) * x_mdd impf.paa = np.arange(0, len(impf.intensity)) / len(impf.intensity) * x_paa
def get_haz_test_file(ds_name): client = Client() test_ds = client.get_dataset_info(name=ds_name, status='test_dataset') _, [haz_test_file] = client.download_dataset(test_ds) return haz_test_file