def test_soil_moisture_get_open_params_schema(self):
store = CDSDataStore(client_class=CDSClientMock,
endpoint_url=_CDS_API_URL,
cds_api_key=_CDS_API_KEY)
data_id = 'satellite-soil-moisture:volumetric:monthly'
schema = store.get_open_data_params_schema(data_id)
schema.to_dict()
def test_invalid_data_id(self):
store = CDSDataStore(cds_api_url=_CDS_API_URL,
cds_api_key=_CDS_API_KEY)
with self.assertRaises(ValueError):
store.open_data('this-data-id-does-not-exist',
variable_names=['2m_temperature'],
hours=[0],
months=[1],
years=[2019])
def test_soil_moisture_empty_variables_list(self):
store = CDSDataStore(endpoint_url=_CDS_API_URL,
cds_api_key=_CDS_API_KEY)
dataset = store.open_data('satellite-soil-moisture:volumetric:10-day',
variable_names=[],
time_range=['1981-06-14', '1982-02-13'])
self.assertEqual(len(dataset.data_vars), 0)
self.assertEqual(26, len(dataset.variables['time']))
self.assertEqual(1441, len(dataset.variables['lon']))
def test_request_parameter_out_of_range(self):
store = CDSDataStore(cds_api_url=_CDS_API_URL,
cds_api_key=_CDS_API_KEY)
with self.assertRaises(ValidationError):
store.open_data('reanalysis-era5-single-levels:ensemble_mean',
variable_names=['2m_temperature'],
bbox=[-1, -1, 181, 1],
spatial_res=0.25,
time_period='1M',
time_range=['2019-01-01', '2020-12-31'])
def test_list_and_describe_data_ids(self):
store = CDSDataStore(cds_api_url=_CDS_API_URL,
cds_api_key=_CDS_API_KEY)
data_ids = store.get_data_ids()
self.assertIsInstance(data_ids, Iterator)
for data_id in data_ids:
self.assertIsInstance(data_id, tuple)
self.assertTrue(1 <= len(data_id) <= 2)
for element in data_id:
self.assertIsInstance(element, str)
descriptor = store.describe_data(data_id[0])
self.assertIsInstance(descriptor, DataDescriptor)
def test_open_data_empty_variables_list_2(self):
store = CDSDataStore(cds_api_url=_CDS_API_URL,
cds_api_key=_CDS_API_KEY)
dataset = store.open_data('satellite-soil-moisture:volumetric:10-day',
variable_names=[],
bbox=[-180, -90, 180, 90],
spatial_res=0.25,
time_period='10D',
time_range=['1981-06-14', '1982-02-13'])
self.assertEqual(len(dataset.data_vars), 0)
self.assertEqual(26, len(dataset.variables['time']))
self.assertEqual(1441, len(dataset.variables['lon']))
def test_open_data_empty_variables_list_1(self):
store = CDSDataStore(cds_api_url=_CDS_API_URL,
cds_api_key=_CDS_API_KEY)
dataset = store.open_data(
'reanalysis-era5-land-monthly-means:monthly_averaged_reanalysis',
variable_names=[],
bbox=[-45, 0, 45, 60],
spatial_res=0.25,
time_period='1M',
time_range=['2015-01-01', '2016-12-31'])
self.assertEqual(len(dataset.data_vars), 0)
self.assertEqual(24, len(dataset.variables['time']))
self.assertEqual(361, len(dataset.variables['lon']))
def test_open_data_null_variables_list(self):
store = CDSDataStore(client_class=CDSClientMock,
cds_api_url=_CDS_API_URL,
cds_api_key=_CDS_API_KEY)
data_id = 'reanalysis-era5-single-levels-monthly-means:'\
'monthly_averaged_reanalysis'
schema = store.get_open_data_params_schema(data_id)
n_vars = len(schema.properties['variable_names'].items.enum)
dataset = store.open_data(data_id,
variable_names=None,
bbox=[-1, -1, 1, 1],
spatial_res=0.25,
time_period='1M',
time_range=['2015-10-15', '2015-10-15'])
self.assertEqual(n_vars, len(dataset.data_vars))
def test_era5_land_hourly(self):
store = CDSDataStore(client_class=CDSClientMock,
cds_api_url=_CDS_API_URL,
cds_api_key=_CDS_API_KEY)
dataset = store.open_data(
'reanalysis-era5-land',
variable_names=['2m_temperature'],
bbox=[9.5, 49.5, 10.5, 50.5],
spatial_res=0.1,
time_period='1H',
time_range=['2015-01-01', '2015-01-02'],
)
self.assertIsNotNone(dataset)
self.assertTrue('t2m' in dataset.variables)
self.assertEqual(48, len(dataset.variables['time']))
def test_era5_land_monthly(self):
store = CDSDataStore(client_class=CDSClientMock,
endpoint_url=_CDS_API_URL,
cds_api_key=_CDS_API_KEY)
dataset = store.open_data(
'reanalysis-era5-land-monthly-means:'
'monthly_averaged_reanalysis',
variable_names=['2m_temperature', '10m_u_component_of_wind'],
bbox=[9.5, 49.5, 10.5, 50.5],
spatial_res=0.1,
time_range=['2015-01-01', '2016-12-31'],
)
self.assertIsNotNone(dataset)
self.assertTrue('t2m' in dataset.variables)
self.assertTrue('u10' in dataset.variables)
def test_normalize_variable_names(self):
store = CDSDataStore(client_class=CDSClientMock, normalize_names=True,
endpoint_url=_CDS_API_URL,
cds_api_key=_CDS_API_KEY)
dataset = store.open_data(
'reanalysis-era5-single-levels-monthly-means:'
'monthly_averaged_reanalysis',
# Should be returned as p54.162, and normalized to p54_162.
variable_names=['vertical_integral_of_temperature'],
bbox=[-2, -2, 2, 2],
spatial_res=1.0,
time_range=['2019-01-01', '2020-12-31'],
)
self.assertIsNotNone(dataset)
self.assertTrue('p54_162' in dataset.variables)
def test_era5_single_levels_hourly(self):
store = CDSDataStore(client_class=CDSClientMock,
endpoint_url=_CDS_API_URL,
cds_api_key=_CDS_API_KEY)
dataset = store.open_data(
'reanalysis-era5-single-levels:'
'reanalysis',
variable_names=['2m_temperature'],
bbox=[9, 49, 11, 51],
spatial_res=0.25,
time_range=['2015-01-01',
'2015-01-02'],
)
self.assertIsNotNone(dataset)
self.assertTrue('t2m' in dataset.variables)
self.assertEqual(48, len(dataset.variables['time']))
def test_soil_moisture_volumetric_minimal_params(self):
store = CDSDataStore(client_class=CDSClientMock,
endpoint_url=_CDS_API_URL,
cds_api_key=_CDS_API_KEY)
data_id = 'satellite-soil-moisture:volumetric:monthly'
dataset = store.open_data(
data_id,
time_range=['2015-01-01', '2015-02-28'],
)
self.assertTrue('sm' in dataset.variables)
self.assertEqual(2, len(dataset.variables['time']))
self.assertEqual('2014-12-31T12:00:00Z',
dataset.attrs['time_coverage_start'])
self.assertEqual('2015-02-28T12:00:00Z',
dataset.attrs['time_coverage_end'])
description = store.describe_data(data_id)
self.assertCountEqual(description.data_vars.keys(),
map(str, dataset.data_vars))
def test_soil_moisture_saturation_10_day(self):
store = CDSDataStore(client_class=CDSClientMock,
endpoint_url=_CDS_API_URL,
cds_api_key=_CDS_API_KEY)
data_id = 'satellite-soil-moisture:saturation:10-day'
dataset = store.open_data(
data_id,
time_range=['2015-04-01', '2015-04-11'],
)
self.assertTrue('sm' in dataset.variables)
self.assertEqual(2, len(dataset.variables['time']))
self.assertEqual('2015-03-31T12:00:00Z',
dataset.attrs['time_coverage_start'])
self.assertEqual('2015-04-20T12:00:00Z',
dataset.attrs['time_coverage_end'])
description = store.describe_data(data_id)
self.assertCountEqual(description.data_vars.keys(),
map(str, dataset.data_vars))
def test_soil_moisture_without_optional_parameters(self):
store = CDSDataStore(client_class=CDSClientMock,
cds_api_url=_CDS_API_URL,
cds_api_key=_CDS_API_KEY)
data_id = 'satellite-soil-moisture:volumetric:monthly'
dataset = store.open_data(
data_id,
variable_names=['volumetric_surface_soil_moisture'],
time_range=['2015-01-01', '2015-02-28'],
)
self.assertTrue('sm' in dataset.variables)
self.assertEqual(2, len(dataset.variables['time']))
self.assertEqual('2014-12-31T12:00:00Z',
dataset.attrs['time_coverage_start'])
self.assertEqual('2015-02-28T12:00:00Z',
dataset.attrs['time_coverage_end'])
description = store.describe_data(data_id)
self.assertEqual(sorted([dv.name for dv in description.data_vars]),
sorted(map(str, dataset.data_vars)))
def test_copy_on_open(self):
store = CDSDataStore(client_class=CDSClientMock,
endpoint_url=_CDS_API_URL,
cds_api_key=_CDS_API_KEY)
data_id = 'satellite-soil-moisture:volumetric:monthly'
with tempfile.TemporaryDirectory() as temp_dir:
request_path = os.path.join(temp_dir, 'request.json')
result_path = os.path.join(temp_dir, 'result')
zarr_path = os.path.join(temp_dir, 'result.zarr')
store.open_data(
data_id,
_save_request_to=request_path,
_save_file_to=result_path,
_save_zarr_to=zarr_path,
variable_names=['volumetric_surface_soil_moisture'],
time_range=['2015-01-01', '2015-02-28'],
)
self.assertTrue(os.path.isfile(request_path))
self.assertTrue(os.path.isfile(result_path))
self.assertTrue(os.path.isdir(zarr_path))
def test_era5_describe_data(self):
store = CDSDataStore(endpoint_url=_CDS_API_URL,
cds_api_key=_CDS_API_KEY)
descriptor = store.describe_data(
'reanalysis-era5-single-levels:reanalysis')
self.assertEqual(265, len(descriptor.data_vars))
self.assertEqual('WGS84', descriptor.crs)
self.assertTupleEqual((-180, -90, 180, 90), descriptor.bbox)
# We don't exhaustively check all 260 variables, but we check one
# fully and make sure that the rest have correct type and dimensions.
expected_vd = VariableDescriptor(
name='u100',
dtype='float32',
dims=('time', 'latitude', 'longitude'),
attrs=dict(units='m s**-1', long_name='100 metre U wind component'))
self.assertDictEqual(expected_vd.__dict__,
descriptor.data_vars['u100'].__dict__)
for vd in descriptor.data_vars.values():
self.assertEqual('float32', vd.dtype)
self.assertTupleEqual(('time', 'latitude', 'longitude'),
vd.dims)
def test_get_data_store_params_schema(self):
self.assertDictEqual(
{
'type': 'object',
'properties': {
'normalize_names': {
'type': 'boolean',
'default': False
},
'num_retries': {
'type': 'integer',
'default': 200,
'minimum': 0
}
},
'additionalProperties': False
},
CDSDataStore.get_data_store_params_schema().to_dict())
def test_get_data_ids(self):
store = CDSDataStore(client_class=CDSClientMock,
cds_api_url=_CDS_API_URL,
cds_api_key=_CDS_API_KEY)
self.assertEqual([], list(store.get_data_ids('unsupported_type_spec')))
self.assertEqual([],
list(store.get_data_ids('dataset[unsupported_flag]')))
# The number of available datasets is expected to increase over time,
# so to avoid overfitting the test we just check that more than a few
# datasets and/or cubes are available. "a few" is semi-arbitrarily
# defined to be 5.
minimum_expected_datasets = 5
self.assertGreater(len(list(store.get_data_ids('dataset'))),
minimum_expected_datasets)
self.assertGreater(len(list(store.get_data_ids('dataset[cube]'))),
minimum_expected_datasets)
def test_search_data_valid_id(self):
store = CDSDataStore(cds_api_url=_CDS_API_URL,
cds_api_key=_CDS_API_KEY)
result = list(store.search_data('dataset'))
self.assertTrue(len(result) > 0)
def test_get_data_opener_ids_with_default_arguments(self):
self.assertTupleEqual((CDS_DATA_OPENER_ID, ),
CDSDataStore().get_data_opener_ids())
def test_get_store_open_params_schema_without_data_id(self):
self.assertIsInstance(CDSDataStore().get_open_data_params_schema(),
xcube.util.jsonschema.JsonObjectSchema)
def test_get_data_opener_ids_invalid_type_id(self):
with self.assertRaises(DataStoreError):
CDSDataStore().get_data_opener_ids(CDS_DATA_OPENER_ID,
'this is an invalid ID')
def test_get_data_opener_ids_invalid_opener_id(self):
with self.assertRaises(ValueError):
CDSDataStore().get_data_opener_ids('this is an invalid ID',
TYPE_SPECIFIER_DATASET)
def test_has_data_false(self):
self.assertFalse(CDSDataStore().has_data('nonexistent data ID'))
def test_has_data_true(self):
self.assertTrue(CDSDataStore().has_data('reanalysis-era5-land'))
def test_get_type_specifiers_for_data(self):
store = CDSDataStore()
self.assertEqual(
(TYPE_SPECIFIER_CUBE, ),
store.get_type_specifiers_for_data('reanalysis-era5-land'))
def test_get_type_specifiers(self):
self.assertTupleEqual((TYPE_SPECIFIER_CUBE, ),
CDSDataStore.get_type_specifiers())
def test_search_data_invalid_id(self):
store = CDSDataStore(cds_api_url=_CDS_API_URL,
cds_api_key=_CDS_API_KEY)
with self.assertRaises(DataStoreError):
store.search_data('This is an invalid ID.')