def test_get_dataset_tile_with_time_dim(self):
ctx = new_test_service_context()
tile = get_dataset_tile(ctx, 'demo', 'conc_tsm', '0', '0', '0', RequestParamsMock(time='2017-01-26'))
self.assertIsInstance(tile, bytes)
ctx = new_test_service_context()
tile = get_dataset_tile(ctx, 'demo', 'conc_tsm', '0', '0', '0', RequestParamsMock(time='2017-01-26/2017-01-27'))
self.assertIsInstance(tile, bytes)
ctx = new_test_service_context()
tile = get_dataset_tile(ctx, 'demo', 'conc_tsm', '0', '0', '0', RequestParamsMock(time='current'))
self.assertIsInstance(tile, bytes)
def test_get_rgb_color_mapping(self):
ctx = new_test_service_context()
rgb_cm = ctx.get_rgb_color_mapping('demo')
self.assertEqual(([None, None, None], [(0.0, 1.0), (0.0, 1.0),
(0.0, 1.0)]), rgb_cm)
rgb_cm = ctx.get_rgb_color_mapping('demo', norm_range=(1.0, 2.5))
self.assertEqual(([None, None, None], [(1.0, 2.5), (1.0, 2.5),
(1.0, 2.5)]), rgb_cm)
ctx = new_test_service_context('config-rgb.yml')
rgb_cm = ctx.get_rgb_color_mapping('demo-rgb')
self.assertEqual((['conc_chl', 'conc_tsm', 'kd489'], [(0.0, 24.0),
(0.0, 100.0),
(0.0, 6.0)]),
rgb_cm)
def test_get_dataset_tile_grid(self):
self.maxDiff = None
ctx = new_test_service_context()
tile_grid = get_dataset_tile_grid(ctx, 'demo', 'conc_chl', 'ol4',
'http://bibo')
self.assertEqual(
{
'projection':
'EPSG:4326',
'tileGrid': {
'extent': [0, 50, 5, 52.5],
'origin': [0, 52.5],
'resolutions': [0.01, 0.005, 0.0025],
'sizes': [[2, 1], [4, 2], [8, 4]],
'tileSize': [250, 250]
},
'url':
'http://bibo/datasets/demo/vars/conc_chl/tiles/{z}/{x}/{y}.png'
}, tile_grid)
with self.assertRaises(NotImplementedError):
get_dataset_tile_grid(ctx, 'demo', 'conc_chl', 'cesium',
'http://bibo')
with self.assertRaises(ServiceBadRequestError) as cm:
get_dataset_tile_grid(ctx, 'demo', 'conc_chl', 'ol2.json',
'http://bibo')
self.assertEqual(400, cm.exception.status_code)
self.assertEqual('Unknown tile client "ol2.json"', cm.exception.reason)
def test_get_time_series_for_point_out_of_bounds(self):
ctx = new_test_service_context()
actual_result = get_time_series(
ctx, 'demo', 'conc_tsm',
dict(type="Point", coordinates=[-150.0, -30.0]))
expected_result = []
self.assertEqual(expected_result, actual_result)
def test_get_time_series_for_point_collection(self):
ctx = new_test_service_context()
actual_result = get_time_series(
ctx,
'demo',
'conc_tsm',
dict(type="GeometryCollection",
geometries=[dict(type="Point", coordinates=[2.1, 51.4])]),
agg_methods=['mean', 'count'],
start_date=np.datetime64('2017-01-15'),
end_date=np.datetime64('2017-01-29'))
expected_result = [[{
'mean': 3.534773588180542,
'time': '2017-01-16T10:09:22Z'
}, {
'mean': None,
'time': '2017-01-25T09:35:51Z'
}, {
'mean': None,
'time': '2017-01-26T10:50:17Z'
}, {
'mean': 20.12085723876953,
'time': '2017-01-28T09:58:11Z'
}]]
self.assertAlmostEqualDeep(expected_result, actual_result)
def test_get_time_series_for_polygon_only_valids(self):
ctx = new_test_service_context()
actual_result = get_time_series(ctx,
'demo',
'conc_tsm',
dict(type="Polygon",
coordinates=[[[1.,
51.], [2., 51.],
[2.,
52.], [1., 52.],
[1., 51.]]]),
agg_methods=['mean', 'count'],
max_valids=-1)
expected_result = [{
'count': 122392,
'count_tot': 159600,
'mean': 56.12519223634024,
'time': '2017-01-16T10:09:22Z'
}, {
'count': 132066,
'count_tot': 159600,
'mean': 49.70755256053988,
'time': '2017-01-28T09:58:11Z'
}]
self.assertAlmostEqualDeep(expected_result, actual_result)
def test_dataset_with_point_and_details(self):
ctx = new_test_service_context()
response = get_datasets(ctx,
point=(1.7, 51.2),
details=True,
base_url="http://test")
self.assertIsInstance(response, dict)
self.assertIn("datasets", response)
self.assertIsInstance(response["datasets"], list)
self.assertEqual(2, len(response["datasets"]))
dataset = response["datasets"][0]
self.assertIsInstance(dataset, dict)
self.assertIn("id", dataset)
self.assertIn("title", dataset)
self.assertIn("variables", dataset)
self.assertIn("dimensions", dataset)
response = get_datasets(ctx,
point=(1.7, 58.0),
details=True,
base_url="http://test")
self.assertIsInstance(response, dict)
self.assertIn("datasets", response)
self.assertIsInstance(response["datasets"], list)
self.assertEqual(0, len(response["datasets"]))
def test_dataset_with_details_and_rgb_schema(self):
ctx = new_test_service_context('config-rgb.yml')
response = get_datasets(ctx, details=True, base_url="http://test")
datasets = self._assert_datasets(response, 1)
dataset = datasets[0]
self.assertIsInstance(dataset, dict)
self.assertEqual({'varNames': ['conc_chl', 'conc_tsm', 'kd489'],
'normRanges': [(0.0, 24.0), (0.0, 100.0), (0.0, 6.0)]},
dataset.get("rgbSchema"))
response = get_datasets(ctx, details=True, client='ol4', base_url="http://test")
datasets = self._assert_datasets(response, 1)
dataset = datasets[0]
self.assertIsInstance(dataset, dict)
self.assertEqual({'varNames': ['conc_chl', 'conc_tsm', 'kd489'],
'normRanges': [(0.0, 24.0), (0.0, 100.0), (0.0, 6.0)],
'tileSourceOptions': {
'url': 'http://test/datasets/demo-rgb/vars/rgb/tiles/{z}/{x}/{y}.png',
'maxZoom': 2,
'minZoom': 0,
'projection': 'EPSG:4326',
'tileGrid': {'extent': [0, 50, 5, 52.5],
'origin': [0, 52.5],
'resolutions': [0.01, 0.005, 0.0025],
'tileSize': [250, 250]}
}},
dataset.get("rgbSchema"))
def test_get_time_series_for_point_with_uncertainty(self):
ctx = new_test_service_context()
actual_result = get_time_series_for_point(
ctx,
'demo-1w',
'conc_tsm',
lon=2.1,
lat=51.4,
start_date=np.datetime64('2017-01-15'),
end_date=np.datetime64('2017-01-29'))
expected_result = {
'results': [{
'date': '2017-01-22T00:00:00Z',
'result': {
'average': 3.534773588180542,
'totalCount': 1,
'validCount': 1
}
}, {
'date': '2017-01-29T00:00:00Z',
'result': {
'average': 20.12085723876953,
'totalCount': 1,
'validCount': 1
}
}]
}
self.assertAlmostEqualDeep(expected_result, actual_result)
def test_get_time_series_for_point_only_valids(self):
ctx = new_test_service_context()
actual_result = get_time_series_for_point(
ctx,
'demo',
'conc_tsm',
lon=2.1,
lat=51.4,
start_date=np.datetime64('2017-01-15'),
end_date=np.datetime64('2017-01-29'),
max_valids=-1)
expected_result = {
'results': [{
'date': '2017-01-16T10:09:22Z',
'result': {
'average': 3.534773588180542,
'totalCount': 1,
'validCount': 1
}
}, {
'date': '2017-01-28T09:58:11Z',
'result': {
'average': 20.12085723876953,
'totalCount': 1,
'validCount': 1
}
}]
}
self.assertAlmostEqualDeep(expected_result, actual_result)
def test_get_dataset_tile_grid(self):
self.maxDiff = None
ctx = new_test_service_context()
tile_grid = get_dataset_tile_grid(ctx, 'demo', 'conc_chl', 'ol4', 'http://bibo')
self.assertEqual(
{
'maxZoom': 2,
'minZoom': 0,
'projection': 'EPSG:4326',
'tileGrid': {'extent': [0, 50, 5, 52.5],
'origin': [0, 52.5],
'resolutions': [0.01, 0.005, 0.0025],
'tileSize': [250, 250]},
'url': 'http://bibo/datasets/demo/vars/conc_chl/tiles/{z}/{x}/{y}.png'
},
tile_grid)
tile_grid = get_dataset_tile_grid(ctx, 'demo', 'conc_chl', 'cesium', 'http://bibo')
self.assertEqual({
'url': self.base_url + '/datasets/demo/vars/conc_chl/tiles/{z}/{x}/{y}.png',
'rectangle': dict(west=0.0, south=50.0, east=5.0, north=52.5),
'minimumLevel': 0,
'maximumLevel': 2,
'tileWidth': 250,
'tileHeight': 250,
'tilingScheme': {'rectangle': dict(west=0.0, south=50.0, east=5.0, north=52.5),
'numberOfLevelZeroTilesX': 2,
'numberOfLevelZeroTilesY': 1},
}, tile_grid)
with self.assertRaises(ServiceBadRequestError) as cm:
get_dataset_tile_grid(ctx, 'demo', 'conc_chl', 'ol2.json', 'http://bibo')
self.assertEqual(400, cm.exception.status_code)
self.assertEqual('Unknown tile client "ol2.json"', cm.exception.reason)
def test_get_time_series_for_geometry_polygon_only_valids(self):
ctx = new_test_service_context()
actual_result = get_time_series_for_geometry(
ctx,
'demo',
'conc_tsm',
dict(type="Polygon",
coordinates=[[[1., 51.], [2., 51.], [2., 52.], [1., 52.],
[1., 51.]]]),
include_count=True,
max_valids=-1)
expected_result = {
'results': [{
'date': '2017-01-16T10:09:22Z',
'result': {
'average': 56.12519223634024,
'totalCount': 159600,
'validCount': 122392
}
}, {
'date': '2017-01-28T09:58:11Z',
'result': {
'average': 49.70755256053988,
'totalCount': 159600,
'validCount': 132066
}
}]
}
self.assertAlmostEqualDeep(expected_result, actual_result)
def test_s3_store_already_existing(self):
ctx = new_test_service_context()
dataset_config_1 = {
'Identifier': 'seven',
'Title': 'Test 7',
'FileSystem': 's3',
'Endpoint': 'https://s3.eu-central-1.amazonaws.com',
'Path': 'xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr',
'Region': 'eu-central-1'
}
dataset_config_2 = {
'Identifier': 'seven_a',
'Title': 'Test 7 a',
'FileSystem': 's3',
'Endpoint': 'https://s3.eu-central-1.amazonaws.com',
'Path': 'xcube-examples/OLCI-SNS-RAW-CUBE-3.zarr',
'Region': 'eu-central-1'
}
ctx.config['Datasets'] = [dataset_config_1, dataset_config_2]
dataset_configs = ctx.get_dataset_configs()
self.assertEqual(dataset_configs[0]['StoreInstanceId'],
dataset_configs[1]['StoreInstanceId'])
def test_get_time_series_for_polygon_collection(self):
ctx = new_test_service_context()
actual_result = get_time_series(ctx,
'demo', 'conc_tsm',
dict(type="GeometryCollection",
geometries=[dict(type="Polygon", coordinates=[[
[1., 51.], [2., 51.], [2., 52.], [1., 52.],
[1., 51.]
]])]),
agg_methods=['mean', 'count'])
expected_result = [[{'count': 122392,
'count_tot': 159600,
'mean': 56.12519223634024,
'time': '2017-01-16T10:09:22Z'},
{'count': 0,
'count_tot': 159600,
'mean': None,
'time': '2017-01-25T09:35:51Z'},
{'count': 0,
'count_tot': 159600,
'mean': None,
'time': '2017-01-26T10:50:17Z'},
{'count': 132066,
'count_tot': 159600,
'mean': 49.70755256053988,
'time': '2017-01-28T09:58:11Z'},
{'count': 0,
'count_tot': 159600,
'mean': None,
'time': '2017-01-30T10:46:34Z'}]]
self.assertAlmostEqualDeep(expected_result, actual_result)
def test_get_dataset_rgb_tile_invalid_b(self):
ctx = new_test_service_context('config-rgb.yml')
with self.assertRaises(ServiceBadRequestError) as cm:
get_dataset_tile(ctx, 'demo-rgb', 'rgb', '0', '0', '0', RequestParamsMock(b='refl_3'))
self.assertEqual(400, cm.exception.status_code)
self.assertEqual("Variable 'refl_3' not found in dataset 'demo-rgb'",
cm.exception.reason)
def test_get_dataset_rgb_tile_no_vars(self):
ctx = new_test_service_context()
with self.assertRaises(ServiceBadRequestError) as cm:
get_dataset_tile(ctx, 'demo', 'rgb', '0', '0', '0', RequestParamsMock())
self.assertEqual(400, cm.exception.status_code)
self.assertEqual("No variable in dataset 'demo' specified for RGB",
cm.exception.reason)
def test_obs(self):
# this test tests backwards compatibility.
# TODO please remove when support for file systems 'local' and 'obs'
# has ended
ctx = new_test_service_context()
dataset_config = {
'Identifier': 'two',
'Title': 'Test 2',
'FileSystem': 'obs',
'Endpoint': 'https://s3.eu-central-1.amazonaws.com',
'Path': 'xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr',
'Region': 'eu-central-1'
}
ctx.config['Datasets'] = [dataset_config]
dataset_config = ctx.get_dataset_configs()[0]
self.assertEqual([
'Identifier', 'Title', 'FileSystem', 'Endpoint', 'Path', 'Region',
'StoreInstanceId'
], list(dataset_config.keys()))
self.assertEqual('two', dataset_config['Identifier'])
self.assertEqual('Test 2', dataset_config['Title'])
self.assertEqual('obs', dataset_config['FileSystem'])
self.assertEqual('https://s3.eu-central-1.amazonaws.com',
dataset_config['Endpoint'])
self.assertEqual('OLCI-SNS-RAW-CUBE-2.zarr', dataset_config['Path'])
self.assertEqual('eu-central-1', dataset_config['Region'])
self.assertEqual('s3_1', dataset_config['StoreInstanceId'])
def test_get_dataset_tile_with_invalid_time_dim(self):
ctx = new_test_service_context()
with self.assertRaises(ServiceBadRequestError) as cm:
get_dataset_tile(ctx, 'demo', 'conc_tsm', '0', '0', '0', RequestParamsMock(time='Gnaaark!'))
self.assertEqual(400, cm.exception.status_code)
self.assertEqual("'Gnaaark!' is not a valid value for dimension 'time'",
cm.exception.reason)
def test_get_dataset_tile(self):
ctx = new_test_service_context()
tile = get_dataset_tile(ctx, 'demo', 'conc_tsm', '0', '0', '0', RequestParamsMock())
self.assertIsInstance(tile, bytes)
tile = get_dataset_tile(ctx, 'demo', 'conc_tsm', '-20', '0', '0', RequestParamsMock())
self.assertIsInstance(tile, bytes)
def test_config_and_dataset_cache(self):
ctx = new_test_service_context()
self.assertNotIn('demo', ctx.dataset_cache)
ctx.get_dataset('demo')
self.assertIn('demo', ctx.dataset_cache)
ctx.config = dict(Datasets=[
dict(Identifier='demo',
Path="../../../../examples/serve/demo/cube.nc"),
dict(Identifier='demo2',
Path="../../../../examples/serve/demo/cube.nc"),
])
self.assertNotIn('demo', ctx.dataset_cache)
self.assertNotIn('demo2', ctx.dataset_cache)
ctx.get_dataset('demo2')
self.assertNotIn('demo', ctx.dataset_cache)
self.assertIn('demo2', ctx.dataset_cache)
ctx.config = dict(Datasets=[
dict(Identifier='demo2',
Path="../../../../examples/serve/demo/cube.nc"),
])
self.assertNotIn('demo', ctx.dataset_cache)
self.assertNotIn('demo2', ctx.dataset_cache)
def test_s3(self):
ctx = new_test_service_context()
dataset_config = {
'Identifier': 'two',
'Title': 'Test 2',
'FileSystem': 's3',
'Endpoint': 'https://s3.eu-central-1.amazonaws.com',
'Path': 'xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr',
'Region': 'eu-central-1'
}
ctx.config['Datasets'] = [dataset_config]
dataset_config = ctx.get_dataset_configs()[0]
self.assertEqual([
'Identifier', 'Title', 'FileSystem', 'Endpoint', 'Path', 'Region',
'StoreInstanceId'
], list(dataset_config.keys()))
self.assertEqual('two', dataset_config['Identifier'])
self.assertEqual('Test 2', dataset_config['Title'])
self.assertEqual('s3', dataset_config['FileSystem'])
self.assertEqual('https://s3.eu-central-1.amazonaws.com',
dataset_config['Endpoint'])
self.assertEqual('OLCI-SNS-RAW-CUBE-2.zarr', dataset_config['Path'])
self.assertEqual('eu-central-1', dataset_config['Region'])
self.assertEqual('s3_1', dataset_config['StoreInstanceId'])
def test_dataset_with_details(self):
ctx = new_test_service_context()
response = get_datasets(ctx, details=True, base_url="http://test")
datasets = self._assert_datasets(response, 2)
demo_dataset = None
demo_1w_dataset = None
for dataset in datasets:
self.assertIsInstance(dataset, dict)
self.assertIn("id", dataset)
self.assertIn("title", dataset)
self.assertIn("attributions", dataset)
self.assertIn("variables", dataset)
self.assertIn("dimensions", dataset)
self.assertNotIn("rgbSchema", dataset)
if dataset["id"] == "demo":
demo_dataset = dataset
if dataset["id"] == "demo-1w":
demo_1w_dataset = dataset
self.assertIsNotNone(demo_dataset)
self.assertIsNotNone(demo_1w_dataset)
self.assertEqual(["© by EU H2020 CyanoAlert project"], demo_dataset['attributions'])
self.assertEqual(["© by Brockmann Consult GmbH 2020, "
"contains modified Copernicus Data 2019, processed by ESA"], demo_1w_dataset['attributions'])
def test_get_time_series_for_polygon_with_all_agg_methods(self):
ctx = new_test_service_context()
actual_result = get_time_series(
ctx,
'demo',
'conc_tsm',
dict(type="Polygon",
coordinates=[[[1., 51.], [2., 51.], [2., 52.], [1., 52.],
[1., 51.]]]),
agg_methods=['mean', 'median', 'std', 'min', 'max', 'count'])
expected_result = [{
'count': 122392,
'count_tot': 159600,
'max': 166.57278442382812,
'mean': 56.12519223634024,
'median': 48.009796142578125,
'min': 0.02251400426030159,
'std': 40.78859862094861,
'time': '2017-01-16T10:09:22Z'
}, {
'count': 0,
'count_tot': 159600,
'max': None,
'mean': None,
'median': None,
'min': None,
'std': None,
'time': '2017-01-25T09:35:51Z'
}, {
'count': 0,
'count_tot': 159600,
'max': None,
'mean': None,
'median': None,
'min': None,
'std': None,
'time': '2017-01-26T10:50:17Z'
}, {
'count': 132066,
'count_tot': 159600,
'max': 158.7908477783203,
'mean': 49.70755256053988,
'median': 39.326446533203125,
'min': 0.02607600949704647,
'std': 34.98868194514787,
'time': '2017-01-28T09:58:11Z'
}, {
'count': 0,
'count_tot': 159600,
'max': None,
'mean': None,
'median': None,
'min': None,
'std': None,
'time': '2017-01-30T10:46:34Z'
}]
self.assertAlmostEqualDeep(expected_result, actual_result)
def test_get_dataset_tile_invalid_variable(self):
ctx = new_test_service_context()
with self.assertRaises(ServiceResourceNotFoundError) as cm:
get_dataset_tile(ctx, 'demo', 'conc_tdi', '0', '0', '0',
RequestParamsMock())
self.assertEqual(404, cm.exception.status_code)
self.assertEqual('Variable "conc_tdi" not found in dataset "demo"',
f'{cm.exception.reason}')
def test_get_time_series_info(self):
self.maxDiff = None
ctx = new_test_service_context()
info = get_time_series_info(ctx)
expected_result = self._get_expected_info_dict()
self.assertEqual(expected_result, info)
def test_get_time_series_for_point_invalid_lat_and_lon(self):
ctx = new_test_service_context()
actual_result = get_time_series_for_point(ctx,
'demo',
'conc_tsm',
lon=-150.0,
lat=-30.0)
expected_result = {'results': []}
self.assertEqual(expected_result, actual_result)
def test_get_dataset_tile_with_all_params(self):
ctx = new_test_service_context()
tile = get_dataset_tile(
ctx, 'demo', 'conc_tsm', '0', '0', '0',
RequestParamsMock(time='current',
cbar='plasma',
vmin='0.1',
vmax='0.3'))
self.assertIsInstance(tile, bytes)
def test_get_time_series_for_geometry_polygon_with_stdev(self):
ctx = new_test_service_context()
actual_result = get_time_series_for_geometry(
ctx,
'demo',
'conc_tsm',
dict(type="Polygon",
coordinates=[[[1., 51.], [2., 51.], [2., 52.], [1., 52.],
[1., 51.]]]),
include_count=True,
include_stdev=True)
expected_result = {
'results': [{
'date': '2017-01-16T10:09:22Z',
'result': {
'average': 56.12519223634024,
'totalCount': 159600,
'uncertainty': 40.78859862094861,
'validCount': 122392
}
}, {
'date': '2017-01-25T09:35:51Z',
'result': {
'average': None,
'totalCount': 159600,
'uncertainty': None,
'validCount': 0
}
}, {
'date': '2017-01-26T10:50:17Z',
'result': {
'average': None,
'totalCount': 159600,
'uncertainty': None,
'validCount': 0
}
}, {
'date': '2017-01-28T09:58:11Z',
'result': {
'average': 49.70755256053988,
'totalCount': 159600,
'uncertainty': 34.98868194514786,
'validCount': 132066
}
}, {
'date': '2017-01-30T10:46:34Z',
'result': {
'average': None,
'totalCount': 159600,
'uncertainty': None,
'validCount': 0
}
}]
}
self.assertAlmostEqualDeep(expected_result, actual_result)
def test_get_s3_bucket_mapping(self):
ctx = new_test_service_context()
bucket_mapping = ctx.get_s3_bucket_mapping()
self.assertEqual(['demo'], list(bucket_mapping.keys()))
path = bucket_mapping['demo']
self.assertTrue(os.path.isabs(path))
self.assertTrue(
path.replace(
'\\', '/').endswith('examples/serve/demo/cube-1-250-250.zarr'))
def test_get_dataset_configs_from_stores(self):
ctx = new_test_service_context(
config_file_name='config-datastores.yml')
dataset_configs_from_stores = ctx.get_dataset_configs_from_stores()
self.assertIsNotNone(dataset_configs_from_stores)
self.assertEqual(2, len(dataset_configs_from_stores))
ids = [config['Identifier'] for config in dataset_configs_from_stores]
self.assertIn('test~cube-1-250-250.zarr', ids)
self.assertIn('test~cube-5-100-200.zarr', ids)
