def test_get_dataset_tile_with_time_dim(self): ctx = new_test_service_context() tile = get_dataset_tile(ctx, 'demo', 'conc_tsm', '0', '0', '0', RequestParamsMock(time='2017-01-26')) self.assertIsInstance(tile, bytes) ctx = new_test_service_context() tile = get_dataset_tile(ctx, 'demo', 'conc_tsm', '0', '0', '0', RequestParamsMock(time='2017-01-26/2017-01-27')) self.assertIsInstance(tile, bytes) ctx = new_test_service_context() tile = get_dataset_tile(ctx, 'demo', 'conc_tsm', '0', '0', '0', RequestParamsMock(time='current')) self.assertIsInstance(tile, bytes)
def test_get_rgb_color_mapping(self): ctx = new_test_service_context() rgb_cm = ctx.get_rgb_color_mapping('demo') self.assertEqual(([None, None, None], [(0.0, 1.0), (0.0, 1.0), (0.0, 1.0)]), rgb_cm) rgb_cm = ctx.get_rgb_color_mapping('demo', norm_range=(1.0, 2.5)) self.assertEqual(([None, None, None], [(1.0, 2.5), (1.0, 2.5), (1.0, 2.5)]), rgb_cm) ctx = new_test_service_context('config-rgb.yml') rgb_cm = ctx.get_rgb_color_mapping('demo-rgb') self.assertEqual((['conc_chl', 'conc_tsm', 'kd489'], [(0.0, 24.0), (0.0, 100.0), (0.0, 6.0)]), rgb_cm)
def test_get_dataset_tile_grid(self): self.maxDiff = None ctx = new_test_service_context() tile_grid = get_dataset_tile_grid(ctx, 'demo', 'conc_chl', 'ol4', 'http://bibo') self.assertEqual( { 'projection': 'EPSG:4326', 'tileGrid': { 'extent': [0, 50, 5, 52.5], 'origin': [0, 52.5], 'resolutions': [0.01, 0.005, 0.0025], 'sizes': [[2, 1], [4, 2], [8, 4]], 'tileSize': [250, 250] }, 'url': 'http://bibo/datasets/demo/vars/conc_chl/tiles/{z}/{x}/{y}.png' }, tile_grid) with self.assertRaises(NotImplementedError): get_dataset_tile_grid(ctx, 'demo', 'conc_chl', 'cesium', 'http://bibo') with self.assertRaises(ServiceBadRequestError) as cm: get_dataset_tile_grid(ctx, 'demo', 'conc_chl', 'ol2.json', 'http://bibo') self.assertEqual(400, cm.exception.status_code) self.assertEqual('Unknown tile client "ol2.json"', cm.exception.reason)
def test_get_time_series_for_point_out_of_bounds(self): ctx = new_test_service_context() actual_result = get_time_series( ctx, 'demo', 'conc_tsm', dict(type="Point", coordinates=[-150.0, -30.0])) expected_result = [] self.assertEqual(expected_result, actual_result)
def test_get_time_series_for_point_collection(self): ctx = new_test_service_context() actual_result = get_time_series( ctx, 'demo', 'conc_tsm', dict(type="GeometryCollection", geometries=[dict(type="Point", coordinates=[2.1, 51.4])]), agg_methods=['mean', 'count'], start_date=np.datetime64('2017-01-15'), end_date=np.datetime64('2017-01-29')) expected_result = [[{ 'mean': 3.534773588180542, 'time': '2017-01-16T10:09:22Z' }, { 'mean': None, 'time': '2017-01-25T09:35:51Z' }, { 'mean': None, 'time': '2017-01-26T10:50:17Z' }, { 'mean': 20.12085723876953, 'time': '2017-01-28T09:58:11Z' }]] self.assertAlmostEqualDeep(expected_result, actual_result)
def test_get_time_series_for_polygon_only_valids(self): ctx = new_test_service_context() actual_result = get_time_series(ctx, 'demo', 'conc_tsm', dict(type="Polygon", coordinates=[[[1., 51.], [2., 51.], [2., 52.], [1., 52.], [1., 51.]]]), agg_methods=['mean', 'count'], max_valids=-1) expected_result = [{ 'count': 122392, 'count_tot': 159600, 'mean': 56.12519223634024, 'time': '2017-01-16T10:09:22Z' }, { 'count': 132066, 'count_tot': 159600, 'mean': 49.70755256053988, 'time': '2017-01-28T09:58:11Z' }] self.assertAlmostEqualDeep(expected_result, actual_result)
def test_dataset_with_point_and_details(self): ctx = new_test_service_context() response = get_datasets(ctx, point=(1.7, 51.2), details=True, base_url="http://test") self.assertIsInstance(response, dict) self.assertIn("datasets", response) self.assertIsInstance(response["datasets"], list) self.assertEqual(2, len(response["datasets"])) dataset = response["datasets"][0] self.assertIsInstance(dataset, dict) self.assertIn("id", dataset) self.assertIn("title", dataset) self.assertIn("variables", dataset) self.assertIn("dimensions", dataset) response = get_datasets(ctx, point=(1.7, 58.0), details=True, base_url="http://test") self.assertIsInstance(response, dict) self.assertIn("datasets", response) self.assertIsInstance(response["datasets"], list) self.assertEqual(0, len(response["datasets"]))
def test_dataset_with_details_and_rgb_schema(self): ctx = new_test_service_context('config-rgb.yml') response = get_datasets(ctx, details=True, base_url="http://test") datasets = self._assert_datasets(response, 1) dataset = datasets[0] self.assertIsInstance(dataset, dict) self.assertEqual({'varNames': ['conc_chl', 'conc_tsm', 'kd489'], 'normRanges': [(0.0, 24.0), (0.0, 100.0), (0.0, 6.0)]}, dataset.get("rgbSchema")) response = get_datasets(ctx, details=True, client='ol4', base_url="http://test") datasets = self._assert_datasets(response, 1) dataset = datasets[0] self.assertIsInstance(dataset, dict) self.assertEqual({'varNames': ['conc_chl', 'conc_tsm', 'kd489'], 'normRanges': [(0.0, 24.0), (0.0, 100.0), (0.0, 6.0)], 'tileSourceOptions': { 'url': 'http://test/datasets/demo-rgb/vars/rgb/tiles/{z}/{x}/{y}.png', 'maxZoom': 2, 'minZoom': 0, 'projection': 'EPSG:4326', 'tileGrid': {'extent': [0, 50, 5, 52.5], 'origin': [0, 52.5], 'resolutions': [0.01, 0.005, 0.0025], 'tileSize': [250, 250]} }}, dataset.get("rgbSchema"))
def test_get_time_series_for_point_with_uncertainty(self): ctx = new_test_service_context() actual_result = get_time_series_for_point( ctx, 'demo-1w', 'conc_tsm', lon=2.1, lat=51.4, start_date=np.datetime64('2017-01-15'), end_date=np.datetime64('2017-01-29')) expected_result = { 'results': [{ 'date': '2017-01-22T00:00:00Z', 'result': { 'average': 3.534773588180542, 'totalCount': 1, 'validCount': 1 } }, { 'date': '2017-01-29T00:00:00Z', 'result': { 'average': 20.12085723876953, 'totalCount': 1, 'validCount': 1 } }] } self.assertAlmostEqualDeep(expected_result, actual_result)
def test_get_time_series_for_point_only_valids(self): ctx = new_test_service_context() actual_result = get_time_series_for_point( ctx, 'demo', 'conc_tsm', lon=2.1, lat=51.4, start_date=np.datetime64('2017-01-15'), end_date=np.datetime64('2017-01-29'), max_valids=-1) expected_result = { 'results': [{ 'date': '2017-01-16T10:09:22Z', 'result': { 'average': 3.534773588180542, 'totalCount': 1, 'validCount': 1 } }, { 'date': '2017-01-28T09:58:11Z', 'result': { 'average': 20.12085723876953, 'totalCount': 1, 'validCount': 1 } }] } self.assertAlmostEqualDeep(expected_result, actual_result)
def test_get_dataset_tile_grid(self): self.maxDiff = None ctx = new_test_service_context() tile_grid = get_dataset_tile_grid(ctx, 'demo', 'conc_chl', 'ol4', 'http://bibo') self.assertEqual( { 'maxZoom': 2, 'minZoom': 0, 'projection': 'EPSG:4326', 'tileGrid': {'extent': [0, 50, 5, 52.5], 'origin': [0, 52.5], 'resolutions': [0.01, 0.005, 0.0025], 'tileSize': [250, 250]}, 'url': 'http://bibo/datasets/demo/vars/conc_chl/tiles/{z}/{x}/{y}.png' }, tile_grid) tile_grid = get_dataset_tile_grid(ctx, 'demo', 'conc_chl', 'cesium', 'http://bibo') self.assertEqual({ 'url': self.base_url + '/datasets/demo/vars/conc_chl/tiles/{z}/{x}/{y}.png', 'rectangle': dict(west=0.0, south=50.0, east=5.0, north=52.5), 'minimumLevel': 0, 'maximumLevel': 2, 'tileWidth': 250, 'tileHeight': 250, 'tilingScheme': {'rectangle': dict(west=0.0, south=50.0, east=5.0, north=52.5), 'numberOfLevelZeroTilesX': 2, 'numberOfLevelZeroTilesY': 1}, }, tile_grid) with self.assertRaises(ServiceBadRequestError) as cm: get_dataset_tile_grid(ctx, 'demo', 'conc_chl', 'ol2.json', 'http://bibo') self.assertEqual(400, cm.exception.status_code) self.assertEqual('Unknown tile client "ol2.json"', cm.exception.reason)
def test_get_time_series_for_geometry_polygon_only_valids(self): ctx = new_test_service_context() actual_result = get_time_series_for_geometry( ctx, 'demo', 'conc_tsm', dict(type="Polygon", coordinates=[[[1., 51.], [2., 51.], [2., 52.], [1., 52.], [1., 51.]]]), include_count=True, max_valids=-1) expected_result = { 'results': [{ 'date': '2017-01-16T10:09:22Z', 'result': { 'average': 56.12519223634024, 'totalCount': 159600, 'validCount': 122392 } }, { 'date': '2017-01-28T09:58:11Z', 'result': { 'average': 49.70755256053988, 'totalCount': 159600, 'validCount': 132066 } }] } self.assertAlmostEqualDeep(expected_result, actual_result)
def test_s3_store_already_existing(self): ctx = new_test_service_context() dataset_config_1 = { 'Identifier': 'seven', 'Title': 'Test 7', 'FileSystem': 's3', 'Endpoint': 'https://s3.eu-central-1.amazonaws.com', 'Path': 'xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr', 'Region': 'eu-central-1' } dataset_config_2 = { 'Identifier': 'seven_a', 'Title': 'Test 7 a', 'FileSystem': 's3', 'Endpoint': 'https://s3.eu-central-1.amazonaws.com', 'Path': 'xcube-examples/OLCI-SNS-RAW-CUBE-3.zarr', 'Region': 'eu-central-1' } ctx.config['Datasets'] = [dataset_config_1, dataset_config_2] dataset_configs = ctx.get_dataset_configs() self.assertEqual(dataset_configs[0]['StoreInstanceId'], dataset_configs[1]['StoreInstanceId'])
def test_get_time_series_for_polygon_collection(self): ctx = new_test_service_context() actual_result = get_time_series(ctx, 'demo', 'conc_tsm', dict(type="GeometryCollection", geometries=[dict(type="Polygon", coordinates=[[ [1., 51.], [2., 51.], [2., 52.], [1., 52.], [1., 51.] ]])]), agg_methods=['mean', 'count']) expected_result = [[{'count': 122392, 'count_tot': 159600, 'mean': 56.12519223634024, 'time': '2017-01-16T10:09:22Z'}, {'count': 0, 'count_tot': 159600, 'mean': None, 'time': '2017-01-25T09:35:51Z'}, {'count': 0, 'count_tot': 159600, 'mean': None, 'time': '2017-01-26T10:50:17Z'}, {'count': 132066, 'count_tot': 159600, 'mean': 49.70755256053988, 'time': '2017-01-28T09:58:11Z'}, {'count': 0, 'count_tot': 159600, 'mean': None, 'time': '2017-01-30T10:46:34Z'}]] self.assertAlmostEqualDeep(expected_result, actual_result)
def test_get_dataset_rgb_tile_invalid_b(self): ctx = new_test_service_context('config-rgb.yml') with self.assertRaises(ServiceBadRequestError) as cm: get_dataset_tile(ctx, 'demo-rgb', 'rgb', '0', '0', '0', RequestParamsMock(b='refl_3')) self.assertEqual(400, cm.exception.status_code) self.assertEqual("Variable 'refl_3' not found in dataset 'demo-rgb'", cm.exception.reason)
def test_get_dataset_rgb_tile_no_vars(self): ctx = new_test_service_context() with self.assertRaises(ServiceBadRequestError) as cm: get_dataset_tile(ctx, 'demo', 'rgb', '0', '0', '0', RequestParamsMock()) self.assertEqual(400, cm.exception.status_code) self.assertEqual("No variable in dataset 'demo' specified for RGB", cm.exception.reason)
def test_obs(self): # this test tests backwards compatibility. # TODO please remove when support for file systems 'local' and 'obs' # has ended ctx = new_test_service_context() dataset_config = { 'Identifier': 'two', 'Title': 'Test 2', 'FileSystem': 'obs', 'Endpoint': 'https://s3.eu-central-1.amazonaws.com', 'Path': 'xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr', 'Region': 'eu-central-1' } ctx.config['Datasets'] = [dataset_config] dataset_config = ctx.get_dataset_configs()[0] self.assertEqual([ 'Identifier', 'Title', 'FileSystem', 'Endpoint', 'Path', 'Region', 'StoreInstanceId' ], list(dataset_config.keys())) self.assertEqual('two', dataset_config['Identifier']) self.assertEqual('Test 2', dataset_config['Title']) self.assertEqual('obs', dataset_config['FileSystem']) self.assertEqual('https://s3.eu-central-1.amazonaws.com', dataset_config['Endpoint']) self.assertEqual('OLCI-SNS-RAW-CUBE-2.zarr', dataset_config['Path']) self.assertEqual('eu-central-1', dataset_config['Region']) self.assertEqual('s3_1', dataset_config['StoreInstanceId'])
def test_get_dataset_tile_with_invalid_time_dim(self): ctx = new_test_service_context() with self.assertRaises(ServiceBadRequestError) as cm: get_dataset_tile(ctx, 'demo', 'conc_tsm', '0', '0', '0', RequestParamsMock(time='Gnaaark!')) self.assertEqual(400, cm.exception.status_code) self.assertEqual("'Gnaaark!' is not a valid value for dimension 'time'", cm.exception.reason)
def test_get_dataset_tile(self): ctx = new_test_service_context() tile = get_dataset_tile(ctx, 'demo', 'conc_tsm', '0', '0', '0', RequestParamsMock()) self.assertIsInstance(tile, bytes) tile = get_dataset_tile(ctx, 'demo', 'conc_tsm', '-20', '0', '0', RequestParamsMock()) self.assertIsInstance(tile, bytes)
def test_config_and_dataset_cache(self): ctx = new_test_service_context() self.assertNotIn('demo', ctx.dataset_cache) ctx.get_dataset('demo') self.assertIn('demo', ctx.dataset_cache) ctx.config = dict(Datasets=[ dict(Identifier='demo', Path="../../../../examples/serve/demo/cube.nc"), dict(Identifier='demo2', Path="../../../../examples/serve/demo/cube.nc"), ]) self.assertNotIn('demo', ctx.dataset_cache) self.assertNotIn('demo2', ctx.dataset_cache) ctx.get_dataset('demo2') self.assertNotIn('demo', ctx.dataset_cache) self.assertIn('demo2', ctx.dataset_cache) ctx.config = dict(Datasets=[ dict(Identifier='demo2', Path="../../../../examples/serve/demo/cube.nc"), ]) self.assertNotIn('demo', ctx.dataset_cache) self.assertNotIn('demo2', ctx.dataset_cache)
def test_s3(self): ctx = new_test_service_context() dataset_config = { 'Identifier': 'two', 'Title': 'Test 2', 'FileSystem': 's3', 'Endpoint': 'https://s3.eu-central-1.amazonaws.com', 'Path': 'xcube-examples/OLCI-SNS-RAW-CUBE-2.zarr', 'Region': 'eu-central-1' } ctx.config['Datasets'] = [dataset_config] dataset_config = ctx.get_dataset_configs()[0] self.assertEqual([ 'Identifier', 'Title', 'FileSystem', 'Endpoint', 'Path', 'Region', 'StoreInstanceId' ], list(dataset_config.keys())) self.assertEqual('two', dataset_config['Identifier']) self.assertEqual('Test 2', dataset_config['Title']) self.assertEqual('s3', dataset_config['FileSystem']) self.assertEqual('https://s3.eu-central-1.amazonaws.com', dataset_config['Endpoint']) self.assertEqual('OLCI-SNS-RAW-CUBE-2.zarr', dataset_config['Path']) self.assertEqual('eu-central-1', dataset_config['Region']) self.assertEqual('s3_1', dataset_config['StoreInstanceId'])
def test_dataset_with_details(self): ctx = new_test_service_context() response = get_datasets(ctx, details=True, base_url="http://test") datasets = self._assert_datasets(response, 2) demo_dataset = None demo_1w_dataset = None for dataset in datasets: self.assertIsInstance(dataset, dict) self.assertIn("id", dataset) self.assertIn("title", dataset) self.assertIn("attributions", dataset) self.assertIn("variables", dataset) self.assertIn("dimensions", dataset) self.assertNotIn("rgbSchema", dataset) if dataset["id"] == "demo": demo_dataset = dataset if dataset["id"] == "demo-1w": demo_1w_dataset = dataset self.assertIsNotNone(demo_dataset) self.assertIsNotNone(demo_1w_dataset) self.assertEqual(["© by EU H2020 CyanoAlert project"], demo_dataset['attributions']) self.assertEqual(["© by Brockmann Consult GmbH 2020, " "contains modified Copernicus Data 2019, processed by ESA"], demo_1w_dataset['attributions'])
def test_get_time_series_for_polygon_with_all_agg_methods(self): ctx = new_test_service_context() actual_result = get_time_series( ctx, 'demo', 'conc_tsm', dict(type="Polygon", coordinates=[[[1., 51.], [2., 51.], [2., 52.], [1., 52.], [1., 51.]]]), agg_methods=['mean', 'median', 'std', 'min', 'max', 'count']) expected_result = [{ 'count': 122392, 'count_tot': 159600, 'max': 166.57278442382812, 'mean': 56.12519223634024, 'median': 48.009796142578125, 'min': 0.02251400426030159, 'std': 40.78859862094861, 'time': '2017-01-16T10:09:22Z' }, { 'count': 0, 'count_tot': 159600, 'max': None, 'mean': None, 'median': None, 'min': None, 'std': None, 'time': '2017-01-25T09:35:51Z' }, { 'count': 0, 'count_tot': 159600, 'max': None, 'mean': None, 'median': None, 'min': None, 'std': None, 'time': '2017-01-26T10:50:17Z' }, { 'count': 132066, 'count_tot': 159600, 'max': 158.7908477783203, 'mean': 49.70755256053988, 'median': 39.326446533203125, 'min': 0.02607600949704647, 'std': 34.98868194514787, 'time': '2017-01-28T09:58:11Z' }, { 'count': 0, 'count_tot': 159600, 'max': None, 'mean': None, 'median': None, 'min': None, 'std': None, 'time': '2017-01-30T10:46:34Z' }] self.assertAlmostEqualDeep(expected_result, actual_result)
def test_get_dataset_tile_invalid_variable(self): ctx = new_test_service_context() with self.assertRaises(ServiceResourceNotFoundError) as cm: get_dataset_tile(ctx, 'demo', 'conc_tdi', '0', '0', '0', RequestParamsMock()) self.assertEqual(404, cm.exception.status_code) self.assertEqual('Variable "conc_tdi" not found in dataset "demo"', f'{cm.exception.reason}')
def test_get_time_series_info(self): self.maxDiff = None ctx = new_test_service_context() info = get_time_series_info(ctx) expected_result = self._get_expected_info_dict() self.assertEqual(expected_result, info)
def test_get_time_series_for_point_invalid_lat_and_lon(self): ctx = new_test_service_context() actual_result = get_time_series_for_point(ctx, 'demo', 'conc_tsm', lon=-150.0, lat=-30.0) expected_result = {'results': []} self.assertEqual(expected_result, actual_result)
def test_get_dataset_tile_with_all_params(self): ctx = new_test_service_context() tile = get_dataset_tile( ctx, 'demo', 'conc_tsm', '0', '0', '0', RequestParamsMock(time='current', cbar='plasma', vmin='0.1', vmax='0.3')) self.assertIsInstance(tile, bytes)
def test_get_time_series_for_geometry_polygon_with_stdev(self): ctx = new_test_service_context() actual_result = get_time_series_for_geometry( ctx, 'demo', 'conc_tsm', dict(type="Polygon", coordinates=[[[1., 51.], [2., 51.], [2., 52.], [1., 52.], [1., 51.]]]), include_count=True, include_stdev=True) expected_result = { 'results': [{ 'date': '2017-01-16T10:09:22Z', 'result': { 'average': 56.12519223634024, 'totalCount': 159600, 'uncertainty': 40.78859862094861, 'validCount': 122392 } }, { 'date': '2017-01-25T09:35:51Z', 'result': { 'average': None, 'totalCount': 159600, 'uncertainty': None, 'validCount': 0 } }, { 'date': '2017-01-26T10:50:17Z', 'result': { 'average': None, 'totalCount': 159600, 'uncertainty': None, 'validCount': 0 } }, { 'date': '2017-01-28T09:58:11Z', 'result': { 'average': 49.70755256053988, 'totalCount': 159600, 'uncertainty': 34.98868194514786, 'validCount': 132066 } }, { 'date': '2017-01-30T10:46:34Z', 'result': { 'average': None, 'totalCount': 159600, 'uncertainty': None, 'validCount': 0 } }] } self.assertAlmostEqualDeep(expected_result, actual_result)
def test_get_s3_bucket_mapping(self): ctx = new_test_service_context() bucket_mapping = ctx.get_s3_bucket_mapping() self.assertEqual(['demo'], list(bucket_mapping.keys())) path = bucket_mapping['demo'] self.assertTrue(os.path.isabs(path)) self.assertTrue( path.replace( '\\', '/').endswith('examples/serve/demo/cube-1-250-250.zarr'))
def test_get_dataset_configs_from_stores(self): ctx = new_test_service_context( config_file_name='config-datastores.yml') dataset_configs_from_stores = ctx.get_dataset_configs_from_stores() self.assertIsNotNone(dataset_configs_from_stores) self.assertEqual(2, len(dataset_configs_from_stores)) ids = [config['Identifier'] for config in dataset_configs_from_stores] self.assertIn('test~cube-1-250-250.zarr', ids) self.assertIn('test~cube-5-100-200.zarr', ids)