def _get_dataset_tile_grid(dataset: xr.Dataset,
num_levels: int = None) -> TileGrid:
geo_extent = get_dataset_bounds(dataset)
inv_y = float(dataset.lat[0]) < float(dataset.lat[-1])
width, height, tile_width, tile_height = _get_cube_spatial_sizes(dataset)
if num_levels is not None and tile_width is not None and tile_height is not None:
width_0 = width
height_0 = height
for i in range(num_levels):
width_0 = (width_0 + 1) // 2
height_0 = (height_0 + 1) // 2
num_level_zero_tiles_x = (width_0 + tile_width - 1) // tile_width
num_level_zero_tiles_y = (height_0 + tile_height - 1) // tile_height
tile_grid = TileGrid(num_levels, num_level_zero_tiles_x,
num_level_zero_tiles_y, tile_width, tile_height,
geo_extent, inv_y)
else:
try:
tile_grid = TileGrid.create(width, height, tile_width, tile_height,
geo_extent, inv_y)
except ValueError:
num_levels = 1
num_level_zero_tiles_x = 1
num_level_zero_tiles_y = 1
tile_grid = TileGrid(num_levels, num_level_zero_tiles_x,
num_level_zero_tiles_y, width, height,
geo_extent, inv_y)
return tile_grid
def test_width_and_height(self):
ts = TileGrid(4, 2, 1, 540, 540, GLOBAL_GEO_EXTENT, inv_y=False)
self.assertEqual(ts.width(2), 4320)
self.assertEqual(ts.height(2), 2160)
self.assertEqual(ts.max_width, 8640)
self.assertEqual(ts.max_height, 4320)
self.assertEqual(ts.min_width, 1080)
self.assertEqual(ts.min_height, 540)
def test_to_json(self):
ts = TileGrid(4, 2, 1, 540, 540, GLOBAL_GEO_EXTENT, inv_y=True)
self.assertEqual(ts.to_json(), {
'numLevels': 4,
'numLevelZeroTilesX': 2,
'numLevelZeroTilesY': 1,
'tileHeight': 540,
'tileWidth': 540,
'invY': True,
'extent': {'west': -180.,
'south': -90.,
'east': 180.,
'north': 90.,
},
})
def test_it(self):
ds = _get_test_dataset()
ml_ds1 = BaseMultiLevelDataset(ds)
def input_ml_dataset_getter(ds_id):
if ds_id == "ml_ds1":
return ml_ds1
self.fail(f"unexpected ds_id={ds_id!r}")
ml_ds2 = ComputedMultiLevelDataset(os.path.join(os.path.dirname(__file__),
"..", "webapi", "res", "test", "script.py"),
"compute_dataset",
["ml_ds1"],
input_ml_dataset_getter,
input_parameters=dict(period='1W'),
ds_id="ml_ds2")
self.assertEqual(3, ml_ds2.num_levels)
self.assertEqual(TileGrid(3, 2, 1, 180, 180, (-180, -90, 180, 90), inv_y=False),
ml_ds2.tile_grid)
ds0 = ml_ds2.get_dataset(0)
self.assertEqual({'time': 3, 'lat': 720, 'lon': 1440, 'bnds': 2}, ds0.dims)
ds1 = ml_ds2.get_dataset(1)
self.assertEqual({'time': 3, 'lat': 360, 'lon': 720}, ds1.dims)
ds2 = ml_ds2.get_dataset(2)
self.assertEqual({'time': 3, 'lat': 180, 'lon': 360}, ds2.dims)
self.assertEqual([ds0, ds1, ds2], ml_ds2.datasets)
ml_ds1.close()
ml_ds2.close()
def test_it(self):
ds = _get_test_dataset()
ml_ds = BaseMultiLevelDataset(ds)
self.assertIsInstance(ml_ds.ds_id, str)
self.assertEqual(3, ml_ds.num_levels)
self.assertEqual(
TileGrid(3, 2, 1, 180, 180, (-180, -90, 180, 90), inv_y=False),
ml_ds.tile_grid)
ds0 = ml_ds.get_dataset(0)
self.assertIs(ds, ds0)
ds1 = ml_ds.get_dataset(1)
self.assertIsNot(ds, ds1)
self.assertEqual({'time': 14, 'lat': 360, 'lon': 720}, ds1.dims)
ds2 = ml_ds.get_dataset(2)
self.assertIsNot(ds, ds2)
self.assertEqual({'time': 14, 'lat': 180, 'lon': 360}, ds2.dims)
self.assertEqual([ds0, ds1, ds2], ml_ds.datasets)
ml_ds.close()
def create_from_image(cls,
source_image: TiledImage,
level_transformer: LevelTransformer,
geo_extent: GeoExtent = None,
**kwargs) -> 'ImagePyramid':
"""
Create an image pyramid build from a single, max-resolution source image of type TiledImage.
The given source image will be returned for highest resolution level in the pyramid.
Other level images are created from the given level_image_factory function.
:param source_image: the high-resolution source image, see TiledImage interface
:param level_transformer: transforms level z+1 into level z. Called like:
level_images[z_index] = level_transformer(source_image, level_images[z_index+1], z_index, **kwargs)
:param geo_extent: the geographical extent.
:param kwargs: keyword arguments passed to the level_image_factory function
:return: a new ImagePyramid instance
"""
if geo_extent is None:
geo_extent = GLOBAL_GEO_EXTENT
tile_grid = TileGrid.create(source_image.size[0], source_image.size[1],
source_image.tile_size[0],
source_image.tile_size[1], geo_extent)
level_images = [None] * tile_grid.num_levels
z_index_max = tile_grid.num_levels - 1
level_images[z_index_max] = source_image
level_image = source_image
for i in range(1, tile_grid.num_levels):
z_index = z_index_max - i
image_id = '%s/%d' % (source_image.id, z_index)
level_images[z_index] = level_image = level_transformer(
source_image, level_image, i, image_id=image_id, **kwargs)
return ImagePyramid(tile_grid, level_images)
def test_it(self):
ml_ds_1 = BaseMultiLevelDataset(_get_test_dataset(('noise_1', 'noise_2')))
ml_ds_2 = BaseMultiLevelDataset(_get_test_dataset(('noise_3', 'noise_4')))
ml_ds_3 = BaseMultiLevelDataset(_get_test_dataset(('noise_5', 'noise_6')))
ml_ds = CombinedMultiLevelDataset([ml_ds_1, ml_ds_2, ml_ds_3])
self.assertEqual(3, ml_ds.num_levels)
self.assertEqual(TileGrid(3, 2, 1, 180, 180, (-180, -90, 180, 90), inv_y=False),
ml_ds.tile_grid)
expected_var_names = {'noise_1', 'noise_2',
'noise_3', 'noise_4',
'noise_5', 'noise_6'}
ds0 = ml_ds.get_dataset(0)
self.assertEqual({'time': 14, 'lat': 720, 'lon': 1440, 'bnds': 2}, ds0.dims)
self.assertEqual(expected_var_names, set(map(str, ds0.data_vars)))
self.assertTrue(all(v.dims == ('time', 'lat', 'lon') for v in ds0.data_vars.values()))
ds1 = ml_ds.get_dataset(1)
self.assertEqual({'time': 14, 'lat': 360, 'lon': 720}, ds1.dims)
self.assertEqual(expected_var_names, set(map(str, ds1.data_vars)))
self.assertTrue(all(v.dims == ('time', 'lat', 'lon') for v in ds1.data_vars.values()))
ds2 = ml_ds.get_dataset(2)
self.assertEqual({'time': 14, 'lat': 180, 'lon': 360}, ds2.dims)
self.assertEqual(expected_var_names, set(map(str, ds2.data_vars)))
self.assertTrue(all(v.dims == ('time', 'lat', 'lon') for v in ds2.data_vars.values()))
self.assertEqual([ds0, ds1, ds2], ml_ds.datasets)
ml_ds.close()
def get_dataset_tile_grid(dataset: xr.Dataset,
num_levels: int = None) -> TileGrid:
"""
Compute the tile grid for the given *dataset* and an optional number of resolution
levels *num_levels*, if given.
:param dataset: The dataset.
:param num_levels: The number of resolution levels.
:return: A TileGrid object
"""
geo_extent = get_dataset_bounds(dataset)
inv_y = float(dataset.lat[0]) < float(dataset.lat[-1])
width, height, tile_width, tile_height = _get_cube_spatial_sizes(dataset)
if num_levels is not None and tile_width is not None and tile_height is not None:
width_0 = width
height_0 = height
for i in range(num_levels - 1):
width_0 = (width_0 + 1) // 2
height_0 = (height_0 + 1) // 2
num_level_zero_tiles_x = (width_0 + tile_width - 1) // tile_width
num_level_zero_tiles_y = (height_0 + tile_height - 1) // tile_height
tile_grid = TileGrid(num_levels, num_level_zero_tiles_x,
num_level_zero_tiles_y, tile_width, tile_height,
geo_extent, inv_y)
else:
try:
tile_grid = TileGrid.create(width, height, tile_width, tile_height,
geo_extent, inv_y)
except ValueError:
num_levels = 1
num_level_zero_tiles_x = 1
num_level_zero_tiles_y = 1
tile_grid = TileGrid(num_levels, num_level_zero_tiles_x,
num_level_zero_tiles_y, width, height,
geo_extent, inv_y)
if tile_width is not None and tile_width != tile_grid.tile_width:
warnings.warn(
f'FIXME: wanted tile_width={tile_width} as of chunking, but will use {tile_grid.tile_width}. '
f'This is inefficient.')
if tile_height is not None and tile_height != tile_grid.tile_height:
warnings.warn(
f'FIXME: wanted tile_height={tile_width} as of chunking, but will use {tile_grid.tile_height}. '
f'This is inefficient.')
return tile_grid
def test_create_cci_ecv(self):
# 72, 8, 85, 17
# Soilmoisture CCI - daily L3S
self.assertEqual(TileGrid.create(1440, 720, 500, 500, GLOBAL_GEO_EXTENT, inv_y=False),
TileGrid(2, 2, 1, 360, 360, GLOBAL_GEO_EXTENT, inv_y=False))
# Aerosol CCI - monthly
self.assertEqual(TileGrid.create(7200, 3600, 500, 500, GLOBAL_GEO_EXTENT, inv_y=False),
TileGrid(4, 2, 1, 450, 450, GLOBAL_GEO_EXTENT, inv_y=False))
# Cloud CCI - monthly
self.assertEqual(TileGrid.create(720, 360, 500, 500, GLOBAL_GEO_EXTENT, inv_y=True),
TileGrid(1, 2, 1, 360, 360, GLOBAL_GEO_EXTENT, inv_y=True))
# SST CCI - daily L4
self.assertEqual(TileGrid.create(8640, 4320, 500, 500, GLOBAL_GEO_EXTENT, inv_y=True),
TileGrid(4, 2, 1, 540, 540, GLOBAL_GEO_EXTENT, inv_y=True))
# Land Cover CCI
self.assertEqual(TileGrid.create(129600, 64800, 500, 500, GLOBAL_GEO_EXTENT, inv_y=False),
TileGrid(6, 6, 3, 675, 675, GLOBAL_GEO_EXTENT, inv_y=False))
def test_create_illegal_geo_extent(self):
# legal - explains why the next must fail
self.assertEqual(TileGrid.create(50, 25, 5, 5, (0.0, 77.5, 25.0, 90.0), inv_y=True),
TileGrid(2, 5, 2, 5, 7, (0.0, 76.0, 25.0, 90.0), inv_y=True))
with self.assertRaises(ValueError):
TileGrid.create(50, 25, 5, 5, (0.0, 77.5, 25.0, 90.0), inv_y=False)
# legal - explains why the next must fail
self.assertEqual(TileGrid.create(50, 25, 5, 5, (0., -90.0, 25., -77.5), inv_y=False),
TileGrid(2, 5, 2, 5, 7, (0.0, -90.0, 25.0, -76.0), inv_y=False))
with self.assertRaises(ValueError):
TileGrid.create(50, 25, 5, 5, (0., -90.0, 25., -77.5), inv_y=True)
def test_str(self):
ts = TileGrid(4, 2, 1, 540, 540, GLOBAL_GEO_EXTENT, inv_y=False)
self.assertEqual(str(ts),
'number of pyramid levels: 4\n'
'number of tiles at level zero: 2 x 1\n'
'pyramid tile size: 540 x 540\n'
'image size at level zero: 1080 x 540\n'
'image size at level 3: 8640 x 4320\n'
'geographic extent: (-180.0, -90.0, 180.0, 90.0)\n'
'y-axis points down: yes')
def _tile_grid_to_ol4x_xyz_source_options(tile_grid: TileGrid, url: str):
"""
Convert TileGrid into options to be used with ol.source.XYZ(options) of OpenLayers 4.x.
See
* https://openlayers.org/en/latest/apidoc/ol.source.XYZ.html
* https://openlayers.org/en/latest/examples/xyz.html
:param tile_grid: tile grid
:param url: source url
:return:
"""
west, south, east, north = tile_grid.geo_extent
delta_x = east - west + (0 if east >= west else 360)
delta_y = north - south
width = tile_grid.width(0)
height = tile_grid.height(0)
res0_x = delta_x / width
res0_y = delta_y / height
res0 = max(res0_x, res0_y)
if abs(res0_y - res0_x) >= 1.e-5:
warnings.warn(
f'spatial resolutions in x and y direction differ significantly:'
f' {res0_x} and {res0_y} degrees, using maximum {res0}')
# https://openlayers.org/en/latest/examples/xyz.html
# https://openlayers.org/en/latest/apidoc/ol.source.XYZ.html
return dict(
url=url,
projection='EPSG:4326',
minZoom=0,
maxZoom=tile_grid.num_levels - 1,
tileGrid=dict(
extent=[west, south, east, north],
origin=[west, south if tile_grid.inv_y else north],
tileSize=[tile_grid.tile_size[0], tile_grid.tile_size[1]],
resolutions=[res0 / (2**i) for i in range(tile_grid.num_levels)]))
def test_create_subsets(self):
self.assertEqual(TileGrid.create(4000, 3000, 500, 500, (-20., 10., 60., 70.), inv_y=True),
TileGrid(4, 1, 1, 500, 375, (-20., 10., 60., 70.), inv_y=True))
self.assertEqual(TileGrid.create(4012, 3009, 500, 500, (-20., 10., 60., 70.), inv_y=True),
TileGrid(2, 3, 5, 669, 301,
(-20.0, 9.980059820538386, 60.03988035892323, 70.), inv_y=True))
self.assertEqual(TileGrid.create(4000, 3000, 500, 500, (170., 10., -160., 70.), inv_y=True),
TileGrid(4, 1, 1, 500, 375, (170.0, 10.0, -160.0, 70.0), inv_y=True))
def test_illegal_init(self):
with self.assertRaises(ValueError):
TileGrid(0, 2, 1, 540, 540, (0.0, 80., 20.0, 90.0), inv_y=True)
with self.assertRaises(ValueError):
TileGrid(4, 0, 1, 540, 540, (0.0, 80., 20.0, 90.0), inv_y=True)
with self.assertRaises(ValueError):
TileGrid(4, 2, 0, 540, 540, (0.0, 80., 20.0, 90.0), inv_y=True)
with self.assertRaises(ValueError):
TileGrid(4, 2, 1, 0, 540, (0.0, 80., 20.0, 90.0), inv_y=True)
with self.assertRaises(ValueError):
TileGrid(4, 2, 1, 540, 0, (0.0, 80., 20.0, 90.0), inv_y=True)
with self.assertRaises(ValueError):
TileGrid(4, 2, 1, 540, 540, (0.0, 80., 20.0, 90.01), inv_y=True)
def get_pyramid() -> ImagePyramid:
"""
Return an instance of a 'Natural Earth v2' image pyramid:
* global coverage
* JPEG RGB format
* 3 levels of detail: 0 to 2
* tile size: 256 pixels
* 2 x 1 tiles on level zero
"""
dir_path = os.path.join(os.path.dirname(__file__), 'res', 'ne2')
return ImagePyramid(
TileGrid(NaturalEarth2Image.NUM_LEVELS,
NaturalEarth2Image.NUM_LEVEL_0_TILES_X,
NaturalEarth2Image.NUM_LEVEL_0_TILES_Y,
NaturalEarth2Image.TILE_SIZE,
NaturalEarth2Image.TILE_SIZE,
GLOBAL_GEO_EXTENT,
inv_y=False), [
NaturalEarth2Image(dir_path, level)
for level in range(NaturalEarth2Image.NUM_LEVELS)
])
def test_repr(self):
ts = TileGrid(4, 2, 1, 540, 540, GLOBAL_GEO_EXTENT, inv_y=False)
self.assertEqual(repr(ts), 'TileGrid(4, 2, 1, 540, 540, (-180.0, -90.0, 180.0, 90.0), inv_y=False)')
ts = TileGrid(4, 2, 1, 540, 540, GLOBAL_GEO_EXTENT, inv_y=True)
self.assertEqual(repr(ts), 'TileGrid(4, 2, 1, 540, 540, (-180.0, -90.0, 180.0, 90.0), inv_y=True)')
def test_num_tiles(self):
ts = TileGrid(4, 2, 1, 540, 540, GLOBAL_GEO_EXTENT, inv_y=False)
self.assertEqual(ts.num_tiles_x(0), 2)
self.assertEqual(ts.num_tiles_y(0), 1)
self.assertEqual(ts.num_tiles_x(3), 16)
self.assertEqual(ts.num_tiles_y(3), 8)
def test_create_cci_ecv_subsets(self):
# Soilmoisture CCI - daily L3S - use case #6
self.assertEqual(TileGrid.create(52, 36, 500, 500, (72, 8, 85, 17)),
TileGrid(1, 1, 1, 52, 36, (72., 8., 85., 17.)))
