def test_invalid(self):
with self.assertRaises(ValueError) as cm:
parse_non_spatial_labels(dict(time='jetzt'),
dims=self.dims,
coords=self.coords)
self.assertEqual("'jetzt' is not a valid value for dimension 'time'",
f'{cm.exception}')
def test_use_last_if_current(self):
labels = parse_non_spatial_labels(dict(time='current'),
dims=self.dims,
coords=self.coords)
self.assertEqual(
dict(time=np.array('2021-06-09 11:10:21', dtype='datetime64')),
labels)
def test_use_first_not_given(self):
labels = parse_non_spatial_labels(dict(),
dims=self.dims,
coords=self.coords)
self.assertEqual(
dict(time=np.array('2021-06-07 10:20:45', dtype='datetime64')),
labels)
def test_ensure_timezone_naive(self):
da_tznaive = xr.DataArray(np.zeros((3, 3, 3)),
coords=self.coords,
dims=self.dims)
labels = parse_non_spatial_labels(dict(time='2000-01-02T00:00:00Z'),
dims=da_tznaive.dims,
coords=da_tznaive.coords,
var=da_tznaive)
self.assertIsNone(pd.Timestamp(labels['time']).tzinfo)
def test_use_slice_if_range_given(self):
labels = parse_non_spatial_labels(dict(time='2021-06-06/2021-06-08'),
dims=self.dims,
coords=self.coords,
allow_slices=True)
self.assertEqual(
dict(time=slice(np.array('2021-06-06', dtype='datetime64'),
np.array('2021-06-08', dtype='datetime64'))),
labels)
def get_dataset_tile(ctx: ServiceContext, ds_id: str, var_name: str, x: str,
y: str, z: str, params: RequestParams):
x = RequestParams.to_int('x', x)
y = RequestParams.to_int('y', y)
z = RequestParams.to_int('z', z)
tile_comp_mode = params.get_query_argument_int('mode', ctx.tile_comp_mode)
trace_perf = params.get_query_argument_int('debug', ctx.trace_perf) != 0
cmap_name = params.get_query_argument('cbar', default=None)
cmap_vmin = params.get_query_argument_float('vmin', default=None)
cmap_vmax = params.get_query_argument_float('vmax', default=None)
if cmap_name is None or cmap_vmin is None or cmap_vmax is None:
default_cmap_name, default_cmap_vmin, default_cmap_vmax = ctx.get_color_mapping(
ds_id, var_name)
cmap_name = cmap_name or default_cmap_name
cmap_vmin = cmap_vmin or default_cmap_vmin
cmap_vmax = cmap_vmax or default_cmap_vmax
ml_dataset = ctx.get_ml_dataset(ds_id)
var = ml_dataset.base_dataset[var_name]
labels = parse_non_spatial_labels(params.get_query_arguments(),
var.dims,
var.coords,
allow_slices=False,
exception_type=ServiceBadRequestError)
return get_ml_dataset_tile(ml_dataset,
var_name,
x,
y,
z,
labels=labels,
cmap_name=cmap_name,
cmap_vmin=cmap_vmin,
cmap_vmax=cmap_vmax,
image_cache=ctx.image_cache,
tile_cache=ctx.tile_cache,
tile_comp_mode=tile_comp_mode,
trace_perf=trace_perf,
exception_type=ServiceBadRequestError)
def get_dataset_tile(ctx: ServiceContext, ds_id: str, var_name: str, x: str,
y: str, z: str, params: RequestParams):
x = RequestParams.to_int('x', x)
y = RequestParams.to_int('y', y)
z = RequestParams.to_int('z', z)
tile_comp_mode = params.get_query_argument_int('mode', ctx.tile_comp_mode)
trace_perf = params.get_query_argument_int('debug', ctx.trace_perf) != 0
ml_dataset = ctx.get_ml_dataset(ds_id)
if var_name == 'rgb':
norm_vmin = params.get_query_argument_float('vmin', default=0.0)
norm_vmax = params.get_query_argument_float('vmax', default=1.0)
var_names, norm_ranges = ctx.get_rgb_color_mapping(
ds_id, norm_range=(norm_vmin, norm_vmax))
components = ('r', 'g', 'b')
for i in range(3):
c = components[i]
var_names[i] = params.get_query_argument(c, default=var_names[i])
norm_ranges[i] = params.get_query_argument_float(f'{c}vmin', default=norm_ranges[i][0]), \
params.get_query_argument_float(f'{c}vmax', default=norm_ranges[i][1])
cmap_name = tuple(var_names)
cmap_range = tuple(norm_ranges)
for name in var_names:
if name and name not in ml_dataset.base_dataset:
raise ServiceBadRequestError(
f'Variable {name!r} not found in dataset {ds_id!r}')
var = None
for name in var_names:
if name and name in ml_dataset.base_dataset:
var = ml_dataset.base_dataset[name]
break
if var is None:
raise ServiceBadRequestError(
f'No variable in dataset {ds_id!r} specified for RGB')
else:
cmap_name = params.get_query_argument('cbar', default=None)
cmap_vmin = params.get_query_argument_float('vmin', default=None)
cmap_vmax = params.get_query_argument_float('vmax', default=None)
if cmap_name is None or cmap_vmin is None or cmap_vmax is None:
default_cmap_name, (default_cmap_vmin,
default_cmap_vmax) = ctx.get_color_mapping(
ds_id, var_name)
cmap_name = cmap_name or default_cmap_name
cmap_vmin = cmap_vmin or default_cmap_vmin
cmap_vmax = cmap_vmax or default_cmap_vmax
cmap_range = cmap_vmin, cmap_vmax
if var_name not in ml_dataset.base_dataset:
raise ServiceBadRequestError(
f'Variable {var_name!r} not found in dataset {ds_id!r}')
var = ml_dataset.base_dataset[var_name]
labels = parse_non_spatial_labels(params.get_query_arguments(),
var.dims,
var.coords,
allow_slices=False,
exception_type=ServiceBadRequestError)
return get_ml_dataset_tile(ml_dataset,
var_name,
x,
y,
z,
labels=labels,
cmap_name=cmap_name,
cmap_range=cmap_range,
image_cache=ctx.image_cache,
tile_cache=ctx.tile_cache,
tile_comp_mode=tile_comp_mode,
trace_perf=trace_perf,
exception_type=ServiceBadRequestError)
def test_use_average_if_range_given(self):
labels = parse_non_spatial_labels(dict(time='2021-06-06/2021-06-08'),
dims=self.dims,
coords=self.coords)
self.assertEqual(dict(time=np.array('2021-06-07', dtype='datetime64')),
labels)
def tile(cube: str,
variables: Optional[str],
labels: Optional[str],
tile_size: Optional[str],
config_path: Optional[str],
style_id: Optional[str],
output_path: Optional[str],
verbose: List[bool],
dry_run: bool):
"""
Create RGBA tiles from CUBE.
Color bars and value ranges for variables can be specified in a CONFIG file.
Here the color mappings are defined for a style named "ocean_color":
\b
Styles:
- Identifier: ocean_color
ColorMappings:
conc_chl:
ColorBar: "plasma"
ValueRange: [0., 24.]
conc_tsm:
ColorBar: "PuBuGn"
ValueRange: [0., 100.]
kd489:
ColorBar: "jet"
ValueRange: [0., 6.]
This is the same styles syntax as the configuration file for "xcube serve",
hence its configuration can be reused.
"""
import fractions
import itertools
import json
import os.path
# noinspection PyPackageRequirements
import yaml
import xarray as xr
import numpy as np
from xcube.core.mldataset import open_ml_dataset
from xcube.core.mldataset import MultiLevelDataset
from xcube.core.schema import CubeSchema
from xcube.core.tile import get_ml_dataset_tile
from xcube.core.tile import get_var_valid_range
from xcube.core.tile import get_var_cmap_params
from xcube.core.tile import parse_non_spatial_labels
from xcube.core.select import select_variables_subset
from xcube.cli.common import parse_cli_kwargs
from xcube.cli.common import parse_cli_sequence
from xcube.cli.common import assert_positive_int_item
from xcube.util.tilegrid import TileGrid
from xcube.util.tiledimage import DEFAULT_COLOR_MAP_NUM_COLORS
# noinspection PyShadowingNames
def write_tile_map_resource(path: str,
resolutions: List[fractions.Fraction],
tile_grid: TileGrid,
title='',
abstract='',
srs='CRS:84'):
num_levels = len(resolutions)
z_and_upp = zip(range(num_levels), map(float, resolutions))
x1, y1, x2, y2 = tile_grid.geo_extent
xml = [f'<TileMap version="1.0.0" tilemapservice="http://tms.osgeo.org/1.0.0">',
f' <Title>{title}</Title>',
f' <Abstract>{abstract}</Abstract>',
f' <SRS>{srs}</SRS>',
f' <BoundingBox minx="{x1}" miny="{y1}" maxx="{x2}" maxy="{y2}"/>',
f' <Origin x="{x1}" y="{y1}"/>',
f' <TileFormat width="{tile_grid.tile_width}" height="{tile_grid.tile_height}"'
f' mime-type="image/png" extension="png"/>',
f' <TileSets profile="local">'] + [
f' <TileSet href="{z}" order="{z}" units-per-pixel="{upp}"/>' for z, upp in z_and_upp] + [
f' </TileSets>',
f'</TileMap>']
with open(path, 'w') as fp:
fp.write('\n'.join(xml))
# noinspection PyShadowingNames
def _convert_coord_var(coord_var: xr.DataArray):
values = coord_var.values
if np.issubdtype(values.dtype, np.datetime64):
return list(np.datetime_as_string(values, timezone='UTC'))
elif np.issubdtype(values.dtype, np.integer):
return [int(value) for value in values]
else:
return [float(value) for value in values]
# noinspection PyShadowingNames
def _get_color_mappings(ml_dataset: MultiLevelDataset,
var_name: str,
config: Mapping[str, Any],
style_id: str):
cmap_name = None
cmap_range = None, None
if config:
style_id = style_id or 'default'
styles = config.get('Styles')
if styles:
color_mappings = None
for style in styles:
if style.get('Identifier') == style_id:
color_mappings = style.get('ColorMappings')
break
if color_mappings:
color_mapping = color_mappings.get(var_name)
if color_mapping:
cmap_name = color_mapping.get('ColorBar')
cmap_vmin, cmap_vmax = color_mapping.get('ValueRange', (None, None))
cmap_range = cmap_vmin, cmap_vmax
if cmap_name is not None and None not in cmap_range:
return cmap_name, cmap_range
var = ml_dataset.base_dataset[var_name]
valid_range = get_var_valid_range(var)
return get_var_cmap_params(var, cmap_name, cmap_range, valid_range)
variables = parse_cli_sequence(variables, metavar='VARIABLES', num_items_min=1,
item_plural_name='variables')
tile_size = parse_cli_sequence(tile_size, num_items=2, metavar='TILE_SIZE',
item_parser=int,
item_validator=assert_positive_int_item,
item_plural_name='tile sizes')
labels = parse_cli_kwargs(labels, metavar='LABELS')
verbosity = len(verbose)
config = {}
if config_path:
if verbosity:
print(f'Opening {config_path}...')
with open(config_path, 'r') as fp:
config = yaml.safe_load(fp)
if verbosity:
print(f'Opening {cube}...')
ml_dataset = open_ml_dataset(cube, chunks='auto')
tile_grid = ml_dataset.tile_grid
base_dataset = ml_dataset.base_dataset
schema = CubeSchema.new(base_dataset)
spatial_dims = schema.x_dim, schema.y_dim
if tile_size:
tile_width, tile_height = tile_size
else:
if verbosity:
print(f'Warning: using default tile sizes derived from CUBE')
tile_width, tile_height = tile_grid.tile_width, tile_grid.tile_height
indexers = None
if labels:
indexers = parse_non_spatial_labels(labels,
schema.dims,
schema.coords,
allow_slices=True,
exception_type=click.ClickException)
def transform(ds: xr.Dataset) -> xr.Dataset:
if variables:
ds = select_variables_subset(ds, var_names=variables)
if indexers:
ds = ds.sel(**indexers)
chunk_sizes = {dim: 1 for dim in ds.dims}
chunk_sizes[spatial_dims[0]] = tile_width
chunk_sizes[spatial_dims[1]] = tile_height
return ds.chunk(chunk_sizes)
ml_dataset = ml_dataset.apply(transform)
tile_grid = ml_dataset.tile_grid
base_dataset = ml_dataset.base_dataset
schema = CubeSchema.new(base_dataset)
spatial_dims = schema.x_dim, schema.y_dim
x1, _, x2, _ = tile_grid.geo_extent
num_levels = tile_grid.num_levels
resolutions = [fractions.Fraction(fractions.Fraction(x2 - x1), tile_grid.width(z))
for z in range(num_levels)]
if verbosity:
print(f'Writing tile sets...')
print(f' Zoom levels: {num_levels}')
print(f' Resolutions: {", ".join(map(str, resolutions))} units/pixel')
print(f' Tile size: {tile_width} x {tile_height} pixels')
image_cache = {}
for var_name, var in base_dataset.data_vars.items():
color_bar, (value_min, value_max) = _get_color_mappings(ml_dataset, str(var_name), config, style_id)
label_names = []
label_indexes = []
for dim in var.dims:
if dim not in spatial_dims:
label_names.append(dim)
label_indexes.append(list(range(var[dim].size)))
var_path = os.path.join(output_path, str(var_name))
metadata_path = os.path.join(var_path, 'metadata.json')
metadata = dict(name=str(var_name),
attrs={name: value
for name, value in var.attrs.items()},
dims=[str(dim)
for dim in var.dims],
dim_sizes={dim: int(var[dim].size)
for dim in var.dims},
color_mapping=dict(color_bar=color_bar,
value_min=value_min,
value_max=value_max,
num_colors=DEFAULT_COLOR_MAP_NUM_COLORS),
coordinates={name: _convert_coord_var(coord_var)
for name, coord_var in var.coords.items() if coord_var.ndim == 1})
if verbosity:
print(f'Writing {metadata_path}')
if not dry_run:
os.makedirs(var_path, exist_ok=True)
with open(metadata_path, 'w') as fp:
json.dump(metadata, fp, indent=2)
for label_index in itertools.product(*label_indexes):
labels = {name: index for name, index in zip(label_names, label_index)}
tilemap_path = os.path.join(var_path, *[str(l) for l in label_index])
tilemap_resource_path = os.path.join(tilemap_path, 'tilemapresource.xml')
if verbosity > 1:
print(f'Writing {tilemap_resource_path}')
if not dry_run:
os.makedirs(tilemap_path, exist_ok=True)
write_tile_map_resource(tilemap_resource_path, resolutions, tile_grid, title=f'{var_name}')
for z in range(num_levels):
num_tiles_x = tile_grid.num_tiles_x(z)
num_tiles_y = tile_grid.num_tiles_y(z)
tile_z_path = os.path.join(tilemap_path, str(z))
if not dry_run and not os.path.exists(tile_z_path):
os.mkdir(tile_z_path)
for x in range(num_tiles_x):
tile_zx_path = os.path.join(tile_z_path, str(x))
if not dry_run and not os.path.exists(tile_zx_path):
os.mkdir(tile_zx_path)
for y in range(num_tiles_y):
tile_bytes = get_ml_dataset_tile(ml_dataset,
str(var_name),
x, y, z,
labels=labels,
labels_are_indices=True,
cmap_name=color_bar,
cmap_range=(value_min, value_max),
image_cache=image_cache,
trace_perf=True,
exception_type=click.ClickException)
tile_path = os.path.join(tile_zx_path, f'{num_tiles_y - 1 - y}.png')
if verbosity > 2:
print(f'Writing tile {tile_path}')
if not dry_run:
with open(tile_path, 'wb') as fp:
fp.write(tile_bytes)
print(f'Done writing tile sets.')