def rectify(dataset: str,
xy_var_names: str = None,
var_names: str = None,
output_path: str = None,
output_format: str = None,
output_size: str = None,
output_tile_size: str = None,
output_point: str = None,
output_res: float = None,
delta: float = DEFAULT_DELTA,
dry_run: bool = DEFAULT_DRY_RUN):
"""
Rectify a dataset to WGS-84 using its per-pixel geo-locations.
"""
input_path = dataset
xy_var_names = parse_cli_sequence(xy_var_names,
metavar='VARIABLES',
num_items=2,
item_plural_name='names')
var_names = parse_cli_sequence(var_names,
metavar='VARIABLES',
item_plural_name='names')
output_size = parse_cli_sequence(output_size,
metavar='SIZE',
num_items=2,
item_plural_name='sizes',
item_parser=int,
item_validator=assert_positive_int_item)
output_tile_size = parse_cli_sequence(
output_size,
metavar='TILE_SIZE',
num_items=2,
item_plural_name='tile sizes',
item_parser=int,
item_validator=assert_positive_int_item)
output_point = parse_cli_sequence(output_point,
metavar='POINT',
num_items=2,
item_plural_name='coordinates',
item_parser=float)
# noinspection PyBroadException
_rectify(input_path,
xy_var_names,
var_names,
output_path,
output_format,
output_size,
output_tile_size,
output_point,
output_res,
delta,
dry_run=dry_run,
monitor=print)
return 0
def level(input, output, link, tile_size, num_levels_max):
"""
Generate multi-resolution levels.
Transform the given dataset by INPUT into the levels of a multi-level pyramid with spatial resolution
decreasing by a factor of two in both spatial dimensions and write the result to directory OUTPUT.
"""
import time
import os
from xcube.cli.common import parse_cli_sequence
from xcube.cli.common import assert_positive_int_item
from xcube.core.level import write_levels
input_path = input
output_path = output
link_input = link
if num_levels_max is not None and num_levels_max < 1:
raise click.ClickException(
f"NUM_LEVELS_MAX must be a positive integer")
if not output_path:
basename, ext = os.path.splitext(input_path)
output_path = os.path.join(os.path.dirname(input_path),
basename + ".levels")
if os.path.exists(output_path):
raise click.ClickException(f"output {output_path!r} already exists")
spatial_tile_shape = None
if tile_size is not None:
tile_size = parse_cli_sequence(tile_size,
metavar='TILE_SIZE',
num_items=2,
item_plural_name='tile sizes',
item_parser=int,
item_validator=assert_positive_int_item)
spatial_tile_shape = tile_size[1], tile_size[0]
start_time = t0 = time.perf_counter()
# noinspection PyUnusedLocal
def progress_monitor(dataset, index, num_levels):
nonlocal t0
print(
f"Level {index + 1} of {num_levels} written after {time.perf_counter() - t0} seconds"
)
t0 = time.perf_counter()
levels = write_levels(output_path,
input_path=input_path,
link_input=link_input,
progress_monitor=progress_monitor,
spatial_tile_shape=spatial_tile_shape,
num_levels_max=num_levels_max)
print(
f"{len(levels)} level(s) written into {output_path} after {time.perf_counter() - start_time} seconds"
)
def test_parse_cli_sequence(self):
self.assertEqual((512, 1024),
parse_cli_sequence('512, 1024',
metavar='X',
item_parser=int))
self.assertEqual((11, 11, 11),
parse_cli_sequence('11',
metavar='X',
item_parser=int,
num_items=3))
self.assertEqual((11, 12, 13),
parse_cli_sequence(['11', '12 ', ' 13'],
metavar='X',
item_parser=int))
self.assertEqual(None, parse_cli_sequence(None, allow_none=True))
with self.assertRaises(click.ClickException) as cm:
parse_cli_sequence(None, metavar='X', allow_none=False)
self.assertEqual("X must be given", f'{cm.exception}')
with self.assertRaises(click.ClickException) as cm:
parse_cli_sequence('11',
metavar='X',
item_parser=int,
num_items_min=3,
item_plural_name='lollies')
self.assertEqual("X must have at least 3 lollies separated by ','",
f'{cm.exception}')
with self.assertRaises(click.ClickException) as cm:
parse_cli_sequence('11,12,13',
metavar='X',
item_parser=int,
num_items_max=2,
item_plural_name='lollies')
self.assertEqual("X must have no more than 2 lollies separated by ','",
f'{cm.exception}')
with self.assertRaises(click.ClickException) as cm:
parse_cli_sequence('11,12,13',
metavar='X',
item_parser=int,
num_items=2,
item_plural_name='lollies')
self.assertEqual("X must have 2 lollies separated by ','",
f'{cm.exception}')
with self.assertRaises(click.ClickException) as cm:
parse_cli_sequence('11,,13',
metavar='X',
item_parser=int,
num_items=3,
item_plural_name='lollies',
allow_empty_items=False)
self.assertEqual("lollies in X must not be empty", f'{cm.exception}')
with self.assertRaises(click.ClickException) as cm:
parse_cli_sequence('11,12,-13',
metavar='X',
item_parser=int,
item_validator=assert_positive_int_item,
num_items=3,
item_plural_name='lollies')
self.assertEqual(
"Invalid lollies in X found: all items must be positive integer numbers",
f'{cm.exception}')
with self.assertRaises(ValueError) as cm:
parse_cli_sequence('11,12,-13',
metavar='X',
item_parser=int,
item_validator=assert_positive_int_item,
num_items=3,
item_plural_name='lollies',
error_type=ValueError)
self.assertEqual(
"Invalid lollies in X found: all items must be positive integer numbers",
f'{cm.exception}')
def tile(cube: str,
variables: Optional[str],
labels: Optional[str],
tile_size: Optional[str],
config_path: Optional[str],
style_id: Optional[str],
output_path: Optional[str],
verbose: List[bool],
dry_run: bool):
"""
Create RGBA tiles from CUBE.
Color bars and value ranges for variables can be specified in a CONFIG file.
Here the color mappings are defined for a style named "ocean_color":
\b
Styles:
- Identifier: ocean_color
ColorMappings:
conc_chl:
ColorBar: "plasma"
ValueRange: [0., 24.]
conc_tsm:
ColorBar: "PuBuGn"
ValueRange: [0., 100.]
kd489:
ColorBar: "jet"
ValueRange: [0., 6.]
This is the same styles syntax as the configuration file for "xcube serve",
hence its configuration can be reused.
"""
import fractions
import itertools
import json
import os.path
# noinspection PyPackageRequirements
import yaml
import xarray as xr
import numpy as np
from xcube.core.mldataset import open_ml_dataset
from xcube.core.mldataset import MultiLevelDataset
from xcube.core.schema import CubeSchema
from xcube.core.tile import get_ml_dataset_tile
from xcube.core.tile import get_var_valid_range
from xcube.core.tile import get_var_cmap_params
from xcube.core.tile import parse_non_spatial_labels
from xcube.core.select import select_variables_subset
from xcube.cli.common import parse_cli_kwargs
from xcube.cli.common import parse_cli_sequence
from xcube.cli.common import assert_positive_int_item
from xcube.util.tilegrid import TileGrid
from xcube.util.tiledimage import DEFAULT_COLOR_MAP_NUM_COLORS
# noinspection PyShadowingNames
def write_tile_map_resource(path: str,
resolutions: List[fractions.Fraction],
tile_grid: TileGrid,
title='',
abstract='',
srs='CRS:84'):
num_levels = len(resolutions)
z_and_upp = zip(range(num_levels), map(float, resolutions))
x1, y1, x2, y2 = tile_grid.geo_extent
xml = [f'<TileMap version="1.0.0" tilemapservice="http://tms.osgeo.org/1.0.0">',
f' <Title>{title}</Title>',
f' <Abstract>{abstract}</Abstract>',
f' <SRS>{srs}</SRS>',
f' <BoundingBox minx="{x1}" miny="{y1}" maxx="{x2}" maxy="{y2}"/>',
f' <Origin x="{x1}" y="{y1}"/>',
f' <TileFormat width="{tile_grid.tile_width}" height="{tile_grid.tile_height}"'
f' mime-type="image/png" extension="png"/>',
f' <TileSets profile="local">'] + [
f' <TileSet href="{z}" order="{z}" units-per-pixel="{upp}"/>' for z, upp in z_and_upp] + [
f' </TileSets>',
f'</TileMap>']
with open(path, 'w') as fp:
fp.write('\n'.join(xml))
# noinspection PyShadowingNames
def _convert_coord_var(coord_var: xr.DataArray):
values = coord_var.values
if np.issubdtype(values.dtype, np.datetime64):
return list(np.datetime_as_string(values, timezone='UTC'))
elif np.issubdtype(values.dtype, np.integer):
return [int(value) for value in values]
else:
return [float(value) for value in values]
# noinspection PyShadowingNames
def _get_color_mappings(ml_dataset: MultiLevelDataset,
var_name: str,
config: Mapping[str, Any],
style_id: str):
cmap_name = None
cmap_range = None, None
if config:
style_id = style_id or 'default'
styles = config.get('Styles')
if styles:
color_mappings = None
for style in styles:
if style.get('Identifier') == style_id:
color_mappings = style.get('ColorMappings')
break
if color_mappings:
color_mapping = color_mappings.get(var_name)
if color_mapping:
cmap_name = color_mapping.get('ColorBar')
cmap_vmin, cmap_vmax = color_mapping.get('ValueRange', (None, None))
cmap_range = cmap_vmin, cmap_vmax
if cmap_name is not None and None not in cmap_range:
return cmap_name, cmap_range
var = ml_dataset.base_dataset[var_name]
valid_range = get_var_valid_range(var)
return get_var_cmap_params(var, cmap_name, cmap_range, valid_range)
variables = parse_cli_sequence(variables, metavar='VARIABLES', num_items_min=1,
item_plural_name='variables')
tile_size = parse_cli_sequence(tile_size, num_items=2, metavar='TILE_SIZE',
item_parser=int,
item_validator=assert_positive_int_item,
item_plural_name='tile sizes')
labels = parse_cli_kwargs(labels, metavar='LABELS')
verbosity = len(verbose)
config = {}
if config_path:
if verbosity:
print(f'Opening {config_path}...')
with open(config_path, 'r') as fp:
config = yaml.safe_load(fp)
if verbosity:
print(f'Opening {cube}...')
ml_dataset = open_ml_dataset(cube, chunks='auto')
tile_grid = ml_dataset.tile_grid
base_dataset = ml_dataset.base_dataset
schema = CubeSchema.new(base_dataset)
spatial_dims = schema.x_dim, schema.y_dim
if tile_size:
tile_width, tile_height = tile_size
else:
if verbosity:
print(f'Warning: using default tile sizes derived from CUBE')
tile_width, tile_height = tile_grid.tile_width, tile_grid.tile_height
indexers = None
if labels:
indexers = parse_non_spatial_labels(labels,
schema.dims,
schema.coords,
allow_slices=True,
exception_type=click.ClickException)
def transform(ds: xr.Dataset) -> xr.Dataset:
if variables:
ds = select_variables_subset(ds, var_names=variables)
if indexers:
ds = ds.sel(**indexers)
chunk_sizes = {dim: 1 for dim in ds.dims}
chunk_sizes[spatial_dims[0]] = tile_width
chunk_sizes[spatial_dims[1]] = tile_height
return ds.chunk(chunk_sizes)
ml_dataset = ml_dataset.apply(transform)
tile_grid = ml_dataset.tile_grid
base_dataset = ml_dataset.base_dataset
schema = CubeSchema.new(base_dataset)
spatial_dims = schema.x_dim, schema.y_dim
x1, _, x2, _ = tile_grid.geo_extent
num_levels = tile_grid.num_levels
resolutions = [fractions.Fraction(fractions.Fraction(x2 - x1), tile_grid.width(z))
for z in range(num_levels)]
if verbosity:
print(f'Writing tile sets...')
print(f' Zoom levels: {num_levels}')
print(f' Resolutions: {", ".join(map(str, resolutions))} units/pixel')
print(f' Tile size: {tile_width} x {tile_height} pixels')
image_cache = {}
for var_name, var in base_dataset.data_vars.items():
color_bar, (value_min, value_max) = _get_color_mappings(ml_dataset, str(var_name), config, style_id)
label_names = []
label_indexes = []
for dim in var.dims:
if dim not in spatial_dims:
label_names.append(dim)
label_indexes.append(list(range(var[dim].size)))
var_path = os.path.join(output_path, str(var_name))
metadata_path = os.path.join(var_path, 'metadata.json')
metadata = dict(name=str(var_name),
attrs={name: value
for name, value in var.attrs.items()},
dims=[str(dim)
for dim in var.dims],
dim_sizes={dim: int(var[dim].size)
for dim in var.dims},
color_mapping=dict(color_bar=color_bar,
value_min=value_min,
value_max=value_max,
num_colors=DEFAULT_COLOR_MAP_NUM_COLORS),
coordinates={name: _convert_coord_var(coord_var)
for name, coord_var in var.coords.items() if coord_var.ndim == 1})
if verbosity:
print(f'Writing {metadata_path}')
if not dry_run:
os.makedirs(var_path, exist_ok=True)
with open(metadata_path, 'w') as fp:
json.dump(metadata, fp, indent=2)
for label_index in itertools.product(*label_indexes):
labels = {name: index for name, index in zip(label_names, label_index)}
tilemap_path = os.path.join(var_path, *[str(l) for l in label_index])
tilemap_resource_path = os.path.join(tilemap_path, 'tilemapresource.xml')
if verbosity > 1:
print(f'Writing {tilemap_resource_path}')
if not dry_run:
os.makedirs(tilemap_path, exist_ok=True)
write_tile_map_resource(tilemap_resource_path, resolutions, tile_grid, title=f'{var_name}')
for z in range(num_levels):
num_tiles_x = tile_grid.num_tiles_x(z)
num_tiles_y = tile_grid.num_tiles_y(z)
tile_z_path = os.path.join(tilemap_path, str(z))
if not dry_run and not os.path.exists(tile_z_path):
os.mkdir(tile_z_path)
for x in range(num_tiles_x):
tile_zx_path = os.path.join(tile_z_path, str(x))
if not dry_run and not os.path.exists(tile_zx_path):
os.mkdir(tile_zx_path)
for y in range(num_tiles_y):
tile_bytes = get_ml_dataset_tile(ml_dataset,
str(var_name),
x, y, z,
labels=labels,
labels_are_indices=True,
cmap_name=color_bar,
cmap_range=(value_min, value_max),
image_cache=image_cache,
trace_perf=True,
exception_type=click.ClickException)
tile_path = os.path.join(tile_zx_path, f'{num_tiles_y - 1 - y}.png')
if verbosity > 2:
print(f'Writing tile {tile_path}')
if not dry_run:
with open(tile_path, 'wb') as fp:
fp.write(tile_bytes)
print(f'Done writing tile sets.')