def rectify(dataset: str, xy_var_names: str = None, var_names: str = None, output_path: str = None, output_format: str = None, output_size: str = None, output_tile_size: str = None, output_point: str = None, output_res: float = None, delta: float = DEFAULT_DELTA, dry_run: bool = DEFAULT_DRY_RUN): """ Rectify a dataset to WGS-84 using its per-pixel geo-locations. """ input_path = dataset xy_var_names = parse_cli_sequence(xy_var_names, metavar='VARIABLES', num_items=2, item_plural_name='names') var_names = parse_cli_sequence(var_names, metavar='VARIABLES', item_plural_name='names') output_size = parse_cli_sequence(output_size, metavar='SIZE', num_items=2, item_plural_name='sizes', item_parser=int, item_validator=assert_positive_int_item) output_tile_size = parse_cli_sequence( output_size, metavar='TILE_SIZE', num_items=2, item_plural_name='tile sizes', item_parser=int, item_validator=assert_positive_int_item) output_point = parse_cli_sequence(output_point, metavar='POINT', num_items=2, item_plural_name='coordinates', item_parser=float) # noinspection PyBroadException _rectify(input_path, xy_var_names, var_names, output_path, output_format, output_size, output_tile_size, output_point, output_res, delta, dry_run=dry_run, monitor=print) return 0
def level(input, output, link, tile_size, num_levels_max): """ Generate multi-resolution levels. Transform the given dataset by INPUT into the levels of a multi-level pyramid with spatial resolution decreasing by a factor of two in both spatial dimensions and write the result to directory OUTPUT. """ import time import os from xcube.cli.common import parse_cli_sequence from xcube.cli.common import assert_positive_int_item from xcube.core.level import write_levels input_path = input output_path = output link_input = link if num_levels_max is not None and num_levels_max < 1: raise click.ClickException( f"NUM_LEVELS_MAX must be a positive integer") if not output_path: basename, ext = os.path.splitext(input_path) output_path = os.path.join(os.path.dirname(input_path), basename + ".levels") if os.path.exists(output_path): raise click.ClickException(f"output {output_path!r} already exists") spatial_tile_shape = None if tile_size is not None: tile_size = parse_cli_sequence(tile_size, metavar='TILE_SIZE', num_items=2, item_plural_name='tile sizes', item_parser=int, item_validator=assert_positive_int_item) spatial_tile_shape = tile_size[1], tile_size[0] start_time = t0 = time.perf_counter() # noinspection PyUnusedLocal def progress_monitor(dataset, index, num_levels): nonlocal t0 print( f"Level {index + 1} of {num_levels} written after {time.perf_counter() - t0} seconds" ) t0 = time.perf_counter() levels = write_levels(output_path, input_path=input_path, link_input=link_input, progress_monitor=progress_monitor, spatial_tile_shape=spatial_tile_shape, num_levels_max=num_levels_max) print( f"{len(levels)} level(s) written into {output_path} after {time.perf_counter() - start_time} seconds" )
def test_parse_cli_sequence(self): self.assertEqual((512, 1024), parse_cli_sequence('512, 1024', metavar='X', item_parser=int)) self.assertEqual((11, 11, 11), parse_cli_sequence('11', metavar='X', item_parser=int, num_items=3)) self.assertEqual((11, 12, 13), parse_cli_sequence(['11', '12 ', ' 13'], metavar='X', item_parser=int)) self.assertEqual(None, parse_cli_sequence(None, allow_none=True)) with self.assertRaises(click.ClickException) as cm: parse_cli_sequence(None, metavar='X', allow_none=False) self.assertEqual("X must be given", f'{cm.exception}') with self.assertRaises(click.ClickException) as cm: parse_cli_sequence('11', metavar='X', item_parser=int, num_items_min=3, item_plural_name='lollies') self.assertEqual("X must have at least 3 lollies separated by ','", f'{cm.exception}') with self.assertRaises(click.ClickException) as cm: parse_cli_sequence('11,12,13', metavar='X', item_parser=int, num_items_max=2, item_plural_name='lollies') self.assertEqual("X must have no more than 2 lollies separated by ','", f'{cm.exception}') with self.assertRaises(click.ClickException) as cm: parse_cli_sequence('11,12,13', metavar='X', item_parser=int, num_items=2, item_plural_name='lollies') self.assertEqual("X must have 2 lollies separated by ','", f'{cm.exception}') with self.assertRaises(click.ClickException) as cm: parse_cli_sequence('11,,13', metavar='X', item_parser=int, num_items=3, item_plural_name='lollies', allow_empty_items=False) self.assertEqual("lollies in X must not be empty", f'{cm.exception}') with self.assertRaises(click.ClickException) as cm: parse_cli_sequence('11,12,-13', metavar='X', item_parser=int, item_validator=assert_positive_int_item, num_items=3, item_plural_name='lollies') self.assertEqual( "Invalid lollies in X found: all items must be positive integer numbers", f'{cm.exception}') with self.assertRaises(ValueError) as cm: parse_cli_sequence('11,12,-13', metavar='X', item_parser=int, item_validator=assert_positive_int_item, num_items=3, item_plural_name='lollies', error_type=ValueError) self.assertEqual( "Invalid lollies in X found: all items must be positive integer numbers", f'{cm.exception}')
def tile(cube: str, variables: Optional[str], labels: Optional[str], tile_size: Optional[str], config_path: Optional[str], style_id: Optional[str], output_path: Optional[str], verbose: List[bool], dry_run: bool): """ Create RGBA tiles from CUBE. Color bars and value ranges for variables can be specified in a CONFIG file. Here the color mappings are defined for a style named "ocean_color": \b Styles: - Identifier: ocean_color ColorMappings: conc_chl: ColorBar: "plasma" ValueRange: [0., 24.] conc_tsm: ColorBar: "PuBuGn" ValueRange: [0., 100.] kd489: ColorBar: "jet" ValueRange: [0., 6.] This is the same styles syntax as the configuration file for "xcube serve", hence its configuration can be reused. """ import fractions import itertools import json import os.path # noinspection PyPackageRequirements import yaml import xarray as xr import numpy as np from xcube.core.mldataset import open_ml_dataset from xcube.core.mldataset import MultiLevelDataset from xcube.core.schema import CubeSchema from xcube.core.tile import get_ml_dataset_tile from xcube.core.tile import get_var_valid_range from xcube.core.tile import get_var_cmap_params from xcube.core.tile import parse_non_spatial_labels from xcube.core.select import select_variables_subset from xcube.cli.common import parse_cli_kwargs from xcube.cli.common import parse_cli_sequence from xcube.cli.common import assert_positive_int_item from xcube.util.tilegrid import TileGrid from xcube.util.tiledimage import DEFAULT_COLOR_MAP_NUM_COLORS # noinspection PyShadowingNames def write_tile_map_resource(path: str, resolutions: List[fractions.Fraction], tile_grid: TileGrid, title='', abstract='', srs='CRS:84'): num_levels = len(resolutions) z_and_upp = zip(range(num_levels), map(float, resolutions)) x1, y1, x2, y2 = tile_grid.geo_extent xml = [f'<TileMap version="1.0.0" tilemapservice="http://tms.osgeo.org/1.0.0">', f' <Title>{title}</Title>', f' <Abstract>{abstract}</Abstract>', f' <SRS>{srs}</SRS>', f' <BoundingBox minx="{x1}" miny="{y1}" maxx="{x2}" maxy="{y2}"/>', f' <Origin x="{x1}" y="{y1}"/>', f' <TileFormat width="{tile_grid.tile_width}" height="{tile_grid.tile_height}"' f' mime-type="image/png" extension="png"/>', f' <TileSets profile="local">'] + [ f' <TileSet href="{z}" order="{z}" units-per-pixel="{upp}"/>' for z, upp in z_and_upp] + [ f' </TileSets>', f'</TileMap>'] with open(path, 'w') as fp: fp.write('\n'.join(xml)) # noinspection PyShadowingNames def _convert_coord_var(coord_var: xr.DataArray): values = coord_var.values if np.issubdtype(values.dtype, np.datetime64): return list(np.datetime_as_string(values, timezone='UTC')) elif np.issubdtype(values.dtype, np.integer): return [int(value) for value in values] else: return [float(value) for value in values] # noinspection PyShadowingNames def _get_color_mappings(ml_dataset: MultiLevelDataset, var_name: str, config: Mapping[str, Any], style_id: str): cmap_name = None cmap_range = None, None if config: style_id = style_id or 'default' styles = config.get('Styles') if styles: color_mappings = None for style in styles: if style.get('Identifier') == style_id: color_mappings = style.get('ColorMappings') break if color_mappings: color_mapping = color_mappings.get(var_name) if color_mapping: cmap_name = color_mapping.get('ColorBar') cmap_vmin, cmap_vmax = color_mapping.get('ValueRange', (None, None)) cmap_range = cmap_vmin, cmap_vmax if cmap_name is not None and None not in cmap_range: return cmap_name, cmap_range var = ml_dataset.base_dataset[var_name] valid_range = get_var_valid_range(var) return get_var_cmap_params(var, cmap_name, cmap_range, valid_range) variables = parse_cli_sequence(variables, metavar='VARIABLES', num_items_min=1, item_plural_name='variables') tile_size = parse_cli_sequence(tile_size, num_items=2, metavar='TILE_SIZE', item_parser=int, item_validator=assert_positive_int_item, item_plural_name='tile sizes') labels = parse_cli_kwargs(labels, metavar='LABELS') verbosity = len(verbose) config = {} if config_path: if verbosity: print(f'Opening {config_path}...') with open(config_path, 'r') as fp: config = yaml.safe_load(fp) if verbosity: print(f'Opening {cube}...') ml_dataset = open_ml_dataset(cube, chunks='auto') tile_grid = ml_dataset.tile_grid base_dataset = ml_dataset.base_dataset schema = CubeSchema.new(base_dataset) spatial_dims = schema.x_dim, schema.y_dim if tile_size: tile_width, tile_height = tile_size else: if verbosity: print(f'Warning: using default tile sizes derived from CUBE') tile_width, tile_height = tile_grid.tile_width, tile_grid.tile_height indexers = None if labels: indexers = parse_non_spatial_labels(labels, schema.dims, schema.coords, allow_slices=True, exception_type=click.ClickException) def transform(ds: xr.Dataset) -> xr.Dataset: if variables: ds = select_variables_subset(ds, var_names=variables) if indexers: ds = ds.sel(**indexers) chunk_sizes = {dim: 1 for dim in ds.dims} chunk_sizes[spatial_dims[0]] = tile_width chunk_sizes[spatial_dims[1]] = tile_height return ds.chunk(chunk_sizes) ml_dataset = ml_dataset.apply(transform) tile_grid = ml_dataset.tile_grid base_dataset = ml_dataset.base_dataset schema = CubeSchema.new(base_dataset) spatial_dims = schema.x_dim, schema.y_dim x1, _, x2, _ = tile_grid.geo_extent num_levels = tile_grid.num_levels resolutions = [fractions.Fraction(fractions.Fraction(x2 - x1), tile_grid.width(z)) for z in range(num_levels)] if verbosity: print(f'Writing tile sets...') print(f' Zoom levels: {num_levels}') print(f' Resolutions: {", ".join(map(str, resolutions))} units/pixel') print(f' Tile size: {tile_width} x {tile_height} pixels') image_cache = {} for var_name, var in base_dataset.data_vars.items(): color_bar, (value_min, value_max) = _get_color_mappings(ml_dataset, str(var_name), config, style_id) label_names = [] label_indexes = [] for dim in var.dims: if dim not in spatial_dims: label_names.append(dim) label_indexes.append(list(range(var[dim].size))) var_path = os.path.join(output_path, str(var_name)) metadata_path = os.path.join(var_path, 'metadata.json') metadata = dict(name=str(var_name), attrs={name: value for name, value in var.attrs.items()}, dims=[str(dim) for dim in var.dims], dim_sizes={dim: int(var[dim].size) for dim in var.dims}, color_mapping=dict(color_bar=color_bar, value_min=value_min, value_max=value_max, num_colors=DEFAULT_COLOR_MAP_NUM_COLORS), coordinates={name: _convert_coord_var(coord_var) for name, coord_var in var.coords.items() if coord_var.ndim == 1}) if verbosity: print(f'Writing {metadata_path}') if not dry_run: os.makedirs(var_path, exist_ok=True) with open(metadata_path, 'w') as fp: json.dump(metadata, fp, indent=2) for label_index in itertools.product(*label_indexes): labels = {name: index for name, index in zip(label_names, label_index)} tilemap_path = os.path.join(var_path, *[str(l) for l in label_index]) tilemap_resource_path = os.path.join(tilemap_path, 'tilemapresource.xml') if verbosity > 1: print(f'Writing {tilemap_resource_path}') if not dry_run: os.makedirs(tilemap_path, exist_ok=True) write_tile_map_resource(tilemap_resource_path, resolutions, tile_grid, title=f'{var_name}') for z in range(num_levels): num_tiles_x = tile_grid.num_tiles_x(z) num_tiles_y = tile_grid.num_tiles_y(z) tile_z_path = os.path.join(tilemap_path, str(z)) if not dry_run and not os.path.exists(tile_z_path): os.mkdir(tile_z_path) for x in range(num_tiles_x): tile_zx_path = os.path.join(tile_z_path, str(x)) if not dry_run and not os.path.exists(tile_zx_path): os.mkdir(tile_zx_path) for y in range(num_tiles_y): tile_bytes = get_ml_dataset_tile(ml_dataset, str(var_name), x, y, z, labels=labels, labels_are_indices=True, cmap_name=color_bar, cmap_range=(value_min, value_max), image_cache=image_cache, trace_perf=True, exception_type=click.ClickException) tile_path = os.path.join(tile_zx_path, f'{num_tiles_y - 1 - y}.png') if verbosity > 2: print(f'Writing tile {tile_path}') if not dry_run: with open(tile_path, 'wb') as fp: fp.write(tile_bytes) print(f'Done writing tile sets.')