def get_time_series_info(ctx: ServiceContext) -> Dict:
"""
Get time-series meta-information for variables.
:param ctx: Service context object
:return: a dictionary with a single entry "layers" which is a list of entries that are
dictionaries containing a variable's "name", "dates", and "bounds".
"""
time_series_info = {'layers': []}
descriptors = ctx.get_dataset_descriptors()
for descriptor in descriptors:
if 'Identifier' in descriptor:
if descriptor.get('Hidden'):
continue
dataset = ctx.get_dataset(descriptor['Identifier'])
if 'time' not in dataset.variables:
continue
xmin, ymin, xmax, ymax = get_dataset_bounds(dataset)
time_data = dataset.variables['time'].data
time_stamps = []
for time in time_data:
time_stamps.append(timestamp_to_iso_string(time))
var_names = sorted(dataset.data_vars)
for var_name in var_names:
ds_id = descriptor['Identifier']
variable_dict = {
'name': f'{ds_id}.{var_name}',
'dates': time_stamps,
'bounds': dict(xmin=xmin, ymin=ymin, xmax=xmax, ymax=ymax)
}
time_series_info['layers'].append(variable_dict)
return time_series_info
def _get_dataset_tile_grid(dataset: xr.Dataset,
num_levels: int = None) -> TileGrid:
geo_extent = get_dataset_bounds(dataset)
inv_y = float(dataset.lat[0]) < float(dataset.lat[-1])
width, height, tile_width, tile_height = _get_cube_spatial_sizes(dataset)
if num_levels is not None and tile_width is not None and tile_height is not None:
width_0 = width
height_0 = height
for i in range(num_levels):
width_0 = (width_0 + 1) // 2
height_0 = (height_0 + 1) // 2
num_level_zero_tiles_x = (width_0 + tile_width - 1) // tile_width
num_level_zero_tiles_y = (height_0 + tile_height - 1) // tile_height
tile_grid = TileGrid(num_levels, num_level_zero_tiles_x,
num_level_zero_tiles_y, tile_width, tile_height,
geo_extent, inv_y)
else:
try:
tile_grid = TileGrid.create(width, height, tile_width, tile_height,
geo_extent, inv_y)
except ValueError:
num_levels = 1
num_level_zero_tiles_x = 1
num_level_zero_tiles_y = 1
tile_grid = TileGrid(num_levels, num_level_zero_tiles_x,
num_level_zero_tiles_y, width, height,
geo_extent, inv_y)
return tile_grid
def _determine_bbox(data: xr.Dataset) \
-> Optional[Tuple[float, float, float, float]]:
try:
return get_dataset_bounds(data)
except ValueError:
if 'geospatial_lon_min' in data.attrs and \
'geospatial_lat_min' in data.attrs and \
'geospatial_lon_max' in data.attrs and \
'geospatial_lat_max' in data.attrs:
return (data.geospatial_lat_min, data.geospatial_lon_min,
data.geospatial_lat_max, data.geospatial_lon_max)
def get_datasets(ctx: ServiceContext,
details: bool = False,
client: str = None,
point: Tuple[float, float] = None,
base_url: str = None) -> Dict:
dataset_descriptors = ctx.get_dataset_descriptors()
dataset_dicts = list()
for dataset_descriptor in dataset_descriptors:
if dataset_descriptor.get('Hidden'):
continue
ds_id = dataset_descriptor['Identifier']
dataset_dict = dict(id=ds_id)
if 'Title' in dataset_descriptor:
ds_title = dataset_descriptor['Title']
if ds_title and isinstance(ds_title, str):
dataset_dict['title'] = ds_title
else:
dataset_dict['title'] = ds_id
if 'BoundingBox' in dataset_descriptor:
ds_bbox = dataset_descriptor['BoundingBox']
if ds_bbox \
and len(ds_bbox) == 4 \
and all(map(lambda c: isinstance(c, float) or isinstance(c, int), ds_bbox)):
dataset_dict['bbox'] = ds_bbox
dataset_dicts.append(dataset_dict)
if details or point:
for dataset_dict in dataset_dicts:
ds_id = dataset_dict["id"]
if point:
ds = ctx.get_dataset(ds_id)
if "bbox" not in dataset_dict:
dataset_dict["bbox"] = list(get_dataset_bounds(ds))
if details:
dataset_dict.update(get_dataset(ctx, ds_id, client, base_url))
if point:
is_point_in_dataset_bbox = functools.partial(_is_point_in_dataset_bbox,
point)
# noinspection PyTypeChecker
dataset_dicts = list(filter(is_point_in_dataset_bbox, dataset_dicts))
return dict(datasets=dataset_dicts)
def get_dataset_tile_grid(dataset: xr.Dataset,
num_levels: int = None) -> TileGrid:
"""
Compute the tile grid for the given *dataset* and an optional number of resolution
levels *num_levels*, if given.
:param dataset: The dataset.
:param num_levels: The number of resolution levels.
:return: A TileGrid object
"""
geo_extent = get_dataset_bounds(dataset)
inv_y = float(dataset.lat[0]) < float(dataset.lat[-1])
width, height, tile_width, tile_height = _get_cube_spatial_sizes(dataset)
if num_levels is not None and tile_width is not None and tile_height is not None:
width_0 = width
height_0 = height
for i in range(num_levels - 1):
width_0 = (width_0 + 1) // 2
height_0 = (height_0 + 1) // 2
num_level_zero_tiles_x = (width_0 + tile_width - 1) // tile_width
num_level_zero_tiles_y = (height_0 + tile_height - 1) // tile_height
tile_grid = TileGrid(num_levels, num_level_zero_tiles_x,
num_level_zero_tiles_y, tile_width, tile_height,
geo_extent, inv_y)
else:
try:
tile_grid = TileGrid.create(width, height, tile_width, tile_height,
geo_extent, inv_y)
except ValueError:
num_levels = 1
num_level_zero_tiles_x = 1
num_level_zero_tiles_y = 1
tile_grid = TileGrid(num_levels, num_level_zero_tiles_x,
num_level_zero_tiles_y, width, height,
geo_extent, inv_y)
if tile_width is not None and tile_width != tile_grid.tile_width:
warnings.warn(
f'FIXME: wanted tile_width={tile_width} as of chunking, but will use {tile_grid.tile_width}. '
f'This is inefficient.')
if tile_height is not None and tile_height != tile_grid.tile_height:
warnings.warn(
f'FIXME: wanted tile_height={tile_width} as of chunking, but will use {tile_grid.tile_height}. '
f'This is inefficient.')
return tile_grid
def get_dataset(ctx: ServiceContext,
ds_id: str,
client=None,
base_url: str = None,
granted_scopes: Set[str] = None) -> Dict:
granted_scopes = granted_scopes or set()
dataset_descriptor = ctx.get_dataset_descriptor(ds_id)
ds_id = dataset_descriptor['Identifier']
if 'read:dataset:*' not in granted_scopes:
required_scopes = ctx.get_required_dataset_scopes(dataset_descriptor)
assert_scopes(required_scopes, granted_scopes or set())
ds_title = dataset_descriptor['Title']
dataset_dict = dict(id=ds_id, title=ds_title)
ds = ctx.get_dataset(ds_id)
if "bbox" not in dataset_dict:
dataset_dict["bbox"] = list(get_dataset_bounds(ds))
variable_dicts = []
for var_name in ds.data_vars:
var = ds.data_vars[var_name]
dims = var.dims
if len(dims) < 3 or dims[0] != 'time' or dims[-2] != 'lat' or dims[
-1] != 'lon':
continue
if 'read:variable:*' not in granted_scopes:
required_scopes = ctx.get_required_variable_scopes(
dataset_descriptor, var_name)
if not check_scopes(required_scopes, granted_scopes):
continue
variable_dict = dict(id=f'{ds_id}.{var_name}',
name=var_name,
dims=list(dims),
shape=list(var.shape),
dtype=str(var.dtype),
units=var.attrs.get('units', ''),
title=var.attrs.get(
'title', var.attrs.get('long_name',
var_name)))
if client is not None:
tile_grid = ctx.get_tile_grid(ds_id)
tile_xyz_source_options = get_tile_source_options(
tile_grid,
get_dataset_tile_url(ctx, ds_id, var_name, base_url),
client=client)
variable_dict["tileSourceOptions"] = tile_xyz_source_options
cmap_name, (cmap_vmin,
cmap_vmax) = ctx.get_color_mapping(ds_id, var_name)
variable_dict["colorBarName"] = cmap_name
variable_dict["colorBarMin"] = cmap_vmin
variable_dict["colorBarMax"] = cmap_vmax
if hasattr(var.data, '_repr_html_'):
variable_dict["htmlRepr"] = var.data._repr_html_()
variable_dict["attrs"] = {
key: var.attrs[key]
for key in sorted(list(var.attrs.keys()))
}
variable_dicts.append(variable_dict)
ctx.get_rgb_color_mapping(ds_id)
dataset_dict["variables"] = variable_dicts
rgb_var_names, rgb_norm_ranges = ctx.get_rgb_color_mapping(ds_id)
if any(rgb_var_names):
rgb_schema = {'varNames': rgb_var_names, 'normRanges': rgb_norm_ranges}
if client is not None:
tile_grid = ctx.get_tile_grid(ds_id)
tile_xyz_source_options = get_tile_source_options(
tile_grid,
get_dataset_tile_url(ctx, ds_id, 'rgb', base_url),
client=client)
rgb_schema["tileSourceOptions"] = tile_xyz_source_options
dataset_dict["rgbSchema"] = rgb_schema
dim_names = ds.data_vars[list(
ds.data_vars)[0]].dims if len(ds.data_vars) > 0 else ds.dims.keys()
dataset_dict["dimensions"] = [
get_dataset_coordinates(ctx, ds_id, dim_name) for dim_name in dim_names
]
dataset_dict["attrs"] = {
key: ds.attrs[key]
for key in sorted(list(ds.attrs.keys()))
}
dataset_attributions = dataset_descriptor.get(
'DatasetAttribution', ctx.config.get('DatasetAttribution'))
if dataset_attributions is not None:
if isinstance(dataset_attributions, str):
dataset_attributions = [dataset_attributions]
dataset_dict['attributions'] = dataset_attributions
place_groups = ctx.get_dataset_place_groups(ds_id, base_url)
if place_groups:
dataset_dict["placeGroups"] = _filter_place_groups(place_groups,
del_features=True)
return dataset_dict
def get_datasets(ctx: ServiceContext,
details: bool = False,
client: str = None,
point: Tuple[float, float] = None,
base_url: str = None,
granted_scopes: Set[str] = None) -> Dict:
granted_scopes = granted_scopes or set()
dataset_descriptors = ctx.get_dataset_descriptors()
dataset_dicts = list()
for dataset_descriptor in dataset_descriptors:
ds_id = dataset_descriptor['Identifier']
if dataset_descriptor.get('Hidden'):
continue
if 'read:dataset:*' not in granted_scopes:
required_scopes = ctx.get_required_dataset_scopes(
dataset_descriptor)
is_substitute = dataset_descriptor.get('AccessControl', {}).get(
'IsSubstitute', False)
if not check_scopes(required_scopes,
granted_scopes,
is_substitute=is_substitute):
continue
dataset_dict = dict(id=ds_id)
if 'Title' in dataset_descriptor:
ds_title = dataset_descriptor['Title']
if ds_title and isinstance(ds_title, str):
dataset_dict['title'] = ds_title
else:
dataset_dict['title'] = ds_id
if 'BoundingBox' in dataset_descriptor:
ds_bbox = dataset_descriptor['BoundingBox']
if ds_bbox \
and len(ds_bbox) == 4 \
and all(map(lambda c: isinstance(c, float) or isinstance(c, int), ds_bbox)):
dataset_dict['bbox'] = ds_bbox
dataset_dicts.append(dataset_dict)
if details or point:
for dataset_dict in dataset_dicts:
ds_id = dataset_dict["id"]
if point:
ds = ctx.get_dataset(ds_id)
if "bbox" not in dataset_dict:
dataset_dict["bbox"] = list(get_dataset_bounds(ds))
if details:
dataset_dict.update(
get_dataset(ctx,
ds_id,
client,
base_url,
granted_scopes=granted_scopes))
if point:
is_point_in_dataset_bbox = functools.partial(_is_point_in_dataset_bbox,
point)
# noinspection PyTypeChecker
dataset_dicts = list(filter(is_point_in_dataset_bbox, dataset_dicts))
return dict(datasets=dataset_dicts)
def get_dataset(ctx: ServiceContext,
ds_id: str,
client=None,
base_url: str = None) -> GeoJsonFeatureCollection:
dataset_descriptor = ctx.get_dataset_descriptor(ds_id)
ds_id = dataset_descriptor['Identifier']
ds_title = dataset_descriptor['Title']
dataset_dict = dict(id=ds_id, title=ds_title)
ds = ctx.get_dataset(ds_id)
if "bbox" not in dataset_dict:
dataset_dict["bbox"] = list(get_dataset_bounds(ds))
variable_dicts = []
for var_name in ds.data_vars:
var = ds.data_vars[var_name]
dims = var.dims
if len(dims) < 3 or dims[0] != 'time' or dims[-2] != 'lat' or dims[
-1] != 'lon':
continue
variable_dict = dict(id=f'{ds_id}.{var_name}',
name=var_name,
dims=list(dims),
shape=list(var.shape),
dtype=str(var.dtype),
units=var.attrs.get('units', ''),
title=var.attrs.get(
'title', var.attrs.get('long_name',
var_name)))
if client is not None:
tile_grid = ctx.get_tile_grid(ds_id)
tile_xyz_source_options = get_tile_source_options(
tile_grid,
get_dataset_tile_url(ctx, ds_id, var_name, base_url),
client=client)
variable_dict["tileSourceOptions"] = tile_xyz_source_options
cbar, vmin, vmax = ctx.get_color_mapping(ds_id, var_name)
variable_dict["colorBarName"] = cbar
variable_dict["colorBarMin"] = vmin
variable_dict["colorBarMax"] = vmax
variable_dicts.append(variable_dict)
dataset_dict["variables"] = variable_dicts
dim_names = ds.data_vars[list(
ds.data_vars)[0]].dims if len(ds.data_vars) > 0 else ds.dims.keys()
dataset_dict["dimensions"] = [
get_dataset_coordinates(ctx, ds_id, dim_name) for dim_name in dim_names
]
place_groups = ctx.get_dataset_place_groups(ds_id)
if place_groups:
dataset_dict["placeGroups"] = _filter_place_groups(place_groups,
del_features=True)
return dataset_dict
def test_anti_meridian(self):
ds1, ds2 = _get_antimeridian_datasets()
bounds = get_dataset_bounds(ds1)
self.assertEqual((165.0, -15.0, -155.0, 15.0), bounds)
bounds = get_dataset_bounds(ds2)
self.assertEqual((165.0, -15.0, -155.0, 15.0), bounds)
def test_inv_y_wrong_order_bounds(self):
ds1, ds2 = _get_inv_y_wrong_order_bounds_datasets()
bounds = get_dataset_bounds(ds1)
self.assertEqual((-25.0, -15.0, 15.0, 15.0), bounds)
bounds = get_dataset_bounds(ds2)
self.assertEqual((-25.0, -15.0, 15.0, 15.0), bounds)
def test_nominal(self):
ds1, ds2 = _get_nominal_datasets()
bounds = get_dataset_bounds(ds1)
self.assertEqual((-25.0, -15.0, 15.0, 15.0), bounds)
bounds = get_dataset_bounds(ds2)
self.assertEqual((-25.0, -15.0, 15.0, 15.0), bounds)
def get_datasets(ctx: ServiceContext,
details: bool = False,
client: str = None,
point: Tuple[float, float] = None,
base_url: str = None,
granted_scopes: Set[str] = None) -> Dict:
granted_scopes = granted_scopes or set()
dataset_configs = list(ctx.get_dataset_configs())
dataset_dicts = list()
for dataset_config in dataset_configs:
ds_id = dataset_config['Identifier']
if dataset_config.get('Hidden'):
continue
if 'read:dataset:*' not in granted_scopes:
required_scopes = ctx.get_required_dataset_scopes(dataset_config)
is_substitute = dataset_config \
.get('AccessControl', {}) \
.get('IsSubstitute', False)
if not check_scopes(required_scopes,
granted_scopes,
is_substitute=is_substitute):
continue
dataset_dict = dict(id=ds_id)
dataset_dict['title'] = ds_id
if 'Title' in dataset_config:
ds_title = dataset_config['Title']
if ds_title and isinstance(ds_title, str):
dataset_dict['title'] = ds_title
if 'BoundingBox' in dataset_config:
ds_bbox = dataset_config['BoundingBox']
if ds_bbox \
and len(ds_bbox) == 4 \
and all(map(lambda c: isinstance(c, float)
or isinstance(c, int),
ds_bbox)):
dataset_dict['bbox'] = ds_bbox
dataset_dicts.append(dataset_dict)
# Important note:
# the "point" parameter is used by
# the CyanoAlert app only
if details or point:
filtered_dataset_dicts = []
for dataset_dict in dataset_dicts:
ds_id = dataset_dict["id"]
try:
if point:
ds = ctx.get_dataset(ds_id)
if "bbox" not in dataset_dict:
dataset_dict["bbox"] = list(get_dataset_bounds(ds))
if details:
dataset_dict.update(
get_dataset(ctx,
ds_id,
client,
base_url,
granted_scopes=granted_scopes))
filtered_dataset_dicts.append(dataset_dict)
except (DatasetIsNotACubeError, CubeIsNotDisplayable) as e:
LOG.warning(f'skipping dataset {ds_id}: {e}')
dataset_dicts = filtered_dataset_dicts
if point:
is_point_in_dataset_bbox = functools.partial(_is_point_in_dataset_bbox,
point)
# noinspection PyTypeChecker
dataset_dicts = list(filter(is_point_in_dataset_bbox, dataset_dicts))
return dict(datasets=dataset_dicts)
def get_dataset(ctx: ServiceContext,
ds_id: str,
client=None,
base_url: str = None,
granted_scopes: Set[str] = None) -> Dict:
granted_scopes = granted_scopes or set()
dataset_config = ctx.get_dataset_config(ds_id)
ds_id = dataset_config['Identifier']
if 'read:dataset:*' not in granted_scopes:
required_scopes = ctx.get_required_dataset_scopes(dataset_config)
assert_scopes(required_scopes, granted_scopes or set())
try:
ml_ds = ctx.get_ml_dataset(ds_id)
except (ValueError, DataStoreError) as e:
raise DatasetIsNotACubeError(f'could not open dataset: {e}') from e
grid_mapping = ml_ds.grid_mapping
if not grid_mapping.crs.is_geographic:
raise CubeIsNotDisplayable(f'CRS is not geographic:'
f' {grid_mapping.crs.srs}')
if not math.isclose(grid_mapping.x_res, grid_mapping.y_res,
rel_tol=0.01): # we allow up to 1% dev
raise CubeIsNotDisplayable(f'spatial resolutions are'
f' different in x, y:'
f' {grid_mapping.x_res}'
f' and {grid_mapping.y_res}')
try:
# Make sure we have a valid tile grid
tile_grid = ml_ds.tile_grid
assert_instance(tile_grid, TileGrid)
except ValueError as e:
raise CubeIsNotDisplayable(f'could not create tile grid: {e}')
ds = ml_ds.get_dataset(0)
ds_title = dataset_config.get(
'Title', ds.attrs.get('title', ds.attrs.get('name', ds_id)))
dataset_dict = dict(id=ds_id, title=ds_title)
if "bbox" not in dataset_dict:
dataset_dict["bbox"] = list(get_dataset_bounds(ds))
variable_dicts = []
for var_name in ds.data_vars:
var = ds.data_vars[var_name]
dims = var.dims
if len(dims) < 3 \
or dims[0] != 'time' \
or dims[-2] != 'lat' \
or dims[-1] != 'lon':
continue
if 'read:variable:*' not in granted_scopes:
required_scopes = ctx.get_required_variable_scopes(
dataset_config, var_name)
if not check_scopes(required_scopes, granted_scopes):
continue
variable_dict = dict(id=f'{ds_id}.{var_name}',
name=var_name,
dims=list(dims),
shape=list(var.shape),
dtype=str(var.dtype),
units=var.attrs.get('units', ''),
title=var.attrs.get(
'title', var.attrs.get('long_name',
var_name)))
if client is not None:
tile_grid = ctx.get_tile_grid(ds_id)
tile_xyz_source_options = get_tile_source_options(
tile_grid,
get_dataset_tile_url(ctx, ds_id, var_name, base_url),
client=client)
variable_dict["tileSourceOptions"] = tile_xyz_source_options
cmap_name, (cmap_vmin,
cmap_vmax) = ctx.get_color_mapping(ds_id, var_name)
variable_dict["colorBarName"] = cmap_name
variable_dict["colorBarMin"] = cmap_vmin
variable_dict["colorBarMax"] = cmap_vmax
if hasattr(var.data, '_repr_html_'):
# noinspection PyProtectedMember
variable_dict["htmlRepr"] = var.data._repr_html_()
variable_dict["attrs"] = {
key:
("NaN" if isinstance(value, float) and np.isnan(value) else value)
for key, value in var.attrs.items()
}
variable_dicts.append(variable_dict)
dataset_dict["variables"] = variable_dicts
rgb_var_names, rgb_norm_ranges = ctx.get_rgb_color_mapping(ds_id)
if any(rgb_var_names):
rgb_schema = {'varNames': rgb_var_names, 'normRanges': rgb_norm_ranges}
if client is not None:
tile_grid = ctx.get_tile_grid(ds_id)
tile_xyz_source_options = get_tile_source_options(
tile_grid,
get_dataset_tile_url(ctx, ds_id, 'rgb', base_url),
client=client)
rgb_schema["tileSourceOptions"] = tile_xyz_source_options
dataset_dict["rgbSchema"] = rgb_schema
dim_names = ds.data_vars[list(ds.data_vars)[0]].dims \
if len(ds.data_vars) > 0 else ds.dims.keys()
dataset_dict["dimensions"] = [
get_dataset_coordinates(ctx, ds_id, dim_name) for dim_name in dim_names
]
dataset_dict["attrs"] = {
key: ds.attrs[key]
for key in sorted(list(ds.attrs.keys()))
}
dataset_attributions = dataset_config.get(
'DatasetAttribution', ctx.config.get('DatasetAttribution'))
if dataset_attributions is not None:
if isinstance(dataset_attributions, str):
dataset_attributions = [dataset_attributions]
dataset_dict['attributions'] = dataset_attributions
place_groups = ctx.get_dataset_place_groups(ds_id, base_url)
if place_groups:
dataset_dict["placeGroups"] = _filter_place_groups(place_groups,
del_features=True)
return dataset_dict
