def get_dataset(ctx: ServiceContext,
ds_id: str,
client=None,
base_url: str = None,
granted_scopes: Set[str] = None) -> Dict:
granted_scopes = granted_scopes or set()
dataset_descriptor = ctx.get_dataset_descriptor(ds_id)
ds_id = dataset_descriptor['Identifier']
if 'read:dataset:*' not in granted_scopes:
required_scopes = ctx.get_required_dataset_scopes(dataset_descriptor)
assert_scopes(required_scopes, granted_scopes or set())
ds_title = dataset_descriptor['Title']
dataset_dict = dict(id=ds_id, title=ds_title)
ds = ctx.get_dataset(ds_id)
if "bbox" not in dataset_dict:
dataset_dict["bbox"] = list(get_dataset_bounds(ds))
variable_dicts = []
for var_name in ds.data_vars:
var = ds.data_vars[var_name]
dims = var.dims
if len(dims) < 3 or dims[0] != 'time' or dims[-2] != 'lat' or dims[
-1] != 'lon':
continue
if 'read:variable:*' not in granted_scopes:
required_scopes = ctx.get_required_variable_scopes(
dataset_descriptor, var_name)
if not check_scopes(required_scopes, granted_scopes):
continue
variable_dict = dict(id=f'{ds_id}.{var_name}',
name=var_name,
dims=list(dims),
shape=list(var.shape),
dtype=str(var.dtype),
units=var.attrs.get('units', ''),
title=var.attrs.get(
'title', var.attrs.get('long_name',
var_name)))
if client is not None:
tile_grid = ctx.get_tile_grid(ds_id)
tile_xyz_source_options = get_tile_source_options(
tile_grid,
get_dataset_tile_url(ctx, ds_id, var_name, base_url),
client=client)
variable_dict["tileSourceOptions"] = tile_xyz_source_options
cmap_name, (cmap_vmin,
cmap_vmax) = ctx.get_color_mapping(ds_id, var_name)
variable_dict["colorBarName"] = cmap_name
variable_dict["colorBarMin"] = cmap_vmin
variable_dict["colorBarMax"] = cmap_vmax
if hasattr(var.data, '_repr_html_'):
variable_dict["htmlRepr"] = var.data._repr_html_()
variable_dict["attrs"] = {
key: var.attrs[key]
for key in sorted(list(var.attrs.keys()))
}
variable_dicts.append(variable_dict)
ctx.get_rgb_color_mapping(ds_id)
dataset_dict["variables"] = variable_dicts
rgb_var_names, rgb_norm_ranges = ctx.get_rgb_color_mapping(ds_id)
if any(rgb_var_names):
rgb_schema = {'varNames': rgb_var_names, 'normRanges': rgb_norm_ranges}
if client is not None:
tile_grid = ctx.get_tile_grid(ds_id)
tile_xyz_source_options = get_tile_source_options(
tile_grid,
get_dataset_tile_url(ctx, ds_id, 'rgb', base_url),
client=client)
rgb_schema["tileSourceOptions"] = tile_xyz_source_options
dataset_dict["rgbSchema"] = rgb_schema
dim_names = ds.data_vars[list(
ds.data_vars)[0]].dims if len(ds.data_vars) > 0 else ds.dims.keys()
dataset_dict["dimensions"] = [
get_dataset_coordinates(ctx, ds_id, dim_name) for dim_name in dim_names
]
dataset_dict["attrs"] = {
key: ds.attrs[key]
for key in sorted(list(ds.attrs.keys()))
}
dataset_attributions = dataset_descriptor.get(
'DatasetAttribution', ctx.config.get('DatasetAttribution'))
if dataset_attributions is not None:
if isinstance(dataset_attributions, str):
dataset_attributions = [dataset_attributions]
dataset_dict['attributions'] = dataset_attributions
place_groups = ctx.get_dataset_place_groups(ds_id, base_url)
if place_groups:
dataset_dict["placeGroups"] = _filter_place_groups(place_groups,
del_features=True)
return dataset_dict
def get_datasets(ctx: ServiceContext,
details: bool = False,
client: str = None,
point: Tuple[float, float] = None,
base_url: str = None,
granted_scopes: Set[str] = None) -> Dict:
granted_scopes = granted_scopes or set()
dataset_configs = list(ctx.get_dataset_configs())
dataset_dicts = list()
for dataset_config in dataset_configs:
ds_id = dataset_config['Identifier']
if dataset_config.get('Hidden'):
continue
if 'read:dataset:*' not in granted_scopes:
required_scopes = ctx.get_required_dataset_scopes(dataset_config)
is_substitute = dataset_config \
.get('AccessControl', {}) \
.get('IsSubstitute', False)
if not check_scopes(required_scopes,
granted_scopes,
is_substitute=is_substitute):
continue
dataset_dict = dict(id=ds_id)
dataset_dict['title'] = ds_id
if 'Title' in dataset_config:
ds_title = dataset_config['Title']
if ds_title and isinstance(ds_title, str):
dataset_dict['title'] = ds_title
if 'BoundingBox' in dataset_config:
ds_bbox = dataset_config['BoundingBox']
if ds_bbox \
and len(ds_bbox) == 4 \
and all(map(lambda c: isinstance(c, float)
or isinstance(c, int),
ds_bbox)):
dataset_dict['bbox'] = ds_bbox
dataset_dicts.append(dataset_dict)
# Important note:
# the "point" parameter is used by
# the CyanoAlert app only
if details or point:
filtered_dataset_dicts = []
for dataset_dict in dataset_dicts:
ds_id = dataset_dict["id"]
try:
if point:
ds = ctx.get_dataset(ds_id)
if "bbox" not in dataset_dict:
dataset_dict["bbox"] = list(get_dataset_bounds(ds))
if details:
dataset_dict.update(
get_dataset(ctx,
ds_id,
client,
base_url,
granted_scopes=granted_scopes))
filtered_dataset_dicts.append(dataset_dict)
except (DatasetIsNotACubeError, CubeIsNotDisplayable) as e:
LOG.warning(f'skipping dataset {ds_id}: {e}')
dataset_dicts = filtered_dataset_dicts
if point:
is_point_in_dataset_bbox = functools.partial(_is_point_in_dataset_bbox,
point)
# noinspection PyTypeChecker
dataset_dicts = list(filter(is_point_in_dataset_bbox, dataset_dicts))
return dict(datasets=dataset_dicts)
def get_datasets(ctx: ServiceContext,
details: bool = False,
client: str = None,
point: Tuple[float, float] = None,
base_url: str = None,
granted_scopes: Set[str] = None) -> Dict:
granted_scopes = granted_scopes or set()
dataset_descriptors = ctx.get_dataset_descriptors()
dataset_dicts = list()
for dataset_descriptor in dataset_descriptors:
ds_id = dataset_descriptor['Identifier']
if dataset_descriptor.get('Hidden'):
continue
if 'read:dataset:*' not in granted_scopes:
required_scopes = ctx.get_required_dataset_scopes(
dataset_descriptor)
is_substitute = dataset_descriptor.get('AccessControl', {}).get(
'IsSubstitute', False)
if not check_scopes(required_scopes,
granted_scopes,
is_substitute=is_substitute):
continue
dataset_dict = dict(id=ds_id)
if 'Title' in dataset_descriptor:
ds_title = dataset_descriptor['Title']
if ds_title and isinstance(ds_title, str):
dataset_dict['title'] = ds_title
else:
dataset_dict['title'] = ds_id
if 'BoundingBox' in dataset_descriptor:
ds_bbox = dataset_descriptor['BoundingBox']
if ds_bbox \
and len(ds_bbox) == 4 \
and all(map(lambda c: isinstance(c, float) or isinstance(c, int), ds_bbox)):
dataset_dict['bbox'] = ds_bbox
dataset_dicts.append(dataset_dict)
if details or point:
for dataset_dict in dataset_dicts:
ds_id = dataset_dict["id"]
if point:
ds = ctx.get_dataset(ds_id)
if "bbox" not in dataset_dict:
dataset_dict["bbox"] = list(get_dataset_bounds(ds))
if details:
dataset_dict.update(
get_dataset(ctx,
ds_id,
client,
base_url,
granted_scopes=granted_scopes))
if point:
is_point_in_dataset_bbox = functools.partial(_is_point_in_dataset_bbox,
point)
# noinspection PyTypeChecker
dataset_dicts = list(filter(is_point_in_dataset_bbox, dataset_dicts))
return dict(datasets=dataset_dicts)
def get_dataset(ctx: ServiceContext,
ds_id: str,
client=None,
base_url: str = None,
granted_scopes: Set[str] = None) -> Dict:
granted_scopes = granted_scopes or set()
dataset_config = ctx.get_dataset_config(ds_id)
ds_id = dataset_config['Identifier']
if 'read:dataset:*' not in granted_scopes:
required_scopes = ctx.get_required_dataset_scopes(dataset_config)
assert_scopes(required_scopes, granted_scopes or set())
try:
ml_ds = ctx.get_ml_dataset(ds_id)
except (ValueError, DataStoreError) as e:
raise DatasetIsNotACubeError(f'could not open dataset: {e}') from e
grid_mapping = ml_ds.grid_mapping
if not grid_mapping.crs.is_geographic:
raise CubeIsNotDisplayable(f'CRS is not geographic:'
f' {grid_mapping.crs.srs}')
if not math.isclose(grid_mapping.x_res, grid_mapping.y_res,
rel_tol=0.01): # we allow up to 1% dev
raise CubeIsNotDisplayable(f'spatial resolutions are'
f' different in x, y:'
f' {grid_mapping.x_res}'
f' and {grid_mapping.y_res}')
try:
# Make sure we have a valid tile grid
tile_grid = ml_ds.tile_grid
assert_instance(tile_grid, TileGrid)
except ValueError as e:
raise CubeIsNotDisplayable(f'could not create tile grid: {e}')
ds = ml_ds.get_dataset(0)
ds_title = dataset_config.get(
'Title', ds.attrs.get('title', ds.attrs.get('name', ds_id)))
dataset_dict = dict(id=ds_id, title=ds_title)
if "bbox" not in dataset_dict:
dataset_dict["bbox"] = list(get_dataset_bounds(ds))
variable_dicts = []
for var_name in ds.data_vars:
var = ds.data_vars[var_name]
dims = var.dims
if len(dims) < 3 \
or dims[0] != 'time' \
or dims[-2] != 'lat' \
or dims[-1] != 'lon':
continue
if 'read:variable:*' not in granted_scopes:
required_scopes = ctx.get_required_variable_scopes(
dataset_config, var_name)
if not check_scopes(required_scopes, granted_scopes):
continue
variable_dict = dict(id=f'{ds_id}.{var_name}',
name=var_name,
dims=list(dims),
shape=list(var.shape),
dtype=str(var.dtype),
units=var.attrs.get('units', ''),
title=var.attrs.get(
'title', var.attrs.get('long_name',
var_name)))
if client is not None:
tile_grid = ctx.get_tile_grid(ds_id)
tile_xyz_source_options = get_tile_source_options(
tile_grid,
get_dataset_tile_url(ctx, ds_id, var_name, base_url),
client=client)
variable_dict["tileSourceOptions"] = tile_xyz_source_options
cmap_name, (cmap_vmin,
cmap_vmax) = ctx.get_color_mapping(ds_id, var_name)
variable_dict["colorBarName"] = cmap_name
variable_dict["colorBarMin"] = cmap_vmin
variable_dict["colorBarMax"] = cmap_vmax
if hasattr(var.data, '_repr_html_'):
# noinspection PyProtectedMember
variable_dict["htmlRepr"] = var.data._repr_html_()
variable_dict["attrs"] = {
key:
("NaN" if isinstance(value, float) and np.isnan(value) else value)
for key, value in var.attrs.items()
}
variable_dicts.append(variable_dict)
dataset_dict["variables"] = variable_dicts
rgb_var_names, rgb_norm_ranges = ctx.get_rgb_color_mapping(ds_id)
if any(rgb_var_names):
rgb_schema = {'varNames': rgb_var_names, 'normRanges': rgb_norm_ranges}
if client is not None:
tile_grid = ctx.get_tile_grid(ds_id)
tile_xyz_source_options = get_tile_source_options(
tile_grid,
get_dataset_tile_url(ctx, ds_id, 'rgb', base_url),
client=client)
rgb_schema["tileSourceOptions"] = tile_xyz_source_options
dataset_dict["rgbSchema"] = rgb_schema
dim_names = ds.data_vars[list(ds.data_vars)[0]].dims \
if len(ds.data_vars) > 0 else ds.dims.keys()
dataset_dict["dimensions"] = [
get_dataset_coordinates(ctx, ds_id, dim_name) for dim_name in dim_names
]
dataset_dict["attrs"] = {
key: ds.attrs[key]
for key in sorted(list(ds.attrs.keys()))
}
dataset_attributions = dataset_config.get(
'DatasetAttribution', ctx.config.get('DatasetAttribution'))
if dataset_attributions is not None:
if isinstance(dataset_attributions, str):
dataset_attributions = [dataset_attributions]
dataset_dict['attributions'] = dataset_attributions
place_groups = ctx.get_dataset_place_groups(ds_id, base_url)
if place_groups:
dataset_dict["placeGroups"] = _filter_place_groups(place_groups,
del_features=True)
return dataset_dict