def test_check_scopes_fails(self): self.assertEqual( False, check_scopes({'read:dataset:test1.zarr'}, {'read:dataset:test1.zarr'}, is_substitute=True)) self.assertEqual( False, check_scopes({'read:dataset:test2.zarr'}, {'read:dataset:test1.zarr'})) self.assertEqual( False, check_scopes({'read:dataset:test2.zarr'}, {'read:dataset:test1.zarr'}, is_substitute=True))
def test_check_scopes_ok(self): self.assertEqual( True, check_scopes({'read:dataset:test1.zarr'}, set(), is_substitute=False)) self.assertEqual( True, check_scopes({'read:dataset:test1.zarr'}, set(), is_substitute=True)) self.assertEqual( True, check_scopes({'read:dataset:test1.zarr'}, {'read:dataset:test1.zarr'})) self.assertEqual( True, check_scopes({'read:dataset:test1.zarr'}, {'read:dataset:test?.zarr'})) self.assertEqual( True, check_scopes({'read:dataset:test1.zarr'}, {'read:dataset:test1.*'}))
def get_dataset(ctx: ServiceContext, ds_id: str, client=None, base_url: str = None, granted_scopes: Set[str] = None) -> Dict: granted_scopes = granted_scopes or set() dataset_descriptor = ctx.get_dataset_descriptor(ds_id) ds_id = dataset_descriptor['Identifier'] if 'read:dataset:*' not in granted_scopes: required_scopes = ctx.get_required_dataset_scopes(dataset_descriptor) assert_scopes(required_scopes, granted_scopes or set()) ds_title = dataset_descriptor['Title'] dataset_dict = dict(id=ds_id, title=ds_title) ds = ctx.get_dataset(ds_id) if "bbox" not in dataset_dict: dataset_dict["bbox"] = list(get_dataset_bounds(ds)) variable_dicts = [] for var_name in ds.data_vars: var = ds.data_vars[var_name] dims = var.dims if len(dims) < 3 or dims[0] != 'time' or dims[-2] != 'lat' or dims[ -1] != 'lon': continue if 'read:variable:*' not in granted_scopes: required_scopes = ctx.get_required_variable_scopes( dataset_descriptor, var_name) if not check_scopes(required_scopes, granted_scopes): continue variable_dict = dict(id=f'{ds_id}.{var_name}', name=var_name, dims=list(dims), shape=list(var.shape), dtype=str(var.dtype), units=var.attrs.get('units', ''), title=var.attrs.get( 'title', var.attrs.get('long_name', var_name))) if client is not None: tile_grid = ctx.get_tile_grid(ds_id) tile_xyz_source_options = get_tile_source_options( tile_grid, get_dataset_tile_url(ctx, ds_id, var_name, base_url), client=client) variable_dict["tileSourceOptions"] = tile_xyz_source_options cmap_name, (cmap_vmin, cmap_vmax) = ctx.get_color_mapping(ds_id, var_name) variable_dict["colorBarName"] = cmap_name variable_dict["colorBarMin"] = cmap_vmin variable_dict["colorBarMax"] = cmap_vmax if hasattr(var.data, '_repr_html_'): variable_dict["htmlRepr"] = var.data._repr_html_() variable_dict["attrs"] = { key: var.attrs[key] for key in sorted(list(var.attrs.keys())) } variable_dicts.append(variable_dict) ctx.get_rgb_color_mapping(ds_id) dataset_dict["variables"] = variable_dicts rgb_var_names, rgb_norm_ranges = ctx.get_rgb_color_mapping(ds_id) if any(rgb_var_names): rgb_schema = {'varNames': rgb_var_names, 'normRanges': rgb_norm_ranges} if client is not None: tile_grid = ctx.get_tile_grid(ds_id) tile_xyz_source_options = get_tile_source_options( tile_grid, get_dataset_tile_url(ctx, ds_id, 'rgb', base_url), client=client) rgb_schema["tileSourceOptions"] = tile_xyz_source_options dataset_dict["rgbSchema"] = rgb_schema dim_names = ds.data_vars[list( ds.data_vars)[0]].dims if len(ds.data_vars) > 0 else ds.dims.keys() dataset_dict["dimensions"] = [ get_dataset_coordinates(ctx, ds_id, dim_name) for dim_name in dim_names ] dataset_dict["attrs"] = { key: ds.attrs[key] for key in sorted(list(ds.attrs.keys())) } dataset_attributions = dataset_descriptor.get( 'DatasetAttribution', ctx.config.get('DatasetAttribution')) if dataset_attributions is not None: if isinstance(dataset_attributions, str): dataset_attributions = [dataset_attributions] dataset_dict['attributions'] = dataset_attributions place_groups = ctx.get_dataset_place_groups(ds_id, base_url) if place_groups: dataset_dict["placeGroups"] = _filter_place_groups(place_groups, del_features=True) return dataset_dict
def get_datasets(ctx: ServiceContext, details: bool = False, client: str = None, point: Tuple[float, float] = None, base_url: str = None, granted_scopes: Set[str] = None) -> Dict: granted_scopes = granted_scopes or set() dataset_descriptors = ctx.get_dataset_descriptors() dataset_dicts = list() for dataset_descriptor in dataset_descriptors: ds_id = dataset_descriptor['Identifier'] if dataset_descriptor.get('Hidden'): continue if 'read:dataset:*' not in granted_scopes: required_scopes = ctx.get_required_dataset_scopes( dataset_descriptor) is_substitute = dataset_descriptor.get('AccessControl', {}).get( 'IsSubstitute', False) if not check_scopes(required_scopes, granted_scopes, is_substitute=is_substitute): continue dataset_dict = dict(id=ds_id) if 'Title' in dataset_descriptor: ds_title = dataset_descriptor['Title'] if ds_title and isinstance(ds_title, str): dataset_dict['title'] = ds_title else: dataset_dict['title'] = ds_id if 'BoundingBox' in dataset_descriptor: ds_bbox = dataset_descriptor['BoundingBox'] if ds_bbox \ and len(ds_bbox) == 4 \ and all(map(lambda c: isinstance(c, float) or isinstance(c, int), ds_bbox)): dataset_dict['bbox'] = ds_bbox dataset_dicts.append(dataset_dict) if details or point: for dataset_dict in dataset_dicts: ds_id = dataset_dict["id"] if point: ds = ctx.get_dataset(ds_id) if "bbox" not in dataset_dict: dataset_dict["bbox"] = list(get_dataset_bounds(ds)) if details: dataset_dict.update( get_dataset(ctx, ds_id, client, base_url, granted_scopes=granted_scopes)) if point: is_point_in_dataset_bbox = functools.partial(_is_point_in_dataset_bbox, point) # noinspection PyTypeChecker dataset_dicts = list(filter(is_point_in_dataset_bbox, dataset_dicts)) return dict(datasets=dataset_dicts)
def get_datasets(ctx: ServiceContext, details: bool = False, client: str = None, point: Tuple[float, float] = None, base_url: str = None, granted_scopes: Set[str] = None) -> Dict: granted_scopes = granted_scopes or set() dataset_configs = list(ctx.get_dataset_configs()) dataset_dicts = list() for dataset_config in dataset_configs: ds_id = dataset_config['Identifier'] if dataset_config.get('Hidden'): continue if 'read:dataset:*' not in granted_scopes: required_scopes = ctx.get_required_dataset_scopes(dataset_config) is_substitute = dataset_config \ .get('AccessControl', {}) \ .get('IsSubstitute', False) if not check_scopes(required_scopes, granted_scopes, is_substitute=is_substitute): continue dataset_dict = dict(id=ds_id) dataset_dict['title'] = ds_id if 'Title' in dataset_config: ds_title = dataset_config['Title'] if ds_title and isinstance(ds_title, str): dataset_dict['title'] = ds_title if 'BoundingBox' in dataset_config: ds_bbox = dataset_config['BoundingBox'] if ds_bbox \ and len(ds_bbox) == 4 \ and all(map(lambda c: isinstance(c, float) or isinstance(c, int), ds_bbox)): dataset_dict['bbox'] = ds_bbox dataset_dicts.append(dataset_dict) # Important note: # the "point" parameter is used by # the CyanoAlert app only if details or point: filtered_dataset_dicts = [] for dataset_dict in dataset_dicts: ds_id = dataset_dict["id"] try: if point: ds = ctx.get_dataset(ds_id) if "bbox" not in dataset_dict: dataset_dict["bbox"] = list(get_dataset_bounds(ds)) if details: dataset_dict.update( get_dataset(ctx, ds_id, client, base_url, granted_scopes=granted_scopes)) filtered_dataset_dicts.append(dataset_dict) except (DatasetIsNotACubeError, CubeIsNotDisplayable) as e: LOG.warning(f'skipping dataset {ds_id}: {e}') dataset_dicts = filtered_dataset_dicts if point: is_point_in_dataset_bbox = functools.partial(_is_point_in_dataset_bbox, point) # noinspection PyTypeChecker dataset_dicts = list(filter(is_point_in_dataset_bbox, dataset_dicts)) return dict(datasets=dataset_dicts)
def get_dataset(ctx: ServiceContext, ds_id: str, client=None, base_url: str = None, granted_scopes: Set[str] = None) -> Dict: granted_scopes = granted_scopes or set() dataset_config = ctx.get_dataset_config(ds_id) ds_id = dataset_config['Identifier'] if 'read:dataset:*' not in granted_scopes: required_scopes = ctx.get_required_dataset_scopes(dataset_config) assert_scopes(required_scopes, granted_scopes or set()) try: ml_ds = ctx.get_ml_dataset(ds_id) except (ValueError, DataStoreError) as e: raise DatasetIsNotACubeError(f'could not open dataset: {e}') from e grid_mapping = ml_ds.grid_mapping if not grid_mapping.crs.is_geographic: raise CubeIsNotDisplayable(f'CRS is not geographic:' f' {grid_mapping.crs.srs}') if not math.isclose(grid_mapping.x_res, grid_mapping.y_res, rel_tol=0.01): # we allow up to 1% dev raise CubeIsNotDisplayable(f'spatial resolutions are' f' different in x, y:' f' {grid_mapping.x_res}' f' and {grid_mapping.y_res}') try: # Make sure we have a valid tile grid tile_grid = ml_ds.tile_grid assert_instance(tile_grid, TileGrid) except ValueError as e: raise CubeIsNotDisplayable(f'could not create tile grid: {e}') ds = ml_ds.get_dataset(0) ds_title = dataset_config.get( 'Title', ds.attrs.get('title', ds.attrs.get('name', ds_id))) dataset_dict = dict(id=ds_id, title=ds_title) if "bbox" not in dataset_dict: dataset_dict["bbox"] = list(get_dataset_bounds(ds)) variable_dicts = [] for var_name in ds.data_vars: var = ds.data_vars[var_name] dims = var.dims if len(dims) < 3 \ or dims[0] != 'time' \ or dims[-2] != 'lat' \ or dims[-1] != 'lon': continue if 'read:variable:*' not in granted_scopes: required_scopes = ctx.get_required_variable_scopes( dataset_config, var_name) if not check_scopes(required_scopes, granted_scopes): continue variable_dict = dict(id=f'{ds_id}.{var_name}', name=var_name, dims=list(dims), shape=list(var.shape), dtype=str(var.dtype), units=var.attrs.get('units', ''), title=var.attrs.get( 'title', var.attrs.get('long_name', var_name))) if client is not None: tile_grid = ctx.get_tile_grid(ds_id) tile_xyz_source_options = get_tile_source_options( tile_grid, get_dataset_tile_url(ctx, ds_id, var_name, base_url), client=client) variable_dict["tileSourceOptions"] = tile_xyz_source_options cmap_name, (cmap_vmin, cmap_vmax) = ctx.get_color_mapping(ds_id, var_name) variable_dict["colorBarName"] = cmap_name variable_dict["colorBarMin"] = cmap_vmin variable_dict["colorBarMax"] = cmap_vmax if hasattr(var.data, '_repr_html_'): # noinspection PyProtectedMember variable_dict["htmlRepr"] = var.data._repr_html_() variable_dict["attrs"] = { key: ("NaN" if isinstance(value, float) and np.isnan(value) else value) for key, value in var.attrs.items() } variable_dicts.append(variable_dict) dataset_dict["variables"] = variable_dicts rgb_var_names, rgb_norm_ranges = ctx.get_rgb_color_mapping(ds_id) if any(rgb_var_names): rgb_schema = {'varNames': rgb_var_names, 'normRanges': rgb_norm_ranges} if client is not None: tile_grid = ctx.get_tile_grid(ds_id) tile_xyz_source_options = get_tile_source_options( tile_grid, get_dataset_tile_url(ctx, ds_id, 'rgb', base_url), client=client) rgb_schema["tileSourceOptions"] = tile_xyz_source_options dataset_dict["rgbSchema"] = rgb_schema dim_names = ds.data_vars[list(ds.data_vars)[0]].dims \ if len(ds.data_vars) > 0 else ds.dims.keys() dataset_dict["dimensions"] = [ get_dataset_coordinates(ctx, ds_id, dim_name) for dim_name in dim_names ] dataset_dict["attrs"] = { key: ds.attrs[key] for key in sorted(list(ds.attrs.keys())) } dataset_attributions = dataset_config.get( 'DatasetAttribution', ctx.config.get('DatasetAttribution')) if dataset_attributions is not None: if isinstance(dataset_attributions, str): dataset_attributions = [dataset_attributions] dataset_dict['attributions'] = dataset_attributions place_groups = ctx.get_dataset_place_groups(ds_id, base_url) if place_groups: dataset_dict["placeGroups"] = _filter_place_groups(place_groups, del_features=True) return dataset_dict