def get_time_series_info(ctx: ServiceContext) -> Dict: """ Get time-series meta-information for variables. :param ctx: Service context object :return: a dictionary with a single entry "layers" which is a list of entries that are dictionaries containing a variable's "name", "dates", and "bounds". """ time_series_info = {'layers': []} descriptors = ctx.get_dataset_descriptors() for descriptor in descriptors: if 'Identifier' in descriptor: if descriptor.get('Hidden'): continue dataset = ctx.get_dataset(descriptor['Identifier']) if 'time' not in dataset.variables: continue xmin, ymin, xmax, ymax = get_dataset_bounds(dataset) time_data = dataset.variables['time'].data time_stamps = [] for time in time_data: time_stamps.append(timestamp_to_iso_string(time)) var_names = sorted(dataset.data_vars) for var_name in var_names: ds_id = descriptor['Identifier'] variable_dict = { 'name': f'{ds_id}.{var_name}', 'dates': time_stamps, 'bounds': dict(xmin=xmin, ymin=ymin, xmax=xmax, ymax=ymax) } time_series_info['layers'].append(variable_dict) return time_series_info
def _get_dataset_tile_grid(dataset: xr.Dataset, num_levels: int = None) -> TileGrid: geo_extent = get_dataset_bounds(dataset) inv_y = float(dataset.lat[0]) < float(dataset.lat[-1]) width, height, tile_width, tile_height = _get_cube_spatial_sizes(dataset) if num_levels is not None and tile_width is not None and tile_height is not None: width_0 = width height_0 = height for i in range(num_levels): width_0 = (width_0 + 1) // 2 height_0 = (height_0 + 1) // 2 num_level_zero_tiles_x = (width_0 + tile_width - 1) // tile_width num_level_zero_tiles_y = (height_0 + tile_height - 1) // tile_height tile_grid = TileGrid(num_levels, num_level_zero_tiles_x, num_level_zero_tiles_y, tile_width, tile_height, geo_extent, inv_y) else: try: tile_grid = TileGrid.create(width, height, tile_width, tile_height, geo_extent, inv_y) except ValueError: num_levels = 1 num_level_zero_tiles_x = 1 num_level_zero_tiles_y = 1 tile_grid = TileGrid(num_levels, num_level_zero_tiles_x, num_level_zero_tiles_y, width, height, geo_extent, inv_y) return tile_grid
def _determine_bbox(data: xr.Dataset) \ -> Optional[Tuple[float, float, float, float]]: try: return get_dataset_bounds(data) except ValueError: if 'geospatial_lon_min' in data.attrs and \ 'geospatial_lat_min' in data.attrs and \ 'geospatial_lon_max' in data.attrs and \ 'geospatial_lat_max' in data.attrs: return (data.geospatial_lat_min, data.geospatial_lon_min, data.geospatial_lat_max, data.geospatial_lon_max)
def get_datasets(ctx: ServiceContext, details: bool = False, client: str = None, point: Tuple[float, float] = None, base_url: str = None) -> Dict: dataset_descriptors = ctx.get_dataset_descriptors() dataset_dicts = list() for dataset_descriptor in dataset_descriptors: if dataset_descriptor.get('Hidden'): continue ds_id = dataset_descriptor['Identifier'] dataset_dict = dict(id=ds_id) if 'Title' in dataset_descriptor: ds_title = dataset_descriptor['Title'] if ds_title and isinstance(ds_title, str): dataset_dict['title'] = ds_title else: dataset_dict['title'] = ds_id if 'BoundingBox' in dataset_descriptor: ds_bbox = dataset_descriptor['BoundingBox'] if ds_bbox \ and len(ds_bbox) == 4 \ and all(map(lambda c: isinstance(c, float) or isinstance(c, int), ds_bbox)): dataset_dict['bbox'] = ds_bbox dataset_dicts.append(dataset_dict) if details or point: for dataset_dict in dataset_dicts: ds_id = dataset_dict["id"] if point: ds = ctx.get_dataset(ds_id) if "bbox" not in dataset_dict: dataset_dict["bbox"] = list(get_dataset_bounds(ds)) if details: dataset_dict.update(get_dataset(ctx, ds_id, client, base_url)) if point: is_point_in_dataset_bbox = functools.partial(_is_point_in_dataset_bbox, point) # noinspection PyTypeChecker dataset_dicts = list(filter(is_point_in_dataset_bbox, dataset_dicts)) return dict(datasets=dataset_dicts)
def get_dataset_tile_grid(dataset: xr.Dataset, num_levels: int = None) -> TileGrid: """ Compute the tile grid for the given *dataset* and an optional number of resolution levels *num_levels*, if given. :param dataset: The dataset. :param num_levels: The number of resolution levels. :return: A TileGrid object """ geo_extent = get_dataset_bounds(dataset) inv_y = float(dataset.lat[0]) < float(dataset.lat[-1]) width, height, tile_width, tile_height = _get_cube_spatial_sizes(dataset) if num_levels is not None and tile_width is not None and tile_height is not None: width_0 = width height_0 = height for i in range(num_levels - 1): width_0 = (width_0 + 1) // 2 height_0 = (height_0 + 1) // 2 num_level_zero_tiles_x = (width_0 + tile_width - 1) // tile_width num_level_zero_tiles_y = (height_0 + tile_height - 1) // tile_height tile_grid = TileGrid(num_levels, num_level_zero_tiles_x, num_level_zero_tiles_y, tile_width, tile_height, geo_extent, inv_y) else: try: tile_grid = TileGrid.create(width, height, tile_width, tile_height, geo_extent, inv_y) except ValueError: num_levels = 1 num_level_zero_tiles_x = 1 num_level_zero_tiles_y = 1 tile_grid = TileGrid(num_levels, num_level_zero_tiles_x, num_level_zero_tiles_y, width, height, geo_extent, inv_y) if tile_width is not None and tile_width != tile_grid.tile_width: warnings.warn( f'FIXME: wanted tile_width={tile_width} as of chunking, but will use {tile_grid.tile_width}. ' f'This is inefficient.') if tile_height is not None and tile_height != tile_grid.tile_height: warnings.warn( f'FIXME: wanted tile_height={tile_width} as of chunking, but will use {tile_grid.tile_height}. ' f'This is inefficient.') return tile_grid
def get_dataset(ctx: ServiceContext, ds_id: str, client=None, base_url: str = None, granted_scopes: Set[str] = None) -> Dict: granted_scopes = granted_scopes or set() dataset_descriptor = ctx.get_dataset_descriptor(ds_id) ds_id = dataset_descriptor['Identifier'] if 'read:dataset:*' not in granted_scopes: required_scopes = ctx.get_required_dataset_scopes(dataset_descriptor) assert_scopes(required_scopes, granted_scopes or set()) ds_title = dataset_descriptor['Title'] dataset_dict = dict(id=ds_id, title=ds_title) ds = ctx.get_dataset(ds_id) if "bbox" not in dataset_dict: dataset_dict["bbox"] = list(get_dataset_bounds(ds)) variable_dicts = [] for var_name in ds.data_vars: var = ds.data_vars[var_name] dims = var.dims if len(dims) < 3 or dims[0] != 'time' or dims[-2] != 'lat' or dims[ -1] != 'lon': continue if 'read:variable:*' not in granted_scopes: required_scopes = ctx.get_required_variable_scopes( dataset_descriptor, var_name) if not check_scopes(required_scopes, granted_scopes): continue variable_dict = dict(id=f'{ds_id}.{var_name}', name=var_name, dims=list(dims), shape=list(var.shape), dtype=str(var.dtype), units=var.attrs.get('units', ''), title=var.attrs.get( 'title', var.attrs.get('long_name', var_name))) if client is not None: tile_grid = ctx.get_tile_grid(ds_id) tile_xyz_source_options = get_tile_source_options( tile_grid, get_dataset_tile_url(ctx, ds_id, var_name, base_url), client=client) variable_dict["tileSourceOptions"] = tile_xyz_source_options cmap_name, (cmap_vmin, cmap_vmax) = ctx.get_color_mapping(ds_id, var_name) variable_dict["colorBarName"] = cmap_name variable_dict["colorBarMin"] = cmap_vmin variable_dict["colorBarMax"] = cmap_vmax if hasattr(var.data, '_repr_html_'): variable_dict["htmlRepr"] = var.data._repr_html_() variable_dict["attrs"] = { key: var.attrs[key] for key in sorted(list(var.attrs.keys())) } variable_dicts.append(variable_dict) ctx.get_rgb_color_mapping(ds_id) dataset_dict["variables"] = variable_dicts rgb_var_names, rgb_norm_ranges = ctx.get_rgb_color_mapping(ds_id) if any(rgb_var_names): rgb_schema = {'varNames': rgb_var_names, 'normRanges': rgb_norm_ranges} if client is not None: tile_grid = ctx.get_tile_grid(ds_id) tile_xyz_source_options = get_tile_source_options( tile_grid, get_dataset_tile_url(ctx, ds_id, 'rgb', base_url), client=client) rgb_schema["tileSourceOptions"] = tile_xyz_source_options dataset_dict["rgbSchema"] = rgb_schema dim_names = ds.data_vars[list( ds.data_vars)[0]].dims if len(ds.data_vars) > 0 else ds.dims.keys() dataset_dict["dimensions"] = [ get_dataset_coordinates(ctx, ds_id, dim_name) for dim_name in dim_names ] dataset_dict["attrs"] = { key: ds.attrs[key] for key in sorted(list(ds.attrs.keys())) } dataset_attributions = dataset_descriptor.get( 'DatasetAttribution', ctx.config.get('DatasetAttribution')) if dataset_attributions is not None: if isinstance(dataset_attributions, str): dataset_attributions = [dataset_attributions] dataset_dict['attributions'] = dataset_attributions place_groups = ctx.get_dataset_place_groups(ds_id, base_url) if place_groups: dataset_dict["placeGroups"] = _filter_place_groups(place_groups, del_features=True) return dataset_dict
def get_datasets(ctx: ServiceContext, details: bool = False, client: str = None, point: Tuple[float, float] = None, base_url: str = None, granted_scopes: Set[str] = None) -> Dict: granted_scopes = granted_scopes or set() dataset_descriptors = ctx.get_dataset_descriptors() dataset_dicts = list() for dataset_descriptor in dataset_descriptors: ds_id = dataset_descriptor['Identifier'] if dataset_descriptor.get('Hidden'): continue if 'read:dataset:*' not in granted_scopes: required_scopes = ctx.get_required_dataset_scopes( dataset_descriptor) is_substitute = dataset_descriptor.get('AccessControl', {}).get( 'IsSubstitute', False) if not check_scopes(required_scopes, granted_scopes, is_substitute=is_substitute): continue dataset_dict = dict(id=ds_id) if 'Title' in dataset_descriptor: ds_title = dataset_descriptor['Title'] if ds_title and isinstance(ds_title, str): dataset_dict['title'] = ds_title else: dataset_dict['title'] = ds_id if 'BoundingBox' in dataset_descriptor: ds_bbox = dataset_descriptor['BoundingBox'] if ds_bbox \ and len(ds_bbox) == 4 \ and all(map(lambda c: isinstance(c, float) or isinstance(c, int), ds_bbox)): dataset_dict['bbox'] = ds_bbox dataset_dicts.append(dataset_dict) if details or point: for dataset_dict in dataset_dicts: ds_id = dataset_dict["id"] if point: ds = ctx.get_dataset(ds_id) if "bbox" not in dataset_dict: dataset_dict["bbox"] = list(get_dataset_bounds(ds)) if details: dataset_dict.update( get_dataset(ctx, ds_id, client, base_url, granted_scopes=granted_scopes)) if point: is_point_in_dataset_bbox = functools.partial(_is_point_in_dataset_bbox, point) # noinspection PyTypeChecker dataset_dicts = list(filter(is_point_in_dataset_bbox, dataset_dicts)) return dict(datasets=dataset_dicts)
def get_dataset(ctx: ServiceContext, ds_id: str, client=None, base_url: str = None) -> GeoJsonFeatureCollection: dataset_descriptor = ctx.get_dataset_descriptor(ds_id) ds_id = dataset_descriptor['Identifier'] ds_title = dataset_descriptor['Title'] dataset_dict = dict(id=ds_id, title=ds_title) ds = ctx.get_dataset(ds_id) if "bbox" not in dataset_dict: dataset_dict["bbox"] = list(get_dataset_bounds(ds)) variable_dicts = [] for var_name in ds.data_vars: var = ds.data_vars[var_name] dims = var.dims if len(dims) < 3 or dims[0] != 'time' or dims[-2] != 'lat' or dims[ -1] != 'lon': continue variable_dict = dict(id=f'{ds_id}.{var_name}', name=var_name, dims=list(dims), shape=list(var.shape), dtype=str(var.dtype), units=var.attrs.get('units', ''), title=var.attrs.get( 'title', var.attrs.get('long_name', var_name))) if client is not None: tile_grid = ctx.get_tile_grid(ds_id) tile_xyz_source_options = get_tile_source_options( tile_grid, get_dataset_tile_url(ctx, ds_id, var_name, base_url), client=client) variable_dict["tileSourceOptions"] = tile_xyz_source_options cbar, vmin, vmax = ctx.get_color_mapping(ds_id, var_name) variable_dict["colorBarName"] = cbar variable_dict["colorBarMin"] = vmin variable_dict["colorBarMax"] = vmax variable_dicts.append(variable_dict) dataset_dict["variables"] = variable_dicts dim_names = ds.data_vars[list( ds.data_vars)[0]].dims if len(ds.data_vars) > 0 else ds.dims.keys() dataset_dict["dimensions"] = [ get_dataset_coordinates(ctx, ds_id, dim_name) for dim_name in dim_names ] place_groups = ctx.get_dataset_place_groups(ds_id) if place_groups: dataset_dict["placeGroups"] = _filter_place_groups(place_groups, del_features=True) return dataset_dict
def test_anti_meridian(self): ds1, ds2 = _get_antimeridian_datasets() bounds = get_dataset_bounds(ds1) self.assertEqual((165.0, -15.0, -155.0, 15.0), bounds) bounds = get_dataset_bounds(ds2) self.assertEqual((165.0, -15.0, -155.0, 15.0), bounds)
def test_inv_y_wrong_order_bounds(self): ds1, ds2 = _get_inv_y_wrong_order_bounds_datasets() bounds = get_dataset_bounds(ds1) self.assertEqual((-25.0, -15.0, 15.0, 15.0), bounds) bounds = get_dataset_bounds(ds2) self.assertEqual((-25.0, -15.0, 15.0, 15.0), bounds)
def test_nominal(self): ds1, ds2 = _get_nominal_datasets() bounds = get_dataset_bounds(ds1) self.assertEqual((-25.0, -15.0, 15.0, 15.0), bounds) bounds = get_dataset_bounds(ds2) self.assertEqual((-25.0, -15.0, 15.0, 15.0), bounds)
def get_datasets(ctx: ServiceContext, details: bool = False, client: str = None, point: Tuple[float, float] = None, base_url: str = None, granted_scopes: Set[str] = None) -> Dict: granted_scopes = granted_scopes or set() dataset_configs = list(ctx.get_dataset_configs()) dataset_dicts = list() for dataset_config in dataset_configs: ds_id = dataset_config['Identifier'] if dataset_config.get('Hidden'): continue if 'read:dataset:*' not in granted_scopes: required_scopes = ctx.get_required_dataset_scopes(dataset_config) is_substitute = dataset_config \ .get('AccessControl', {}) \ .get('IsSubstitute', False) if not check_scopes(required_scopes, granted_scopes, is_substitute=is_substitute): continue dataset_dict = dict(id=ds_id) dataset_dict['title'] = ds_id if 'Title' in dataset_config: ds_title = dataset_config['Title'] if ds_title and isinstance(ds_title, str): dataset_dict['title'] = ds_title if 'BoundingBox' in dataset_config: ds_bbox = dataset_config['BoundingBox'] if ds_bbox \ and len(ds_bbox) == 4 \ and all(map(lambda c: isinstance(c, float) or isinstance(c, int), ds_bbox)): dataset_dict['bbox'] = ds_bbox dataset_dicts.append(dataset_dict) # Important note: # the "point" parameter is used by # the CyanoAlert app only if details or point: filtered_dataset_dicts = [] for dataset_dict in dataset_dicts: ds_id = dataset_dict["id"] try: if point: ds = ctx.get_dataset(ds_id) if "bbox" not in dataset_dict: dataset_dict["bbox"] = list(get_dataset_bounds(ds)) if details: dataset_dict.update( get_dataset(ctx, ds_id, client, base_url, granted_scopes=granted_scopes)) filtered_dataset_dicts.append(dataset_dict) except (DatasetIsNotACubeError, CubeIsNotDisplayable) as e: LOG.warning(f'skipping dataset {ds_id}: {e}') dataset_dicts = filtered_dataset_dicts if point: is_point_in_dataset_bbox = functools.partial(_is_point_in_dataset_bbox, point) # noinspection PyTypeChecker dataset_dicts = list(filter(is_point_in_dataset_bbox, dataset_dicts)) return dict(datasets=dataset_dicts)
def get_dataset(ctx: ServiceContext, ds_id: str, client=None, base_url: str = None, granted_scopes: Set[str] = None) -> Dict: granted_scopes = granted_scopes or set() dataset_config = ctx.get_dataset_config(ds_id) ds_id = dataset_config['Identifier'] if 'read:dataset:*' not in granted_scopes: required_scopes = ctx.get_required_dataset_scopes(dataset_config) assert_scopes(required_scopes, granted_scopes or set()) try: ml_ds = ctx.get_ml_dataset(ds_id) except (ValueError, DataStoreError) as e: raise DatasetIsNotACubeError(f'could not open dataset: {e}') from e grid_mapping = ml_ds.grid_mapping if not grid_mapping.crs.is_geographic: raise CubeIsNotDisplayable(f'CRS is not geographic:' f' {grid_mapping.crs.srs}') if not math.isclose(grid_mapping.x_res, grid_mapping.y_res, rel_tol=0.01): # we allow up to 1% dev raise CubeIsNotDisplayable(f'spatial resolutions are' f' different in x, y:' f' {grid_mapping.x_res}' f' and {grid_mapping.y_res}') try: # Make sure we have a valid tile grid tile_grid = ml_ds.tile_grid assert_instance(tile_grid, TileGrid) except ValueError as e: raise CubeIsNotDisplayable(f'could not create tile grid: {e}') ds = ml_ds.get_dataset(0) ds_title = dataset_config.get( 'Title', ds.attrs.get('title', ds.attrs.get('name', ds_id))) dataset_dict = dict(id=ds_id, title=ds_title) if "bbox" not in dataset_dict: dataset_dict["bbox"] = list(get_dataset_bounds(ds)) variable_dicts = [] for var_name in ds.data_vars: var = ds.data_vars[var_name] dims = var.dims if len(dims) < 3 \ or dims[0] != 'time' \ or dims[-2] != 'lat' \ or dims[-1] != 'lon': continue if 'read:variable:*' not in granted_scopes: required_scopes = ctx.get_required_variable_scopes( dataset_config, var_name) if not check_scopes(required_scopes, granted_scopes): continue variable_dict = dict(id=f'{ds_id}.{var_name}', name=var_name, dims=list(dims), shape=list(var.shape), dtype=str(var.dtype), units=var.attrs.get('units', ''), title=var.attrs.get( 'title', var.attrs.get('long_name', var_name))) if client is not None: tile_grid = ctx.get_tile_grid(ds_id) tile_xyz_source_options = get_tile_source_options( tile_grid, get_dataset_tile_url(ctx, ds_id, var_name, base_url), client=client) variable_dict["tileSourceOptions"] = tile_xyz_source_options cmap_name, (cmap_vmin, cmap_vmax) = ctx.get_color_mapping(ds_id, var_name) variable_dict["colorBarName"] = cmap_name variable_dict["colorBarMin"] = cmap_vmin variable_dict["colorBarMax"] = cmap_vmax if hasattr(var.data, '_repr_html_'): # noinspection PyProtectedMember variable_dict["htmlRepr"] = var.data._repr_html_() variable_dict["attrs"] = { key: ("NaN" if isinstance(value, float) and np.isnan(value) else value) for key, value in var.attrs.items() } variable_dicts.append(variable_dict) dataset_dict["variables"] = variable_dicts rgb_var_names, rgb_norm_ranges = ctx.get_rgb_color_mapping(ds_id) if any(rgb_var_names): rgb_schema = {'varNames': rgb_var_names, 'normRanges': rgb_norm_ranges} if client is not None: tile_grid = ctx.get_tile_grid(ds_id) tile_xyz_source_options = get_tile_source_options( tile_grid, get_dataset_tile_url(ctx, ds_id, 'rgb', base_url), client=client) rgb_schema["tileSourceOptions"] = tile_xyz_source_options dataset_dict["rgbSchema"] = rgb_schema dim_names = ds.data_vars[list(ds.data_vars)[0]].dims \ if len(ds.data_vars) > 0 else ds.dims.keys() dataset_dict["dimensions"] = [ get_dataset_coordinates(ctx, ds_id, dim_name) for dim_name in dim_names ] dataset_dict["attrs"] = { key: ds.attrs[key] for key in sorted(list(ds.attrs.keys())) } dataset_attributions = dataset_config.get( 'DatasetAttribution', ctx.config.get('DatasetAttribution')) if dataset_attributions is not None: if isinstance(dataset_attributions, str): dataset_attributions = [dataset_attributions] dataset_dict['attributions'] = dataset_attributions place_groups = ctx.get_dataset_place_groups(ds_id, base_url) if place_groups: dataset_dict["placeGroups"] = _filter_place_groups(place_groups, del_features=True) return dataset_dict