def _get_time_series_dataset(ctx: ServiceContext,
ds_name: str,
var_name: str = None):
descriptor = ctx.get_dataset_descriptor(ds_name)
ts_ds_name = descriptor.get('TimeSeriesDataset', ds_name)
return ctx.get_dataset(ts_ds_name,
expected_var_names=[var_name] if var_name else None)
def __find_places(ctx: ServiceContext,
place_group_id: str,
base_url: str,
query_geometry: shapely.geometry.base.BaseGeometry = None,
query_expr: Any = None,
comb_op: str = "and") -> GeoJsonFeatureCollection:
if comb_op is not None and comb_op != "and":
raise NotImplementedError("comb_op not yet supported")
if place_group_id == ALL_PLACES:
place_groups = ctx.get_global_place_groups(base_url, load_features=True)
features = []
for place_group in place_groups:
features.extend(place_group['features'])
feature_collection = dict(type="FeatureCollection", features=features)
else:
feature_collection = ctx.get_global_place_group(place_group_id, base_url, load_features=True)
feature_collection = dict(type="FeatureCollection", features=feature_collection['features'])
if query_geometry is None:
if query_expr is None:
return feature_collection
else:
raise NotImplementedError()
else:
matching_places = []
if query_expr is None:
for feature in feature_collection['features']:
geometry = shapely.geometry.shape(feature['geometry'])
if geometry.intersects(query_geometry):
matching_places.append(feature)
else:
raise NotImplementedError()
return dict(type="FeatureCollection", features=matching_places)
def get_time_series_info(ctx: ServiceContext) -> Dict:
"""
Get time-series meta-information for variables.
:param ctx: Service context object
:return: a dictionary with a single entry "layers" which is a list of entries that are
dictionaries containing a variable's "name", "dates", and "bounds".
"""
time_series_info = {'layers': []}
descriptors = ctx.get_dataset_descriptors()
for descriptor in descriptors:
if 'Identifier' in descriptor:
if descriptor.get('Hidden'):
continue
dataset = ctx.get_dataset(descriptor['Identifier'])
if 'time' not in dataset.variables:
continue
xmin, ymin, xmax, ymax = get_dataset_bounds(dataset)
time_data = dataset.variables['time'].data
time_stamps = []
for time in time_data:
time_stamps.append(timestamp_to_iso_string(time))
var_names = sorted(dataset.data_vars)
for var_name in var_names:
ds_id = descriptor['Identifier']
variable_dict = {
'name': f'{ds_id}.{var_name}',
'dates': time_stamps,
'bounds': dict(xmin=xmin, ymin=ymin, xmax=xmax, ymax=ymax)
}
time_series_info['layers'].append(variable_dict)
return time_series_info
def new_test_service_context(config_file_name: str = 'config.yml',
ml_dataset_openers: Dict[str, MultiLevelDatasetOpener] = None) -> ServiceContext:
ctx = ServiceContext(base_dir=get_res_test_dir(), ml_dataset_openers=ml_dataset_openers)
config_file = os.path.join(ctx.base_dir, config_file_name)
with open(config_file, encoding='utf-8') as fp:
ctx.config = yaml.safe_load(fp)
return ctx
def new_demo_service_context(
ml_dataset_openers: Dict[str, MultiLevelDatasetOpener] = None
) -> ServiceContext:
ctx = ServiceContext(base_dir=get_res_demo_dir(),
ml_dataset_openers=ml_dataset_openers)
config_file = os.path.join(ctx.base_dir, 'config.yml')
with open(config_file) as fp:
ctx.config = yaml.safe_load(fp)
return ctx
def _get_time_series_dataset(ctx: ServiceContext,
ds_id: str,
var_name: str = None):
descriptor = ctx.get_dataset_descriptor(ds_id)
ts_ds_name = descriptor.get('TimeSeriesDataset', ds_id)
try:
# Try to get more efficient, time-chunked dataset
return ctx.get_dataset(
ts_ds_name, expected_var_names=[var_name] if var_name else None)
except ServiceResourceNotFoundError:
# This happens, if the dataset pointed to by 'TimeSeriesDataset'
# does not contain the variable given by var_name.
return ctx.get_dataset(
ds_id, expected_var_names=[var_name] if var_name else None)
def get_legend(ctx: ServiceContext, ds_id: str, var_name: str,
params: RequestParams):
cmap_name = params.get_query_argument('cbar', default=None)
cmap_vmin = params.get_query_argument_float('vmin', default=None)
cmap_vmax = params.get_query_argument_float('vmax', default=None)
cmap_w = params.get_query_argument_int('width', default=None)
cmap_h = params.get_query_argument_int('height', default=None)
if cmap_name is None or cmap_vmin is None or cmap_vmax is None or cmap_w is None or cmap_h is None:
default_cmap_cbar, (default_cmap_vmin,
default_cmap_vmax) = ctx.get_color_mapping(
ds_id, var_name)
cmap_name = cmap_name or default_cmap_cbar
cmap_vmin = cmap_vmin or default_cmap_vmin
cmap_vmax = cmap_vmax or default_cmap_vmax
cmap_w = cmap_w or DEFAULT_CMAP_WIDTH
cmap_h = cmap_h or DEFAULT_CMAP_HEIGHT
try:
_, cmap = get_cmap(cmap_name)
except ValueError:
raise ServiceResourceNotFoundError(
f"color bar {cmap_name!r} not found")
fig = matplotlib.figure.Figure(figsize=(cmap_w, cmap_h))
ax1 = fig.add_subplot(1, 1, 1)
if '.cpd' in cmap_name:
norm, ticks = get_norm(cmap_name)
else:
norm = matplotlib.colors.Normalize(vmin=cmap_vmin, vmax=cmap_vmax)
ticks = None
image_legend = matplotlib.colorbar.ColorbarBase(ax1,
format='%.1f',
ticks=ticks,
cmap=cmap,
norm=norm,
orientation='vertical')
image_legend_label = ctx.get_legend_label(ds_id, var_name)
if image_legend_label is not None:
image_legend.set_label(image_legend_label)
fig.patch.set_facecolor('white')
fig.patch.set_alpha(0.0)
fig.tight_layout()
buffer = io.BytesIO()
fig.savefig(buffer, format='png')
return buffer.getvalue()
def get_datasets(ctx: ServiceContext,
details: bool = False,
client: str = None,
point: Tuple[float, float] = None,
base_url: str = None) -> Dict:
dataset_descriptors = ctx.get_dataset_descriptors()
dataset_dicts = list()
for dataset_descriptor in dataset_descriptors:
if dataset_descriptor.get('Hidden'):
continue
ds_id = dataset_descriptor['Identifier']
dataset_dict = dict(id=ds_id)
if 'Title' in dataset_descriptor:
ds_title = dataset_descriptor['Title']
if ds_title and isinstance(ds_title, str):
dataset_dict['title'] = ds_title
else:
dataset_dict['title'] = ds_id
if 'BoundingBox' in dataset_descriptor:
ds_bbox = dataset_descriptor['BoundingBox']
if ds_bbox \
and len(ds_bbox) == 4 \
and all(map(lambda c: isinstance(c, float) or isinstance(c, int), ds_bbox)):
dataset_dict['bbox'] = ds_bbox
dataset_dicts.append(dataset_dict)
if details or point:
for dataset_dict in dataset_dicts:
ds_id = dataset_dict["id"]
if point:
ds = ctx.get_dataset(ds_id)
if "bbox" not in dataset_dict:
dataset_dict["bbox"] = list(get_dataset_bounds(ds))
if details:
dataset_dict.update(get_dataset(ctx, ds_id, client, base_url))
if point:
is_point_in_dataset_bbox = functools.partial(_is_point_in_dataset_bbox,
point)
# noinspection PyTypeChecker
dataset_dicts = list(filter(is_point_in_dataset_bbox, dataset_dicts))
return dict(datasets=dataset_dicts)
def get_dataset_place_groups(ctx: ServiceContext, ds_id: str,
base_url: str) -> List[GeoJsonFeatureCollection]:
# Do not load or return features, just place group (metadata).
place_groups = ctx.get_dataset_place_groups(ds_id,
base_url,
load_features=False)
return _filter_place_groups(place_groups, del_features=True)
def get_dataset_place_group(ctx: ServiceContext, ds_id: str,
place_group_id: str) -> GeoJsonFeatureCollection:
# Load and return features for specific place group.
place_group = ctx.get_dataset_place_group(ds_id,
place_group_id,
load_features=True)
return _filter_place_group(place_group, del_features=False)
def get_dataset_tile_grid(ctx: ServiceContext, ds_id: str, var_name: str,
tile_client: str, base_url: str) -> Dict[str, Any]:
tile_grid = ctx.get_tile_grid(ds_id)
if tile_client == 'ol4' or tile_client == 'cesium':
return get_tile_source_options(tile_grid,
get_dataset_tile_url(
ctx, ds_id, var_name, base_url),
client=tile_client)
else:
raise ServiceBadRequestError(f'Unknown tile client "{tile_client}"')
def get_dataset_tile_url(ctx: ServiceContext,
ds_id: str,
var_name: str,
base_url: str):
return ctx.get_service_url(base_url,
'datasets',
urllib.parse.quote_plus(ds_id),
'vars',
urllib.parse.quote_plus(var_name),
'tiles',
'{z}/{x}/{y}.png')
def get_time_series_for_feature_collection(ctx: ServiceContext,
ds_name: str,
var_name: str,
feature_collection: Dict,
start_date: np.datetime64 = None,
end_date: np.datetime64 = None,
include_count: bool = False,
include_stdev: bool = False,
max_valids: int = None) -> Dict:
"""
Get the time-series for the geometries of a given *feature_collection*.
:param ctx: Service context object
:param ds_name: The dataset identifier.
:param var_name: The variable name.
:param feature_collection: The feature collection.
:param start_date: An optional start date.
:param end_date: An optional end date.
:param include_count: Whether to include the valid number of observations in the result.
:param include_stdev: Whether to include the standard deviation in the result.
:param max_valids: Optional number of valid points.
If it is None (default), also missing values are returned as NaN;
if it is -1 only valid values are returned;
if it is a positive integer, the most recent valid values are returned.
:return: Time-series data structure.
"""
dataset = ctx.get_time_series_dataset(ds_name, var_name=var_name)
features = GeoJSON.get_feature_collection_features(feature_collection)
if features is None:
raise ServiceBadRequestError("Invalid GeoJSON feature collection")
shapes = []
for feature in features:
geometry = GeoJSON.get_feature_geometry(feature)
try:
geometry = shapely.geometry.shape(geometry)
except (TypeError, ValueError) as e:
raise ServiceBadRequestError(
"Invalid GeoJSON feature collection") from e
shapes.append(geometry)
with measure_time() as time_result:
result = _get_time_series_for_geometries(dataset,
var_name,
shapes,
start_date=start_date,
end_date=end_date,
include_count=include_count,
include_stdev=include_stdev,
max_valids=max_valids)
if ctx.trace_perf:
LOG.info(
f'get_time_series_for_feature_collection: dataset id {ds_name}, variable {var_name},'
f'size={len(result["results"])}, took {time_result.duration} seconds'
)
return result
def get_dataset_tile(ctx: ServiceContext, ds_id: str, var_name: str, x: str,
y: str, z: str, params: RequestParams):
x = RequestParams.to_int('x', x)
y = RequestParams.to_int('y', y)
z = RequestParams.to_int('z', z)
tile_comp_mode = params.get_query_argument_int('mode', ctx.tile_comp_mode)
trace_perf = params.get_query_argument_int('debug', ctx.trace_perf) != 0
cmap_name = params.get_query_argument('cbar', default=None)
cmap_vmin = params.get_query_argument_float('vmin', default=None)
cmap_vmax = params.get_query_argument_float('vmax', default=None)
if cmap_name is None or cmap_vmin is None or cmap_vmax is None:
default_cmap_name, default_cmap_vmin, default_cmap_vmax = ctx.get_color_mapping(
ds_id, var_name)
cmap_name = cmap_name or default_cmap_name
cmap_vmin = cmap_vmin or default_cmap_vmin
cmap_vmax = cmap_vmax or default_cmap_vmax
ml_dataset = ctx.get_ml_dataset(ds_id)
var = ml_dataset.base_dataset[var_name]
labels = parse_non_spatial_labels(params.get_query_arguments(),
var.dims,
var.coords,
allow_slices=False,
exception_type=ServiceBadRequestError)
return get_ml_dataset_tile(ml_dataset,
var_name,
x,
y,
z,
labels=labels,
cmap_name=cmap_name,
cmap_vmin=cmap_vmin,
cmap_vmax=cmap_vmax,
image_cache=ctx.image_cache,
tile_cache=ctx.tile_cache,
tile_comp_mode=tile_comp_mode,
trace_perf=trace_perf,
exception_type=ServiceBadRequestError)
def find_dataset_places(ctx: ServiceContext,
place_group_id: str,
ds_id: str,
query_expr: Any = None,
comb_op: str = "and") -> GeoJsonFeatureCollection:
dataset = ctx.get_dataset(ds_id)
query_geometry = get_dataset_geometry(dataset)
return _find_places(ctx,
place_group_id,
query_geometry=query_geometry,
query_expr=query_expr,
comb_op=comb_op)
def get_time_series_for_geometry(ctx: ServiceContext,
ds_name: str,
var_name: str,
geometry: Dict,
start_date: np.datetime64 = None,
end_date: np.datetime64 = None,
include_count: bool = False,
include_stdev: bool = False,
max_valids: int = None) -> Dict:
"""
Get the time-series for a given *geometry*.
:param ctx: Service context object
:param ds_name: The dataset identifier.
:param var_name: The variable name.
:param geometry: The geometry, usually a point or polygon.
:param start_date: An optional start date.
:param end_date: An optional end date.
:param include_count: Whether to include the valid number of observations in the result.
:param include_stdev: Whether to include the standard deviation in the result.
:param max_valids: Optional number of valid points.
If it is None (default), also missing values are returned as NaN;
if it is -1 only valid values are returned;
if it is a positive integer, the most recent valid values are returned.
:return: Time-series data structure.
"""
dataset = ctx.get_time_series_dataset(ds_name, var_name=var_name)
if not GeoJSON.is_geometry(geometry):
raise ServiceBadRequestError("Invalid GeoJSON geometry")
if isinstance(geometry, dict):
geometry = shapely.geometry.shape(geometry)
with measure_time() as time_result:
result = _get_time_series_for_geometry(dataset,
var_name,
geometry,
start_date=start_date,
end_date=end_date,
include_count=include_count,
include_stdev=include_stdev,
max_valids=max_valids)
if ctx.trace_perf:
LOG.info(
f'get_time_series_for_geometry: dataset id {ds_name}, variable {var_name}, '
f'geometry type {geometry},'
f'size={len(result["results"])}, took {time_result.duration} seconds'
)
return result
def get_dataset_coordinates(ctx: ServiceContext, ds_id: str,
dim_name: str) -> Dict:
ds, var = ctx.get_dataset_and_coord_variable(ds_id, dim_name)
values = list()
if np.issubdtype(var.dtype, np.floating):
converter = float
elif np.issubdtype(var.dtype, np.integer):
converter = int
else:
converter = timestamp_to_iso_string
for value in var.values:
values.append(converter(value))
return dict(name=dim_name,
size=len(values),
dtype=str(var.dtype),
coordinates=values)
def test_get_colorbars(self):
ctx = ServiceContext()
response = get_color_bars(ctx, 'application/json')
self.assertIsInstance(response, str)
self.assertTrue(len(response) > 40)
self.assertEqual('[\n [\n "Perceptually Uniform Sequenti', response[0:40])
response = get_color_bars(ctx, 'text/html')
self.assertIsInstance(response, str)
self.assertTrue(len(response) > 40)
self.assertEqual('<!DOCTYPE html>\n<html lang="en">\n<head><', response[0:40])
with self.assertRaises(ServiceBadRequestError) as cm:
get_color_bars(ctx, 'text/xml')
self.assertEqual(400, cm.exception.status_code)
self.assertEqual("Format 'text/xml' not supported for color bars", cm.exception.reason)
def test_get_ne2_tile_grid(self):
ctx = ServiceContext()
tile_grid = get_ne2_tile_grid(ctx, 'ol4', 'http://bibo')
self.assertEqual({
'url': self.base_url + '/ne2/tiles/{z}/{x}/{y}.jpg',
'projection': 'EPSG:4326',
'minZoom': 0,
'maxZoom': 2,
'tileGrid': {'extent': [-180.0, -90.0, 180.0, 90.0],
'origin': [-180.0, 90.0],
'resolutions': [0.703125, 0.3515625, 0.17578125],
'tileSize': [256, 256]},
}, tile_grid)
with self.assertRaises(ServiceBadRequestError) as cm:
get_ne2_tile_grid(ctx, 'cesium', 'http://bibo')
self.assertEqual(400, cm.exception.status_code)
self.assertEqual("Unknown tile client 'cesium'", cm.exception.reason)
def test_point_ts_perf(self):
TEST_CUBE = 'ts_test.zarr'
if not os.path.isdir(TEST_CUBE):
from xcube.core.new import new_cube
cube = new_cube(time_periods=2000, variables=dict(analysed_sst=280.4))
cube = cube.chunk(dict(time=1, lon=90, lat=90))
cube.to_zarr(TEST_CUBE)
ctx = ServiceContext(
base_dir='.',
config=dict(
Datasets=[
dict(Identifier='ts_test',
FileSystem='file',
Path=TEST_CUBE,
Format='zarr')
]
))
N = 5
import random
import time
time_sum = 0.0
for i in range(N):
lon = -180 + 360 * random.random()
lat = -90 + 180 * random.random()
t1 = time.perf_counter()
result = get_time_series(ctx, 'ts_test', 'analysed_sst', dict(type='Point', coordinates=[lon, lat]))
t2 = time.perf_counter()
self.assertIsInstance(result, list)
self.assertEqual(2000, len(result))
time_delta = t2 - t1
time_sum += time_delta
print(f'test {i + 1} took {time_delta} seconds')
print(f'all tests took {time_sum / N} seconds in average')
def get_time_series_for_point(ctx: ServiceContext,
ds_name: str,
var_name: str,
lon: float,
lat: float,
start_date: np.datetime64 = None,
end_date: np.datetime64 = None,
max_valids: int = None) -> Dict:
"""
Get the time-series for a given point.
:param ctx: Service context object
:param ds_name: The dataset identifier.
:param var_name: The variable name.
:param lon: The point's longitude in decimal degrees.
:param lat: The point's latitude in decimal degrees.
:param start_date: An optional start date.
:param end_date: An optional end date.
:param max_valids: Optional number of valid points.
If it is None (default), also missing values are returned as NaN;
if it is -1 only valid values are returned;
if it is a positive integer, the most recent valid values are returned.
:return: Time-series data structure.
"""
dataset = ctx.get_time_series_dataset(ds_name, var_name=var_name)
with measure_time() as time_result:
result = _get_time_series_for_point(dataset,
var_name,
shapely.geometry.Point(lon, lat),
start_date=start_date,
end_date=end_date,
max_valids=max_valids)
if ctx.trace_perf:
LOG.info(
f'get_time_series_for_point:: dataset id {ds_name}, variable {var_name}, '
f'geometry type {shapely.geometry.Point(lon, lat)}, size={len(result["results"])}, '
f'took {time_result.duration} seconds')
return result
def get_dataset(ctx: ServiceContext,
ds_id: str,
client=None,
base_url: str = None) -> GeoJsonFeatureCollection:
dataset_descriptor = ctx.get_dataset_descriptor(ds_id)
ds_id = dataset_descriptor['Identifier']
ds_title = dataset_descriptor['Title']
dataset_dict = dict(id=ds_id, title=ds_title)
ds = ctx.get_dataset(ds_id)
if "bbox" not in dataset_dict:
dataset_dict["bbox"] = list(get_dataset_bounds(ds))
variable_dicts = []
for var_name in ds.data_vars:
var = ds.data_vars[var_name]
dims = var.dims
if len(dims) < 3 or dims[0] != 'time' or dims[-2] != 'lat' or dims[
-1] != 'lon':
continue
variable_dict = dict(id=f'{ds_id}.{var_name}',
name=var_name,
dims=list(dims),
shape=list(var.shape),
dtype=str(var.dtype),
units=var.attrs.get('units', ''),
title=var.attrs.get(
'title', var.attrs.get('long_name',
var_name)))
if client is not None:
tile_grid = ctx.get_tile_grid(ds_id)
tile_xyz_source_options = get_tile_source_options(
tile_grid,
get_dataset_tile_url(ctx, ds_id, var_name, base_url),
client=client)
variable_dict["tileSourceOptions"] = tile_xyz_source_options
cbar, vmin, vmax = ctx.get_color_mapping(ds_id, var_name)
variable_dict["colorBarName"] = cbar
variable_dict["colorBarMin"] = vmin
variable_dict["colorBarMax"] = vmax
variable_dicts.append(variable_dict)
dataset_dict["variables"] = variable_dicts
dim_names = ds.data_vars[list(
ds.data_vars)[0]].dims if len(ds.data_vars) > 0 else ds.dims.keys()
dataset_dict["dimensions"] = [
get_dataset_coordinates(ctx, ds_id, dim_name) for dim_name in dim_names
]
place_groups = ctx.get_dataset_place_groups(ds_id)
if place_groups:
dataset_dict["placeGroups"] = _filter_place_groups(place_groups,
del_features=True)
return dataset_dict
def get_dataset(ctx: ServiceContext,
ds_id: str,
client=None,
base_url: str = None,
granted_scopes: Set[str] = None) -> Dict:
granted_scopes = granted_scopes or set()
dataset_descriptor = ctx.get_dataset_descriptor(ds_id)
ds_id = dataset_descriptor['Identifier']
if 'read:dataset:*' not in granted_scopes:
required_scopes = ctx.get_required_dataset_scopes(dataset_descriptor)
assert_scopes(required_scopes, granted_scopes or set())
ds_title = dataset_descriptor['Title']
dataset_dict = dict(id=ds_id, title=ds_title)
ds = ctx.get_dataset(ds_id)
if "bbox" not in dataset_dict:
dataset_dict["bbox"] = list(get_dataset_bounds(ds))
variable_dicts = []
for var_name in ds.data_vars:
var = ds.data_vars[var_name]
dims = var.dims
if len(dims) < 3 or dims[0] != 'time' or dims[-2] != 'lat' or dims[
-1] != 'lon':
continue
if 'read:variable:*' not in granted_scopes:
required_scopes = ctx.get_required_variable_scopes(
dataset_descriptor, var_name)
if not check_scopes(required_scopes, granted_scopes):
continue
variable_dict = dict(id=f'{ds_id}.{var_name}',
name=var_name,
dims=list(dims),
shape=list(var.shape),
dtype=str(var.dtype),
units=var.attrs.get('units', ''),
title=var.attrs.get(
'title', var.attrs.get('long_name',
var_name)))
if client is not None:
tile_grid = ctx.get_tile_grid(ds_id)
tile_xyz_source_options = get_tile_source_options(
tile_grid,
get_dataset_tile_url(ctx, ds_id, var_name, base_url),
client=client)
variable_dict["tileSourceOptions"] = tile_xyz_source_options
cmap_name, (cmap_vmin,
cmap_vmax) = ctx.get_color_mapping(ds_id, var_name)
variable_dict["colorBarName"] = cmap_name
variable_dict["colorBarMin"] = cmap_vmin
variable_dict["colorBarMax"] = cmap_vmax
if hasattr(var.data, '_repr_html_'):
variable_dict["htmlRepr"] = var.data._repr_html_()
variable_dict["attrs"] = {
key: var.attrs[key]
for key in sorted(list(var.attrs.keys()))
}
variable_dicts.append(variable_dict)
ctx.get_rgb_color_mapping(ds_id)
dataset_dict["variables"] = variable_dicts
rgb_var_names, rgb_norm_ranges = ctx.get_rgb_color_mapping(ds_id)
if any(rgb_var_names):
rgb_schema = {'varNames': rgb_var_names, 'normRanges': rgb_norm_ranges}
if client is not None:
tile_grid = ctx.get_tile_grid(ds_id)
tile_xyz_source_options = get_tile_source_options(
tile_grid,
get_dataset_tile_url(ctx, ds_id, 'rgb', base_url),
client=client)
rgb_schema["tileSourceOptions"] = tile_xyz_source_options
dataset_dict["rgbSchema"] = rgb_schema
dim_names = ds.data_vars[list(
ds.data_vars)[0]].dims if len(ds.data_vars) > 0 else ds.dims.keys()
dataset_dict["dimensions"] = [
get_dataset_coordinates(ctx, ds_id, dim_name) for dim_name in dim_names
]
dataset_dict["attrs"] = {
key: ds.attrs[key]
for key in sorted(list(ds.attrs.keys()))
}
dataset_attributions = dataset_descriptor.get(
'DatasetAttribution', ctx.config.get('DatasetAttribution'))
if dataset_attributions is not None:
if isinstance(dataset_attributions, str):
dataset_attributions = [dataset_attributions]
dataset_dict['attributions'] = dataset_attributions
place_groups = ctx.get_dataset_place_groups(ds_id, base_url)
if place_groups:
dataset_dict["placeGroups"] = _filter_place_groups(place_groups,
del_features=True)
return dataset_dict
class Service:
"""
A web service that provides a remote API to some application.
"""
def __init__(self,
application: Application,
prefix: str = None,
address: str = DEFAULT_ADDRESS,
port: int = DEFAULT_PORT,
cube_paths: List[str] = None,
styles: Dict[str, Tuple] = None,
config_file: Optional[str] = None,
base_dir: Optional[str] = None,
tile_cache_size: Optional[str] = DEFAULT_TILE_CACHE_SIZE,
tile_comp_mode: int = DEFAULT_TILE_COMP_MODE,
update_period: Optional[float] = DEFAULT_UPDATE_PERIOD,
trace_perf: bool = DEFAULT_TRACE_PERF,
log_file_prefix: str = DEFAULT_LOG_PREFIX,
log_to_stderr: bool = False,
aws_prof: str = None,
aws_env: bool = False) -> None:
"""
Start a tile service.
The *service_info_file*, if given, represents the service in the filesystem, similar to
the ``/var/run/`` directory on Linux systems.
If the service file exist and its information is compatible with the requested *port*, *address*, *caller*, then
this function simply returns without taking any other actions.
:param application: The Tornado web application
:param address: the address
:param port: the port number
:param cube_paths: optional list of cube paths
:param config_file: optional configuration file
:param base_dir: optional base directory
:param update_period: if not-None, time of idleness in seconds before service is updated
:param log_file_prefix: Log file prefix, default is "xcube-serve.log"
:param log_to_stderr: Whether logging should be shown on stderr
:return: service information dictionary
"""
if config_file and cube_paths:
raise ValueError("config_file and cube_paths cannot be given both")
if config_file and styles:
raise ValueError("config_file and styles cannot be given both")
if config_file and aws_prof:
raise ValueError("config_file and aws_profile cannot be given both")
if config_file and aws_env:
raise ValueError("config_file and aws_env cannot be given both")
global SNAP_CPD_LIST
if config_file:
SNAP_CPD_LIST = _get_custom_color_list(config_file)
log_dir = os.path.dirname(log_file_prefix)
if log_dir and not os.path.isdir(log_dir):
os.makedirs(log_dir, exist_ok=True)
options = tornado.options.options
options.log_file_prefix = log_file_prefix or DEFAULT_LOG_PREFIX
options.log_to_stderr = log_to_stderr
enable_pretty_logging()
tile_cache_capacity = parse_mem_size(tile_cache_size)
config = None
if cube_paths:
config = new_default_config(cube_paths, styles, aws_prof=aws_prof, aws_env=aws_env)
self.config_file = os.path.abspath(config_file) if config_file else None
self.update_period = update_period
self.update_timer = None
self.config_error = None
self.service_info = dict(port=port,
address=address,
started=datetime.now().isoformat(sep=' '),
pid=os.getpid())
self.context = ServiceContext(prefix=prefix,
config=config,
base_dir=base_dir,
trace_perf=trace_perf,
tile_comp_mode=tile_comp_mode,
tile_cache_capacity=tile_cache_capacity)
self._maybe_load_config()
application.service_context = self.context
application.time_of_last_activity = time.process_time()
self.application = application
# Register handlers for common termination signals
signal.signal(signal.SIGINT, self._sig_handler)
signal.signal(signal.SIGTERM, self._sig_handler)
self.server = application.listen(port, address=address or 'localhost')
# Ensure we have the same event loop in all threads
asyncio.set_event_loop_policy(_GlobalEventLoopPolicy(asyncio.get_event_loop()))
self._maybe_load_config()
self._maybe_install_update_check()
self._shutdown_requested = False
def start(self):
address = self.service_info['address']
port = self.service_info['port']
test_url = self.context.get_service_url(f"http://{address}:{port}", "datasets")
LOG.info(f'service running, listening on {address}:{port}, try {test_url}')
LOG.info(f'press CTRL+C to stop service')
if not self.context.config.get('Datasets', []) \
and not self.context.config.get('DataStores', []):
LOG.warning('no datasets or data stores configured')
tornado.ioloop.PeriodicCallback(self._try_shutdown, 100).start()
IOLoop.current().start()
def stop(self, kill=False):
"""
Stops the Tornado web server.
"""
if kill:
sys.exit(0)
else:
IOLoop.current().add_callback(self._on_shutdown)
def _on_shutdown(self):
LOG.info('stopping service...')
# noinspection PyUnresolvedReferences,PyBroadException
try:
self.update_timer.cancel()
except Exception:
pass
if self.server:
self.server.stop()
self.server = None
IOLoop.current().stop()
LOG.info('service stopped.')
def _try_shutdown(self):
if self._shutdown_requested:
self._on_shutdown()
# noinspection PyUnusedLocal
def _sig_handler(self, sig, frame):
LOG.warning(f'caught signal {sig}')
self._shutdown_requested = True
def _maybe_install_update_check(self):
if self.config_file is None or self.update_period is None or self.update_period <= 0:
return
IOLoop.current().call_later(self.update_period, self._maybe_check_for_updates)
def _maybe_check_for_updates(self):
self._maybe_load_config()
self._maybe_install_update_check()
def _maybe_load_config(self):
config_file = self.config_file
if config_file is None:
return
try:
stat = os.stat(config_file)
except OSError as e:
if self.config_error is None:
LOG.error(f'configuration file {config_file!r}: {e}')
self.config_error = e
return
if self.context.config_mtime != stat.st_mtime:
self.context.config_mtime = stat.st_mtime
try:
self.context.config = load_json_or_yaml_config(config_file)
self.config_error = None
LOG.info(f'configuration file {config_file!r} successfully loaded')
except ValueError as e:
if self.config_error is None:
LOG.error(f'configuration file {config_file!r}: {e}')
self.config_error = e
def get_datasets(ctx: ServiceContext,
details: bool = False,
client: str = None,
point: Tuple[float, float] = None,
base_url: str = None,
granted_scopes: Set[str] = None) -> Dict:
granted_scopes = granted_scopes or set()
dataset_descriptors = ctx.get_dataset_descriptors()
dataset_dicts = list()
for dataset_descriptor in dataset_descriptors:
ds_id = dataset_descriptor['Identifier']
if dataset_descriptor.get('Hidden'):
continue
if 'read:dataset:*' not in granted_scopes:
required_scopes = ctx.get_required_dataset_scopes(
dataset_descriptor)
is_substitute = dataset_descriptor.get('AccessControl', {}).get(
'IsSubstitute', False)
if not check_scopes(required_scopes,
granted_scopes,
is_substitute=is_substitute):
continue
dataset_dict = dict(id=ds_id)
if 'Title' in dataset_descriptor:
ds_title = dataset_descriptor['Title']
if ds_title and isinstance(ds_title, str):
dataset_dict['title'] = ds_title
else:
dataset_dict['title'] = ds_id
if 'BoundingBox' in dataset_descriptor:
ds_bbox = dataset_descriptor['BoundingBox']
if ds_bbox \
and len(ds_bbox) == 4 \
and all(map(lambda c: isinstance(c, float) or isinstance(c, int), ds_bbox)):
dataset_dict['bbox'] = ds_bbox
dataset_dicts.append(dataset_dict)
if details or point:
for dataset_dict in dataset_dicts:
ds_id = dataset_dict["id"]
if point:
ds = ctx.get_dataset(ds_id)
if "bbox" not in dataset_dict:
dataset_dict["bbox"] = list(get_dataset_bounds(ds))
if details:
dataset_dict.update(
get_dataset(ctx,
ds_id,
client,
base_url,
granted_scopes=granted_scopes))
if point:
is_point_in_dataset_bbox = functools.partial(_is_point_in_dataset_bbox,
point)
# noinspection PyTypeChecker
dataset_dicts = list(filter(is_point_in_dataset_bbox, dataset_dicts))
return dict(datasets=dataset_dicts)
def get_ne2_tile_url(ctx: ServiceContext, base_url: str):
return ctx.get_service_url(base_url, 'ne2', 'tiles', '{z}/{x}/{y}.jpg')
def get_time_series(
ctx: ServiceContext,
ds_name: str,
var_name: str,
geo_json: GeoJsonObj,
agg_methods: Union[str, Sequence[str]] = None,
start_date: np.datetime64 = None,
end_date: np.datetime64 = None,
max_valids: int = None,
incl_ancillary_vars: bool = False
) -> Union[TimeSeries, TimeSeriesCollection]:
"""
Get the time-series for a given GeoJSON object *geo_json*.
If *geo_json* is a single geometry or feature a list of time-series values is returned.
If *geo_json* is a single geometry collection or feature collection a collection of lists of time-series values
is returned so that geometry/feature collection and time-series collection elements are corresponding
at same indices.
A time series value always contains the key "time" whose value is an ISO date/time string. Other entries
are values with varying keys depending on the geometry type and *agg_methods*. Their values may be
either a bool, int, float or None.
For point geometries the second key is "value".
For non-point geometries that cover spatial areas, there will be values for all keys given by *agg_methods*.
:param ctx: Service context object
:param ds_name: The dataset identifier.
:param var_name: The variable name.
:param geo_json: The GeoJSON object that is a or has a geometry or collection of geometryies.
:param start_date: An optional start date.
:param end_date: An optional end date.
:param agg_methods: Spatial aggregation methods for geometries that cover a spatial area.
:param max_valids: Optional number of valid points.
If it is None (default), also missing values are returned as NaN;
if it is -1 only valid values are returned;
if it is a positive integer, the most recent valid values are returned.
:param incl_ancillary_vars: For point geometries, include values of ancillary variables, if any.
:return: Time-series data structure.
"""
agg_methods = timeseries.normalize_agg_methods(
agg_methods, exception_type=ServiceBadRequestError)
dataset = ctx.get_time_series_dataset(ds_name, var_name=var_name)
geo_json_geometries, is_collection = _to_geo_json_geometries(geo_json)
geometries = _to_shapely_geometries(geo_json_geometries)
with measure_time() as time_result:
results = _get_time_series_for_geometries(
dataset,
var_name,
geometries,
start_date=start_date,
end_date=end_date,
agg_methods=agg_methods,
max_valids=max_valids,
incl_ancillary_vars=incl_ancillary_vars)
if ctx.trace_perf:
LOG.info(
f'get_time_series: dataset id {ds_name}, variable {var_name}, '
f'{len(results)} x {len(results[0])} values, took {time_result.duration} seconds'
)
return results[0] if not is_collection and len(results) == 1 else results
def get_dataset_tile(ctx: ServiceContext, ds_id: str, var_name: str, x: str,
y: str, z: str, params: RequestParams):
x = RequestParams.to_int('x', x)
y = RequestParams.to_int('y', y)
z = RequestParams.to_int('z', z)
tile_comp_mode = params.get_query_argument_int('mode', ctx.tile_comp_mode)
trace_perf = params.get_query_argument_int('debug', ctx.trace_perf) != 0
ml_dataset = ctx.get_ml_dataset(ds_id)
if var_name == 'rgb':
norm_vmin = params.get_query_argument_float('vmin', default=0.0)
norm_vmax = params.get_query_argument_float('vmax', default=1.0)
var_names, norm_ranges = ctx.get_rgb_color_mapping(
ds_id, norm_range=(norm_vmin, norm_vmax))
components = ('r', 'g', 'b')
for i in range(3):
c = components[i]
var_names[i] = params.get_query_argument(c, default=var_names[i])
norm_ranges[i] = params.get_query_argument_float(f'{c}vmin', default=norm_ranges[i][0]), \
params.get_query_argument_float(f'{c}vmax', default=norm_ranges[i][1])
cmap_name = tuple(var_names)
cmap_range = tuple(norm_ranges)
for name in var_names:
if name and name not in ml_dataset.base_dataset:
raise ServiceBadRequestError(
f'Variable {name!r} not found in dataset {ds_id!r}')
var = None
for name in var_names:
if name and name in ml_dataset.base_dataset:
var = ml_dataset.base_dataset[name]
break
if var is None:
raise ServiceBadRequestError(
f'No variable in dataset {ds_id!r} specified for RGB')
else:
cmap_name = params.get_query_argument('cbar', default=None)
cmap_vmin = params.get_query_argument_float('vmin', default=None)
cmap_vmax = params.get_query_argument_float('vmax', default=None)
if cmap_name is None or cmap_vmin is None or cmap_vmax is None:
default_cmap_name, (default_cmap_vmin,
default_cmap_vmax) = ctx.get_color_mapping(
ds_id, var_name)
cmap_name = cmap_name or default_cmap_name
cmap_vmin = cmap_vmin or default_cmap_vmin
cmap_vmax = cmap_vmax or default_cmap_vmax
cmap_range = cmap_vmin, cmap_vmax
if var_name not in ml_dataset.base_dataset:
raise ServiceBadRequestError(
f'Variable {var_name!r} not found in dataset {ds_id!r}')
var = ml_dataset.base_dataset[var_name]
labels = parse_non_spatial_labels(params.get_query_arguments(),
var.dims,
var.coords,
allow_slices=False,
exception_type=ServiceBadRequestError)
return get_ml_dataset_tile(ml_dataset,
var_name,
x,
y,
z,
labels=labels,
cmap_name=cmap_name,
cmap_range=cmap_range,
image_cache=ctx.image_cache,
tile_cache=ctx.tile_cache,
tile_comp_mode=tile_comp_mode,
trace_perf=trace_perf,
exception_type=ServiceBadRequestError)
def __init__(self,
application: Application,
prefix: str = None,
address: str = DEFAULT_ADDRESS,
port: int = DEFAULT_PORT,
cube_paths: List[str] = None,
styles: Dict[str, Tuple] = None,
config_file: Optional[str] = None,
base_dir: Optional[str] = None,
tile_cache_size: Optional[str] = DEFAULT_TILE_CACHE_SIZE,
tile_comp_mode: int = DEFAULT_TILE_COMP_MODE,
update_period: Optional[float] = DEFAULT_UPDATE_PERIOD,
trace_perf: bool = DEFAULT_TRACE_PERF,
log_file_prefix: str = DEFAULT_LOG_PREFIX,
log_to_stderr: bool = False,
aws_prof: str = None,
aws_env: bool = False) -> None:
"""
Start a tile service.
The *service_info_file*, if given, represents the service in the filesystem, similar to
the ``/var/run/`` directory on Linux systems.
If the service file exist and its information is compatible with the requested *port*, *address*, *caller*, then
this function simply returns without taking any other actions.
:param application: The Tornado web application
:param address: the address
:param port: the port number
:param cube_paths: optional list of cube paths
:param config_file: optional configuration file
:param base_dir: optional base directory
:param update_period: if not-None, time of idleness in seconds before service is updated
:param log_file_prefix: Log file prefix, default is "xcube-serve.log"
:param log_to_stderr: Whether logging should be shown on stderr
:return: service information dictionary
"""
if config_file and cube_paths:
raise ValueError("config_file and cube_paths cannot be given both")
if config_file and styles:
raise ValueError("config_file and styles cannot be given both")
if config_file and aws_prof:
raise ValueError("config_file and aws_profile cannot be given both")
if config_file and aws_env:
raise ValueError("config_file and aws_env cannot be given both")
global SNAP_CPD_LIST
if config_file:
SNAP_CPD_LIST = _get_custom_color_list(config_file)
log_dir = os.path.dirname(log_file_prefix)
if log_dir and not os.path.isdir(log_dir):
os.makedirs(log_dir, exist_ok=True)
options = tornado.options.options
options.log_file_prefix = log_file_prefix or DEFAULT_LOG_PREFIX
options.log_to_stderr = log_to_stderr
enable_pretty_logging()
tile_cache_capacity = parse_mem_size(tile_cache_size)
config = None
if cube_paths:
config = new_default_config(cube_paths, styles, aws_prof=aws_prof, aws_env=aws_env)
self.config_file = os.path.abspath(config_file) if config_file else None
self.update_period = update_period
self.update_timer = None
self.config_error = None
self.service_info = dict(port=port,
address=address,
started=datetime.now().isoformat(sep=' '),
pid=os.getpid())
self.context = ServiceContext(prefix=prefix,
config=config,
base_dir=base_dir,
trace_perf=trace_perf,
tile_comp_mode=tile_comp_mode,
tile_cache_capacity=tile_cache_capacity)
self._maybe_load_config()
application.service_context = self.context
application.time_of_last_activity = time.process_time()
self.application = application
# Register handlers for common termination signals
signal.signal(signal.SIGINT, self._sig_handler)
signal.signal(signal.SIGTERM, self._sig_handler)
self.server = application.listen(port, address=address or 'localhost')
# Ensure we have the same event loop in all threads
asyncio.set_event_loop_policy(_GlobalEventLoopPolicy(asyncio.get_event_loop()))
self._maybe_load_config()
self._maybe_install_update_check()
self._shutdown_requested = False
def get_dataset_tile_url(ctx: ServiceContext, ds_id: str, var_name: str,
base_url: str):
return ctx.get_service_url(base_url, 'datasets', ds_id, 'vars', var_name,
'tiles', '{z}/{x}/{y}.png')