def add_ranges(dc, product_names, summary=False, merge_only=False):
odc_products = {}
ows_multiproducts = []
for pname in product_names:
dc_product = None
ows_product = get_config().product_index.get(pname)
if not ows_product:
ows_product = get_config().native_product_index.get(pname)
if ows_product:
for dc_pname in ows_product.product_names:
if dc_pname in odc_products:
odc_products[dc_pname]["ows"].append(ows_product)
else:
odc_products[dc_pname] = { "ows": [ows_product]}
print("OWS Layer %s maps to ODC Product(s): %s" % (
ows_product.name,
repr(ows_product.product_names)
))
if ows_product.multi_product:
ows_multiproducts.append(ows_product)
if not ows_product:
dc_product = dc.index.products.get_by_name(pname)
if dc_product:
print("ODC Layer: %s" % pname)
if pname in odc_products:
odc_products[pname]["ows"].append(None)
else:
odc_products[pname] = { "ows": [None]}
else:
print("Unrecognised product name:", pname)
continue
if ows_multiproducts and merge_only:
print("Merge-only: Skipping range update of products:", repr(list(odc_products.keys())))
else:
for pname, ows_prods in odc_products.items():
dc_product = dc.index.products.get_by_name(pname)
if dc_product is None:
print("Could not find ODC product:", pname)
elif datasets_exist(dc, dc_product.name):
prod_summary = summary
for ows_prod in ows_prods["ows"]:
if ows_prod:
prod_summary = not ows_prod.is_raw_time_res
break
create_range_entry(dc, dc_product, get_crses(), prod_summary)
else:
print("Could not find any datasets for: ", pname)
for mp in ows_multiproducts:
create_multiprod_range_entry(dc, mp, get_crses())
print("Done.")
def method_specific_init(self, args):
# Validate Format parameter
self.format = get_arg(args, "format", "image format",
errcode=WMSException.INVALID_FORMAT,
lower=True,
permitted_values=["image/png"])
# Styles
self.styles = args.get("styles", "").split(",")
if len(self.styles) != 1:
raise WMSException("Multi-layer GetMap requests not supported")
style_r = self.styles[0]
if not style_r:
style_r = self.product.default_style.name
self.style = self.product.style_index.get(style_r)
if not self.style:
raise WMSException("Style %s is not defined" % style_r,
WMSException.STYLE_NOT_DEFINED,
locator="Style parameter")
cfg = get_config()
if self.geobox.width > cfg.wms_max_width:
raise WMSException(f"Width {self.geobox.width} exceeds supported maximum {self.cfg.wms_max_width}.",
locator="Width parameter")
if self.geobox.height > cfg.wms_max_height:
raise WMSException(f"Width {self.geobox.height} exceeds supported maximum {self.cfg.wms_max_height}.",
locator="Height parameter")
# Zoom factor
self.zf = zoom_factor(args, self.crs)
# TODO: Do we need to make resampling method configurable?
self.resampling = Resampling.nearest
def desc_coverages(args):
# Note: Only WCS v1.0.0 is fully supported at this stage, so no version negotiation is necessary
# Extract layer metadata from Datacube.
cfg = get_config()
coverages = args.get("coverage")
products = []
if coverages:
coverages = coverages.split(",")
for c in coverages:
p = cfg.product_index.get(c)
if p and p.wcs:
products.append(p)
else:
raise WCS1Exception("Invalid coverage: %s" % c,
WCS1Exception.COVERAGE_NOT_DEFINED,
locator="Coverage parameter")
else:
for p in cfg.product_index.values():
if p.wcs:
products.append(p)
return (render_template("wcs_desc_coverage.xml",
cfg=cfg,
products=products), 200,
cfg.response_headers({
"Content-Type": "application/xml",
"Cache-Control": "max-age=10"
}))
def __init__(self, args):
# Version
self.version = get_arg(args,
"version",
"WMS version",
permitted_values=['1.1.1', '1.3.0'])
# CRS
if self.version == '1.1.1':
crs_arg = "srs"
else:
crs_arg = "crs"
self.crsid = get_arg(
args,
crs_arg,
"Coordinate Reference System",
errcode=WMSException.INVALID_CRS,
permitted_values=get_config().published_CRSs.keys())
self.crs = geometry.CRS(self.crsid)
# Layers
self.product = self.get_product(args)
self.raw_product = self.get_raw_product(args)
self.geometry = _get_polygon(args, self.crs)
# BBox, height and width parameters
self.geobox = _get_geobox(args, self.crs)
# Time parameter
self.times = get_times(args, self.product, self.raw_product)
self.method_specific_init(args)
def img_coords_to_geopoint(geobox, i, j):
cfg = get_config()
h_coord = cfg.published_CRSs[geobox.crs.crs_str]["horizontal_coord"]
v_coord = cfg.published_CRSs[geobox.crs.crs_str]["vertical_coord"]
return geometry.point(geobox.coordinates[h_coord].values[int(i)],
geobox.coordinates[v_coord].values[int(j)],
geobox.crs)
def test_count():
cfg = get_config()
lyr = list(cfg.product_index.values())[0]
with cube() as dc:
count = mv_search_datasets(dc.index, MVSelectOpts.COUNT, layer=lyr)
ids = mv_search_datasets(dc.index, MVSelectOpts.IDS, layer=lyr)
assert len(ids) == count
def update_all_ranges(dc):
i = 0
u = 0
p = 0
si = 0
su = 0
sp = 0
mi = 0
mu = 0
mp = 0
multiproducts = set()
for prod in get_config().product_index.values():
if prod.multi_product:
multiproducts.add(prod)
else:
stats = update_single_range(dc, prod)
p += stats[0]
u += stats[1]
i += stats[2]
sp += stats[3]
su += stats[4]
si += stats[5]
for mprod in multiproducts:
stats = update_multi_range(dc, mprod, follow_dependencies=False)
mp += stats[0]
mu += stats[1]
mi += stats[2]
return p, u, i, sp, su, si, mp, mu, mi
def no_db(monkeypatch):
monkeypatch.setenv("DATACUBE_OWS_CFG", "tests.cfg.minimal_cfg.ows_cfg")
monkeypatch.setenv("DB_USERNAME", "fakeuser")
monkeypatch.setenv("DB_PASSWORD", "password")
monkeypatch.setenv("DB_HOSTNAME", "localhost")
from datacube_ows.ows_configuration import get_config
cfg = get_config(refresh=True)
def get_capabilities(args):
# TODO: Handle updatesequence request parameter for cache consistency.
# Note: Only WMS v1.0.0 exists at this stage, so no version negotiation is necessary
# Extract layer metadata from Datacube.
cfg = get_config()
url = args.get('Host', args['url_root'])
base_url = get_service_base_url(cfg.allowed_urls, url)
section = args.get("section")
if section:
section = section.lower()
show_service_id = False
show_service_provider = False
show_ops_metadata = False
show_contents = False
show_themes = False
if section is None:
show_service_id = True
show_service_provider = True
show_ops_metadata = True
show_contents = True
show_themes = True
else:
sections = section.split(",")
for s in sections:
if s == "all":
show_service_id = True
show_service_provider = True
show_ops_metadata = True
show_contents = True
show_themes = True
elif s == "serviceidentification":
show_service_id = True
elif s == "serviceprovider":
show_service_provider = True
elif s == "operationsmetadata":
show_ops_metadata = True
elif s == "contents":
show_contents = True
elif s == "themes":
show_themes = True
else:
raise WMTSException("Invalid section: %s" % section,
WMTSException.INVALID_PARAMETER_VALUE,
locator="Section parameter")
return (
render_template(
"wmts_capabilities.xml",
cfg=cfg,
base_url=base_url,
show_service_id = show_service_id,
show_service_provider = show_service_provider,
show_ops_metadata = show_ops_metadata,
show_contents = show_contents,
show_themes = show_themes,
webmerc_ss = WebMercScaleSet),
200,
cfg.response_headers(
{"Content-Type": "application/xml", "Cache-Control": "max-age=10"}
)
)
def png_response(body, cfg=None, extra_headers=None):
if not cfg:
cfg = get_config()
if extra_headers is None:
extra_headers = {}
headers = {"Content-Type": "image/png"}
headers.update(extra_headers)
headers = cfg.response_headers(headers)
return body, 200, cfg.response_headers(headers)
def test_datasets():
cfg = get_config()
lyr = list(cfg.product_index.values())[0]
with cube() as dc:
dss = mv_search_datasets(dc.index, MVSelectOpts.DATASETS, layer=lyr)
ids = mv_search_datasets(dc.index, MVSelectOpts.IDS, layer=lyr)
assert len(ids) == len(dss)
for ds in dss:
assert str(ds.id) in ids
def get_coverage_data(req):
#pylint: disable=too-many-locals, protected-access
with cube() as dc:
if not dc:
raise WCS1Exception("Database connectivity failure")
stacker = DataStacker(req.product,
req.geobox,
req.times,
bands=req.bands)
datasets = stacker.datasets(dc.index)
if not datasets:
# TODO: Return an empty coverage file with full metadata?
cfg = get_config()
x_range = (req.minx, req.maxx)
y_range = (req.miny, req.maxy)
xname = cfg.published_CRSs[req.request_crsid]["horizontal_coord"]
yname = cfg.published_CRSs[req.request_crsid]["vertical_coord"]
xvals = numpy.linspace(x_range[0], x_range[1], num=req.width)
yvals = numpy.linspace(y_range[0], y_range[1], num=req.height)
if cfg.published_CRSs[req.request_crsid]["vertical_coord_first"]:
nparrays = {
band: (("time", yname, xname),
numpy.full((len(req.times), len(yvals), len(xvals)),
req.product.nodata_dict[band]))
for band in req.bands
}
else:
nparrays = {
band: (("time", xname, yname),
numpy.full((len(req.times), len(xvals), len(yvals)),
req.product.nodata_dict[band]))
for band in req.bands
}
data = xarray.Dataset(nparrays,
coords={
"time": req.times,
xname: xvals,
yname: yvals,
}).astype("int16")
return data
n_datasets = datasets_in_xarray(datasets)
if req.product.max_datasets_wcs > 0 and n_datasets > req.product.max_datasets_wcs:
raise WCS1Exception(
"This request processes too much data to be served in a reasonable amount of time."
"Please reduce the bounds of your request and try again."
"(max: %d, this request requires: %d)" %
(req.product.max_datasets_wcs, n_datasets))
stacker = DataStacker(req.product,
req.geobox,
req.times,
bands=req.bands)
output = stacker.data(datasets, skip_corrections=True)
return output
def legend(layer, style):
cfg = get_config()
product = cfg.product_index.get(layer)
if not product:
return ("Unknown Layer", 404,
resp_headers({"Content-Type": "text/plain"}))
img = create_legend_for_style(product, style)
if not img:
return ("Unknown Style", 404,
resp_headers({"Content-Type": "text/plain"}))
return img
def get_coverage(args):
# Note: Only WCS v1.0.0 is fully supported at this stage, so no version negotiation is necessary
cfg = get_config()
req = WCS1GetCoverageRequest(args)
data = get_coverage_data(req)
return (req.format["renderer"](req, data), 200,
cfg.response_headers({
"Content-Type":
req.format["mime"],
'content-disposition':
'attachment; filename=%s.%s' %
(req.product_name, req.format["extension"])
}))
def get_capabilities(args):
# TODO: Handle updatesequence request parameter for cache consistency.
# Note: Only WMS v1.3.0 is fully supported at this stage, so no version negotiation is necessary
# Extract layer metadata from Datacube.
cfg = get_config()
url = args.get('Host', args['url_root'])
base_url = get_service_base_url(cfg.allowed_urls, url)
return (render_template("wms_capabilities.xml", cfg=cfg,
base_url=base_url), 200,
cfg.response_headers({
"Content-Type": "application/xml",
"Cache-Control": "no-cache,max-age=0"
}))
def add_all(dc):
multi_products = set()
for product_cfg in get_config().product_index.values():
product_name = product_cfg.product_name
if product_cfg.multi_product:
multi_products.add(product_cfg)
else:
print("Adding range for:", product_name)
add_product_range(dc, product_name)
for p in multi_products:
print("Adding multiproduct range for:", p.name)
add_multiproduct_range(dc, p, follow_dependencies=False)
def get_tiff(req, data):
"""Uses rasterio MemoryFiles in order to return a streamable GeoTiff response"""
# Copied from CEOS. Does not seem to support multi-time dimension data - is this even possible in GeoTiff?
supported_dtype_map = {
'uint8': 1,
'uint16': 2,
'int16': 3,
'uint32': 4,
'int32': 5,
'float32': 6,
'float64': 7,
'complex': 9,
'complex64': 10,
'complex128': 11,
}
dtype_list = [data[array].dtype for array in data.data_vars]
dtype = str(max(dtype_list, key=lambda d: supported_dtype_map[str(d)]))
data = data.squeeze(dim="time", drop=True)
data = data.astype(dtype)
cfg = get_config()
xname = cfg.published_CRSs[req.request_crsid]["horizontal_coord"]
yname = cfg.published_CRSs[req.request_crsid]["vertical_coord"]
nodata = 0
for band in data.data_vars:
nodata = req.product.band_idx.nodata_val(band)
with MemoryFile() as memfile:
#pylint: disable=protected-access, bad-continuation
with memfile.open(driver="GTiff",
width=data.dims[xname],
height=data.dims[yname],
count=len(data.data_vars),
transform=req.affine,
crs=req.response_crsid,
nodata=nodata,
tiled=True,
compress="lzw",
interleave="band",
dtype=dtype) as dst:
for idx, band in enumerate(data.data_vars, start=1):
dst.write(data[band].values, idx)
dst.set_band_description(idx,
req.product.band_idx.band_label(band))
dst.update_tags(idx,
STATISTICS_MINIMUM=data[band].values.min())
dst.update_tags(idx,
STATISTICS_MAXIMUM=data[band].values.max())
dst.update_tags(idx, STATISTICS_MEAN=data[band].values.mean())
dst.update_tags(idx, STATISTICS_STDDEV=data[band].values.std())
return memfile.read()
def legend(layer, style, dates=None):
cfg = get_config()
product = cfg.product_index.get(layer)
if not product:
return ("Unknown Layer", 404, resp_headers({"Content-Type": "text/plain"}))
if dates is None:
args = lower_get_args()
ndates = int(args.get("ndates", 0))
else:
ndates = len(dates)
img = create_legend_for_style(product, style, ndates)
if not img:
return ("Unknown Style", 404, resp_headers({"Content-Type": "text/plain"}))
return img
def get_product_from_arg(args, argname="layers"):
layers = args.get(argname, "").split(",")
if len(layers) != 1:
raise WMSException("Multi-layer requests not supported")
layer = layers[0]
layer_chunks = layer.split("__")
layer = layer_chunks[0]
cfg = get_config()
product = cfg.product_index.get(layer)
if not product:
raise WMSException("Layer %s is not defined" % layer,
WMSException.LAYER_NOT_DEFINED,
locator="Layer parameter")
return product
def test_time_search():
cfg = get_config()
lyr = list(cfg.product_index.values())[0]
time = lyr.ranges["times"][-1]
geom = box(lyr.bboxes["EPSG:4326"]["bottom"],
lyr.bboxes["EPSG:4326"]["left"], lyr.bboxes["EPSG:4326"]["top"],
lyr.bboxes["EPSG:4326"]["right"], "EPSG:4326")
time_rng = local_solar_date_range(MockGeobox(geom), time)
with cube() as dc:
sel = mv_search_datasets(dc.index,
MVSelectOpts.COUNT,
times=[time_rng],
layer=lyr)
assert sel > 0
def update_range(dc, product, multi=False, follow_dependencies=True):
if multi:
product = get_config().product_index.get(product)
else:
product = dc.index.products.get_by_name(product)
if product is None:
raise Exception("Requested product not found.")
if multi:
return update_multi_range(dc,
product,
follow_dependencies=follow_dependencies)
else:
return update_single_range(dc, product)
def add_multiproduct_range(dc, product, follow_dependencies=True):
if isinstance(product, str):
product = get_config().product_index.get(product)
assert product is not None
assert product.multi_product
if follow_dependencies:
for product_name in product.product_names:
dc_prod = dc.index.products.get_by_name(product_name)
add_product_range(dc, dc_prod)
if not check_datasets_exist(dc, dc_prod):
print("Could not find any datasets for: ", product_name)
# Actually merge and store!
create_multiprod_range_entry(dc, product, get_crses())
def get_tile(args):
cfg = get_config()
wms_args = wmts_args_to_wms(args, cfg)
try:
return get_map(wms_args)
except WMSException as wmse:
first_error = wmse.errors[0]
e = WMTSException(first_error["msg"],
code=first_error["code"],
locator=first_error["locator"],
http_response=wmse.http_response)
for error in wmse.errors[1:]:
e.add_error(error["msg"],
code=error["code"],
locator=error["locator"])
raise e
def get_coverage(args):
cfg = get_config()
req = WCS1GetCoverageRequest(args)
if req.format.multi_time and len(req.times) > 1:
raise WCS1Exception(
"Multi-time requests are not currently supported for WCS1",
WCS1Exception.INVALID_PARAMETER_VALUE,
locator="Time parameter")
data = get_coverage_data(req)
return (req.format.renderer(req.version)(req, data), 200,
cfg.response_headers({
"Content-Type":
req.format.mime,
'content-disposition':
'attachment; filename=%s.%s' %
(req.product_name, req.format.extension)
}))
def ogc_impl():
#pylint: disable=too-many-branches
nocase_args = lower_get_args()
nocase_args = capture_headers(request, nocase_args)
service = nocase_args.get("service", "").upper()
if service:
return ogc_svc_impl(service.lower())
# create dummy env if not exists
try:
# service argument is only required (in fact only defined) by OGC for
# GetCapabilities requests. As long as we are persisting with a single
# routing end point for all services, we must derive the service from the request
# parameter.
# This is a quick hack to fix #64. Service and operation routing could be
# handled more elegantly.
op = nocase_args.get("request", "").upper()
if op in WMS_REQUESTS:
return ogc_svc_impl("wms")
elif op in WCS_REQUESTS:
return ogc_svc_impl("wcs")
elif op:
# Should we return a WMS or WCS exception if there is no service specified?
# Defaulting to WMS because that's what we already have.
raise WMSException("Invalid service and/or request", locator="Service and request parameters")
else:
cfg = get_config()
url = nocase_args.get('Host', nocase_args['url_root'])
base_url = get_service_base_url(cfg.allowed_urls, url)
return (render_template(
"index.html",
cfg=cfg,
supported=OWS_SUPPORTED,
base_url=base_url,
version=__version__,
),
200,
resp_headers({"Content-Type": "text/html"}))
except OGCException as e:
_LOG.error("Handled Error: %s", repr(e.errors))
return e.exception_response()
except Exception as e: # pylint: disable=broad-except
tb = sys.exc_info()[2]
ogc_e = WMSException("Unexpected server error: %s" % str(e), http_response=500)
return ogc_e.exception_response(traceback=traceback.extract_tb(tb))
def uniform_crs(cfg, crs):
" Helper function to transform a URL style EPSG definition to an 'EPSG:nnn' one "
if crs.startswith('http://www.opengis.net/def/crs/EPSG/'):
code = crs.rpartition('/')[-1]
crs = 'EPSG:%s' % code
elif crs.startswith('urn:ogc:def:crs:EPSG:'):
code = crs.rpartition(':')[-1]
crs = 'EPSG:%s' % code
elif crs.startswith('EPSG'):
pass
elif crs in cfg.published_CRSs:
pass
else:
raise WCS2Exception("Not a CRS: %s" % crs,
WCS2Exception.NOT_A_CRS,
locator=crs,
valid_keys=list(get_config().published_CRSs))
return crs
def test_extent_and_spatial():
cfg = get_config()
lyr = list(cfg.product_index.values())[0]
layer_ext_bbx = (
lyr.bboxes["EPSG:4326"]["left"],
lyr.bboxes["EPSG:4326"]["bottom"],
lyr.bboxes["EPSG:4326"]["right"],
lyr.bboxes["EPSG:4326"]["top"],
)
small_bbox = pytest.helpers.enclosed_bbox(layer_ext_bbx)
layer_ext_geom = box(
layer_ext_bbx[0],
layer_ext_bbx[1],
layer_ext_bbx[2],
layer_ext_bbx[3],
"EPSG:4326",
)
small_geom = box(small_bbox[0], small_bbox[1], small_bbox[2],
small_bbox[3], "EPSG:4326")
with cube() as dc:
all_ext = mv_search_datasets(dc.index,
MVSelectOpts.EXTENT,
geom=layer_ext_geom,
layer=lyr)
small_ext = mv_search_datasets(dc.index,
MVSelectOpts.EXTENT,
geom=small_geom,
layer=lyr)
assert layer_ext_geom.contains(all_ext)
assert small_geom.contains(small_ext)
assert all_ext.contains(small_ext)
assert small_ext.area < all_ext.area
all_count = mv_search_datasets(dc.index,
MVSelectOpts.COUNT,
geom=layer_ext_geom,
layer=lyr)
small_count = mv_search_datasets(dc.index,
MVSelectOpts.COUNT,
geom=small_geom,
layer=lyr)
assert small_count <= all_count
def get_odc_products(dc, any_product, odc_only=False):
if isinstance(any_product, OWSNamedLayer):
return any_product.products
elif isinstance(any_product, str):
dc_product = dc.index.products.get_by_name(any_product)
if odc_only:
ows_product = None
else:
ows_product = get_config().product_index.get(any_product)
if ows_product:
if dc_product and [dc_product] == ows_product.products:
# The same!
return [dc_product]
print(
"Updating OWS product %s (ODC Products: []). If you meant the ODC product %s, please use the --odc-only flag."
)
return ows_product.products
else:
# Assume ODC product
return [any_product]
def desc_coverages(args):
cfg = get_config()
request_obj = kvp_decode_describe_coverage(args)
products = []
for coverage_id in request_obj.coverage_ids:
product = cfg.product_index.get(coverage_id)
if product and product.wcs:
products.append(product)
else:
raise WCS2Exception("Invalid coverage: %s" % coverage_id,
WCS2Exception.NO_SUCH_COVERAGE,
locator=coverage_id)
coverage_descriptions = [
create_coverage_description(cfg, product)
for product in products
]
version = request_obj.version
if version == (2, 0):
result = encoders_v20.xml_encode_coverage_descriptions(coverage_descriptions)
elif version == (2, 1):
result = encoders_v21.xml_encode_coverage_descriptions(coverage_descriptions)
else:
raise WCS2Exception("Unsupported version: %s" % version,
WCS2Exception.INVALID_PARAMETER_VALUE,
locator="version")
return (
result.value,
200,
resp_headers({
"Content-Type": result.content_type,
"Cache-Control": "no-cache, max-age=0"
})
)
def get_capabilities(args):
# TODO: Handle updatesequence request parameter for cache consistency.
section = args.get("section")
if section:
section = section.lower()
show_service = False
show_capability = False
show_content_metadata = False
if section is None or section == "/":
show_service = True
show_capability = True
show_content_metadata = True
elif section == "/wcs_capabilities/service":
show_service = True
elif section == "/wcs_capabilities/capability":
show_capability = True
elif section == "/wcs_capabilities/contentmetadata":
show_content_metadata = True
else:
raise WCS1Exception("Invalid section: %s" % section,
WCS1Exception.INVALID_PARAMETER_VALUE,
locator="Section parameter")
# Extract layer metadata from Datacube.
cfg = get_config()
url = args.get('Host', args['url_root'])
base_url = get_service_base_url(cfg.allowed_urls, url)
return (render_template("wcs_capabilities.xml",
show_service=show_service,
show_capability=show_capability,
show_content_metadata=show_content_metadata,
cfg=cfg,
base_url=base_url), 200,
cfg.response_headers({
"Content-Type": "application/xml",
"Cache-Control": "no-cache, max-age=0"
}))
