def add_ranges(dc, product_names, summary=False, merge_only=False): odc_products = {} ows_multiproducts = [] for pname in product_names: dc_product = None ows_product = get_config().product_index.get(pname) if not ows_product: ows_product = get_config().native_product_index.get(pname) if ows_product: for dc_pname in ows_product.product_names: if dc_pname in odc_products: odc_products[dc_pname]["ows"].append(ows_product) else: odc_products[dc_pname] = { "ows": [ows_product]} print("OWS Layer %s maps to ODC Product(s): %s" % ( ows_product.name, repr(ows_product.product_names) )) if ows_product.multi_product: ows_multiproducts.append(ows_product) if not ows_product: dc_product = dc.index.products.get_by_name(pname) if dc_product: print("ODC Layer: %s" % pname) if pname in odc_products: odc_products[pname]["ows"].append(None) else: odc_products[pname] = { "ows": [None]} else: print("Unrecognised product name:", pname) continue if ows_multiproducts and merge_only: print("Merge-only: Skipping range update of products:", repr(list(odc_products.keys()))) else: for pname, ows_prods in odc_products.items(): dc_product = dc.index.products.get_by_name(pname) if dc_product is None: print("Could not find ODC product:", pname) elif datasets_exist(dc, dc_product.name): prod_summary = summary for ows_prod in ows_prods["ows"]: if ows_prod: prod_summary = not ows_prod.is_raw_time_res break create_range_entry(dc, dc_product, get_crses(), prod_summary) else: print("Could not find any datasets for: ", pname) for mp in ows_multiproducts: create_multiprod_range_entry(dc, mp, get_crses()) print("Done.")
def method_specific_init(self, args): # Validate Format parameter self.format = get_arg(args, "format", "image format", errcode=WMSException.INVALID_FORMAT, lower=True, permitted_values=["image/png"]) # Styles self.styles = args.get("styles", "").split(",") if len(self.styles) != 1: raise WMSException("Multi-layer GetMap requests not supported") style_r = self.styles[0] if not style_r: style_r = self.product.default_style.name self.style = self.product.style_index.get(style_r) if not self.style: raise WMSException("Style %s is not defined" % style_r, WMSException.STYLE_NOT_DEFINED, locator="Style parameter") cfg = get_config() if self.geobox.width > cfg.wms_max_width: raise WMSException(f"Width {self.geobox.width} exceeds supported maximum {self.cfg.wms_max_width}.", locator="Width parameter") if self.geobox.height > cfg.wms_max_height: raise WMSException(f"Width {self.geobox.height} exceeds supported maximum {self.cfg.wms_max_height}.", locator="Height parameter") # Zoom factor self.zf = zoom_factor(args, self.crs) # TODO: Do we need to make resampling method configurable? self.resampling = Resampling.nearest
def desc_coverages(args): # Note: Only WCS v1.0.0 is fully supported at this stage, so no version negotiation is necessary # Extract layer metadata from Datacube. cfg = get_config() coverages = args.get("coverage") products = [] if coverages: coverages = coverages.split(",") for c in coverages: p = cfg.product_index.get(c) if p and p.wcs: products.append(p) else: raise WCS1Exception("Invalid coverage: %s" % c, WCS1Exception.COVERAGE_NOT_DEFINED, locator="Coverage parameter") else: for p in cfg.product_index.values(): if p.wcs: products.append(p) return (render_template("wcs_desc_coverage.xml", cfg=cfg, products=products), 200, cfg.response_headers({ "Content-Type": "application/xml", "Cache-Control": "max-age=10" }))
def __init__(self, args): # Version self.version = get_arg(args, "version", "WMS version", permitted_values=['1.1.1', '1.3.0']) # CRS if self.version == '1.1.1': crs_arg = "srs" else: crs_arg = "crs" self.crsid = get_arg( args, crs_arg, "Coordinate Reference System", errcode=WMSException.INVALID_CRS, permitted_values=get_config().published_CRSs.keys()) self.crs = geometry.CRS(self.crsid) # Layers self.product = self.get_product(args) self.raw_product = self.get_raw_product(args) self.geometry = _get_polygon(args, self.crs) # BBox, height and width parameters self.geobox = _get_geobox(args, self.crs) # Time parameter self.times = get_times(args, self.product, self.raw_product) self.method_specific_init(args)
def img_coords_to_geopoint(geobox, i, j): cfg = get_config() h_coord = cfg.published_CRSs[geobox.crs.crs_str]["horizontal_coord"] v_coord = cfg.published_CRSs[geobox.crs.crs_str]["vertical_coord"] return geometry.point(geobox.coordinates[h_coord].values[int(i)], geobox.coordinates[v_coord].values[int(j)], geobox.crs)
def test_count(): cfg = get_config() lyr = list(cfg.product_index.values())[0] with cube() as dc: count = mv_search_datasets(dc.index, MVSelectOpts.COUNT, layer=lyr) ids = mv_search_datasets(dc.index, MVSelectOpts.IDS, layer=lyr) assert len(ids) == count
def update_all_ranges(dc): i = 0 u = 0 p = 0 si = 0 su = 0 sp = 0 mi = 0 mu = 0 mp = 0 multiproducts = set() for prod in get_config().product_index.values(): if prod.multi_product: multiproducts.add(prod) else: stats = update_single_range(dc, prod) p += stats[0] u += stats[1] i += stats[2] sp += stats[3] su += stats[4] si += stats[5] for mprod in multiproducts: stats = update_multi_range(dc, mprod, follow_dependencies=False) mp += stats[0] mu += stats[1] mi += stats[2] return p, u, i, sp, su, si, mp, mu, mi
def no_db(monkeypatch): monkeypatch.setenv("DATACUBE_OWS_CFG", "tests.cfg.minimal_cfg.ows_cfg") monkeypatch.setenv("DB_USERNAME", "fakeuser") monkeypatch.setenv("DB_PASSWORD", "password") monkeypatch.setenv("DB_HOSTNAME", "localhost") from datacube_ows.ows_configuration import get_config cfg = get_config(refresh=True)
def get_capabilities(args): # TODO: Handle updatesequence request parameter for cache consistency. # Note: Only WMS v1.0.0 exists at this stage, so no version negotiation is necessary # Extract layer metadata from Datacube. cfg = get_config() url = args.get('Host', args['url_root']) base_url = get_service_base_url(cfg.allowed_urls, url) section = args.get("section") if section: section = section.lower() show_service_id = False show_service_provider = False show_ops_metadata = False show_contents = False show_themes = False if section is None: show_service_id = True show_service_provider = True show_ops_metadata = True show_contents = True show_themes = True else: sections = section.split(",") for s in sections: if s == "all": show_service_id = True show_service_provider = True show_ops_metadata = True show_contents = True show_themes = True elif s == "serviceidentification": show_service_id = True elif s == "serviceprovider": show_service_provider = True elif s == "operationsmetadata": show_ops_metadata = True elif s == "contents": show_contents = True elif s == "themes": show_themes = True else: raise WMTSException("Invalid section: %s" % section, WMTSException.INVALID_PARAMETER_VALUE, locator="Section parameter") return ( render_template( "wmts_capabilities.xml", cfg=cfg, base_url=base_url, show_service_id = show_service_id, show_service_provider = show_service_provider, show_ops_metadata = show_ops_metadata, show_contents = show_contents, show_themes = show_themes, webmerc_ss = WebMercScaleSet), 200, cfg.response_headers( {"Content-Type": "application/xml", "Cache-Control": "max-age=10"} ) )
def png_response(body, cfg=None, extra_headers=None): if not cfg: cfg = get_config() if extra_headers is None: extra_headers = {} headers = {"Content-Type": "image/png"} headers.update(extra_headers) headers = cfg.response_headers(headers) return body, 200, cfg.response_headers(headers)
def test_datasets(): cfg = get_config() lyr = list(cfg.product_index.values())[0] with cube() as dc: dss = mv_search_datasets(dc.index, MVSelectOpts.DATASETS, layer=lyr) ids = mv_search_datasets(dc.index, MVSelectOpts.IDS, layer=lyr) assert len(ids) == len(dss) for ds in dss: assert str(ds.id) in ids
def get_coverage_data(req): #pylint: disable=too-many-locals, protected-access with cube() as dc: if not dc: raise WCS1Exception("Database connectivity failure") stacker = DataStacker(req.product, req.geobox, req.times, bands=req.bands) datasets = stacker.datasets(dc.index) if not datasets: # TODO: Return an empty coverage file with full metadata? cfg = get_config() x_range = (req.minx, req.maxx) y_range = (req.miny, req.maxy) xname = cfg.published_CRSs[req.request_crsid]["horizontal_coord"] yname = cfg.published_CRSs[req.request_crsid]["vertical_coord"] xvals = numpy.linspace(x_range[0], x_range[1], num=req.width) yvals = numpy.linspace(y_range[0], y_range[1], num=req.height) if cfg.published_CRSs[req.request_crsid]["vertical_coord_first"]: nparrays = { band: (("time", yname, xname), numpy.full((len(req.times), len(yvals), len(xvals)), req.product.nodata_dict[band])) for band in req.bands } else: nparrays = { band: (("time", xname, yname), numpy.full((len(req.times), len(xvals), len(yvals)), req.product.nodata_dict[band])) for band in req.bands } data = xarray.Dataset(nparrays, coords={ "time": req.times, xname: xvals, yname: yvals, }).astype("int16") return data n_datasets = datasets_in_xarray(datasets) if req.product.max_datasets_wcs > 0 and n_datasets > req.product.max_datasets_wcs: raise WCS1Exception( "This request processes too much data to be served in a reasonable amount of time." "Please reduce the bounds of your request and try again." "(max: %d, this request requires: %d)" % (req.product.max_datasets_wcs, n_datasets)) stacker = DataStacker(req.product, req.geobox, req.times, bands=req.bands) output = stacker.data(datasets, skip_corrections=True) return output
def legend(layer, style): cfg = get_config() product = cfg.product_index.get(layer) if not product: return ("Unknown Layer", 404, resp_headers({"Content-Type": "text/plain"})) img = create_legend_for_style(product, style) if not img: return ("Unknown Style", 404, resp_headers({"Content-Type": "text/plain"})) return img
def get_coverage(args): # Note: Only WCS v1.0.0 is fully supported at this stage, so no version negotiation is necessary cfg = get_config() req = WCS1GetCoverageRequest(args) data = get_coverage_data(req) return (req.format["renderer"](req, data), 200, cfg.response_headers({ "Content-Type": req.format["mime"], 'content-disposition': 'attachment; filename=%s.%s' % (req.product_name, req.format["extension"]) }))
def get_capabilities(args): # TODO: Handle updatesequence request parameter for cache consistency. # Note: Only WMS v1.3.0 is fully supported at this stage, so no version negotiation is necessary # Extract layer metadata from Datacube. cfg = get_config() url = args.get('Host', args['url_root']) base_url = get_service_base_url(cfg.allowed_urls, url) return (render_template("wms_capabilities.xml", cfg=cfg, base_url=base_url), 200, cfg.response_headers({ "Content-Type": "application/xml", "Cache-Control": "no-cache,max-age=0" }))
def add_all(dc): multi_products = set() for product_cfg in get_config().product_index.values(): product_name = product_cfg.product_name if product_cfg.multi_product: multi_products.add(product_cfg) else: print("Adding range for:", product_name) add_product_range(dc, product_name) for p in multi_products: print("Adding multiproduct range for:", p.name) add_multiproduct_range(dc, p, follow_dependencies=False)
def get_tiff(req, data): """Uses rasterio MemoryFiles in order to return a streamable GeoTiff response""" # Copied from CEOS. Does not seem to support multi-time dimension data - is this even possible in GeoTiff? supported_dtype_map = { 'uint8': 1, 'uint16': 2, 'int16': 3, 'uint32': 4, 'int32': 5, 'float32': 6, 'float64': 7, 'complex': 9, 'complex64': 10, 'complex128': 11, } dtype_list = [data[array].dtype for array in data.data_vars] dtype = str(max(dtype_list, key=lambda d: supported_dtype_map[str(d)])) data = data.squeeze(dim="time", drop=True) data = data.astype(dtype) cfg = get_config() xname = cfg.published_CRSs[req.request_crsid]["horizontal_coord"] yname = cfg.published_CRSs[req.request_crsid]["vertical_coord"] nodata = 0 for band in data.data_vars: nodata = req.product.band_idx.nodata_val(band) with MemoryFile() as memfile: #pylint: disable=protected-access, bad-continuation with memfile.open(driver="GTiff", width=data.dims[xname], height=data.dims[yname], count=len(data.data_vars), transform=req.affine, crs=req.response_crsid, nodata=nodata, tiled=True, compress="lzw", interleave="band", dtype=dtype) as dst: for idx, band in enumerate(data.data_vars, start=1): dst.write(data[band].values, idx) dst.set_band_description(idx, req.product.band_idx.band_label(band)) dst.update_tags(idx, STATISTICS_MINIMUM=data[band].values.min()) dst.update_tags(idx, STATISTICS_MAXIMUM=data[band].values.max()) dst.update_tags(idx, STATISTICS_MEAN=data[band].values.mean()) dst.update_tags(idx, STATISTICS_STDDEV=data[band].values.std()) return memfile.read()
def legend(layer, style, dates=None): cfg = get_config() product = cfg.product_index.get(layer) if not product: return ("Unknown Layer", 404, resp_headers({"Content-Type": "text/plain"})) if dates is None: args = lower_get_args() ndates = int(args.get("ndates", 0)) else: ndates = len(dates) img = create_legend_for_style(product, style, ndates) if not img: return ("Unknown Style", 404, resp_headers({"Content-Type": "text/plain"})) return img
def get_product_from_arg(args, argname="layers"): layers = args.get(argname, "").split(",") if len(layers) != 1: raise WMSException("Multi-layer requests not supported") layer = layers[0] layer_chunks = layer.split("__") layer = layer_chunks[0] cfg = get_config() product = cfg.product_index.get(layer) if not product: raise WMSException("Layer %s is not defined" % layer, WMSException.LAYER_NOT_DEFINED, locator="Layer parameter") return product
def test_time_search(): cfg = get_config() lyr = list(cfg.product_index.values())[0] time = lyr.ranges["times"][-1] geom = box(lyr.bboxes["EPSG:4326"]["bottom"], lyr.bboxes["EPSG:4326"]["left"], lyr.bboxes["EPSG:4326"]["top"], lyr.bboxes["EPSG:4326"]["right"], "EPSG:4326") time_rng = local_solar_date_range(MockGeobox(geom), time) with cube() as dc: sel = mv_search_datasets(dc.index, MVSelectOpts.COUNT, times=[time_rng], layer=lyr) assert sel > 0
def update_range(dc, product, multi=False, follow_dependencies=True): if multi: product = get_config().product_index.get(product) else: product = dc.index.products.get_by_name(product) if product is None: raise Exception("Requested product not found.") if multi: return update_multi_range(dc, product, follow_dependencies=follow_dependencies) else: return update_single_range(dc, product)
def add_multiproduct_range(dc, product, follow_dependencies=True): if isinstance(product, str): product = get_config().product_index.get(product) assert product is not None assert product.multi_product if follow_dependencies: for product_name in product.product_names: dc_prod = dc.index.products.get_by_name(product_name) add_product_range(dc, dc_prod) if not check_datasets_exist(dc, dc_prod): print("Could not find any datasets for: ", product_name) # Actually merge and store! create_multiprod_range_entry(dc, product, get_crses())
def get_tile(args): cfg = get_config() wms_args = wmts_args_to_wms(args, cfg) try: return get_map(wms_args) except WMSException as wmse: first_error = wmse.errors[0] e = WMTSException(first_error["msg"], code=first_error["code"], locator=first_error["locator"], http_response=wmse.http_response) for error in wmse.errors[1:]: e.add_error(error["msg"], code=error["code"], locator=error["locator"]) raise e
def get_coverage(args): cfg = get_config() req = WCS1GetCoverageRequest(args) if req.format.multi_time and len(req.times) > 1: raise WCS1Exception( "Multi-time requests are not currently supported for WCS1", WCS1Exception.INVALID_PARAMETER_VALUE, locator="Time parameter") data = get_coverage_data(req) return (req.format.renderer(req.version)(req, data), 200, cfg.response_headers({ "Content-Type": req.format.mime, 'content-disposition': 'attachment; filename=%s.%s' % (req.product_name, req.format.extension) }))
def ogc_impl(): #pylint: disable=too-many-branches nocase_args = lower_get_args() nocase_args = capture_headers(request, nocase_args) service = nocase_args.get("service", "").upper() if service: return ogc_svc_impl(service.lower()) # create dummy env if not exists try: # service argument is only required (in fact only defined) by OGC for # GetCapabilities requests. As long as we are persisting with a single # routing end point for all services, we must derive the service from the request # parameter. # This is a quick hack to fix #64. Service and operation routing could be # handled more elegantly. op = nocase_args.get("request", "").upper() if op in WMS_REQUESTS: return ogc_svc_impl("wms") elif op in WCS_REQUESTS: return ogc_svc_impl("wcs") elif op: # Should we return a WMS or WCS exception if there is no service specified? # Defaulting to WMS because that's what we already have. raise WMSException("Invalid service and/or request", locator="Service and request parameters") else: cfg = get_config() url = nocase_args.get('Host', nocase_args['url_root']) base_url = get_service_base_url(cfg.allowed_urls, url) return (render_template( "index.html", cfg=cfg, supported=OWS_SUPPORTED, base_url=base_url, version=__version__, ), 200, resp_headers({"Content-Type": "text/html"})) except OGCException as e: _LOG.error("Handled Error: %s", repr(e.errors)) return e.exception_response() except Exception as e: # pylint: disable=broad-except tb = sys.exc_info()[2] ogc_e = WMSException("Unexpected server error: %s" % str(e), http_response=500) return ogc_e.exception_response(traceback=traceback.extract_tb(tb))
def uniform_crs(cfg, crs): " Helper function to transform a URL style EPSG definition to an 'EPSG:nnn' one " if crs.startswith('http://www.opengis.net/def/crs/EPSG/'): code = crs.rpartition('/')[-1] crs = 'EPSG:%s' % code elif crs.startswith('urn:ogc:def:crs:EPSG:'): code = crs.rpartition(':')[-1] crs = 'EPSG:%s' % code elif crs.startswith('EPSG'): pass elif crs in cfg.published_CRSs: pass else: raise WCS2Exception("Not a CRS: %s" % crs, WCS2Exception.NOT_A_CRS, locator=crs, valid_keys=list(get_config().published_CRSs)) return crs
def test_extent_and_spatial(): cfg = get_config() lyr = list(cfg.product_index.values())[0] layer_ext_bbx = ( lyr.bboxes["EPSG:4326"]["left"], lyr.bboxes["EPSG:4326"]["bottom"], lyr.bboxes["EPSG:4326"]["right"], lyr.bboxes["EPSG:4326"]["top"], ) small_bbox = pytest.helpers.enclosed_bbox(layer_ext_bbx) layer_ext_geom = box( layer_ext_bbx[0], layer_ext_bbx[1], layer_ext_bbx[2], layer_ext_bbx[3], "EPSG:4326", ) small_geom = box(small_bbox[0], small_bbox[1], small_bbox[2], small_bbox[3], "EPSG:4326") with cube() as dc: all_ext = mv_search_datasets(dc.index, MVSelectOpts.EXTENT, geom=layer_ext_geom, layer=lyr) small_ext = mv_search_datasets(dc.index, MVSelectOpts.EXTENT, geom=small_geom, layer=lyr) assert layer_ext_geom.contains(all_ext) assert small_geom.contains(small_ext) assert all_ext.contains(small_ext) assert small_ext.area < all_ext.area all_count = mv_search_datasets(dc.index, MVSelectOpts.COUNT, geom=layer_ext_geom, layer=lyr) small_count = mv_search_datasets(dc.index, MVSelectOpts.COUNT, geom=small_geom, layer=lyr) assert small_count <= all_count
def get_odc_products(dc, any_product, odc_only=False): if isinstance(any_product, OWSNamedLayer): return any_product.products elif isinstance(any_product, str): dc_product = dc.index.products.get_by_name(any_product) if odc_only: ows_product = None else: ows_product = get_config().product_index.get(any_product) if ows_product: if dc_product and [dc_product] == ows_product.products: # The same! return [dc_product] print( "Updating OWS product %s (ODC Products: []). If you meant the ODC product %s, please use the --odc-only flag." ) return ows_product.products else: # Assume ODC product return [any_product]
def desc_coverages(args): cfg = get_config() request_obj = kvp_decode_describe_coverage(args) products = [] for coverage_id in request_obj.coverage_ids: product = cfg.product_index.get(coverage_id) if product and product.wcs: products.append(product) else: raise WCS2Exception("Invalid coverage: %s" % coverage_id, WCS2Exception.NO_SUCH_COVERAGE, locator=coverage_id) coverage_descriptions = [ create_coverage_description(cfg, product) for product in products ] version = request_obj.version if version == (2, 0): result = encoders_v20.xml_encode_coverage_descriptions(coverage_descriptions) elif version == (2, 1): result = encoders_v21.xml_encode_coverage_descriptions(coverage_descriptions) else: raise WCS2Exception("Unsupported version: %s" % version, WCS2Exception.INVALID_PARAMETER_VALUE, locator="version") return ( result.value, 200, resp_headers({ "Content-Type": result.content_type, "Cache-Control": "no-cache, max-age=0" }) )
def get_capabilities(args): # TODO: Handle updatesequence request parameter for cache consistency. section = args.get("section") if section: section = section.lower() show_service = False show_capability = False show_content_metadata = False if section is None or section == "/": show_service = True show_capability = True show_content_metadata = True elif section == "/wcs_capabilities/service": show_service = True elif section == "/wcs_capabilities/capability": show_capability = True elif section == "/wcs_capabilities/contentmetadata": show_content_metadata = True else: raise WCS1Exception("Invalid section: %s" % section, WCS1Exception.INVALID_PARAMETER_VALUE, locator="Section parameter") # Extract layer metadata from Datacube. cfg = get_config() url = args.get('Host', args['url_root']) base_url = get_service_base_url(cfg.allowed_urls, url) return (render_template("wcs_capabilities.xml", show_service=show_service, show_capability=show_capability, show_content_metadata=show_content_metadata, cfg=cfg, base_url=base_url), 200, cfg.response_headers({ "Content-Type": "application/xml", "Cache-Control": "no-cache, max-age=0" }))