def test_RBGAMapped_Masking(product_layer_mask_map, style_cfg_map_mask):
def fake_make_mask(data, **kwargs):
val = kwargs["bar"]
return data == val
band = np.array([0, 0, 1, 1, 2, 2])
da = DataArray(band, name='foo')
ds = Dataset(data_vars={'foo': da})
with patch('datacube_wms.band_mapper.make_mask',
new_callable=lambda: fake_make_mask) as fmm:
style_def = StyleDef(product_layer_mask_map, style_cfg_map_mask)
data = style_def.transform_data(ds, None, None)
r = data["red"]
g = data["green"]
b = data["blue"]
a = data["alpha"]
assert (r[2:3:1] == 0)
assert (g[2:3:1] == 0)
assert (b[2:3:1] == 0)
assert (a[2:3:1] == 0)
assert (r[4:5:1] == 255)
assert (g[4:5:1] == 255)
assert (b[4:5:1] == 255)
assert (a[4:5:1] == 255)
def test_alpha_style_map(product_layer_alpha_map, style_cfg_map_alpha_1,
style_cfg_map_alpha_2, style_cfg_map_alpha_3):
def fake_make_mask(data, **kwargs):
return data
band = np.array([True, True, True])
da = DataArray(band, name='foo')
ds = Dataset(data_vars={'foo': da})
with patch('datacube_wms.band_mapper.make_mask',
new_callable=lambda: fake_make_mask) as fmm:
style_def = StyleDef(product_layer_alpha_map, style_cfg_map_alpha_1)
result = style_def.transform_data(ds, None, None)
alpha_channel = result["alpha"].values
assert (alpha_channel == 0).all()
style_def = StyleDef(product_layer_alpha_map, style_cfg_map_alpha_2)
result = style_def.transform_data(ds, None, None)
alpha_channel = result["alpha"].values
assert (alpha_channel == 127).all()
style_def = StyleDef(product_layer_alpha_map, style_cfg_map_alpha_3)
result = style_def.transform_data(ds, None, None)
alpha_channel = result["alpha"].values
assert (alpha_channel == 255).all()
def test_correct_style_hybrid(product_layer, style_cfg_lin):
style_cfg_lin["component_ratio"] = 1.0
style_cfg_lin["range"] = [1, 2]
style_cfg_lin["index_function"] = lambda x: x
style_def = StyleDef(product_layer, style_cfg_lin)
assert isinstance(style_def, bm.HybridStyleDef)
def test_dynamic_range_compression_scale_range(product_layer, style_cfg_lin):
style_cfg_lin["scale_range"] = [-3000, 3000]
style_def = StyleDef(product_layer, style_cfg_lin)
assert style_def.scale_min == -3000
assert style_def.scale_max == 3000
band = np.zeros(3)
band[0] = -3000
band[1] = 0
band[2] = 3000
compressed = style_def.compress_band(band)
assert compressed[0] == 0
assert compressed[1] == 255 / 2
assert compressed[2] == 255
def __init__(self, product_cfg, platform_def, dc):
self.platform = platform_def
self.name = product_cfg["name"]
self.product_name = product_cfg["product_name"]
if "__" in self.product_name:
raise Exception(
"Product names cannot have a double underscore '__' in them.")
self.product_label = product_cfg["label"]
self.product_type = product_cfg["type"]
self.product_variant = product_cfg["variant"]
self.product = dc.index.products.get_by_name(self.product_name)
self.definition = self.product.definition
self.title = "%s %s %s (%s)" % (platform_def.title,
self.product_variant,
self.product_type, self.product_label)
self.ranges = get_ranges(dc, self.product)
self.sub_ranges = get_sub_ranges(dc, self.product)
self.pq_name = product_cfg.get("pq_dataset")
self.pq_band = product_cfg.get("pq_band")
self.min_zoom = product_cfg.get("min_zoom_factor", 300.0)
self.zoom_fill = product_cfg.get("zoomed_out_fill_colour",
[150, 180, 200])
self.ignore_flags_info = product_cfg.get("ignore_flags_info", [])
self.always_fetch_bands = product_cfg.get("always_fetch_bands", [])
self.data_manual_merge = product_cfg.get("data_manual_merge", False)
self.band_drill = product_cfg.get("band_drill", [])
self.solar_correction = product_cfg.get("apply_solar_corrections",
False)
self.sub_product_extractor = product_cfg.get("sub_product_extractor",
None)
self.sub_product_label = product_cfg.get("sub_product_label", None)
if self.pq_name:
self.pq_product = dc.index.products.get_by_name(self.pq_name)
self.info_mask = ~0
fd = self.pq_product.measurements[self.pq_band]["flags_definition"]
for bitname in self.ignore_flags_info:
bit = fd[bitname]["bits"]
if not isinstance(bit, int):
continue
flag = 1 << bit
self.info_mask &= ~flag
else:
self.pq_product = None
self.time_zone = product_cfg.get("time_zone", 9)
self.styles = product_cfg["styles"]
self.default_style = product_cfg["default_style"]
self.style_index = {s["name"]: StyleDef(self, s) for s in self.styles}
try:
i = iter(product_cfg["extent_mask_func"])
self.extent_mask_func = product_cfg["extent_mask_func"]
except TypeError:
self.extent_mask_func = [product_cfg["extent_mask_func"]]
self.pq_manual_merge = product_cfg.get("pq_manual_merge", False)
def __init__(self, platform_cfg, prod_idx, dc=None):
self.name = platform_cfg["name"]
self.title = platform_cfg["title"]
self.abstract = platform_cfg["abstract"]
self.styles = platform_cfg["styles"]
self.default_style = platform_cfg["default_style"]
self.style_index = {s["name"]: StyleDef(s) for s in self.styles}
self.products = []
for prod_cfg in platform_cfg["products"]:
prod = ProductLayerDef(prod_cfg, self, dc=dc)
self.products.append(prod)
prod_idx[prod.name] = prod
def test_dynamic_range_compression_scale_factor(product_layer, style_cfg_lin):
del style_cfg_lin["scale_range"]
style_cfg_lin["scale_factor"] = 2.5
style_def = StyleDef(product_layer, style_cfg_lin)
assert style_def.scale_min == 0.0
assert style_def.scale_max == 637.5
band = np.zeros(3)
band[0] = -3000
band[1] = 0
band[2] = 3000
def __init__(self, product_cfg, platform_def, dc):
self.platform = platform_def
self.name = product_cfg["name"]
self.product_name = product_cfg["product_name"]
if "__" in self.product_name:
raise Exception(
"Product names cannot have a double underscore '__' in them.")
self.product_label = product_cfg["label"]
self.product_type = product_cfg["type"]
self.product_variant = product_cfg["variant"]
self.product = dc.index.products.get_by_name(self.product_name)
self.definition = self.product.definition
self.title = "%s %s %s (%s)" % (platform_def.title,
self.product_variant,
self.product_type, self.product_label)
self.ranges = get_ranges(dc, self.product)
self.sub_ranges = get_sub_ranges(dc, self.product)
self.pq_name = product_cfg.get("pq_dataset")
self.pq_band = product_cfg.get("pq_band")
self.min_zoom = product_cfg.get("min_zoom_factor", 300.0)
self.max_datasets_wms = product_cfg.get("max_datasets_wms", 0)
self.zoom_fill = product_cfg.get("zoomed_out_fill_colour",
[150, 180, 200])
self.ignore_flags_info = product_cfg.get("ignore_flags_info", [])
self.always_fetch_bands = product_cfg.get("always_fetch_bands", [])
self.data_manual_merge = product_cfg.get("data_manual_merge", False)
self.band_drill = product_cfg.get("band_drill", [])
self.solar_correction = product_cfg.get("apply_solar_corrections",
False)
self.sub_product_extractor = product_cfg.get("sub_product_extractor",
None)
self.sub_product_label = product_cfg.get("sub_product_label", None)
if self.pq_name:
self.pq_product = dc.index.products.get_by_name(self.pq_name)
self.info_mask = ~0
fd = self.pq_product.measurements[self.pq_band]["flags_definition"]
for bitname in self.ignore_flags_info:
bit = fd[bitname]["bits"]
if not isinstance(bit, int):
continue
flag = 1 << bit
self.info_mask &= ~flag
else:
self.pq_product = None
self.time_zone = product_cfg.get("time_zone", 9)
self.styles = product_cfg["styles"]
self.default_style = product_cfg["default_style"]
self.style_index = {s["name"]: StyleDef(self, s) for s in self.styles}
try:
i = iter(product_cfg["extent_mask_func"])
self.extent_mask_func = product_cfg["extent_mask_func"]
except TypeError:
self.extent_mask_func = [product_cfg["extent_mask_func"]]
self.pq_manual_merge = product_cfg.get("pq_manual_merge", False)
# For WCS
svc_cfg = get_service_cfg()
if svc_cfg.wcs:
try:
self.native_CRS = self.product.definition["storage"]["crs"]
if self.native_CRS not in svc_cfg.published_CRSs:
raise Exception(
"Native CRS for product {} ({}) not in published CRSs".
format(self.product_name, self.native_CRS))
self.native_CRS_def = svc_cfg.published_CRSs[self.native_CRS]
data = dc.load(self.product_name, dask_chunks={})
self.grid_high_x = len(data[svc_cfg.published_CRSs[
self.native_CRS]["horizontal_coord"]])
self.grid_high_y = len(data[svc_cfg.published_CRSs[
self.native_CRS]["vertical_coord"]])
self.origin_x = data.affine[3]
self.origin_y = data.affine[5]
self.resolution_x = data.affine[0]
self.resolution_y = data.affine[4]
self.max_datasets_wcs = product_cfg.get("max_datasets_wcs", 0)
except:
self.native_CRS = None
bands = dc.list_measurements().ix[self.product_name]
self.bands = bands.index.values
self.nodata_values = bands['nodata'].values
self.nodata_dict = {
a: b
for a, b in zip(self.bands, self.nodata_values)
}
def test_correct_style_ramp(product_layer, style_cfg_ramp):
style_def = StyleDef(product_layer, style_cfg_ramp)
assert isinstance(style_def, bm.RgbaColorRampDef)
def test_correct_style_map(product_layer, style_cfg_map):
style_def = StyleDef(product_layer, style_cfg_map)
assert isinstance(style_def, bm.RGBAMappedStyleDef)
def test_correct_style_linear(product_layer, style_cfg_lin):
style_def = StyleDef(product_layer, style_cfg_lin)
assert isinstance(style_def, bm.LinearStyleDef)
def __init__(self, product_cfg, platform_def, dc):
self.platform = platform_def
self.name = product_cfg["name"]
self.multi_product = product_cfg.get("multi_product", False)
if self.multi_product:
self.product_names = product_cfg["product_name"]
self.product_name = product_cfg["product_name"][0]
else:
self.product_name = product_cfg["product_name"]
self.product_names = [self.product_name]
for prod_name in self.product_names:
if "__" in prod_name:
raise ProductLayerException(
"Product names cannot have a double underscore '__' in them."
)
self.product_label = product_cfg["label"]
self.product_type = product_cfg["type"]
self.product_variant = product_cfg["variant"]
self.products = []
for pn in self.product_names:
product = dc.index.products.get_by_name(pn)
if product is None:
raise ProductLayerException(
f"Could not find product {pn} in datacube")
self.products.append(product)
self.product = self.products[0]
self.band_idx = BandIndex(self.product, product_cfg.get("bands"), dc)
self.definition = self.product.definition
self.abstract = product_cfg[
"abstract"] if "abstract" in product_cfg else self.definition[
'description']
self.title = "%s %s %s (%s)" % (platform_def.title,
self.product_variant,
self.product_type, self.product_label)
from datacube_wms.product_ranges import get_ranges, get_sub_ranges, merge_ranges
self.ranges = get_ranges(dc, self)
if self.ranges is None:
if self.multi_product:
_LOG.warning(
"Warning: Ranges for multi-product %s not yet in database",
self.name)
else:
_LOG.warning("Could not find ranges for %s in database",
self.product_name)
# TODO: subranges not supported with multi-product
self.sub_ranges = get_sub_ranges(dc, self)
if self.multi_product:
self.pq_names = product_cfg.get("pq_dataset")
else:
self.pq_names = [product_cfg.get("pq_dataset")]
self.pq_name = self.pq_names[0] if self.pq_names is not None and len(
self.pq_names) > 0 else None
self.pq_band = product_cfg.get("pq_band")
self.time_resolution = product_cfg.get("time_resolution", TIMERES_RAW)
if self.time_resolution not in TIMERES_VALS:
raise ProductLayerException("Invalid time resolution value: %s" %
self.time_resolution)
self.min_zoom = product_cfg.get("min_zoom_factor", 300.0)
self.max_datasets_wms = product_cfg.get("max_datasets_wms", 0)
self.zoom_fill = product_cfg.get("zoomed_out_fill_colour",
[150, 180, 200])
self.ignore_flags_info = product_cfg.get("ignore_flags_info", [])
self.feature_info_include_utc_dates = product_cfg.get(
"feature_info_include_utc_dates", False)
self.feature_info_include_custom = product_cfg.get(
"feature_info_include_custom", None)
raw_always_fetch_bands = product_cfg.get("always_fetch_bands", [])
self.always_fetch_bands = []
for b in raw_always_fetch_bands:
self.always_fetch_bands.append(self.band_idx.band(b))
self.data_manual_merge = product_cfg.get("data_manual_merge", False)
self.solar_correction = product_cfg.get("apply_solar_corrections",
False)
if "sub_product_extractor" in product_cfg:
self.sub_product_extractor = FunctionWrapper(
self, product_cfg["sub_product_extractor"])
else:
self.sub_product_extractor = None
self.sub_product_label = product_cfg.get("sub_product_label", None)
self.pq_products = []
if self.pq_names:
for pqn in self.pq_names:
if pqn is not None:
pq_product = dc.index.products.get_by_name(pqn)
if pq_product is None:
raise ProductLayerException(
f"Could not find pq_product {pqn} for {self.name} in database"
)
self.pq_products.append(pq_product)
self.info_mask = ~0
if self.pq_products:
self.pq_product = self.pq_products[0]
fd = self.pq_product.measurements[self.pq_band]["flags_definition"]
for bitname in self.ignore_flags_info:
bit = fd[bitname]["bits"]
if not isinstance(bit, int):
continue
flag = 1 << bit
self.info_mask &= ~flag
else:
self.pq_product = None
self.legend = product_cfg.get("legend", None)
self.styles = product_cfg["styles"]
self.default_style = product_cfg["default_style"]
self.style_index = {s["name"]: StyleDef(self, s) for s in self.styles}
if (isinstance(product_cfg["extent_mask_func"], Mapping)
or callable(product_cfg["extent_mask_func"])
or isinstance(product_cfg["extent_mask_func"], str)):
# Single extent mask function.
self.extent_mask_func = [
FunctionWrapper(self, product_cfg["extent_mask_func"])
]
else:
# Multiple extent mask functions.
self.extent_mask_func = list([
FunctionWrapper(self, f_cfg)
for f_cfg in product_cfg["extent_mask_func"]
])
if "fuse_func" in product_cfg:
self.fuse_func = FunctionWrapper(self, product_cfg["fuse_func"])
else:
self.fuse_func = None
if "pq_fuse_func" in product_cfg:
self.pq_fuse_func = FunctionWrapper(self,
product_cfg["pq_fuse_func"])
else:
self.pq_fuse_func = None
self.pq_manual_merge = product_cfg.get("pq_manual_merge", False)
self.pq_ignore_time = product_cfg.get("pq_ignore_time", False)
self.attribution = AttributionCfg.parse(product_cfg.get("attribution"))
if not self.attribution:
self.attribution = platform_def.attribution
self.identifiers = product_cfg.get("identifiers", {})
svc_cfg = get_service_cfg()
for auth in self.identifiers.keys():
if auth not in svc_cfg.authorities:
raise ProductLayerException(
"Identifier with non-declared authority: %s" % repr(auth))
self.feature_list_urls = SuppURL.parse_list(
product_cfg.get("feature_list_urls"))
self.data_urls = SuppURL.parse_list(product_cfg.get("data_urls"))
# For WCS
if svc_cfg.wcs:
try:
self.native_CRS = self.product.definition["storage"]["crs"]
except KeyError:
self.native_CRS = None
if not self.native_CRS:
self.native_CRS = product_cfg.get("native_wcs_crs")
if self.native_CRS not in svc_cfg.published_CRSs:
logging.warning(
"Native CRS for product %s (%s) not in published CRSs",
self.product_name, self.native_CRS)
self.native_CRS = None
if self.native_CRS:
self.native_CRS_def = svc_cfg.published_CRSs[self.native_CRS]
if svc_cfg.create_grid and not svc_cfg.dummy_grid and self.native_CRS:
data = dc.load(self.product_name, dask_chunks={})
self.grid_high_x = len(data[svc_cfg.published_CRSs[
self.native_CRS]["horizontal_coord"]])
self.grid_high_y = len(data[svc_cfg.published_CRSs[
self.native_CRS]["vertical_coord"]])
self.origin_x = data.affine[3]
self.origin_y = data.affine[5]
self.resolution_x = data.affine[0]
self.resolution_y = data.affine[4]
elif not svc_cfg.dummy_grid and self.native_CRS and self.ranges is not None:
native_bounding_box = self.bboxes[self.native_CRS]
self.origin_x = native_bounding_box["left"]
self.origin_y = native_bounding_box["bottom"]
self.resolution_x, self.resolution_y = product_cfg.get(
"native_wcs_resolution", [25.0, 25.0])
self.grid_high_x = int(
(native_bounding_box["right"] -
native_bounding_box["left"]) / self.resolution_x)
self.grid_high_y = int(
(native_bounding_box["top"] -
native_bounding_box["bottom"]) / self.resolution_y)
self.max_datasets_wcs = product_cfg.get("max_datasets_wcs", 0)
bands = dc.list_measurements().loc[self.product_name]
self.bands = bands.index.values
self.nodata_values = bands['nodata'].values
self.nodata_dict = {
a: b
for a, b in zip(self.bands, self.nodata_values)
}
self.wcs_sole_time = product_cfg.get("wcs_sole_time")
self.wcs_default_bands = [
self.band_idx.band(b) for b in product_cfg["wcs_default_bands"]
]
