def transform_single_date_data(self, data):
# pylint: disable=too-many-locals, too-many-branches
# extent mask data per band to preseve nodata
_LOG.debug("transform begin %s", datetime.now())
#if extent_mask is not None:
# for band in data.data_vars:
## try:
# data[band] = data[band].where(extent_mask, other=data[band].attrs['nodata'])
# except AttributeError:
# data[band] = data[band].where(extent_mask)
imgdata = Dataset(coords=data.coords)
for cfg_band, rules in self.value_map.items():
# Run through each item
band = self.product.band_idx.band(cfg_band)
band_data = Dataset()
bdata = data[band]
if bdata.dtype.kind == 'f':
# Convert back to int for bitmasking
bdata = ColorMapStyleDef.reint(bdata)
for rule in rules:
mask = rule.create_mask(bdata)
masked = ColorMapStyleDef.create_colordata(
bdata, rule.rgb, rule.alpha, mask)
band_data = masked if len(
band_data.data_vars) == 0 else band_data.combine_first(
masked)
imgdata = band_data if len(imgdata.data_vars) == 0 else merge(
[imgdata, band_data])
imgdata *= 255
return imgdata.astype('uint8')
def transform_data(self, data, extent_mask, *masks):
hm_index_data = self._index_function(data)
dims = data[list(self.needed_bands)[0]].dims
imgdata = Dataset()
for band, map_func in [
("red", hm_index_to_red),
("green", hm_index_to_green),
("blue", hm_index_to_blue),
]:
f = numpy.vectorize(
hm_index_func_for_range(map_func, self.range[0],
self.range[1]))
img_band_raw_data = f(hm_index_data)
img_band_data = numpy.clip(img_band_raw_data * 255.0, 0,
255).astype("uint8")
imgdata[band] = (dims, img_band_data)
imgdata = imgdata.where(extent_mask)
if masks:
andmask = None
for mask in masks:
if andmask is None:
andmask = mask
else:
andmask = andmask & mask
if self.pq_mask_invert:
imgdata = imgdata.where(~andmask)
else:
imgdata = imgdata.where(andmask)
imgdata = imgdata.astype("uint8")
return imgdata
def _ml_standard_names(merged: xr.Dataset):
# since ML target is Q1/Q2, dQ1=Q1 and same for moistening
merged["dQ1"] = merged["Q1"]
merged["dQ2"] = merged["Q2"]
return merged.astype(np.float32)
def _one_hot_encode(
self, ds: xr.Dataset, legend: pd.DataFrame, variable: str
) -> xr.Dataset:
"""
Arguments:
---------
: ds: xr.Dataset
The dataset with the `variable` (str) we want to one
hot encode.
: legend: pd.DataFrame
Dataframe with the code / label_text columns acting as a
legend.
: variable: str
the variable in `ds` that we want to one hot encode.
"""
assert all(
np.isin(["code", "label_text"], legend.columns)
), "Need code / label_text columns in legend"
assert variable in list(
ds.data_vars
), f"expect {variable} to be in data vars: {list(ds.data_vars)}"
# ensure categorical is type int
# (prevents rounding errors)
ds = ds.astype("int")
for idx, row in legend.iterrows():
value, label = row["code"], row["label_text"]
ds[f"{label}_one_hot"] = (
ds[variable].where(ds[variable] == value, 0).clip(min=0, max=1)
)
ds = ds.drop(variable)
return ds
def transform_data(self, data, pq_data, extent_mask, *masks):
# pylint: disable=too-many-locals
# extent mask data per band to preseve nodata
_LOG.debug("transform begin %s", datetime.now())
if extent_mask is not None:
for band in data.data_vars:
try:
data[band] = data[band].where(
extent_mask, other=data[band].attrs['nodata'])
except AttributeError:
data[band] = data[band].where(extent_mask)
_LOG.debug("extent mask complete %d", datetime.now())
data = self.apply_masks(data, pq_data)
_LOG.debug("mask complete %d", datetime.now())
imgdata = Dataset()
for band in self.value_map.keys():
band_data = Dataset()
for value in self.value_map[band]:
target = Dataset()
flags = value["flags"]
rgb = Color(value["color"])
dims = data[band].dims
coords = data[band].coords
bdata = data[band]
colors = ["red", "green", "blue"]
for color in colors:
c = numpy.full(data[band].shape, getattr(rgb, color))
target[color] = DataArray(c, dims=dims, coords=coords)
if "or" in flags:
fs = flags["or"]
mask = None
for f in fs.items():
f = {f[0]: f[1]}
if mask is None:
mask = make_mask(bdata, **f)
else:
mask |= make_mask(bdata, **f)
else:
fs = flags if "and" not in flags else flags["and"]
mask = make_mask(bdata, **fs)
masked = target.where(mask)
if len(band_data.data_vars) == 0:
band_data = masked
else:
band_data = band_data.combine_first(masked)
if len(imgdata.data_vars) == 0:
imgdata = band_data
else:
imgdata = merge([imgdata, band_data])
imgdata *= 255
return imgdata.astype('uint8')
def calculate_flammability(
ds: xr.Dataset,
year: int = 2018,
diff: t.Optional[xr.DataArray] = None) -> xr.Dataset:
"""Add flammability variable to a dataset.
There are several reasons to leave the flammability out of tiles:
- redundant with mosaics but using storage
- requires baseline mean of tiles 2001-2015 (cyclic dependency)
- providing `diff` requires previous mosiac for timestep
"""
ds = ds.chunk(dict(time=1)).astype('float32')
masks = get_landcover_masks(year=year)
if diff is None:
diff = ds.lvmc_mean.diff('time')
anomaly = ds.lvmc_mean - get_mean_LMVC()
print('loaded flammability inputs ({})'.format(elapsed_time()))
# Calculate flammability and insert into dataset
# Note: Xarray now supports .where, so we could use higher level code here
flammability = dask.array.full(
shape=diff.shape,
fill_value=np.nan,
dtype='float32',
chunks=(1, ) + ds.lvmc_mean.shape[1:],
)
grass = 0.18 - 0.01 * ds.lvmc_mean + 0.02 * diff - 0.02 * anomaly
shrub = 5.66 - 0.09 * ds.lvmc_mean + 0.005 * diff - 0.28 * anomaly
forest = 1.51 - 0.03 * ds.lvmc_mean + 0.02 * diff - 0.02 * anomaly
print('calculated flammability components ({})'.format(elapsed_time()))
for msk, vals in [(masks.grass, grass), (masks.shrub, shrub),
(masks.forest, forest)]:
flammability = dask.array.where(msk.data, vals.data, flammability)
# Convert to [0..1] index with exponential equation
flammability = 1 / (1 + np.e**-flammability)
# We model this off the shape and coords of the diff, which may or may not
# include the first observation of the year, to avoid shape mismatch later
ds['flammability_index'] = xr.DataArray(
data=flammability,
coords=diff.coords,
dims=diff.dims,
name='flammability_index',
)
ds.flammability_index.attrs = dict(
comment='Unitless index of flammability',
units='unitless',
long_name='Flammability Index',
)
print('done with flammability ({})'.format(elapsed_time()))
return ds.astype('float32')
def transform_data(self, data, extent_mask, *masks):
hm_index_data = self._index_function(data)
hm_mask = hm_index_data != float("nan")
if masks:
for mask in masks:
data = data.where(mask)
dims = data[list(self.needed_bands)[0]].dims
imgdata = Dataset()
for band, map_func in [
("red", hm_index_to_red),
("green", hm_index_to_green),
("blue", hm_index_to_blue),
]:
components = self.components[band]
component_band_data = None
for c_band, c_intensity in components.items():
imgband_component_data = data[c_band] * c_intensity
if component_band_data is not None:
component_band_data += imgband_component_data
else:
component_band_data = imgband_component_data
f = numpy.vectorize(
hm_index_func_for_range(map_func,
self.range[0],
self.range[1],
nan_mask=False))
hmap_raw_data = f(hm_index_data)
unclipped_band_data = hmap_raw_data * 255.0 * (
1.0 - self.component_ratio
) + self.component_ratio / self.scale_factor * imgband_component_data
img_band_data = numpy.clip(unclipped_band_data, 1, 254) + 1
img_band_data = img_band_data.astype("uint8")
imgdata[band] = (dims, img_band_data)
imgdata = imgdata.where(extent_mask)
imgdata = imgdata.where(hm_mask)
if masks:
andmask = None
for mask in masks:
if andmask is None:
andmask = mask
else:
andmask = andmask & mask
if self.pq_mask_invert:
imgdata = imgdata.where(~andmask)
else:
imgdata = imgdata.where(andmask)
imgdata = imgdata.astype("uint8")
return imgdata
def transform_data(self, data, pq_data, extent_mask, *masks):
# pylint: disable=too-many-locals, too-many-branches
# extent mask data per band to preseve nodata
_LOG.debug("transform begin %s", datetime.now())
if extent_mask is not None:
for band in data.data_vars:
try:
data[band] = data[band].where(
extent_mask, other=data[band].attrs['nodata'])
except AttributeError:
data[band] = data[band].where(extent_mask)
_LOG.debug("extent mask complete %d", datetime.now())
data = self.apply_masks(data, pq_data)
_LOG.debug("mask complete %d", datetime.now())
imgdata = Dataset()
for cfg_band, values in self.value_map.items():
# Run through each item
band = self.product.band_idx.band(cfg_band)
band_data = Dataset()
bdata = data[band]
if bdata.dtype.kind == 'f':
# Convert back to int for bitmasking
bdata = RGBAMappedStyleDef.reint(bdata)
for value in values:
flags = value["flags"]
rgb = Color(value["color"])
alpha = value.get("alpha", 1.0)
mask_source_band = value.get("mask", False)
mask = RGBAMappedStyleDef.create_mask(bdata, flags)
if mask_source_band:
# disable checking on the use of ~mask
# pylint: disable=invalid-unary-operand-type
bdata = bdata.where(~mask)
bdata = RGBAMappedStyleDef.reint(bdata)
else:
masked = RGBAMappedStyleDef.create_colordata(
bdata, rgb, alpha, mask)
band_data = masked if len(
band_data.data_vars) == 0 else band_data.combine_first(
masked)
imgdata = band_data if len(imgdata.data_vars) == 0 else merge(
[imgdata, band_data])
imgdata *= 255
return imgdata.astype('uint8')
def transform_data(self, data, pq_data, extent_mask, *masks):
if extent_mask is not None:
data = data.where(extent_mask)
data = self.apply_masks(data, pq_data)
data_bands = self.needed_bands
if self.index_function is not None:
data_bands = ['index_function']
data[data_bands[0]] = (data.dims, self.index_function(data))
imgdata = Dataset()
for data_band in data_bands:
d = data[data_band]
for band, intensity in self.components.items():
imgdata[band] = (d.dims,
self.get_value(d, self.values, intensity))
imgdata *= 255
imgdata = imgdata.astype("uint8")
return imgdata
def transform_data(self, data, pq_data, extent_mask):
hm_index_data = self._index_function(data)
dims = data[list(self.needed_bands)[0]].dims
imgdata = Dataset()
for band, map_func in [
("red", hm_index_to_red),
("green", hm_index_to_green),
("blue", hm_index_to_blue),
]:
f = numpy.vectorize(
hm_index_func_for_range(map_func, self.range[0],
self.range[1]))
img_band_raw_data = f(hm_index_data)
img_band_data = numpy.clip(img_band_raw_data * 255.0, 0,
255).astype("uint8")
imgdata[band] = (dims, img_band_data)
imgdata = imgdata.where(extent_mask)
imgdata = self.apply_masks(imgdata, pq_data)
imgdata = imgdata.astype("uint8")
return imgdata
def transform_data(self, data, pq_data, extent_mask, *masks):
#pylint: disable=too-many-locals
hm_index_data = self._index_function(data)
hm_mask = hm_index_data != float("nan")
data = self.apply_masks(data, pq_data)
dims = data[list(self.needed_bands)[0]].dims
imgdata = Dataset()
for band, map_func in [
("red", hm_index_to_red),
("green", hm_index_to_green),
("blue", hm_index_to_blue),
]:
components = self.components[band]
component_band_data = None
for c_band, c_intensity in components.items():
imgband_component_data = data[c_band] * c_intensity
if component_band_data is not None:
component_band_data += imgband_component_data
else:
component_band_data = imgband_component_data
f = numpy.vectorize(
hm_index_func_for_range(map_func,
self.range[0],
self.range[1],
nan_mask=False))
hmap_raw_data = f(hm_index_data)
component_band_data = self.compress_band(component_band_data)
img_band_data = (hmap_raw_data * 255.0 *
(1.0 - self.component_ratio) +
self.component_ratio * component_band_data)
imgdata[band] = (dims, img_band_data.astype("uint8"))
if extent_mask is not None:
imgdata = imgdata.where(extent_mask)
imgdata = imgdata.where(hm_mask)
imgdata = self.apply_masks(imgdata, pq_data)
imgdata = imgdata.astype("uint8")
return imgdata
