def transform_single_date_data(self, data): # pylint: disable=too-many-locals, too-many-branches # extent mask data per band to preseve nodata _LOG.debug("transform begin %s", datetime.now()) #if extent_mask is not None: # for band in data.data_vars: ## try: # data[band] = data[band].where(extent_mask, other=data[band].attrs['nodata']) # except AttributeError: # data[band] = data[band].where(extent_mask) imgdata = Dataset(coords=data.coords) for cfg_band, rules in self.value_map.items(): # Run through each item band = self.product.band_idx.band(cfg_band) band_data = Dataset() bdata = data[band] if bdata.dtype.kind == 'f': # Convert back to int for bitmasking bdata = ColorMapStyleDef.reint(bdata) for rule in rules: mask = rule.create_mask(bdata) masked = ColorMapStyleDef.create_colordata( bdata, rule.rgb, rule.alpha, mask) band_data = masked if len( band_data.data_vars) == 0 else band_data.combine_first( masked) imgdata = band_data if len(imgdata.data_vars) == 0 else merge( [imgdata, band_data]) imgdata *= 255 return imgdata.astype('uint8')
def transform_data(self, data, extent_mask, *masks): hm_index_data = self._index_function(data) dims = data[list(self.needed_bands)[0]].dims imgdata = Dataset() for band, map_func in [ ("red", hm_index_to_red), ("green", hm_index_to_green), ("blue", hm_index_to_blue), ]: f = numpy.vectorize( hm_index_func_for_range(map_func, self.range[0], self.range[1])) img_band_raw_data = f(hm_index_data) img_band_data = numpy.clip(img_band_raw_data * 255.0, 0, 255).astype("uint8") imgdata[band] = (dims, img_band_data) imgdata = imgdata.where(extent_mask) if masks: andmask = None for mask in masks: if andmask is None: andmask = mask else: andmask = andmask & mask if self.pq_mask_invert: imgdata = imgdata.where(~andmask) else: imgdata = imgdata.where(andmask) imgdata = imgdata.astype("uint8") return imgdata
def _ml_standard_names(merged: xr.Dataset): # since ML target is Q1/Q2, dQ1=Q1 and same for moistening merged["dQ1"] = merged["Q1"] merged["dQ2"] = merged["Q2"] return merged.astype(np.float32)
def _one_hot_encode( self, ds: xr.Dataset, legend: pd.DataFrame, variable: str ) -> xr.Dataset: """ Arguments: --------- : ds: xr.Dataset The dataset with the `variable` (str) we want to one hot encode. : legend: pd.DataFrame Dataframe with the code / label_text columns acting as a legend. : variable: str the variable in `ds` that we want to one hot encode. """ assert all( np.isin(["code", "label_text"], legend.columns) ), "Need code / label_text columns in legend" assert variable in list( ds.data_vars ), f"expect {variable} to be in data vars: {list(ds.data_vars)}" # ensure categorical is type int # (prevents rounding errors) ds = ds.astype("int") for idx, row in legend.iterrows(): value, label = row["code"], row["label_text"] ds[f"{label}_one_hot"] = ( ds[variable].where(ds[variable] == value, 0).clip(min=0, max=1) ) ds = ds.drop(variable) return ds
def transform_data(self, data, pq_data, extent_mask, *masks): # pylint: disable=too-many-locals # extent mask data per band to preseve nodata _LOG.debug("transform begin %s", datetime.now()) if extent_mask is not None: for band in data.data_vars: try: data[band] = data[band].where( extent_mask, other=data[band].attrs['nodata']) except AttributeError: data[band] = data[band].where(extent_mask) _LOG.debug("extent mask complete %d", datetime.now()) data = self.apply_masks(data, pq_data) _LOG.debug("mask complete %d", datetime.now()) imgdata = Dataset() for band in self.value_map.keys(): band_data = Dataset() for value in self.value_map[band]: target = Dataset() flags = value["flags"] rgb = Color(value["color"]) dims = data[band].dims coords = data[band].coords bdata = data[band] colors = ["red", "green", "blue"] for color in colors: c = numpy.full(data[band].shape, getattr(rgb, color)) target[color] = DataArray(c, dims=dims, coords=coords) if "or" in flags: fs = flags["or"] mask = None for f in fs.items(): f = {f[0]: f[1]} if mask is None: mask = make_mask(bdata, **f) else: mask |= make_mask(bdata, **f) else: fs = flags if "and" not in flags else flags["and"] mask = make_mask(bdata, **fs) masked = target.where(mask) if len(band_data.data_vars) == 0: band_data = masked else: band_data = band_data.combine_first(masked) if len(imgdata.data_vars) == 0: imgdata = band_data else: imgdata = merge([imgdata, band_data]) imgdata *= 255 return imgdata.astype('uint8')
def calculate_flammability( ds: xr.Dataset, year: int = 2018, diff: t.Optional[xr.DataArray] = None) -> xr.Dataset: """Add flammability variable to a dataset. There are several reasons to leave the flammability out of tiles: - redundant with mosaics but using storage - requires baseline mean of tiles 2001-2015 (cyclic dependency) - providing `diff` requires previous mosiac for timestep """ ds = ds.chunk(dict(time=1)).astype('float32') masks = get_landcover_masks(year=year) if diff is None: diff = ds.lvmc_mean.diff('time') anomaly = ds.lvmc_mean - get_mean_LMVC() print('loaded flammability inputs ({})'.format(elapsed_time())) # Calculate flammability and insert into dataset # Note: Xarray now supports .where, so we could use higher level code here flammability = dask.array.full( shape=diff.shape, fill_value=np.nan, dtype='float32', chunks=(1, ) + ds.lvmc_mean.shape[1:], ) grass = 0.18 - 0.01 * ds.lvmc_mean + 0.02 * diff - 0.02 * anomaly shrub = 5.66 - 0.09 * ds.lvmc_mean + 0.005 * diff - 0.28 * anomaly forest = 1.51 - 0.03 * ds.lvmc_mean + 0.02 * diff - 0.02 * anomaly print('calculated flammability components ({})'.format(elapsed_time())) for msk, vals in [(masks.grass, grass), (masks.shrub, shrub), (masks.forest, forest)]: flammability = dask.array.where(msk.data, vals.data, flammability) # Convert to [0..1] index with exponential equation flammability = 1 / (1 + np.e**-flammability) # We model this off the shape and coords of the diff, which may or may not # include the first observation of the year, to avoid shape mismatch later ds['flammability_index'] = xr.DataArray( data=flammability, coords=diff.coords, dims=diff.dims, name='flammability_index', ) ds.flammability_index.attrs = dict( comment='Unitless index of flammability', units='unitless', long_name='Flammability Index', ) print('done with flammability ({})'.format(elapsed_time())) return ds.astype('float32')
def transform_data(self, data, extent_mask, *masks): hm_index_data = self._index_function(data) hm_mask = hm_index_data != float("nan") if masks: for mask in masks: data = data.where(mask) dims = data[list(self.needed_bands)[0]].dims imgdata = Dataset() for band, map_func in [ ("red", hm_index_to_red), ("green", hm_index_to_green), ("blue", hm_index_to_blue), ]: components = self.components[band] component_band_data = None for c_band, c_intensity in components.items(): imgband_component_data = data[c_band] * c_intensity if component_band_data is not None: component_band_data += imgband_component_data else: component_band_data = imgband_component_data f = numpy.vectorize( hm_index_func_for_range(map_func, self.range[0], self.range[1], nan_mask=False)) hmap_raw_data = f(hm_index_data) unclipped_band_data = hmap_raw_data * 255.0 * ( 1.0 - self.component_ratio ) + self.component_ratio / self.scale_factor * imgband_component_data img_band_data = numpy.clip(unclipped_band_data, 1, 254) + 1 img_band_data = img_band_data.astype("uint8") imgdata[band] = (dims, img_band_data) imgdata = imgdata.where(extent_mask) imgdata = imgdata.where(hm_mask) if masks: andmask = None for mask in masks: if andmask is None: andmask = mask else: andmask = andmask & mask if self.pq_mask_invert: imgdata = imgdata.where(~andmask) else: imgdata = imgdata.where(andmask) imgdata = imgdata.astype("uint8") return imgdata
def transform_data(self, data, pq_data, extent_mask, *masks): # pylint: disable=too-many-locals, too-many-branches # extent mask data per band to preseve nodata _LOG.debug("transform begin %s", datetime.now()) if extent_mask is not None: for band in data.data_vars: try: data[band] = data[band].where( extent_mask, other=data[band].attrs['nodata']) except AttributeError: data[band] = data[band].where(extent_mask) _LOG.debug("extent mask complete %d", datetime.now()) data = self.apply_masks(data, pq_data) _LOG.debug("mask complete %d", datetime.now()) imgdata = Dataset() for cfg_band, values in self.value_map.items(): # Run through each item band = self.product.band_idx.band(cfg_band) band_data = Dataset() bdata = data[band] if bdata.dtype.kind == 'f': # Convert back to int for bitmasking bdata = RGBAMappedStyleDef.reint(bdata) for value in values: flags = value["flags"] rgb = Color(value["color"]) alpha = value.get("alpha", 1.0) mask_source_band = value.get("mask", False) mask = RGBAMappedStyleDef.create_mask(bdata, flags) if mask_source_band: # disable checking on the use of ~mask # pylint: disable=invalid-unary-operand-type bdata = bdata.where(~mask) bdata = RGBAMappedStyleDef.reint(bdata) else: masked = RGBAMappedStyleDef.create_colordata( bdata, rgb, alpha, mask) band_data = masked if len( band_data.data_vars) == 0 else band_data.combine_first( masked) imgdata = band_data if len(imgdata.data_vars) == 0 else merge( [imgdata, band_data]) imgdata *= 255 return imgdata.astype('uint8')
def transform_data(self, data, pq_data, extent_mask, *masks): if extent_mask is not None: data = data.where(extent_mask) data = self.apply_masks(data, pq_data) data_bands = self.needed_bands if self.index_function is not None: data_bands = ['index_function'] data[data_bands[0]] = (data.dims, self.index_function(data)) imgdata = Dataset() for data_band in data_bands: d = data[data_band] for band, intensity in self.components.items(): imgdata[band] = (d.dims, self.get_value(d, self.values, intensity)) imgdata *= 255 imgdata = imgdata.astype("uint8") return imgdata
def transform_data(self, data, pq_data, extent_mask): hm_index_data = self._index_function(data) dims = data[list(self.needed_bands)[0]].dims imgdata = Dataset() for band, map_func in [ ("red", hm_index_to_red), ("green", hm_index_to_green), ("blue", hm_index_to_blue), ]: f = numpy.vectorize( hm_index_func_for_range(map_func, self.range[0], self.range[1])) img_band_raw_data = f(hm_index_data) img_band_data = numpy.clip(img_band_raw_data * 255.0, 0, 255).astype("uint8") imgdata[band] = (dims, img_band_data) imgdata = imgdata.where(extent_mask) imgdata = self.apply_masks(imgdata, pq_data) imgdata = imgdata.astype("uint8") return imgdata
def transform_data(self, data, pq_data, extent_mask, *masks): #pylint: disable=too-many-locals hm_index_data = self._index_function(data) hm_mask = hm_index_data != float("nan") data = self.apply_masks(data, pq_data) dims = data[list(self.needed_bands)[0]].dims imgdata = Dataset() for band, map_func in [ ("red", hm_index_to_red), ("green", hm_index_to_green), ("blue", hm_index_to_blue), ]: components = self.components[band] component_band_data = None for c_band, c_intensity in components.items(): imgband_component_data = data[c_band] * c_intensity if component_band_data is not None: component_band_data += imgband_component_data else: component_band_data = imgband_component_data f = numpy.vectorize( hm_index_func_for_range(map_func, self.range[0], self.range[1], nan_mask=False)) hmap_raw_data = f(hm_index_data) component_band_data = self.compress_band(component_band_data) img_band_data = (hmap_raw_data * 255.0 * (1.0 - self.component_ratio) + self.component_ratio * component_band_data) imgdata[band] = (dims, img_band_data.astype("uint8")) if extent_mask is not None: imgdata = imgdata.where(extent_mask) imgdata = imgdata.where(hm_mask) imgdata = self.apply_masks(imgdata, pq_data) imgdata = imgdata.astype("uint8") return imgdata