def generate_derived_nbar(self, dataset_types, nbar, pqa, pqa_masks, overwrite=False):
for dataset_type in dataset_types:
filename = self.get_output_filename_derived_nbar(nbar, dataset_type)
_log.info("Generating data from [%s] with pq [%s] and pq mask [%s] to [%s]", nbar.path, pqa and pqa.path or "", pqa and pqa_masks or "", filename)
metadata = get_dataset_metadata(nbar)
data = None
if pqa:
data = get_dataset_data_with_pq(nbar, pqa, pq_masks=pqa_masks)
else:
data = get_dataset_data(nbar)
_log.debug("data is [%s]", data)
if dataset_type == DatasetType.NDVI:
ndvi = calculate_ndvi(data[nbar.bands.RED], data[nbar.bands.NEAR_INFRARED])
raster_create(filename, [ndvi], metadata.transform, metadata.projection, NDV, gdal.GDT_Float32)
elif dataset_type == DatasetType.EVI:
evi = calculate_evi(data[nbar.bands.RED], data[nbar.bands.BLUE], data[nbar.bands.NEAR_INFRARED])
raster_create(filename, [evi], metadata.transform, metadata.projection, NDV, gdal.GDT_Float32)
elif dataset_type == DatasetType.NBR:
nbr = calculate_nbr(data[nbar.bands.NEAR_INFRARED], data[nbar.bands.SHORT_WAVE_INFRARED_2])
raster_create(filename, [nbr], metadata.transform, metadata.projection, NDV, gdal.GDT_Float32)
def test_calculate_ndvi():
nbar = DatasetTile(satellite_id="LS5", type_id="ARG25", path="/data/tmp/cube/data/from.calum/LS5_TM_NBAR_150_-034_2004-01-13T23-22-17.088044.tif")
metadata = get_dataset_metadata(nbar)
band_data = calculate_ndvi(nbar)
raster_create("/data/tmp/cube/data/unit_test/LS5_TM_NDVI_150_-034_2004-01-13T23-22-17.088044.tif",
[band_data.filled(numpy.NaN)],
metadata.transform, metadata.projection, numpy.NaN, gdal.GDT_Float32)
def run(self):
shape = (4000, 4000)
no_data_value = NDV
best_pixel_fc = dict()
for band in Fc25Bands:
# best_pixel_fc[band] = empty_array(shape=shape, dtype=numpy.int16, ndv=INT16_MIN)
best_pixel_fc[band] = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
best_pixel_nbar = dict()
for band in Ls57Arg25Bands:
best_pixel_nbar[band] = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
best_pixel_satellite = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
best_pixel_date = empty_array(shape=shape, dtype=numpy.int32, ndv=NDV)
current_satellite = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
current_date = empty_array(shape=shape, dtype=numpy.int32, ndv=NDV)
SATELLITE_DATA_VALUES = {Satellite.LS5: 5, Satellite.LS7: 7, Satellite.LS8: 8}
metadata_nbar = None
metadata_fc = None
for tile in self.get_tiles():
pqa = tile.datasets[DatasetType.PQ25]
nbar = tile.datasets[DatasetType.ARG25]
fc = tile.datasets[DatasetType.FC25]
wofs = DatasetType.WATER in tile.datasets and tile.datasets[DatasetType.WATER] or None
_log.info("Processing [%s]", fc.path)
data = dict()
# Create an initial "no mask" mask
mask = numpy.ma.make_mask_none((4000, 4000))
# _log.info("### mask is [%s]", mask[1000][1000])
# Add the PQA mask if we are doing PQA masking
if self.mask_pqa_apply:
mask = get_mask_pqa(pqa, pqa_masks=self.mask_pqa_mask, mask=mask)
# _log.info("### mask PQA is [%s]", mask[1000][1000])
# Add the WOFS mask if we are doing WOFS masking
if self.mask_wofs_apply and wofs:
mask = get_mask_wofs(wofs, wofs_masks=self.mask_wofs_mask, mask=mask)
# _log.info("### mask PQA is [%s]", mask[1000][1000])
# Get NBAR dataset
data[DatasetType.ARG25] = get_dataset_data_masked(nbar, mask=mask)
# _log.info("### NBAR/RED is [%s]", data[DatasetType.ARG25][Ls57Arg25Bands.RED][1000][1000])
# Get the NDVI dataset
data[DatasetType.NDVI] = calculate_ndvi(data[DatasetType.ARG25][Ls57Arg25Bands.RED],
data[DatasetType.ARG25][Ls57Arg25Bands.NEAR_INFRARED])
# _log.info("### NDVI is [%s]", data[DatasetType.NDVI][1000][1000])
# Add the NDVI value range mask (to the existing mask)
mask = self.get_mask_range(data[DatasetType.NDVI], min_val=0.0, max_val=0.3, mask=mask)
# _log.info("### mask NDVI is [%s]", mask[1000][1000])
# Get FC25 dataset
data[DatasetType.FC25] = get_dataset_data_masked(fc, mask=mask)
# _log.info("### FC/BS is [%s]", data[DatasetType.FC25][Fc25Bands.BARE_SOIL][1000][1000])
# Add the bare soil value range mask (to the existing mask)
mask = self.get_mask_range(data[DatasetType.FC25][Fc25Bands.BARE_SOIL], min_val=0, max_val=8000, mask=mask)
# _log.info("### mask BS is [%s]", mask[1000][1000])
# Apply the final mask to the FC25 bare soil data
data_bare_soil = numpy.ma.MaskedArray(data=data[DatasetType.FC25][Fc25Bands.BARE_SOIL], mask=mask).filled(NDV)
# _log.info("### bare soil is [%s]", data_bare_soil[1000][1000])
# Compare the bare soil value from this dataset to the current "best" value
best_pixel_fc[Fc25Bands.BARE_SOIL] = numpy.fmax(best_pixel_fc[Fc25Bands.BARE_SOIL], data_bare_soil)
# _log.info("### best pixel bare soil is [%s]", best_pixel_fc[Fc25Bands.BARE_SOIL][1000][1000])
# Now update the other best pixel datasets/bands to grab the pixels we just selected
for band in Ls57Arg25Bands:
best_pixel_nbar[band] = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BARE_SOIL],
data_bare_soil,
data[DatasetType.ARG25][band],
best_pixel_nbar[band])
for band in [Fc25Bands.PHOTOSYNTHETIC_VEGETATION, Fc25Bands.NON_PHOTOSYNTHETIC_VEGETATION, Fc25Bands.UNMIXING_ERROR]:
best_pixel_fc[band] = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BARE_SOIL],
data_bare_soil,
data[DatasetType.FC25][band],
best_pixel_fc[band])
# And now the other "provenance" data
# Satellite "provenance" data
current_satellite.fill(SATELLITE_DATA_VALUES[fc.satellite])
best_pixel_satellite = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BARE_SOIL],
data_bare_soil,
current_satellite,
best_pixel_satellite)
# Date "provenance" data
current_date.fill(date_to_integer(tile.end_datetime))
best_pixel_date = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BARE_SOIL],
data_bare_soil,
current_date,
best_pixel_date)
# Grab the metadata from the input datasets for use later when creating the output datasets
if not metadata_nbar:
metadata_nbar = get_dataset_metadata(nbar)
if not metadata_fc:
metadata_fc = get_dataset_metadata(fc)
# Create the output datasets
# FC composite
raster_create(self.get_dataset_filename("FC"),
[best_pixel_fc[b] for b in Fc25Bands],
metadata_fc.transform, metadata_fc.projection,
metadata_fc.bands[Fc25Bands.BARE_SOIL].no_data_value,
metadata_fc.bands[Fc25Bands.BARE_SOIL].data_type)
# NBAR composite
raster_create(self.get_dataset_filename("NBAR"),
[best_pixel_nbar[b] for b in Ls57Arg25Bands],
metadata_nbar.transform, metadata_nbar.projection,
metadata_nbar.bands[Ls57Arg25Bands.BLUE].no_data_value,
metadata_nbar.bands[Ls57Arg25Bands.BLUE].data_type)
# Satellite "provenance" composites
raster_create(self.get_dataset_filename("SAT"),
[best_pixel_satellite],
metadata_nbar.transform, metadata_nbar.projection, no_data_value,
gdal.GDT_Int16)
# Date "provenance" composites
raster_create(self.get_dataset_filename("DATE"),
[best_pixel_date],
metadata_nbar.transform, metadata_nbar.projection, no_data_value,
gdal.GDT_Int32)
def doit(self):
_log.debug("Bare Soil Cell Task - doit()")
shape = (4000, 4000)
no_data_value = NDV
best_pixel_fc = dict()
for band in Fc25Bands:
best_pixel_fc[band] = empty_array(shape=shape, dtype=numpy.int16, ndv=INT16_MIN)
best_pixel_nbar = dict()
for band in Ls57Arg25Bands:
best_pixel_nbar[band] = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
best_pixel_satellite = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
best_pixel_year = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
best_pixel_month = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
best_pixel_epoch = empty_array(shape=shape, dtype=numpy.int32, ndv=NDV)
current_satellite = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
current_year = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
current_month = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
current_epoch = empty_array(shape=shape, dtype=numpy.int32, ndv=NDV)
SATELLITE_DATA_VALUES = {Satellite.LS5: 5, Satellite.LS7: 7, Satellite.LS8: 8}
metadata_nbar = None
metadata_fc = None
for tile in self.get_tiles():
# Get the PQ mask
pq = tile.datasets[DatasetType.PQ25]
data_pq = get_dataset_data(pq, [Pq25Bands.PQ])[Pq25Bands.PQ]
mask_pq = get_pq_mask(data_pq)
# Get NBAR dataset
nbar = tile.datasets[DatasetType.ARG25]
_log.info("Processing NBAR tile [%s]", nbar.path)
if not metadata_nbar:
metadata_nbar = get_dataset_metadata(nbar)
data_nbar = get_dataset_data_with_pq(nbar, Ls57Arg25Bands, tile.datasets[DatasetType.PQ25])
# Get the NDVI mask
red = data_nbar[Ls57Arg25Bands.RED]
nir = data_nbar[Ls57Arg25Bands.NEAR_INFRARED]
ndvi_data = calculate_ndvi(red, nir)
ndvi_data = numpy.ma.masked_equal(ndvi_data, NDV)
ndvi_data = numpy.ma.masked_outside(ndvi_data, 0, 0.3, copy=False)
mask_ndvi = ndvi_data.mask
# Get FC25 dataset
fc = tile.datasets[DatasetType.FC25]
_log.info("Processing FC tile [%s]", fc.path)
if not metadata_fc:
metadata_fc = get_dataset_metadata(fc)
_log.debug("metadata fc is %s", metadata_fc)
data_fc = get_dataset_data(fc, Fc25Bands)
data_bare_soil = data_fc[Fc25Bands.BS]
data_bare_soil = numpy.ma.masked_equal(data_bare_soil, -999)
data_bare_soil = numpy.ma.masked_outside(data_bare_soil, 0, 8000)
data_bare_soil.mask = (data_bare_soil.mask | mask_pq | mask_ndvi)
data_bare_soil = data_bare_soil.filled(NDV)
# Compare the bare soil value from this dataset to the current "best" value
best_pixel_fc[Fc25Bands.BS] = numpy.fmax(best_pixel_fc[Fc25Bands.BS], data_bare_soil)
# Now update the other best pixel datasets/bands to grab the pixels we just selected
for band in Ls57Arg25Bands:
best_pixel_nbar[band] = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BS],
data_bare_soil,
data_nbar[band],
best_pixel_nbar[band])
for band in [Fc25Bands.PV, Fc25Bands.NPV, Fc25Bands.ERROR]:
best_pixel_fc[band] = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BS],
data_bare_soil,
data_fc[band],
best_pixel_fc[band])
# And now the other "provenance" data
current_satellite.fill(SATELLITE_DATA_VALUES[fc.satellite])
best_pixel_satellite = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BS],
data_bare_soil,
current_satellite,
best_pixel_satellite)
current_year.fill(tile.end_datetime_year)
best_pixel_year = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BS],
data_bare_soil,
current_year,
best_pixel_year)
current_month.fill(tile.end_datetime_month)
best_pixel_month = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BS],
data_bare_soil,
current_month,
best_pixel_month)
current_epoch.fill(calendar.timegm(tile.end_datetime.timetuple()))
best_pixel_epoch = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BS],
data_bare_soil,
current_epoch,
best_pixel_epoch)
# Create the output datasets
# FC composite
raster_create(self.get_dataset_filename("FC"),
[best_pixel_fc[b] for b in Fc25Bands],
metadata_fc.transform, metadata_fc.projection, metadata_fc.bands[Fc25Bands.BS].no_data_value,
metadata_fc.bands[Fc25Bands.BS].data_type)
# NBAR composite
raster_create(self.get_dataset_filename("NBAR"),
[best_pixel_nbar[b] for b in Ls57Arg25Bands],
metadata_nbar.transform, metadata_nbar.projection,
metadata_nbar.bands[Ls57Arg25Bands.BLUE].no_data_value,
metadata_nbar.bands[Ls57Arg25Bands.BLUE].data_type)
# "Provenance" composites
raster_create(self.get_dataset_filename("SAT"),
[best_pixel_satellite],
metadata_nbar.transform, metadata_nbar.projection, no_data_value,
gdal.GDT_Int16)
raster_create(self.get_dataset_filename("YEAR"),
[best_pixel_year],
metadata_nbar.transform, metadata_nbar.projection, no_data_value,
gdal.GDT_Int16)
raster_create(self.get_dataset_filename("MONTH"),
[best_pixel_month],
metadata_nbar.transform, metadata_nbar.projection, no_data_value,
gdal.GDT_Int16)
raster_create(self.get_dataset_filename("EPOCH"),
[best_pixel_epoch],
metadata_nbar.transform, metadata_nbar.projection, no_data_value,
gdal.GDT_Int32)
def classifier(arg25_dataset, pq25_dataset):
"""
Runs the classifier designed by SF.
"""
# Get the metadata
md = get_dataset_metadata(arg25_dataset)
cols, rows = md.shape
# Read the data and mask pixels via the PQ dataset
data = get_dataset_data_with_pq(arg25_dataset, pq25_dataset)
# Get the wetness coefficients and calculate
coef = TCI_COEFFICIENTS[arg25_dataset.satellite][TasselCapIndex.WETNESS]
wetness = calculate_tassel_cap_index(data, coef)
# NDVI
ndvi = calculate_ndvi(data[arg25_dataset.bands.RED],
data[arg25_dataset.bands.NEAR_INFRARED],
output_ndv=numpy.nan)
# Dump the reflectance data, the classifier only needs tc_wetness and ndvi
del data
# Allocate the result
classified = numpy.zeros((rows, cols), dtype='uint8')
# Water
r1 = wetness > 0
classified[r1] = 1
_tmp = ~r1
#r2 = _tmp & ((wetness >= -250) & (wetness < 0))
r2 = (wetness >= -250) & (wetness < 0)
r3 = ndvi <= 0.3
#_tmp2 = _tmp & r2 & ~r3
_tmp2 = _tmp & r2
# non-veg
classified[_tmp2 & r3] = 2
_tmp3 = _tmp2 & ~r3
r4 = ndvi <= 0.45
# saltmarsh
classified[_tmp3 & r4] = 3
_tmp2 = _tmp3 & ~r4
r5 = ndvi <= 0.6
# mangrove/saltmarsh
classified[_tmp2 & r5] = 4
# mangrove
classified[_tmp2 & ~r5] = 5
# finished rhs of r2
_tmp2 = _tmp & ~r2
r6 = wetness < -750
r7 = ndvi >= 0.3
_tmp3 = _tmp2 & r6
# saltmarsh
classified[_tmp3 & r7] = 3
# non-veg
classified[_tmp3 & ~r7] = 2
r8 = ndvi <= 0.3
_tmp3 = _tmp2 & ~r6
# non-veg
classified[_tmp3 & r8] = 2
r9 = ndvi <= 0.45
_tmp2 = _tmp3 & ~r8
# saltmarsh
classified[_tmp2 & r9] = 3
r10 = ndvi <= 0.6
_tmp3 = _tmp2 & ~r9
# mangrove-saltmarsh
classified[_tmp3 & r10] = 4
# mangrove
classified[_tmp3 & ~r10] = 5
# set any nulls
valid = numpy.isfinite(ndvi)
classified[~valid] = 0
return classified
def classifier(arg25_dataset, pq25_dataset):
"""
Runs the classifier designed by SF.
"""
# Get the metadata
md = get_dataset_metadata(arg25_dataset)
cols, rows = md.shape
# Read the data and mask pixels via the PQ dataset
data = get_dataset_data_with_pq(arg25_dataset, pq25_dataset)
# Get the wetness coefficients and calculate
coef = TCI_COEFFICIENTS[arg25_dataset.satellite][TasselCapIndex.WETNESS]
wetness = calculate_tassel_cap_index(data, coef)
# NDVI
ndvi = calculate_ndvi(data[arg25_dataset.bands.RED],
data[arg25_dataset.bands.NEAR_INFRARED],
output_ndv=numpy.nan)
# Dump the reflectance data, the classifier only needs tc_wetness and ndvi
del data
# Allocate the result
classified = numpy.zeros((rows,cols), dtype='uint8')
# Water
r1 = wetness > 0
classified[r1] = 1
_tmp = ~r1
#r2 = _tmp & ((wetness >= -250) & (wetness < 0))
r2 = (wetness >= -250) & (wetness < 0)
r3 = ndvi <= 0.3
#_tmp2 = _tmp & r2 & ~r3
_tmp2 = _tmp & r2
# non-veg
classified[_tmp2 & r3] = 2
_tmp3 = _tmp2 & ~r3
r4 = ndvi <= 0.45
# saltmarsh
classified[_tmp3 & r4] = 3
_tmp2 = _tmp3 & ~r4
r5 = ndvi <= 0.6
# mangrove/saltmarsh
classified[_tmp2 & r5] = 4
# mangrove
classified[_tmp2 & ~r5] = 5
# finished rhs of r2
_tmp2 = _tmp & ~r2
r6 = wetness < -750
r7 = ndvi >= 0.3
_tmp3 = _tmp2 & r6
# saltmarsh
classified[_tmp3 & r7] = 3
# non-veg
classified[_tmp3 & ~r7] = 2
r8 = ndvi <= 0.3
_tmp3 = _tmp2 & ~r6
# non-veg
classified[_tmp3 & r8] = 2
r9 = ndvi <= 0.45
_tmp2 = _tmp3 & ~r8
# saltmarsh
classified[_tmp2 & r9] = 3
r10 = ndvi <= 0.6
_tmp3 = _tmp2 & ~r9
# mangrove-saltmarsh
classified[_tmp3 & r10] = 4
# mangrove
classified[_tmp3 & ~r10] = 5
# set any nulls
valid = numpy.isfinite(ndvi)
classified[~valid] = 0
return classified
