def get_band_name_intersection(dataset_type, satellites):
bands = [b.name for b in get_bands(dataset_type, satellites[0])]
for satellite in satellites[1:]:
for band in bands:
if band not in [b.name for b in get_bands(dataset_type, satellite)]:
bands.remove(band)
return bands
def get_band_name_union(dataset_type, satellites):
bands = [b.name for b in get_bands(dataset_type, satellites[0])]
for satellite in satellites[1:]:
for b in get_bands(dataset_type, satellite):
if b.name not in bands:
bands.append(b.name)
return bands
def process_arguments(self, args):
# Call method on super class
# super(self.__class__, self).process_arguments(args)
workflow.Workflow.process_arguments(self, args)
self.dataset_type = args.dataset_type
self.bands = args.bands
# Verify that all the requested satellites have the requested bands
for satellite in self.satellites:
if not all(item in [b.name for b in get_bands(self.dataset_type, satellite)] for item in self.bands):
_log.error("Requested bands [%s] not ALL present for satellite [%s]", self.bands, satellite)
raise Exception("Not all bands present for all satellites")
self.chunk_size_x = args.chunk_size_x
self.chunk_size_y = args.chunk_size_y
def process_arguments(self, args):
# Call method on super class
# super(self.__class__, self).process_arguments(args)
workflow.Workflow.process_arguments(self, args)
self.dataset_type = args.dataset_type
self.bands = args.bands
# Verify that all the requested satellites have the requested bands
for satellite in self.satellites:
if not all(
item in
[b.name for b in get_bands(self.dataset_type, satellite)]
for item in self.bands):
_log.error(
"Requested bands [%s] not ALL present for satellite [%s]",
self.bands, satellite)
raise Exception("Not all bands present for all satellites")
self.chunk_size_x = args.chunk_size_x
self.chunk_size_y = args.chunk_size_y
def run(self):
self.parse_arguments()
config = Config()
_log.debug(config.to_str())
path = self.get_output_filename(self.dataset_type)
_log.info("Output file is [%s]", path)
if os.path.exists(path):
if self.overwrite:
_log.info("Removing existing output file [%s]", path)
os.remove(path)
else:
_log.error("Output file [%s] exists", path)
raise Exception("Output file [%s] already exists" % path)
# TODO
bands = get_bands(self.dataset_type, self.satellites[0])
# TODO once WOFS is in the cube
tiles = list_tiles_as_list(x=[self.x], y=[self.y], acq_min=self.acq_min, acq_max=self.acq_max,
satellites=[satellite for satellite in self.satellites],
dataset_types=[self.dataset_type],
database=config.get_db_database(),
user=config.get_db_username(),
password=config.get_db_password(),
host=config.get_db_host(), port=config.get_db_port())
raster = None
metadata = None
# TODO - PQ is UNIT16 (others are INT16) and so -999 NDV doesn't work
ndv = self.dataset_type == DatasetType.PQ25 and UINT16_MAX or NDV
_log.debug("Current MAX RSS usage is [%d] MB", resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024)
import itertools
for x, y in itertools.product(range(0, 4000, self.chunk_size_x), range(0, 4000, self.chunk_size_y)):
_log.info("About to read data chunk ({xmin:4d},{ymin:4d}) to ({xmax:4d},{ymax:4d})".format(xmin=x, ymin=y, xmax=x+self.chunk_size_x-1, ymax=y+self.chunk_size_y-1))
_log.debug("Current MAX RSS usage is [%d] MB", resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024)
stack = dict()
for tile in tiles:
if self.list_only:
_log.info("Would summarise dataset [%s]", tile.datasets[self.dataset_type].path)
continue
pqa = None
_log.debug("Reading dataset [%s]", tile.datasets[self.dataset_type].path)
if not metadata:
metadata = get_dataset_metadata(tile.datasets[self.dataset_type])
# Apply PQA if specified
if self.apply_pqa_filter:
data = get_dataset_data_with_pq(tile.datasets[self.dataset_type], tile.datasets[DatasetType.PQ25], bands=bands, x=x, y=y, x_size=self.chunk_size_x, y_size=self.chunk_size_y, pq_masks=self.pqa_mask, ndv=ndv)
else:
data = get_dataset_data(tile.datasets[self.dataset_type], bands=bands, x=x, y=y, x_size=self.chunk_size_x, y_size=self.chunk_size_y)
for band in bands:
if band in stack:
stack[band].append(data[band])
else:
stack[band] = [data[band]]
_log.debug("data[%s] has shape [%s] and MB [%s]", band.name, numpy.shape(data[band]), data[band].nbytes/1000/1000)
_log.debug("stack[%s] has [%s] elements", band.name, len(stack[band]))
# Apply summary method
_log.info("Finished reading {count} datasets for chunk ({xmin:4d},{ymin:4d}) to ({xmax:4d},{ymax:4d}) - about to summarise them".format(count=len(tiles), xmin=x, ymin=y, xmax=x+self.chunk_size_x-1, ymax=y+self.chunk_size_y-1))
_log.debug("Current MAX RSS usage is [%d] MB", resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024)
masked_stack = dict()
for band in bands:
masked_stack[band] = numpy.ma.masked_equal(stack[band], ndv)
_log.debug("masked_stack[%s] is %s", band.name, masked_stack[band])
_log.debug("masked stack[%s] has shape [%s] and MB [%s]", band.name, numpy.shape(masked_stack[band]), masked_stack[band].nbytes/1000/1000)
_log.debug("Current MAX RSS usage is [%d] MB", resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024)
if self.summary_method == TimeSeriesSummaryMethod.MIN:
masked_summary = numpy.min(masked_stack[band], axis=0)
elif self.summary_method == TimeSeriesSummaryMethod.MAX:
masked_summary = numpy.max(masked_stack[band], axis=0)
elif self.summary_method == TimeSeriesSummaryMethod.MEAN:
masked_summary = numpy.mean(masked_stack[band], axis=0)
elif self.summary_method == TimeSeriesSummaryMethod.MEDIAN:
masked_summary = numpy.median(masked_stack[band], axis=0)
# aka 50th percentile
elif self.summary_method == TimeSeriesSummaryMethod.MEDIAN_NON_INTERPOLATED:
masked_sorted = numpy.ma.sort(masked_stack[band], axis=0)
masked_percentile_index = numpy.ma.floor(numpy.ma.count(masked_sorted, axis=0) * 0.95).astype(numpy.int16)
masked_summary = numpy.ma.choose(masked_percentile_index, masked_sorted)
elif self.summary_method == TimeSeriesSummaryMethod.COUNT:
# TODO Need to artificially create masked array here since it is being expected/filled below!!!
masked_summary = numpy.ma.masked_equal(masked_stack[band].count(axis=0), ndv)
elif self.summary_method == TimeSeriesSummaryMethod.SUM:
masked_summary = numpy.sum(masked_stack[band], axis=0)
elif self.summary_method == TimeSeriesSummaryMethod.STANDARD_DEVIATION:
masked_summary = numpy.std(masked_stack[band], axis=0)
elif self.summary_method == TimeSeriesSummaryMethod.VARIANCE:
masked_summary = numpy.var(masked_stack[band], axis=0)
# currently 95th percentile
elif self.summary_method == TimeSeriesSummaryMethod.PERCENTILE:
masked_sorted = numpy.ma.sort(masked_stack[band], axis=0)
masked_percentile_index = numpy.ma.floor(numpy.ma.count(masked_sorted, axis=0) * 0.95).astype(numpy.int16)
masked_summary = numpy.ma.choose(masked_percentile_index, masked_sorted)
elif self.summary_method == TimeSeriesSummaryMethod.YOUNGEST_PIXEL:
# TODO the fact that this is band at a time might be problematic. We really should be considering
# all bands at once (that is what the landsat_mosaic logic did). If PQA is being applied then
# it's probably all good but if not then we might get odd results....
masked_summary = empty_array(shape=(self.chunk_size_x, self.chunk_size_x), dtype=numpy.int16, ndv=ndv)
# Note the reversed as the stack is created oldest first
for d in reversed(stack[band]):
masked_summary = numpy.where(masked_summary == ndv, d, masked_summary)
# If the summary doesn't contain an no data values then we can stop
if not numpy.any(masked_summary == ndv):
break
# TODO Need to artificially create masked array here since it is being expected/filled below!!!
masked_summary = numpy.ma.masked_equal(masked_summary, ndv)
elif self.summary_method == TimeSeriesSummaryMethod.OLDEST_PIXEL:
# TODO the fact that this is band at a time might be problematic. We really should be considering
# all bands at once (that is what the landsat_mosaic logic did). If PQA is being applied then
# it's probably all good but if not then we might get odd results....
masked_summary = empty_array(shape=(self.chunk_size_x, self.chunk_size_x), dtype=numpy.int16, ndv=ndv)
# Note the NOT reversed as the stack is created oldest first
for d in stack[band]:
masked_summary = numpy.where(masked_summary == ndv, d, masked_summary)
# If the summary doesn't contain an no data values then we can stop
if not numpy.any(masked_summary == ndv):
break
# TODO Need to artificially create masked array here since it is being expected/filled below!!!
masked_summary = numpy.ma.masked_equal(masked_summary, ndv)
masked_stack[band] = None
_log.debug("NONE-ing masked stack[%s]", band.name)
_log.debug("Current MAX RSS usage is [%d] MB", resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024)
_log.debug("masked summary is [%s]", masked_summary)
_log.debug("Current MAX RSS usage is [%d] MB", resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024)
# Create the output file
if not os.path.exists(path):
_log.info("Creating raster [%s]", path)
driver = gdal.GetDriverByName("GTiff")
assert driver
raster = driver.Create(path, metadata.shape[0], metadata.shape[1], len(bands), gdal.GDT_Int16)
assert raster
raster.SetGeoTransform(metadata.transform)
raster.SetProjection(metadata.projection)
for b in bands:
raster.GetRasterBand(b.value).SetNoDataValue(ndv)
_log.info("Writing band [%s] data to raster [%s]", band.name, path)
_log.debug("Current MAX RSS usage is [%d] MB", resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024)
raster.GetRasterBand(band.value).WriteArray(masked_summary.filled(ndv), xoff=x, yoff=y)
raster.GetRasterBand(band.value).ComputeStatistics(True)
raster.FlushCache()
masked_summary = None
_log.debug("NONE-ing the masked summary")
_log.debug("Current MAX RSS usage is [%d] MB", resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024)
stack = None
_log.debug("Just NONE-ed the stack")
_log.debug("Current MAX RSS usage is [%d] MB", resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024)
raster = None
_log.debug("Just NONE'd the raster")
_log.debug("Current MAX RSS usage is [%d] MB", resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024)
_log.info("Memory usage was [%d MB]", resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024)
_log.info("CPU time used [%s]", timedelta(seconds=int(resource.getrusage(resource.RUSAGE_SELF).ru_utime)))
def get_dataset_data(dataset, bands=None, x=0, y=0, x_size=None, y_size=None):
# dataset_types_physical = [
# DatasetType.ARG25, DatasetType.PQ25, DatasetType.FC25,
# DatasetType.WATER,
# DatasetType.DSM, DatasetType.DEM, DatasetType.DEM_HYDROLOGICALLY_ENFORCED, DatasetType.DEM_SMOOTHED]
#
# dataset_types_virtual_nbar = [
# DatasetType.NDVI,
# DatasetType.EVI,
# DatasetType.NBR,
# DatasetType.TCI
# ]
# NDVI calculated using RED and NIR from ARG25
if dataset.dataset_type == DatasetType.NDVI:
bands = get_bands(DatasetType.ARG25, dataset.satellite)
band_red = bands[Ls57Arg25Bands.RED.name]
band_nir = bands[Ls57Arg25Bands.NEAR_INFRARED.name]
data = read_dataset_data(dataset, bands=[band_red, band_nir], x=x, y=y, x_size=x_size, y_size=y_size)
data = calculate_ndvi(data[band_red], data[band_nir])
return {NdviBands.NDVI: data}
# EVI calculated using RED, BLUE and NIR from ARG25
elif dataset.dataset_type == DatasetType.EVI:
bands = get_bands(DatasetType.ARG25, dataset.satellite)
band_red = bands[Ls57Arg25Bands.RED.name]
band_blue = bands[Ls57Arg25Bands.BLUE.name]
band_nir = bands[Ls57Arg25Bands.NEAR_INFRARED.name]
data = read_dataset_data(dataset, bands=[band_red, band_blue, band_nir], x=x, y=y, x_size=x_size, y_size=y_size)
data = calculate_evi(data[band_red], data[band_blue], data[band_nir])
return {EviBands.EVI: data}
# NBR calculated using NIR and SWIR-2 from ARG25
elif dataset.dataset_type == DatasetType.NBR:
bands = get_bands(DatasetType.ARG25, dataset.satellite)
band_nir = bands[Ls57Arg25Bands.NEAR_INFRARED.name]
band_swir = bands[Ls57Arg25Bands.SHORT_WAVE_INFRARED_2.name]
data = read_dataset_data(dataset, bands=[band_nir, band_swir], x=x, y=y, x_size=x_size, y_size=y_size)
data = calculate_nbr(data[band_nir], data[band_swir])
return {NbrBands.NBR: data}
# TCI calculated from ARG25
elif dataset.dataset_type == DatasetType.TCI:
bands = get_bands(DatasetType.ARG25, dataset.satellite)
data = read_dataset_data(dataset, bands=bands, x=x, y=y, x_size=x_size, y_size=y_size)
out = dict()
for index in TasselCapIndex:
out[TciBands[index.name]] = calculate_tassel_cap_index(data, TCI_COEFFICIENTS[dataset.satellite][index])
return out
# It is a "physical" dataset so just read it
else:
return read_dataset_data(dataset, bands, x, y, x_size, y_size)
def go(self):
import numpy
from datacube.api.query import list_cells_as_list, list_tiles_as_list
from datacube.config import Config
# Verify that all the requested satellites have the same band combinations
dataset_bands = get_bands(self.dataset_type, self.satellites[0])
_log.info("dataset bands is [%s]", " ".join([b.name for b in dataset_bands]))
for satellite in self.satellites:
if dataset_bands != get_bands(self.dataset_type, satellite):
_log.error("Satellites [%s] have differing bands", " ".join([satellite.name for satellite in self.satellites]))
raise Exception("Satellites with different band combinations selected")
bands = []
dataset_bands_list = list(dataset_bands)
if not self.bands:
bands = dataset_bands_list
else:
for b in self.bands:
bands.append(dataset_bands_list[b - 1])
_log.info("Using bands [%s]", " ".join(band.name for band in bands))
x_min, x_max, y_max, y_min = self.extract_bounds_from_vector()
_log.debug("The bounds are [%s]", (x_min, x_max, y_min, y_max))
cells_vector = self.extract_cells_from_vector()
_log.debug("Intersecting cells_vector are [%d] [%s]", len(cells_vector), cells_vector)
config = Config(os.path.expanduser("~/.datacube/config"))
_log.debug(config.to_str())
x_list = range(x_min, x_max + 1)
y_list = range(y_min, y_max + 1)
_log.debug("x = [%s] y=[%s]", x_list, y_list)
cells_db = list()
for cell in list_cells_as_list(x=x_list, y=y_list, acq_min=self.acq_min, acq_max=self.acq_max,
satellites=[satellite for satellite in self.satellites],
dataset_types=[self.dataset_type]):
cells_db.append((cell.x, cell.y))
_log.debug("Cells from DB are [%d] [%s]", len(cells_db), cells_db)
cells = intersection(cells_vector, cells_db)
_log.debug("Combined cells are [%d] [%s]", len(cells), cells)
for (x, y) in cells:
_log.info("Processing cell [%3d/%4d]", x, y)
tiles = list_tiles_as_list(x=x_list, y=y_list, acq_min=self.acq_min, acq_max=self.acq_max,
satellites=[satellite for satellite in self.satellites],
dataset_types=[self.dataset_type])
_log.info("There are [%d] tiles", len(tiles))
if self.list_only:
for tile in tiles:
_log.info("Would process [%s]", tile.datasets[self.dataset_type].path)
continue
# Calculate the mask for the cell
mask_aoi = self.get_mask_aoi_cell(x, y)
pixel_count = 4000 * 4000
pixel_count_aoi = (mask_aoi == False).sum()
_log.debug("mask_aoi is [%s]\n[%s]", numpy.shape(mask_aoi), mask_aoi)
metadata = None
with self.get_output_file() as csv_file:
csv_writer = csv.writer(csv_file)
import operator
header = reduce(operator.add, [["DATE", "INSTRUMENT", "# PIXELS", "# PIXELS IN AOI"]] + [
["%s - # DATA PIXELS" % b.name,
"%s - # DATA PIXELS AFTER PQA" % b.name,
"%s - # DATA PIXELS AFTER PQA WOFS" % b.name,
"%s - # DATA PIXELS AFTER PQA WOFS AOI" % b.name,
"%s - MIN" % b.name, "%s - MAX" % b.name, "%s - MEAN" % b.name] for b in bands])
csv_writer.writerow(header)
for tile in tiles:
_log.info("Processing tile [%s]", tile.datasets[self.dataset_type].path)
if self.list_only:
continue
if not metadata:
metadata = get_dataset_metadata(tile.datasets[self.dataset_type])
# Apply PQA if specified
pqa = None
mask_pqa = None
if self.mask_pqa_apply and DatasetType.PQ25 in tile.datasets:
pqa = tile.datasets[DatasetType.PQ25]
mask_pqa = get_mask_pqa(pqa, self.mask_pqa_mask)
_log.debug("mask_pqa is [%s]\n[%s]", numpy.shape(mask_pqa), mask_pqa)
# Apply WOFS if specified
wofs = None
mask_wofs = None
if self.mask_wofs_apply and DatasetType.WATER in tile.datasets:
wofs = tile.datasets[DatasetType.WATER]
mask_wofs = get_mask_wofs(wofs, self.mask_wofs_mask)
_log.debug("mask_wofs is [%s]\n[%s]", numpy.shape(mask_wofs), mask_wofs)
data = get_dataset_data(tile.datasets[self.dataset_type], bands=bands)
_log.debug("data is [%s]\n[%s]", numpy.shape(data), data)
pixel_count_data = dict()
pixel_count_data_pqa = dict()
pixel_count_data_pqa_wofs = dict()
pixel_count_data_pqa_wofs_aoi = dict()
mmin = dict()
mmax = dict()
mmean = dict()
for band in bands:
data[band] = numpy.ma.masked_equal(data[band], NDV)
_log.debug("masked data is [%s] [%d]\n[%s]", numpy.shape(data), numpy.ma.count(data), data)
pixel_count_data[band] = numpy.ma.count(data[band])
if pqa:
data[band].mask = numpy.ma.mask_or(data[band].mask, mask_pqa)
_log.debug("PQA masked data is [%s] [%d]\n[%s]", numpy.shape(data[band]), numpy.ma.count(data[band]), data[band])
pixel_count_data_pqa[band] = numpy.ma.count(data[band])
if wofs:
data[band].mask = numpy.ma.mask_or(data[band].mask, mask_wofs)
_log.debug("WOFS masked data is [%s] [%d]\n[%s]", numpy.shape(data[band]), numpy.ma.count(data[band]), data[band])
pixel_count_data_pqa_wofs[band] = numpy.ma.count(data[band])
data[band].mask = numpy.ma.mask_or(data[band].mask, mask_aoi)
_log.debug("AOI masked data is [%s] [%d]\n[%s]", numpy.shape(data[band]), numpy.ma.count(data[band]), data[band])
pixel_count_data_pqa_wofs_aoi[band] = numpy.ma.count(data[band])
mmin[band] = numpy.ma.min(data[band])
mmax[band] = numpy.ma.max(data[band])
mmean[band] = numpy.ma.mean(data[band])
# Convert the mean to an int...which is actually trickier than you would expect due to masking....
if numpy.ma.count(mmean[band]) != 0:
mmean[band] = mmean[band].astype(numpy.int16)
# Should we output if no data values found?
pixel_count_data_pqa_wofs_aoi_all_bands = reduce(operator.add, pixel_count_data_pqa_wofs_aoi.itervalues())
if pixel_count_data_pqa_wofs_aoi_all_bands == 0 and not self.output_no_data:
_log.info("Skipping dataset with no non-masked data values in ANY band")
continue
row = reduce(
operator.add,
[[tile.end_datetime,
self.decode_satellite_as_instrument(tile.datasets[self.dataset_type].satellite),
pixel_count, pixel_count_aoi]] +
[[pixel_count_data[band], pixel_count_data_pqa[band],
pixel_count_data_pqa_wofs[band], pixel_count_data_pqa_wofs_aoi[band],
mmin[band], mmax[band], mmean[band]] for band in bands])
csv_writer.writerow(row)