def run(self):
print "****", self.output().path
dataset = self.tile.datasets[self.dataset_type]
metadata = get_dataset_metadata(dataset)
mask = None
# If doing PQA masking then get PQA mask
if self.mask_pqa_apply and DatasetType.PQ25 in self.tile.datasets:
mask = get_mask_pqa(self.tile.datasets[DatasetType.PQ25], self.mask_pqa_mask, mask=mask)
# If doing WOFS masking then get WOFS mask
if self.mask_wofs_apply and DatasetType.WATER in self.tile.datasets:
mask = get_mask_wofs(self.tile.datasets[DatasetType.WATER], self.mask_wofs_mask, mask=mask)
# TODO - no data value and data type
ndv = get_dataset_ndv(dataset)
data = get_dataset_data_masked(dataset, mask=mask, ndv=ndv)
raster_create(self.output().path, [data[b] for b in dataset.bands],
metadata.transform, metadata.projection, ndv, gdal.GDT_Int16,
dataset_metadata=self.generate_raster_metadata(dataset),
band_ids=[b.name for b in dataset.bands])
def run(self):
ndv = NDV
nbar = self.tile.datasets[DatasetType.ARG25]
_log.info("Processing tile [%s]", nbar.path)
# Apply PQA if specified
pqa = None
if self.mask_pqa_apply and DatasetType.PQ25 in self.tile.datasets:
pqa = self.tile.datasets[DatasetType.PQ25]
mask = None
log_mem("Before get PQA mask")
if pqa:
mask = get_mask_pqa(pqa, self.mask_pqa_mask)
data = get_dataset_data_masked(nbar, mask=mask, ndv=ndv)
log_mem("After get data (masked)")
metadata = get_dataset_metadata(nbar)
data = calculate_tassel_cap_index(data, coefficients=TCI_COEFFICIENTS[nbar.satellite][TasselCapIndex.WETNESS])
raster_create(self.output().path, [data], metadata.transform, metadata.projection, numpy.nan, gdal.GDT_Float32)
def doit(self):
# TODO this is working around the issue of it blowing up?
# First create the wetness output for each tile
for tile in self.get_tiles():
self.create_water_tile(tile)
########
bands = []
metadata = None
for water_tile in self.input():
_log.debug("Processing %s", water_tile.path)
if not metadata:
metadata = raster_get_metadata(water_tile.path, 1)
bands.append(raster_get_data(water_tile.path, 1))
data = numpy.array([band for band in bands])
_log.info("Creating summary raster %s", self.output().path)
raster_create(self.output().path,
[numpy.nanmin(data, axis=0), numpy.nanmax(data, axis=0), numpy.nanmean(data, axis=0)],
metadata.transform, metadata.projection, numpy.nan, gdal.GDT_Float32)
def retrieve_data(dataset, pq, pq_masks, path, x, y, overwrite=False, stack=False):
_log.info("Retrieving data from [%s] with pq [%s] and pq mask [%s] to [%s]", dataset.path, pq and pq.path or "", pq and pq_masks or "", path)
if os.path.exists(path) and not overwrite:
_log.error("Output file [%s] exists", path)
raise Exception("Output file [%s] already exists" % path)
data = None
metadata = get_dataset_metadata(dataset)
if pq:
data = get_dataset_data_with_pq(dataset, pq, pq_masks=pq_masks)
else:
data = get_dataset_data(dataset)
_log.debug("data is [%s]", data)
raster_create(path, [data[b] for b in dataset.bands],
metadata.transform, metadata.projection, NDV, gdal.GDT_Int16)
# If we are creating a stack then also add to a file list file...
if stack:
path_file_list = os.path.join(os.path.dirname(path), get_filename_file_list(dataset.satellite, dataset.dataset_type, x, y))
_log.info("Also going to write file list to [%s]", path_file_list)
with open(path_file_list, "ab") as f:
print >>f, path
def doit(self):
shape = (4000, 4000)
masks = [PQ_MASK_CLEAR,
PQ_MASK_SATURATION_OPTICAL,
PQ_MASK_SATURATION_THERMAL,
PQ_MASK_CONTIGUITY,
PQ_MASK_LAND,
PQ_MASK_CLOUD_ACCA,
PQ_MASK_CLOUD_FMASK,
PQ_MASK_CLOUD_SHADOW_ACCA,
PQ_MASK_CLOUD_SHADOW_FMASK]
observation_count = empty_array(shape=shape, dtype=numpy.int16, ndv=0)
observation_count_clear = dict()
for mask in masks:
observation_count_clear[mask] = empty_array(shape=shape, dtype=numpy.int16, ndv=0)
metadata = None
for tile in self.get_tiles():
# Get the PQ mask
pq = tile.datasets[DatasetType.PQ25]
data = get_dataset_data(pq, [Pq25Bands.PQ])[Pq25Bands.PQ]
#
# Count any pixels that are no NDV - don't think we should actually have any but anyway
#
# Mask out any no data pixels - should actually be none but anyway
pq = numpy.ma.masked_equal(data, NDV)
# Count the data pixels - i.e. pixels that were NOT masked out
observation_count += numpy.where(data.mask, 0, 1)
#
# Count and pixels that are not masked due to pixel quality
#
for mask in masks:
# Apply the particular pixel mask
pqm = numpy.ma.masked_where(numpy.bitwise_and(data, mask) != mask, data)
# Count the pixels that were not masked out
observation_count_clear[mask] += numpy.where(pqm.mask, 0, 1)
if not metadata:
metadata = get_dataset_metadata(pq)
# Create the output datasets
# Observation Count
raster_create(self.output()[0].path, [observation_count] + [observation_count_clear[mask] for mask in masks],
metadata.transform, metadata.projection, NDV, GDT_Int16)
def create_water_tile(self, tile):
arg = tile.datasets[DatasetType.ARG25]
pqa = tile.datasets[DatasetType.PQ25]
_log.info("ARG tile [%s]", arg)
_log.info("PQ tile [%s]", pqa)
filename = os.path.basename(arg.path)
filename = filename.replace("NBAR", "WETNESS")
filename = filename.replace(".vrt", ".tif")
filename = os.path.join(self.output_directory, filename)
metadata = get_dataset_metadata(arg)
data = get_dataset_data_with_pq(arg, Ls57Arg25Bands, pqa)
# Calculate TCI Wetness
tci = calculate_tassel_cap_index(data,
coefficients=TCI_COEFFICIENTS[arg.satellite][TasselCapIndex.WETNESS])
_log.info("TCI shape is %s | min = %s | max = %s", numpy.shape(tci), tci.min(), tci.max())
raster_create(filename,
[tci],
metadata.transform, metadata.projection, numpy.nan, gdal.GDT_Float32)
def test_calculate_ndvi():
nbar = DatasetTile(satellite_id="LS5", type_id="ARG25", path="/data/tmp/cube/data/from.calum/LS5_TM_NBAR_150_-034_2004-01-13T23-22-17.088044.tif")
metadata = get_dataset_metadata(nbar)
band_data = calculate_ndvi(nbar)
raster_create("/data/tmp/cube/data/unit_test/LS5_TM_NDVI_150_-034_2004-01-13T23-22-17.088044.tif",
[band_data.filled(numpy.NaN)],
metadata.transform, metadata.projection, numpy.NaN, gdal.GDT_Float32)
def test_apply_pq():
nbar = DatasetTile(satellite_id="LS5", type_id="ARG25", path="/data/tmp/cube/data/from.calum/LS5_TM_NBAR_150_-034_2004-01-13T23-22-17.088044.tif")
pq = DatasetTile(satellite_id="LS5", type_id="PQ25", path="/data/tmp/cube/data/from.calum/LS5_TM_PQA_150_-034_2004-01-13T23-22-17.088044.tif")
metadata = get_dataset_metadata(nbar)
band_data = raster_get_band_data_with_pq(nbar, Ls57Arg25Bands, pq)
raster_create("/data/tmp/cube/data/from.calum/LS5_TM_NBAR_PQD_150_-034_2004-01-13T23-22-17.088044.tif",
[band_data[b].filled(-999) for b in Ls57Arg25Bands],
metadata.transform, metadata.projection, -999, gdal.GDT_Int16)
def read_write_multi_images(self):
newpat = '*'+'NDWI_'+ str(self.x) +'_'+ str(self.y).zfill(4) + '*'
print 'reading input directory and searching pattern ', self.output_directory , newpat
ndwi =None
newimage=numpy.zeros((4000, 4000), dtype='int16')
heights = ["tide_10","tide_20","tide_30","tide_40","tide_50","tide_60","tide_70","tide_80","tide_90","tide_100"]
shape = (4000, 4000)
band_desc= ["Relative DEM of average tidal offsets for each 10 percent of heights"]
height_offset = dict()
for height in heights:
#height_offset[height] = numpy.zeros(shape, dtype=numpy.int16)
height_offset[height] = 0
_log.info("searching pattern %s" , newpat)
for fl in os.listdir(self.output_directory):
fpath= self.output_directory + '/' + fl
if fnmatch.fnmatch(fpath, newpat):
_log.info("file matched %s" , fl)
dataset = gdal.Open(fpath, GA_ReadOnly)
if dataset is None:
print 'No dataset found for ', fl
_log.info("dataset problem %s" , fl)
continue
for height in heights:
if height in fl:
#get the first ndwi band from the source file
band = dataset.GetRasterBand(1)
ndwi = band.ReadAsArray(0,0, dataset.RasterXSize, dataset.RasterYSize)
mask_nan = numpy.ma.masked_invalid(ndwi)
fill = 0
mask_nan= numpy.ma.filled(mask_nan, fill)
_log.info("shape of ndwi [%s] and data after nan masked %s" , mask_nan.shape, mask_nan)
#ndwi>0 means water (0) else land(1)
if fl.find("tide_100") > 0:
if height.find("tide_100") != -1:
newimage+=numpy.where(mask_nan >= 0, 0,1)
else:
newimage+=numpy.where(mask_nan >= 0, 0, 1)
_log.info("shape of newimage [%s] and data %s" , newimage.shape, newimage)
metadata_info = self.generate_raster_metadata()
_log.info("new binary data %s" , newimage)
srs = osr.SpatialReference()
srs.ImportFromEPSG(4326)
projection = srs.ExportToWkt()
transform = (self.x, 0.00025, 0.0, self.y+1, 0.0, -0.00025)
raster_create(self.output().path, [newimage], transform, projection, NDV, GDT_Int16, dataset_metadata=metadata_info, band_ids=band_desc)
def run(self):
print "****", self.output().path
dataset = self.tile.datasets[DatasetType.TCI]
print "***", dataset.path
transform = (self.x, 0.00025, 0.0, self.y+1, 0.0, -0.00025)
srs = osr.SpatialReference()
srs.ImportFromEPSG(4326)
projection = srs.ExportToWkt()
# metadata = get_dataset_metadata(dataset)
mask = None
# If doing PQA masking then get PQA mask
if self.mask_pqa_apply and DatasetType.PQ25 in self.tile.datasets:
mask = get_mask_pqa(self.tile.datasets[DatasetType.PQ25], self.mask_pqa_mask, mask=mask)
# If doing WOFS masking then get WOFS mask
if self.mask_wofs_apply and DatasetType.WATER in self.tile.datasets:
mask = get_mask_wofs(self.tile.datasets[DatasetType.WATER], self.mask_wofs_mask, mask=mask)
# TODO - no data value and data type
ndv = get_dataset_ndv(dataset)
data = get_dataset_data_masked(dataset, mask=mask, ndv=ndv)
# Create ALL bands raster
# raster_create(self.output().path, [data[b] for b in dataset.bands],
# metadata.transform, metadata.projection, ndv, gdal.GDT_Float32,
# dataset_metadata=self.generate_raster_metadata(dataset),
# band_ids=[b.name for b in dataset.bands])
# Create just the WETNESS band raster
raster_create(self.output().path, [data[TciBands.WETNESS]],
transform, projection, ndv, gdal.GDT_Float32,
dataset_metadata=self.generate_raster_metadata(dataset),
band_ids=[TciBands.WETNESS.name])
def run(self):
shape = (4000, 4000)
extra = [ExtraBands.OBSERVATIONS, ExtraBands.LOW, ExtraBands.MEDIAN, ExtraBands.HIGH]
band_desc = ["TOTAL_OBSERVATIONS","LOWEST TIDE","MEDIAN TIDE", "HIGHEST TIDE"]
extra_fields = dict()
for field in extra:
extra_fields[field] = empty_array(shape=shape, dtype=numpy.int16, fill=0)
metadata = None
low_value = 0
high_value = 0
median_value = 0
observations = 0
cur = self.load_tidal_file()
low=cur[0]
high=cur[1]
observations=cur[2]
#import pdb; pdb.set_trace()
low_value = int(float(low) * 1000)
high_value = int(float(high) * 1000)
median_value = (low_value + high_value)/2
#import pdb; pdb.set_trace()
_log.info("\toffset file low %d high %d median %d lines %d ", low_value, high_value, median_value, observations)
for layer in extra_fields:
lv = layer.value
if lv == 1:
extra_fields[layer][:] = observations
if lv == 2:
extra_fields[layer][:] = low_value
if lv == 3:
extra_fields[layer][:] = median_value
if lv == 4:
extra_fields[layer][:] = high_value
transform = (self.x, 0.00025, 0.0, self.y+1, 0.0, -0.00025)
srs = osr.SpatialReference()
srs.ImportFromEPSG(4326)
projection = srs.ExportToWkt()
# Create the output dataset
metadata_info = self.generate_raster_metadata()
raster_create(self.output().path, [extra_fields[field] for field in extra], transform, projection, NDV, GDT_Int16, dataset_metadata=metadata_info, band_ids=band_desc)
def generate_derived_nbar(self, dataset_types, nbar, pqa, pqa_masks, overwrite=False):
for dataset_type in dataset_types:
filename = self.get_output_filename_derived_nbar(nbar, dataset_type)
_log.info("Generating data from [%s] with pq [%s] and pq mask [%s] to [%s]", nbar.path, pqa and pqa.path or "", pqa and pqa_masks or "", filename)
metadata = get_dataset_metadata(nbar)
data = None
if pqa:
data = get_dataset_data_with_pq(nbar, pqa, pq_masks=pqa_masks)
else:
data = get_dataset_data(nbar)
_log.debug("data is [%s]", data)
if dataset_type == DatasetType.NDVI:
ndvi = calculate_ndvi(data[nbar.bands.RED], data[nbar.bands.NEAR_INFRARED])
raster_create(filename, [ndvi], metadata.transform, metadata.projection, NDV, gdal.GDT_Float32)
elif dataset_type == DatasetType.EVI:
evi = calculate_evi(data[nbar.bands.RED], data[nbar.bands.BLUE], data[nbar.bands.NEAR_INFRARED])
raster_create(filename, [evi], metadata.transform, metadata.projection, NDV, gdal.GDT_Float32)
elif dataset_type == DatasetType.NBR:
nbr = calculate_nbr(data[nbar.bands.NEAR_INFRARED], data[nbar.bands.SHORT_WAVE_INFRARED_2])
raster_create(filename, [nbr], metadata.transform, metadata.projection, NDV, gdal.GDT_Float32)
def read_write_multi_images(self):
print 'reading input directory ', self.output_directory
newpat = '*'+ str(self.x) +'_'+ str(self.y).zfill(4) + '*'
ndwi =None
newimage=numpy.zeros((4000, 4000), dtype='int16')
heights = ["tide_10","tide_20","tide_30","tide_40","tide_50","tide_60","tide_70","tide_80","tide_90", "tide_100"]
shape = (4000, 4000)
band_desc= ["DEM of average tidal offsets for each 10 percent of heights"]
height_offset = dict()
for height in heights:
height_offset[height] = 0
_log.info("searching pattern %s" , newpat)
for fl in os.listdir(self.output_directory):
fpath= self.output_directory + '/' + fl
if fnmatch.fnmatch(fpath, newpat):
_log.info("file matched %s" , fl)
dataset = gdal.Open(fpath, GA_ReadOnly)
if dataset is None:
_log.info("dataset problem %s" , fl)
continue
for height in heights:
if height in fl:
band = dataset.GetRasterBand(1)
ndwi = band.ReadAsArray(0,0, dataset.RasterXSize, dataset.RasterYSize)
mask_nan = numpy.ma.masked_invalid(ndwi)
fill = 0
mask_nan= numpy.ma.filled(mask_nan, fill)
_log.info("shape of ndwi [%s] and data after nan masked %s" , mask_nan.shape, mask_nan)
band=dataset.GetRasterBand(5)
data=band.ReadAsArray(0,0, dataset.RasterXSize, dataset.RasterYSize)
av=data.mean().astype(int)
# set land and water depending on ndwi <0 and >0 respectively
if fl.find("tide_100") > 0:
if height.find("tide_100") != -1:
height_offset[heights[len(heights)-1]]+=av
newimage+=numpy.where(mask_nan >= 0, 0, 1)
else:
height_offset[height]+=av
newimage+=numpy.where(mask_nan >= 0, 0, 1)
_log.info("height_offset for %s is %s avrg %d" , height, height_offset[height] ,av)
_log.info("last binary data %s" , newimage)
for i in range(len(heights)):
if height_offset[heights[i]] != 0:
_log.info("height_offset for index %d is %d" , i, height_offset[heights[i]])
newimage[newimage==i+1] = height_offset[heights[i]]
else:
_log.info("no data for %d", height_offset[heights[i]])
_log.info("mod binary data -- %s" , newimage)
newimage[newimage==0] = -6666
metadata_info = self.generate_raster_metadata()
_log.info("new binary data %s" , newimage)
srs = osr.SpatialReference()
srs.ImportFromEPSG(4326)
projection = srs.ExportToWkt()
transform = (self.x, 0.00025, 0.0, self.y+1, 0.0, -0.00025)
raster_create(self.output().path, [newimage], transform, projection, NDV, GDT_Int16, dataset_metadata=metadata_info, band_ids=band_desc)
def run(self):
shape = (4000, 4000)
extra = [
ExtraBands.OBSERVATIONS, ExtraBands.LOW, ExtraBands.MEDIAN,
ExtraBands.HIGH
]
band_desc = [
"TOTAL_OBSERVATIONS", "LOWEST TIDE", "MEDIAN TIDE", "HIGHEST TIDE"
]
extra_fields = dict()
for field in extra:
extra_fields[field] = empty_array(shape=shape,
dtype=numpy.int16,
fill=0)
metadata = None
low_value = 0
high_value = 0
median_value = 0
observations = 0
cur = self.load_tidal_file()
low = cur[0]
high = cur[1]
observations = cur[2]
#import pdb; pdb.set_trace()
low_value = int(float(low) * 1000)
high_value = int(float(high) * 1000)
median_value = (low_value + high_value) / 2
#import pdb; pdb.set_trace()
_log.info("\toffset file low %d high %d median %d lines %d ",
low_value, high_value, median_value, observations)
for layer in extra_fields:
lv = layer.value
if lv == 1:
extra_fields[layer][:] = observations
if lv == 2:
extra_fields[layer][:] = low_value
if lv == 3:
extra_fields[layer][:] = median_value
if lv == 4:
extra_fields[layer][:] = high_value
transform = (self.x, 0.00025, 0.0, self.y + 1, 0.0, -0.00025)
srs = osr.SpatialReference()
srs.ImportFromEPSG(4326)
projection = srs.ExportToWkt()
# Create the output dataset
metadata_info = self.generate_raster_metadata()
raster_create(self.output().path,
[extra_fields[field] for field in extra],
transform,
projection,
NDV,
GDT_Int16,
dataset_metadata=metadata_info,
band_ids=band_desc)
def read_write_multi_images(self):
newpat = '*' + 'NDWI_' + str(self.x) + '_' + str(self.y).zfill(4) + '*'
print 'reading input directory and searching pattern ', self.output_directory, newpat
ndwi = None
newimage = numpy.zeros((4000, 4000), dtype='int16')
heights = [
"tide_10", "tide_20", "tide_30", "tide_40", "tide_50", "tide_60",
"tide_70", "tide_80", "tide_90", "tide_100"
]
shape = (4000, 4000)
band_desc = [
"Relative DEM of average tidal offsets for each 10 percent of heights"
]
height_offset = dict()
for height in heights:
#height_offset[height] = numpy.zeros(shape, dtype=numpy.int16)
height_offset[height] = 0
_log.info("searching pattern %s", newpat)
for fl in os.listdir(self.output_directory):
fpath = self.output_directory + '/' + fl
if fnmatch.fnmatch(fpath, newpat):
_log.info("file matched %s", fl)
dataset = gdal.Open(fpath, GA_ReadOnly)
if dataset is None:
print 'No dataset found for ', fl
_log.info("dataset problem %s", fl)
continue
for height in heights:
if height in fl:
#get the first ndwi band from the source file
band = dataset.GetRasterBand(1)
ndwi = band.ReadAsArray(0, 0, dataset.RasterXSize,
dataset.RasterYSize)
mask_nan = numpy.ma.masked_invalid(ndwi)
fill = 0
mask_nan = numpy.ma.filled(mask_nan, fill)
_log.info(
"shape of ndwi [%s] and data after nan masked %s",
mask_nan.shape, mask_nan)
#ndwi>0 means water (0) else land(1)
if fl.find("tide_100") > 0:
if height.find("tide_100") != -1:
newimage += numpy.where(mask_nan >= 0, 0, 1)
else:
newimage += numpy.where(mask_nan >= 0, 0, 1)
_log.info("shape of newimage [%s] and data %s",
newimage.shape, newimage)
metadata_info = self.generate_raster_metadata()
_log.info("new binary data %s", newimage)
srs = osr.SpatialReference()
srs.ImportFromEPSG(4326)
projection = srs.ExportToWkt()
transform = (self.x, 0.00025, 0.0, self.y + 1, 0.0, -0.00025)
raster_create(self.output().path, [newimage],
transform,
projection,
NDV,
GDT_Int16,
dataset_metadata=metadata_info,
band_ids=band_desc)
def run(self):
shape = (4000, 4000)
no_data_value = NDV
best_pixel_fc = dict()
for band in Fc25Bands:
# best_pixel_fc[band] = empty_array(shape=shape, dtype=numpy.int16, ndv=INT16_MIN)
best_pixel_fc[band] = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
best_pixel_nbar = dict()
for band in Ls57Arg25Bands:
best_pixel_nbar[band] = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
best_pixel_satellite = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
best_pixel_date = empty_array(shape=shape, dtype=numpy.int32, ndv=NDV)
current_satellite = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
current_date = empty_array(shape=shape, dtype=numpy.int32, ndv=NDV)
SATELLITE_DATA_VALUES = {Satellite.LS5: 5, Satellite.LS7: 7, Satellite.LS8: 8}
metadata_nbar = None
metadata_fc = None
for tile in self.get_tiles():
pqa = tile.datasets[DatasetType.PQ25]
nbar = tile.datasets[DatasetType.ARG25]
fc = tile.datasets[DatasetType.FC25]
wofs = DatasetType.WATER in tile.datasets and tile.datasets[DatasetType.WATER] or None
_log.info("Processing [%s]", fc.path)
data = dict()
# Create an initial "no mask" mask
mask = numpy.ma.make_mask_none((4000, 4000))
# _log.info("### mask is [%s]", mask[1000][1000])
# Add the PQA mask if we are doing PQA masking
if self.mask_pqa_apply:
mask = get_mask_pqa(pqa, pqa_masks=self.mask_pqa_mask, mask=mask)
# _log.info("### mask PQA is [%s]", mask[1000][1000])
# Add the WOFS mask if we are doing WOFS masking
if self.mask_wofs_apply and wofs:
mask = get_mask_wofs(wofs, wofs_masks=self.mask_wofs_mask, mask=mask)
# _log.info("### mask PQA is [%s]", mask[1000][1000])
# Get NBAR dataset
data[DatasetType.ARG25] = get_dataset_data_masked(nbar, mask=mask)
# _log.info("### NBAR/RED is [%s]", data[DatasetType.ARG25][Ls57Arg25Bands.RED][1000][1000])
# Get the NDVI dataset
data[DatasetType.NDVI] = calculate_ndvi(data[DatasetType.ARG25][Ls57Arg25Bands.RED],
data[DatasetType.ARG25][Ls57Arg25Bands.NEAR_INFRARED])
# _log.info("### NDVI is [%s]", data[DatasetType.NDVI][1000][1000])
# Add the NDVI value range mask (to the existing mask)
mask = self.get_mask_range(data[DatasetType.NDVI], min_val=0.0, max_val=0.3, mask=mask)
# _log.info("### mask NDVI is [%s]", mask[1000][1000])
# Get FC25 dataset
data[DatasetType.FC25] = get_dataset_data_masked(fc, mask=mask)
# _log.info("### FC/BS is [%s]", data[DatasetType.FC25][Fc25Bands.BARE_SOIL][1000][1000])
# Add the bare soil value range mask (to the existing mask)
mask = self.get_mask_range(data[DatasetType.FC25][Fc25Bands.BARE_SOIL], min_val=0, max_val=8000, mask=mask)
# _log.info("### mask BS is [%s]", mask[1000][1000])
# Apply the final mask to the FC25 bare soil data
data_bare_soil = numpy.ma.MaskedArray(data=data[DatasetType.FC25][Fc25Bands.BARE_SOIL], mask=mask).filled(NDV)
# _log.info("### bare soil is [%s]", data_bare_soil[1000][1000])
# Compare the bare soil value from this dataset to the current "best" value
best_pixel_fc[Fc25Bands.BARE_SOIL] = numpy.fmax(best_pixel_fc[Fc25Bands.BARE_SOIL], data_bare_soil)
# _log.info("### best pixel bare soil is [%s]", best_pixel_fc[Fc25Bands.BARE_SOIL][1000][1000])
# Now update the other best pixel datasets/bands to grab the pixels we just selected
for band in Ls57Arg25Bands:
best_pixel_nbar[band] = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BARE_SOIL],
data_bare_soil,
data[DatasetType.ARG25][band],
best_pixel_nbar[band])
for band in [Fc25Bands.PHOTOSYNTHETIC_VEGETATION, Fc25Bands.NON_PHOTOSYNTHETIC_VEGETATION, Fc25Bands.UNMIXING_ERROR]:
best_pixel_fc[band] = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BARE_SOIL],
data_bare_soil,
data[DatasetType.FC25][band],
best_pixel_fc[band])
# And now the other "provenance" data
# Satellite "provenance" data
current_satellite.fill(SATELLITE_DATA_VALUES[fc.satellite])
best_pixel_satellite = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BARE_SOIL],
data_bare_soil,
current_satellite,
best_pixel_satellite)
# Date "provenance" data
current_date.fill(date_to_integer(tile.end_datetime))
best_pixel_date = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BARE_SOIL],
data_bare_soil,
current_date,
best_pixel_date)
# Grab the metadata from the input datasets for use later when creating the output datasets
if not metadata_nbar:
metadata_nbar = get_dataset_metadata(nbar)
if not metadata_fc:
metadata_fc = get_dataset_metadata(fc)
# Create the output datasets
# FC composite
raster_create(self.get_dataset_filename("FC"),
[best_pixel_fc[b] for b in Fc25Bands],
metadata_fc.transform, metadata_fc.projection,
metadata_fc.bands[Fc25Bands.BARE_SOIL].no_data_value,
metadata_fc.bands[Fc25Bands.BARE_SOIL].data_type)
# NBAR composite
raster_create(self.get_dataset_filename("NBAR"),
[best_pixel_nbar[b] for b in Ls57Arg25Bands],
metadata_nbar.transform, metadata_nbar.projection,
metadata_nbar.bands[Ls57Arg25Bands.BLUE].no_data_value,
metadata_nbar.bands[Ls57Arg25Bands.BLUE].data_type)
# Satellite "provenance" composites
raster_create(self.get_dataset_filename("SAT"),
[best_pixel_satellite],
metadata_nbar.transform, metadata_nbar.projection, no_data_value,
gdal.GDT_Int16)
# Date "provenance" composites
raster_create(self.get_dataset_filename("DATE"),
[best_pixel_date],
metadata_nbar.transform, metadata_nbar.projection, no_data_value,
gdal.GDT_Int32)
def doit(self):
shape = (4000, 4000)
no_data_value = NDV
best_pixel_data = dict()
# TODO
if Satellite.LS8.value in self.satellites:
bands = Ls8Arg25Bands
else:
bands = Ls57Arg25Bands
for band in bands:
best_pixel_data[band] = empty_array(shape=shape, dtype=numpy.int16, ndv=no_data_value)
best_pixel_satellite = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
# best_pixel_epoch = empty_array(shape=shape, dtype=numpy.int32, ndv=NDV)
best_pixel_date = empty_array(shape=shape, dtype=numpy.int32, ndv=NDV)
current_satellite = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
# current_epoch = empty_array(shape=shape, dtype=numpy.int32, ndv=NDV)
current_date = empty_array(shape=shape, dtype=numpy.int32, ndv=NDV)
metadata = None
SATELLITE_DATA_VALUES = {Satellite.LS5: 5, Satellite.LS7: 7, Satellite.LS8: 8}
for tile in self.get_tiles(sort=SortType.DESC):
# Get ARG25 dataset
dataset = tile.datasets[DatasetType.ARG25]
_log.info("Processing ARG tile [%s]", dataset.path)
if not metadata:
metadata = get_dataset_metadata(dataset)
band_data = None
if self.apply_pq_filter:
band_data = get_dataset_data_with_pq(dataset, tile.datasets[DatasetType.PQ25])
else:
band_data = get_dataset_data(dataset)
# Create the provenance datasets
# NOTE: need to do this BEFORE selecting the pixel since it is actually using the fact that the
# selected pixel currently doesn't have a value
# NOTE: band values are propagated "as a job lot" so can just check any band
# TODO better way than just saying....RED....?
band = bands.RED
# Satellite
current_satellite.fill(SATELLITE_DATA_VALUES[dataset.satellite])
best_pixel_satellite = numpy.where(best_pixel_data[band] == no_data_value, current_satellite, best_pixel_satellite)
# # Epoch dataset
#
# current_epoch.fill(calendar.timegm(tile.end_datetime.timetuple()))
# best_pixel_epoch = numpy.where(best_pixel_data[band] == no_data_value, current_epoch, best_pixel_epoch)
# Date dataset (20150101)
current_date.fill(tile.end_datetime.year * 10000 + tile.end_datetime.month * 100 + tile.end_datetime.day)
best_pixel_date = numpy.where(best_pixel_data[band] == no_data_value, current_date, best_pixel_date)
for band in bands:
data = band_data[band]
# _log.debug("data = \n%s", data)
# Replace any NO DATA best pixels with data pixels
# TODO should I explicitly do the AND data is not NO DATA VALUE?
best_pixel_data[band] = numpy.where(best_pixel_data[band] == no_data_value, data, best_pixel_data[band])
# _log.debug("best pixel = \n%s", best_pixel_data[band])
still_no_data = numpy.any(numpy.array([best_pixel_data[b] for b in bands]) == no_data_value)
# _log.debug("still no data pixels = %s", still_no_data)
if not still_no_data:
break
# Now want to mask out values in the provenance datasets if we haven't actually got a value
# TODO better way than just saying....RED....?
band = bands.RED
mask = numpy.ma.masked_equal(best_pixel_data[band], NDV).mask
best_pixel_satellite = numpy.ma.array(best_pixel_satellite, mask=mask).filled(NDV)
# best_pixel_epoch = numpy.ma.array(best_pixel_epoch, mask=mask).fill(NDV)
best_pixel_date = numpy.ma.array(best_pixel_date, mask=mask).filled(NDV)
# Composite NBAR dataset
raster_create(self.get_output_path("NBAR"), [best_pixel_data[b] for b in bands],
metadata.transform, metadata.projection, NDV, gdal.GDT_Int16)
# Provenance (satellite) dataset
raster_create(self.get_output_path("SAT"),
[best_pixel_satellite],
metadata.transform, metadata.projection, no_data_value,
gdal.GDT_Int16)
# # Provenance (epoch) dataset
#
# raster_create(self.get_output_path("EPOCH"),
# [best_pixel_epoch],
# metadata.transform, metadata.projection, no_data_value,
# gdal.GDT_Int32)
# Provenance (day of month) dataset
raster_create(self.get_output_path("DATE"),
[best_pixel_date],
metadata.transform, metadata.projection, no_data_value,
gdal.GDT_Int32)
def doit(self):
_log.debug("Bare Soil Cell Task - doit()")
shape = (4000, 4000)
no_data_value = NDV
best_pixel_fc = dict()
for band in Fc25Bands:
best_pixel_fc[band] = empty_array(shape=shape, dtype=numpy.int16, ndv=INT16_MIN)
best_pixel_nbar = dict()
for band in Ls57Arg25Bands:
best_pixel_nbar[band] = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
best_pixel_satellite = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
best_pixel_year = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
best_pixel_month = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
best_pixel_epoch = empty_array(shape=shape, dtype=numpy.int32, ndv=NDV)
current_satellite = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
current_year = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
current_month = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
current_epoch = empty_array(shape=shape, dtype=numpy.int32, ndv=NDV)
SATELLITE_DATA_VALUES = {Satellite.LS5: 5, Satellite.LS7: 7, Satellite.LS8: 8}
metadata_nbar = None
metadata_fc = None
for tile in self.get_tiles():
# Get the PQ mask
pq = tile.datasets[DatasetType.PQ25]
data_pq = get_dataset_data(pq, [Pq25Bands.PQ])[Pq25Bands.PQ]
mask_pq = get_pq_mask(data_pq)
# Get NBAR dataset
nbar = tile.datasets[DatasetType.ARG25]
_log.info("Processing NBAR tile [%s]", nbar.path)
if not metadata_nbar:
metadata_nbar = get_dataset_metadata(nbar)
data_nbar = get_dataset_data_with_pq(nbar, Ls57Arg25Bands, tile.datasets[DatasetType.PQ25])
# Get the NDVI mask
red = data_nbar[Ls57Arg25Bands.RED]
nir = data_nbar[Ls57Arg25Bands.NEAR_INFRARED]
ndvi_data = calculate_ndvi(red, nir)
ndvi_data = numpy.ma.masked_equal(ndvi_data, NDV)
ndvi_data = numpy.ma.masked_outside(ndvi_data, 0, 0.3, copy=False)
mask_ndvi = ndvi_data.mask
# Get FC25 dataset
fc = tile.datasets[DatasetType.FC25]
_log.info("Processing FC tile [%s]", fc.path)
if not metadata_fc:
metadata_fc = get_dataset_metadata(fc)
_log.debug("metadata fc is %s", metadata_fc)
data_fc = get_dataset_data(fc, Fc25Bands)
data_bare_soil = data_fc[Fc25Bands.BS]
data_bare_soil = numpy.ma.masked_equal(data_bare_soil, -999)
data_bare_soil = numpy.ma.masked_outside(data_bare_soil, 0, 8000)
data_bare_soil.mask = (data_bare_soil.mask | mask_pq | mask_ndvi)
data_bare_soil = data_bare_soil.filled(NDV)
# Compare the bare soil value from this dataset to the current "best" value
best_pixel_fc[Fc25Bands.BS] = numpy.fmax(best_pixel_fc[Fc25Bands.BS], data_bare_soil)
# Now update the other best pixel datasets/bands to grab the pixels we just selected
for band in Ls57Arg25Bands:
best_pixel_nbar[band] = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BS],
data_bare_soil,
data_nbar[band],
best_pixel_nbar[band])
for band in [Fc25Bands.PV, Fc25Bands.NPV, Fc25Bands.ERROR]:
best_pixel_fc[band] = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BS],
data_bare_soil,
data_fc[band],
best_pixel_fc[band])
# And now the other "provenance" data
current_satellite.fill(SATELLITE_DATA_VALUES[fc.satellite])
best_pixel_satellite = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BS],
data_bare_soil,
current_satellite,
best_pixel_satellite)
current_year.fill(tile.end_datetime_year)
best_pixel_year = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BS],
data_bare_soil,
current_year,
best_pixel_year)
current_month.fill(tile.end_datetime_month)
best_pixel_month = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BS],
data_bare_soil,
current_month,
best_pixel_month)
current_epoch.fill(calendar.timegm(tile.end_datetime.timetuple()))
best_pixel_epoch = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BS],
data_bare_soil,
current_epoch,
best_pixel_epoch)
# Create the output datasets
# FC composite
raster_create(self.get_dataset_filename("FC"),
[best_pixel_fc[b] for b in Fc25Bands],
metadata_fc.transform, metadata_fc.projection, metadata_fc.bands[Fc25Bands.BS].no_data_value,
metadata_fc.bands[Fc25Bands.BS].data_type)
# NBAR composite
raster_create(self.get_dataset_filename("NBAR"),
[best_pixel_nbar[b] for b in Ls57Arg25Bands],
metadata_nbar.transform, metadata_nbar.projection,
metadata_nbar.bands[Ls57Arg25Bands.BLUE].no_data_value,
metadata_nbar.bands[Ls57Arg25Bands.BLUE].data_type)
# "Provenance" composites
raster_create(self.get_dataset_filename("SAT"),
[best_pixel_satellite],
metadata_nbar.transform, metadata_nbar.projection, no_data_value,
gdal.GDT_Int16)
raster_create(self.get_dataset_filename("YEAR"),
[best_pixel_year],
metadata_nbar.transform, metadata_nbar.projection, no_data_value,
gdal.GDT_Int16)
raster_create(self.get_dataset_filename("MONTH"),
[best_pixel_month],
metadata_nbar.transform, metadata_nbar.projection, no_data_value,
gdal.GDT_Int16)
raster_create(self.get_dataset_filename("EPOCH"),
[best_pixel_epoch],
metadata_nbar.transform, metadata_nbar.projection, no_data_value,
gdal.GDT_Int32)
