def run(self):
print "****", self.output().path
dataset = self.tile.datasets[self.dataset_type]
metadata = get_dataset_metadata(dataset)
mask = None
# If doing PQA masking then get PQA mask
if self.mask_pqa_apply and DatasetType.PQ25 in self.tile.datasets:
mask = get_mask_pqa(self.tile.datasets[DatasetType.PQ25], self.mask_pqa_mask, mask=mask)
# If doing WOFS masking then get WOFS mask
if self.mask_wofs_apply and DatasetType.WATER in self.tile.datasets:
mask = get_mask_wofs(self.tile.datasets[DatasetType.WATER], self.mask_wofs_mask, mask=mask)
# TODO - no data value and data type
ndv = get_dataset_ndv(dataset)
data = get_dataset_data_masked(dataset, mask=mask, ndv=ndv)
raster_create(self.output().path, [data[b] for b in dataset.bands],
metadata.transform, metadata.projection, ndv, gdal.GDT_Int16,
dataset_metadata=self.generate_raster_metadata(dataset),
band_ids=[b.name for b in dataset.bands])
def run(self):
ndv = NDV
nbar = self.tile.datasets[DatasetType.ARG25]
_log.info("Processing tile [%s]", nbar.path)
# Apply PQA if specified
pqa = None
if self.mask_pqa_apply and DatasetType.PQ25 in self.tile.datasets:
pqa = self.tile.datasets[DatasetType.PQ25]
mask = None
log_mem("Before get PQA mask")
if pqa:
mask = get_mask_pqa(pqa, self.mask_pqa_mask)
data = get_dataset_data_masked(nbar, mask=mask, ndv=ndv)
log_mem("After get data (masked)")
metadata = get_dataset_metadata(nbar)
data = calculate_tassel_cap_index(data, coefficients=TCI_COEFFICIENTS[nbar.satellite][TasselCapIndex.WETNESS])
raster_create(self.output().path, [data], metadata.transform, metadata.projection, numpy.nan, gdal.GDT_Float32)
def retrieve_pixel_value(dataset, pqa, pqa_masks, wofs, wofs_masks, latitude, longitude, ndv=NDV):
_log.debug(
"Retrieving pixel value(s) at lat=[%f] lon=[%f] from [%s] with pqa [%s] and paq mask [%s] and wofs [%s] and wofs mask [%s]",
latitude, longitude, dataset.path, pqa and pqa.path or "", pqa and pqa_masks or "",
wofs and wofs.path or "", wofs and wofs_masks or "")
metadata = get_dataset_metadata(dataset)
x, y = latlon_to_xy(latitude, longitude, metadata.transform)
_log.info("Retrieving value at x=[%d] y=[%d] from %s", x, y, dataset.path)
x_size = y_size = 1
mask = None
if pqa:
mask = get_mask_pqa(pqa, pqa_masks, x=x, y=y, x_size=x_size, y_size=y_size)
if wofs:
mask = get_mask_wofs(wofs, wofs_masks, x=x, y=y, x_size=x_size, y_size=y_size, mask=mask)
data = get_dataset_data_masked(dataset, x=x, y=y, x_size=x_size, y_size=y_size, mask=mask, ndv=ndv)
_log.debug("data is [%s]", data)
return data
def retrieve_data(x, y, acq_dt, dataset, band_names, pqa, pqa_masks, wofs, wofs_masks, path, output_format,
overwrite=False, data_type=None, ndv=None, mask=None):
_log.info("Retrieving data from [%s] bands [%s] with pq [%s] and pq mask [%s] and wofs [%s] and wofs mask [%s] to [%s] file [%s]",
dataset.path,
band_names,
pqa and pqa.path or "",
pqa and pqa_masks or "",
wofs and wofs.path or "", wofs and wofs_masks or "",
output_format.name, path)
if os.path.exists(path) and not overwrite:
_log.error("Output file [%s] exists", path)
raise Exception("Output file [%s] already exists" % path)
metadata = get_dataset_metadata(dataset)
# mask = None
if pqa:
mask = get_mask_pqa(pqa, pqa_masks, mask=mask)
if wofs:
mask = get_mask_wofs(wofs, wofs_masks, mask=mask)
bands = []
for b in dataset.bands:
if b.name in band_names:
bands.append(b)
ndv = ndv or get_dataset_ndv(dataset)
data = get_dataset_data_masked(dataset, bands=bands, mask=mask, ndv=ndv)
_log.debug("data is [%s]", data)
data_type = data_type or get_dataset_datatype(dataset)
dataset_info = generate_raster_metadata(x, y, acq_dt, dataset, bands,
pqa is not None, pqa_masks,
wofs is not None, wofs_masks)
band_info = [b.name for b in bands]
if output_format == OutputFormat.GEOTIFF:
raster_create_geotiff(path, [data[b] for b in bands], metadata.transform, metadata.projection, ndv,
data_type, dataset_metadata=dataset_info, band_ids=band_info)
elif output_format == OutputFormat.ENVI:
raster_create_envi(path, [data[b] for b in bands], metadata.transform, metadata.projection, ndv,
data_type, dataset_metadata=dataset_info, band_ids=band_info)
def test_retrieve_data_ls5_arg_with_pqa_water_mask_dry(config=None):
filename = "LS5_TM_NBAR_WITH_PQA_WATER_DRY_{x:03d}_{y:04d}_{date}.{x_offset:04d}_{y_offset:04d}.{x_size:04d}x{y_size:04d}.tif".format(x=CELL_X, y=CELL_Y, date=DATE, x_offset=X_OFFSET, y_offset=Y_OFFSET, x_size=X_SIZE, y_size=Y_SIZE)
tiles = list_tiles_as_list(x=[CELL_X], y=[CELL_Y],
acq_min=ACQ_LS5, acq_max=ACQ_LS5,
satellites=[Satellite.LS5],
dataset_types=[ARG_DATASET_TYPE, PQ_DATASET_TYPE, WOFS_DATASET_TYPE],
config=config)
assert len(tiles) == 1
tile = tiles[0]
assert ARG_DATASET_TYPE in tile.datasets
dataset = tile.datasets[ARG_DATASET_TYPE]
assert PQ_DATASET_TYPE in tile.datasets
pqa = tile.datasets[PQ_DATASET_TYPE]
assert WOFS_DATASET_TYPE in tile.datasets
wofs = tile.datasets[WOFS_DATASET_TYPE]
mask = get_mask_pqa(pqa, x=X_OFFSET, y=Y_OFFSET, x_size=X_SIZE, y_size=Y_SIZE)
mask = get_mask_wofs(wofs, wofs_masks=[WofsMask.DRY, WofsMask.NO_DATA, WofsMask.SATURATION_CONTIGUITY,
WofsMask.SEA_WATER, WofsMask.TERRAIN_SHADOW, WofsMask.HIGH_SLOPE,
WofsMask.CLOUD_SHADOW, WofsMask.CLOUD],
x=X_OFFSET, y=Y_OFFSET, x_size=X_SIZE, y_size=Y_SIZE, mask=mask)
data = get_dataset_data_masked(dataset=dataset, x=X_OFFSET, y=Y_OFFSET, x_size=X_SIZE, y_size=Y_SIZE, mask=mask)
assert(data)
_log.info("data is [%s]\n%s", numpy.shape(data), data)
ndv = get_dataset_ndv(dataset)
assert(is_ndv(ndv, ARG_NDV))
data_type = get_dataset_datatype(dataset)
assert(data_type == ARG_DATA_TYPE)
metadata = generate_dataset_metadata(x=CELL_X, y=CELL_Y, acq_dt=ACQ_LS5,
dataset=dataset, bands=None,
mask_pqa_apply=False, mask_pqa_mask=None,
mask_wofs_apply=False, mask_wofs_mask=None)
raster_create_geotiff(filename, [data[b] for b in dataset.bands], CELL_GEO_TRANSFORM, CELL_PROJECTION, ndv, data_type,
dataset_metadata=metadata, band_ids=[b.name for b in dataset.bands])
assert filecmp.cmp(filename, get_test_data_path(filename))
def run(self):
print "****", self.output().path
dataset = self.tile.datasets[DatasetType.TCI]
print "***", dataset.path
transform = (self.x, 0.00025, 0.0, self.y+1, 0.0, -0.00025)
srs = osr.SpatialReference()
srs.ImportFromEPSG(4326)
projection = srs.ExportToWkt()
# metadata = get_dataset_metadata(dataset)
mask = None
# If doing PQA masking then get PQA mask
if self.mask_pqa_apply and DatasetType.PQ25 in self.tile.datasets:
mask = get_mask_pqa(self.tile.datasets[DatasetType.PQ25], self.mask_pqa_mask, mask=mask)
# If doing WOFS masking then get WOFS mask
if self.mask_wofs_apply and DatasetType.WATER in self.tile.datasets:
mask = get_mask_wofs(self.tile.datasets[DatasetType.WATER], self.mask_wofs_mask, mask=mask)
# TODO - no data value and data type
ndv = get_dataset_ndv(dataset)
data = get_dataset_data_masked(dataset, mask=mask, ndv=ndv)
# Create ALL bands raster
# raster_create(self.output().path, [data[b] for b in dataset.bands],
# metadata.transform, metadata.projection, ndv, gdal.GDT_Float32,
# dataset_metadata=self.generate_raster_metadata(dataset),
# band_ids=[b.name for b in dataset.bands])
# Create just the WETNESS band raster
raster_create(self.output().path, [data[TciBands.WETNESS]],
transform, projection, ndv, gdal.GDT_Float32,
dataset_metadata=self.generate_raster_metadata(dataset),
band_ids=[TciBands.WETNESS.name])
def run(self):
# TODO move the dicking around with bands stuff into utils?
import gdal
driver = raster = None
metadata = None
data_type = ndv = None
tiles = self.get_tiles()
_log.info("Total tiles found [%d]", len(tiles))
_log.info("Creating stack for band [%s]", self.band)
relevant_tiles = []
for tile in tiles:
dataset = self.dataset_type in tile.datasets and tile.datasets[self.dataset_type] or None
if not dataset:
_log.info("No applicable [%s] dataset for [%s]", self.dataset_type.name, tile.end_datetime)
continue
if self.band in [b.name for b in tile.datasets[self.dataset_type].bands]:
relevant_tiles.append(tile)
_log.info("Total tiles for band [%s] is [%d]", self.band, len(relevant_tiles))
for index, tile in enumerate(relevant_tiles, start=1):
dataset = tile.datasets[self.dataset_type]
assert dataset
band = dataset.bands[self.band]
assert band
pqa = (self.mask_pqa_apply and DatasetType.PQ25 in tile.datasets) and tile.datasets[DatasetType.PQ25] or None
wofs = (self.mask_wofs_apply and DatasetType.WATER in tile.datasets) and tile.datasets[DatasetType.WATER] or None
if self.dataset_type not in tile.datasets:
_log.debug("No [%s] dataset present for [%s] - skipping", self.dataset_type.name, tile.end_datetime)
continue
filename = self.output().path
if not metadata:
metadata = get_dataset_metadata(dataset)
assert metadata
if not data_type:
data_type = get_dataset_datatype(dataset)
assert data_type
if not ndv:
ndv = get_dataset_ndv(dataset)
assert ndv
if not driver:
if self.output_format == OutputFormat.GEOTIFF:
driver = gdal.GetDriverByName("GTiff")
elif self.output_format == OutputFormat.ENVI:
driver = gdal.GetDriverByName("ENVI")
assert driver
if not raster:
if self.output_format == OutputFormat.GEOTIFF:
raster = driver.Create(filename, metadata.shape[0], metadata.shape[1], len(tiles), data_type, options=["BIGTIFF=YES", "INTERLEAVE=BAND"])
elif self.output_format == OutputFormat.ENVI:
raster = driver.Create(filename, metadata.shape[0], metadata.shape[1], len(tiles), data_type, options=["INTERLEAVE=BSQ"])
assert raster
# NOTE: could do this without the metadata!!
raster.SetGeoTransform(metadata.transform)
raster.SetProjection(metadata.projection)
raster.SetMetadata(self.generate_raster_metadata())
mask = None
if pqa:
mask = get_mask_pqa(pqa, self.mask_pqa_mask, mask=mask)
if wofs:
mask = get_mask_wofs(wofs, self.mask_wofs_mask, mask=mask)
_log.info("Stacking [%s] band data from [%s] with PQA [%s] and PQA mask [%s] and WOFS [%s] and WOFS mask [%s] to [%s]",
band.name, dataset.path,
pqa and pqa.path or "",
pqa and self.mask_pqa_mask or "",
wofs and wofs.path or "", wofs and self.mask_wofs_mask or "",
filename)
data = get_dataset_data_masked(dataset, mask=mask, ndv=ndv)
_log.debug("data is [%s]", data)
stack_band = raster.GetRasterBand(index)
stack_band.SetDescription(os.path.basename(dataset.path))
stack_band.SetNoDataValue(ndv)
stack_band.WriteArray(data[band])
stack_band.ComputeStatistics(True)
stack_band.SetMetadata({"ACQ_DATE": format_date(tile.end_datetime), "SATELLITE": dataset.satellite.name})
stack_band.FlushCache()
del stack_band
if raster:
raster.FlushCache()
raster = None
del raster
def run(self):
# TODO move the dicking around with bands stuff into utils?
import gdal
driver = raster = None
metadata = None
data_type = ndv = None
tiles = self.get_tiles()
_log.info("Total tiles found [%d]", len(tiles))
_log.info("Creating stack for band [%s]", self.band)
relevant_tiles = []
for tile in tiles:
dataset = self.dataset_type in tile.datasets and tile.datasets[
self.dataset_type] or None
if not dataset:
_log.info("No applicable [%s] dataset for [%s]",
self.dataset_type.name, tile.end_datetime)
continue
if self.band in [
b.name for b in tile.datasets[self.dataset_type].bands
]:
relevant_tiles.append(tile)
_log.info("Total tiles for band [%s] is [%d]", self.band,
len(relevant_tiles))
for index, tile in enumerate(relevant_tiles, start=1):
dataset = tile.datasets[self.dataset_type]
assert dataset
band = dataset.bands[self.band]
assert band
pqa = (self.mask_pqa_apply and DatasetType.PQ25 in tile.datasets
) and tile.datasets[DatasetType.PQ25] or None
wofs = (self.mask_wofs_apply and DatasetType.WATER in tile.datasets
) and tile.datasets[DatasetType.WATER] or None
if self.dataset_type not in tile.datasets:
_log.debug("No [%s] dataset present for [%s] - skipping",
self.dataset_type.name, tile.end_datetime)
continue
filename = self.output().path
if not metadata:
metadata = get_dataset_metadata(dataset)
assert metadata
if not data_type:
data_type = get_dataset_datatype(dataset)
assert data_type
if not ndv:
ndv = get_dataset_ndv(dataset)
assert ndv
if not driver:
if self.output_format == OutputFormat.GEOTIFF:
driver = gdal.GetDriverByName("GTiff")
elif self.output_format == OutputFormat.ENVI:
driver = gdal.GetDriverByName("ENVI")
assert driver
if not raster:
if self.output_format == OutputFormat.GEOTIFF:
raster = driver.Create(
filename,
metadata.shape[0],
metadata.shape[1],
len(tiles),
data_type,
options=["BIGTIFF=YES", "INTERLEAVE=BAND"])
elif self.output_format == OutputFormat.ENVI:
raster = driver.Create(filename,
metadata.shape[0],
metadata.shape[1],
len(tiles),
data_type,
options=["INTERLEAVE=BSQ"])
assert raster
# NOTE: could do this without the metadata!!
raster.SetGeoTransform(metadata.transform)
raster.SetProjection(metadata.projection)
raster.SetMetadata(self.generate_raster_metadata())
mask = None
if pqa:
mask = get_mask_pqa(pqa, self.mask_pqa_mask, mask=mask)
if wofs:
mask = get_mask_wofs(wofs, self.mask_wofs_mask, mask=mask)
_log.info(
"Stacking [%s] band data from [%s] with PQA [%s] and PQA mask [%s] and WOFS [%s] and WOFS mask [%s] to [%s]",
band.name, dataset.path, pqa and pqa.path or "",
pqa and self.mask_pqa_mask or "", wofs and wofs.path or "",
wofs and self.mask_wofs_mask or "", filename)
data = get_dataset_data_masked(dataset, mask=mask, ndv=ndv)
_log.debug("data is [%s]", data)
stack_band = raster.GetRasterBand(index)
stack_band.SetDescription(os.path.basename(dataset.path))
stack_band.SetNoDataValue(ndv)
stack_band.WriteArray(data[band])
stack_band.ComputeStatistics(True)
stack_band.SetMetadata({
"ACQ_DATE": format_date(tile.end_datetime),
"SATELLITE": dataset.satellite.name
})
stack_band.FlushCache()
del stack_band
if raster:
raster.FlushCache()
raster = None
del raster
def run(self):
_log.info("Creating stack for band [%s]", self.band.name)
data_type = get_dataset_type_datatype(self.dataset_type)
ndv = get_dataset_type_ndv(self.dataset_type)
metadata = None
driver = None
raster = None
acq_min, acq_max, criteria = build_season_date_criteria(self.acq_min, self.acq_max, self.season,
seasons=SEASONS, extend=True)
_log.info("\tacq %s to %s criteria is %s", acq_min, acq_max, criteria)
dataset_types = [self.dataset_type]
if self.mask_pqa_apply:
dataset_types.append(DatasetType.PQ25)
tiles = list_tiles_as_list(x=[self.x], y=[self.y], satellites=self.satellites,
acq_min=acq_min, acq_max=acq_max,
dataset_types=dataset_types, include=criteria)
for index, tile in enumerate(tiles, start=1):
dataset = tile.datasets[self.dataset_type]
assert dataset
# band = dataset.bands[self.band]
# assert band
band = self.band
pqa = (self.mask_pqa_apply and DatasetType.PQ25 in tile.datasets) and tile.datasets[DatasetType.PQ25] or None
if self.dataset_type not in tile.datasets:
_log.debug("No [%s] dataset present for [%s] - skipping", self.dataset_type.name, tile.end_datetime)
continue
filename = self.output().path
if not metadata:
metadata = get_dataset_metadata(dataset)
assert metadata
if not driver:
if self.output_format == OutputFormat.GEOTIFF:
driver = gdal.GetDriverByName("GTiff")
elif self.output_format == OutputFormat.ENVI:
driver = gdal.GetDriverByName("ENVI")
assert driver
if not raster:
if self.output_format == OutputFormat.GEOTIFF:
raster = driver.Create(filename, metadata.shape[0], metadata.shape[1], len(tiles), data_type, options=["BIGTIFF=YES", "INTERLEAVE=BAND"])
elif self.output_format == OutputFormat.ENVI:
raster = driver.Create(filename, metadata.shape[0], metadata.shape[1], len(tiles), data_type, options=["INTERLEAVE=BSQ"])
assert raster
# NOTE: could do this without the metadata!!
raster.SetGeoTransform(metadata.transform)
raster.SetProjection(metadata.projection)
raster.SetMetadata(self.generate_raster_metadata())
mask = None
if pqa:
mask = get_mask_pqa(pqa, self.mask_pqa_mask, mask=mask)
_log.info("Stacking [%s] band data from [%s] with PQA [%s] and PQA mask [%s] to [%s]",
band.name, dataset.path,
pqa and pqa.path or "", pqa and self.mask_pqa_mask or "",
filename)
data = get_dataset_data_masked(dataset, mask=mask, ndv=ndv)
_log.debug("data is [%s]", data)
stack_band = raster.GetRasterBand(index)
stack_band.SetDescription(os.path.basename(dataset.path))
stack_band.SetNoDataValue(ndv)
stack_band.WriteArray(data[band])
stack_band.ComputeStatistics(True)
stack_band.SetMetadata({"ACQ_DATE": format_date(tile.end_datetime), "SATELLITE": dataset.satellite.name})
stack_band.FlushCache()
del stack_band
if raster:
raster.FlushCache()
del raster
raster = None
def go(self):
# If we are applying a vector mask then calculate it not (once as it is the same for all tiles)
mask_vector = None
if self.mask_vector_apply:
mask_vector = get_mask_vector_for_cell(self.x, self.y, self.mask_vector_file, self.mask_vector_layer, self.mask_vector_feature)
# TODO move the dicking around with bands stuff into utils?
import gdal
if self.output_format == OutputFormat.GEOTIFF:
driver = gdal.GetDriverByName("GTiff")
elif self.output_format == OutputFormat.ENVI:
driver = gdal.GetDriverByName("ENVI")
assert driver
tiles = self.get_tiles()
_log.info("Total tiles found [%d]", len(tiles))
for band_name in self.bands:
_log.info("Creating stack for band [%s]", band_name)
relevant_tiles = []
for tile in tiles:
dataset = self.dataset_type in tile.datasets and tile.datasets[self.dataset_type] or None
if not dataset:
_log.info("No applicable [%s] dataset for [%s]", self.dataset_type.name, tile.end_datetime)
continue
if band_name in [b.name for b in tile.datasets[self.dataset_type].bands]:
relevant_tiles.append(tile)
_log.info("Total tiles for band [%s] is [%d]", band_name, len(relevant_tiles))
filename = None
raster = None
metadata = None
data_type = ndv = None
for index, tile in enumerate(relevant_tiles, start=1):
dataset = tile.datasets[self.dataset_type]
assert dataset
band = dataset.bands[band_name]
assert band
if self.list_only:
_log.info("Would stack band [%s] from dataset [%s]", band.name, dataset.path)
continue
pqa = (self.mask_pqa_apply and DatasetType.PQ25 in tile.datasets) and tile.datasets[DatasetType.PQ25] or None
wofs = (self.mask_wofs_apply and DatasetType.WATER in tile.datasets) and tile.datasets[DatasetType.WATER] or None
if self.dataset_type not in tile.datasets:
_log.debug("No [%s] dataset present for [%s] - skipping", self.dataset_type.name, tile.end_datetime)
continue
if not filename:
filename = os.path.join(self.output_directory,
get_dataset_band_stack_filename(satellites=self.satellites,
dataset_type=self.dataset_type,
band=band,
x=self.x, y=self.y,
acq_min=self.acq_min, acq_max=self.acq_max,
season=self.season,
output_format=self.output_format,
mask_pqa_apply=self.mask_pqa_apply,
mask_wofs_apply=self.mask_wofs_apply,
mask_vector_apply=self.mask_vector_apply))
_log.info("Stacking [%s] band data from [%s] with PQA [%s] and PQA mask [%s] and WOFS [%s] and WOFS mask [%s] to band [%d] of [%s]",
band.name, dataset.path,
pqa and pqa.path or "",
pqa and self.mask_pqa_mask or "",
wofs and wofs.path or "", wofs and self.mask_wofs_mask or "",
index,
filename)
if not metadata:
metadata = get_dataset_metadata(dataset)
assert metadata
if not data_type:
data_type = get_dataset_datatype(dataset)
assert data_type
if not ndv:
ndv = get_dataset_ndv(dataset)
assert ndv
if not raster:
if self.output_format == OutputFormat.GEOTIFF:
raster = driver.Create(filename, metadata.shape[0], metadata.shape[1], len(relevant_tiles), data_type, options=["TILED=YES", "BIGTIFF=YES", "COMPRESS=LZW", "INTERLEAVE=BAND"])
elif self.output_format == OutputFormat.ENVI:
raster = driver.Create(filename, metadata.shape[0], metadata.shape[1], len(relevant_tiles), data_type, options=["INTERLEAVE=BSQ"])
assert raster
# NOTE: could do this without the metadata!!
raster.SetGeoTransform(metadata.transform)
raster.SetProjection(metadata.projection)
raster.SetMetadata(self.generate_raster_metadata())
mask = mask_vector
if pqa:
mask = get_mask_pqa(pqa, self.mask_pqa_mask, mask=mask)
if wofs:
mask = get_mask_wofs(wofs, self.mask_wofs_mask, mask=mask)
# _log.info("mask[3500,3500] is [%s]", mask[3500, 3500])
data = get_dataset_data_masked(dataset, mask=mask, ndv=ndv)
_log.debug("data is [%s]", data)
# _log.info("data[3500,3500] is [%s]", data[band][3500, 3500])
stack_band = raster.GetRasterBand(index)
stack_band.SetDescription(os.path.basename(dataset.path))
stack_band.SetNoDataValue(ndv)
stack_band.WriteArray(data[band])
stack_band.ComputeStatistics(True)
stack_band.SetMetadata({"ACQ_DATE": format_date(tile.end_datetime), "SATELLITE": dataset.satellite.name})
stack_band.FlushCache()
del stack_band
if raster:
raster.FlushCache()
raster = None
del raster
def run(self):
shape = (4000, 4000)
no_data_value = NDV
best_pixel_fc = dict()
for band in Fc25Bands:
# best_pixel_fc[band] = empty_array(shape=shape, dtype=numpy.int16, ndv=INT16_MIN)
best_pixel_fc[band] = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
best_pixel_nbar = dict()
for band in Ls57Arg25Bands:
best_pixel_nbar[band] = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
best_pixel_satellite = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
best_pixel_date = empty_array(shape=shape, dtype=numpy.int32, ndv=NDV)
current_satellite = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV)
current_date = empty_array(shape=shape, dtype=numpy.int32, ndv=NDV)
SATELLITE_DATA_VALUES = {Satellite.LS5: 5, Satellite.LS7: 7, Satellite.LS8: 8}
metadata_nbar = None
metadata_fc = None
for tile in self.get_tiles():
pqa = tile.datasets[DatasetType.PQ25]
nbar = tile.datasets[DatasetType.ARG25]
fc = tile.datasets[DatasetType.FC25]
wofs = DatasetType.WATER in tile.datasets and tile.datasets[DatasetType.WATER] or None
_log.info("Processing [%s]", fc.path)
data = dict()
# Create an initial "no mask" mask
mask = numpy.ma.make_mask_none((4000, 4000))
# _log.info("### mask is [%s]", mask[1000][1000])
# Add the PQA mask if we are doing PQA masking
if self.mask_pqa_apply:
mask = get_mask_pqa(pqa, pqa_masks=self.mask_pqa_mask, mask=mask)
# _log.info("### mask PQA is [%s]", mask[1000][1000])
# Add the WOFS mask if we are doing WOFS masking
if self.mask_wofs_apply and wofs:
mask = get_mask_wofs(wofs, wofs_masks=self.mask_wofs_mask, mask=mask)
# _log.info("### mask PQA is [%s]", mask[1000][1000])
# Get NBAR dataset
data[DatasetType.ARG25] = get_dataset_data_masked(nbar, mask=mask)
# _log.info("### NBAR/RED is [%s]", data[DatasetType.ARG25][Ls57Arg25Bands.RED][1000][1000])
# Get the NDVI dataset
data[DatasetType.NDVI] = calculate_ndvi(data[DatasetType.ARG25][Ls57Arg25Bands.RED],
data[DatasetType.ARG25][Ls57Arg25Bands.NEAR_INFRARED])
# _log.info("### NDVI is [%s]", data[DatasetType.NDVI][1000][1000])
# Add the NDVI value range mask (to the existing mask)
mask = self.get_mask_range(data[DatasetType.NDVI], min_val=0.0, max_val=0.3, mask=mask)
# _log.info("### mask NDVI is [%s]", mask[1000][1000])
# Get FC25 dataset
data[DatasetType.FC25] = get_dataset_data_masked(fc, mask=mask)
# _log.info("### FC/BS is [%s]", data[DatasetType.FC25][Fc25Bands.BARE_SOIL][1000][1000])
# Add the bare soil value range mask (to the existing mask)
mask = self.get_mask_range(data[DatasetType.FC25][Fc25Bands.BARE_SOIL], min_val=0, max_val=8000, mask=mask)
# _log.info("### mask BS is [%s]", mask[1000][1000])
# Apply the final mask to the FC25 bare soil data
data_bare_soil = numpy.ma.MaskedArray(data=data[DatasetType.FC25][Fc25Bands.BARE_SOIL], mask=mask).filled(NDV)
# _log.info("### bare soil is [%s]", data_bare_soil[1000][1000])
# Compare the bare soil value from this dataset to the current "best" value
best_pixel_fc[Fc25Bands.BARE_SOIL] = numpy.fmax(best_pixel_fc[Fc25Bands.BARE_SOIL], data_bare_soil)
# _log.info("### best pixel bare soil is [%s]", best_pixel_fc[Fc25Bands.BARE_SOIL][1000][1000])
# Now update the other best pixel datasets/bands to grab the pixels we just selected
for band in Ls57Arg25Bands:
best_pixel_nbar[band] = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BARE_SOIL],
data_bare_soil,
data[DatasetType.ARG25][band],
best_pixel_nbar[band])
for band in [Fc25Bands.PHOTOSYNTHETIC_VEGETATION, Fc25Bands.NON_PHOTOSYNTHETIC_VEGETATION, Fc25Bands.UNMIXING_ERROR]:
best_pixel_fc[band] = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BARE_SOIL],
data_bare_soil,
data[DatasetType.FC25][band],
best_pixel_fc[band])
# And now the other "provenance" data
# Satellite "provenance" data
current_satellite.fill(SATELLITE_DATA_VALUES[fc.satellite])
best_pixel_satellite = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BARE_SOIL],
data_bare_soil,
current_satellite,
best_pixel_satellite)
# Date "provenance" data
current_date.fill(date_to_integer(tile.end_datetime))
best_pixel_date = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BARE_SOIL],
data_bare_soil,
current_date,
best_pixel_date)
# Grab the metadata from the input datasets for use later when creating the output datasets
if not metadata_nbar:
metadata_nbar = get_dataset_metadata(nbar)
if not metadata_fc:
metadata_fc = get_dataset_metadata(fc)
# Create the output datasets
# FC composite
raster_create(self.get_dataset_filename("FC"),
[best_pixel_fc[b] for b in Fc25Bands],
metadata_fc.transform, metadata_fc.projection,
metadata_fc.bands[Fc25Bands.BARE_SOIL].no_data_value,
metadata_fc.bands[Fc25Bands.BARE_SOIL].data_type)
# NBAR composite
raster_create(self.get_dataset_filename("NBAR"),
[best_pixel_nbar[b] for b in Ls57Arg25Bands],
metadata_nbar.transform, metadata_nbar.projection,
metadata_nbar.bands[Ls57Arg25Bands.BLUE].no_data_value,
metadata_nbar.bands[Ls57Arg25Bands.BLUE].data_type)
# Satellite "provenance" composites
raster_create(self.get_dataset_filename("SAT"),
[best_pixel_satellite],
metadata_nbar.transform, metadata_nbar.projection, no_data_value,
gdal.GDT_Int16)
# Date "provenance" composites
raster_create(self.get_dataset_filename("DATE"),
[best_pixel_date],
metadata_nbar.transform, metadata_nbar.projection, no_data_value,
gdal.GDT_Int32)
def run(self):
_log.info("Creating stack for band [%s]", self.band.name)
data_type = get_dataset_type_datatype(self.dataset_type)
ndv = get_dataset_type_ndv(self.dataset_type)
metadata = None
driver = None
raster = None
acq_min, acq_max, criteria = build_season_date_criteria(
self.acq_min,
self.acq_max,
self.season,
seasons=SEASONS,
extend=True)
_log.info("\tacq %s to %s criteria is %s", acq_min, acq_max, criteria)
dataset_types = [self.dataset_type]
if self.mask_pqa_apply:
dataset_types.append(DatasetType.PQ25)
tiles = list_tiles_as_list(x=[self.x],
y=[self.y],
satellites=self.satellites,
acq_min=acq_min,
acq_max=acq_max,
dataset_types=dataset_types,
include=criteria)
for index, tile in enumerate(tiles, start=1):
dataset = tile.datasets[self.dataset_type]
assert dataset
# band = dataset.bands[self.band]
# assert band
band = self.band
pqa = (self.mask_pqa_apply and DatasetType.PQ25 in tile.datasets
) and tile.datasets[DatasetType.PQ25] or None
if self.dataset_type not in tile.datasets:
_log.debug("No [%s] dataset present for [%s] - skipping",
self.dataset_type.name, tile.end_datetime)
continue
filename = self.output().path
if not metadata:
metadata = get_dataset_metadata(dataset)
assert metadata
if not driver:
if self.output_format == OutputFormat.GEOTIFF:
driver = gdal.GetDriverByName("GTiff")
elif self.output_format == OutputFormat.ENVI:
driver = gdal.GetDriverByName("ENVI")
assert driver
if not raster:
if self.output_format == OutputFormat.GEOTIFF:
raster = driver.Create(
filename,
metadata.shape[0],
metadata.shape[1],
len(tiles),
data_type,
options=["BIGTIFF=YES", "INTERLEAVE=BAND"])
elif self.output_format == OutputFormat.ENVI:
raster = driver.Create(filename,
metadata.shape[0],
metadata.shape[1],
len(tiles),
data_type,
options=["INTERLEAVE=BSQ"])
assert raster
# NOTE: could do this without the metadata!!
raster.SetGeoTransform(metadata.transform)
raster.SetProjection(metadata.projection)
raster.SetMetadata(self.generate_raster_metadata())
mask = None
if pqa:
mask = get_mask_pqa(pqa, self.mask_pqa_mask, mask=mask)
_log.info(
"Stacking [%s] band data from [%s] with PQA [%s] and PQA mask [%s] to [%s]",
band.name, dataset.path, pqa and pqa.path or "",
pqa and self.mask_pqa_mask or "", filename)
data = get_dataset_data_masked(dataset, mask=mask, ndv=ndv)
_log.debug("data is [%s]", data)
stack_band = raster.GetRasterBand(index)
stack_band.SetDescription(os.path.basename(dataset.path))
stack_band.SetNoDataValue(ndv)
stack_band.WriteArray(data[band])
stack_band.ComputeStatistics(True)
stack_band.SetMetadata({
"ACQ_DATE": format_date(tile.end_datetime),
"SATELLITE": dataset.satellite.name
})
stack_band.FlushCache()
del stack_band
if raster:
raster.FlushCache()
del raster
raster = None
def preview_cloudfree_mosaic(x,y,start,end, bands, satellite,iterations=0,xsize=2000,ysize=2000,file_format="GTiff",data_type=gdal.GDT_CInt16):
def resize_array(arr,size):
r = numpy.array(arr).astype(numpy.int16)
i = Image.fromarray(r)
i2 = i.resize(size,Image.NEAREST)
r2 = numpy.array(i2)
del i2
del i
del r
return r2
StartDate = start
EndDate = end
best_data = {}
band_str = "+".join([band.name for band in bands])
sat_str = "+".join([sat.name for sat in satellite])
cache_id = ["preview",str(x),str(y),str(start),str(end),band_str,sat_str,str(xsize),str(ysize),file_format,str(iterations)]
f_name = "_".join(cache_id)
f_name = f_name.replace(" ","_")
c_name = f_name
cached_res = cache.get(c_name)
if cached_res:
return str(cached_res)
f_name = os.path.join("/tilestore/tile_cache",f_name)
tiles = list_tiles(x=[x], y=[y],acq_min=StartDate,acq_max=EndDate,satellites=satellite,dataset_types=[DatasetType.ARG25,DatasetType.PQ25], sort=SortType.ASC)
tile_metadata = None
tile_count = 0
tile_filled = False
for tile in tiles:
if tile_filled:
break
print "merging on tile "+str(tile.x)+", "+str(tile.y)
tile_count+=1
dataset = DatasetType.ARG25 in tile.datasets and tile.datasets[DatasetType.ARG25] or None
if dataset is None:
print "No dataset availible"
tile_count-=1
continue
tile_metadata = get_dataset_metadata(dataset)
if tile_metadata is None:
print "NO METADATA"
tile_count-=1
continue
pqa = DatasetType.PQ25 in tile.datasets and tile.datasets[DatasetType.PQ25] or None
mask = None
mask = get_mask_pqa(pqa,[PqaMask.PQ_MASK_CLEAR],mask=mask)
band_data = get_dataset_data_masked(dataset, mask=mask,bands=bands)
swap_arr = None
for band in band_data:
if not band in best_data:
print "Adding "+band.name
bd = resize_array(band_data[band],(2000,2000))
print bd
best_data[band]=bd
del bd
else:
best = resize_array(best_data[band],(2000,2000))
swap_arr=numpy.in1d(best.ravel(),-999).reshape(best.shape)
b_data = numpy.array(band_data[band])
best[swap_arr]=b_data[swap_arr]
best_data[band]=numpy.copy(best)
del b_data
del best
del swap_arr
if iterations > 0:
if tile_count>iterations:
print "Exiting after "+str(iterations)+" iterations"
break
numberOfBands=len(bands)
if numberOfBands == 0:
return "None"
if bands[0] not in best_data:
print "No data was merged for "+str(x)+", "+str(y)
return "None"
numberOfPixelsInXDirection=len(best_data[bands[0]])
numberOfPixelsInYDirection=len(best_data[bands[0]][0])
if tile_count <1:
print "No tiles found for "+str(x)+", "+str(y)
return "None"
driver = gdal.GetDriverByName(file_format)
if driver is None:
print "No driver found for "+file_format
return "None"
print f_name+'.tif'
raster = driver.Create(f_name+'.tif', numberOfPixelsInXDirection, numberOfPixelsInYDirection, numberOfBands, data_type, options=["BIGTIFF=YES", "INTERLEAVE=BAND"])
gt = tile_metadata.transform
gt2 = (gt[0],gt[1]*2.0,gt[2],gt[3],gt[4],gt[5]*2.0)
tile_metadata.transform = gt2
raster.SetGeoTransform(tile_metadata.transform)
print tile_metadata.transform
raster.SetProjection(tile_metadata.projection)
index = 1
for band in bands:
stack_band = raster.GetRasterBand(index)
stack_band.SetNoDataValue(-999)
stack_band.WriteArray(best_data[band])
stack_band.ComputeStatistics(True)
index+=1
stack_band.FlushCache()
del stack_band
raster.FlushCache()
del raster
cache.set(c_name,f_name+".tif")
return f_name+".tif"
def obtain_cloudfree_mosaic(x,y,start,end, bands, satellite,iterations=0,xsize=4000,ysize=4000,file_format="GTiff",data_type=gdal.GDT_CInt16,months=None):
StartDate = start
EndDate = end
best_data = {}
band_str = "+".join([band.name for band in bands])
sat_str = "+".join([sat.name for sat in satellite])
cache_id = [str(x),str(y),str(start),str(end),band_str,sat_str,str(xsize),str(ysize),file_format,str(iterations)]
f_name = "_".join(cache_id)
f_name = f_name.replace(" ","_")
c_name = f_name
cached_res = cache.get(c_name)
if cached_res:
return str(cached_res)
f_name = os.path.join("/tilestore/tile_cache",f_name)
tiles = list_tiles(x=[x], y=[y],acq_min=StartDate,acq_max=EndDate,satellites=satellite,dataset_types=[DatasetType.ARG25,DatasetType.PQ25], sort=SortType.ASC)
tile_metadata = None
tile_count = 0
tile_filled = False
stats_file = open(f_name+'.csv','w+')
total_ins = 0
for tile in tiles:
if tile_filled:
break
if months:
print tile.start_datetime.month
if not tile.start_datetime.month in months:
continue
#print "merging on tile "+str(tile.x)+", "+str(tile.y)
tile_count+=1
dataset = DatasetType.ARG25 in tile.datasets and tile.datasets[DatasetType.ARG25] or None
if dataset is None:
print "No dataset availible"
tile_count-=1
continue
tile_metadata = get_dataset_metadata(dataset)
if tile_metadata is None:
print "NO METADATA"
tile_count-=1
continue
pqa = DatasetType.PQ25 in tile.datasets and tile.datasets[DatasetType.PQ25] or None
mask = None
mask = get_mask_pqa(pqa,[PqaMask.PQ_MASK_CLEAR],mask=mask)
band_data = get_dataset_data_masked(dataset, mask=mask,bands=bands)
swap_arr = None
best = None
good_ins = None
for band in band_data:
if not band in best_data:
#print "Adding "+band.name
#print band_data[band]
best_data[band]=band_data[band]
best = numpy.array(best_data[band])
swap_arr=numpy.in1d(best.ravel(),-999).reshape(best.shape)
good_ins = len(numpy.where(best[swap_arr]!=-999)[0])
else:
best = numpy.array(best_data[band])
swap_arr=numpy.in1d(best.ravel(),-999).reshape(best.shape)
b_data = numpy.array(band_data[band])
best[swap_arr]=b_data[swap_arr]
best_data[band]=numpy.copy(best)
good_ins = len(numpy.where(b_data[swap_arr]!=-999)[0])
del b_data
total_ins+=good_ins
stats_file.write(str(tile.x)+','+str(tile.y)+','+str(tile.start_datetime.year)+','+str(tile.start_datetime.month)+','+str(len(best[swap_arr]))+','+str(good_ins)+','+str(total_ins)+','+str(tile.dataset)+"\n")
del swap_arr
del best
del good_ins
if iterations > 0:
if tile_count>iterations:
print "Exiting after "+str(iterations)+" iterations"
break
numberOfBands=len(bands)
if numberOfBands == 0:
return "None"
if bands[0] not in best_data:
print "No data was merged for "+str(x)+", "+str(y)
return "None"
numberOfPixelsInXDirection=len(best_data[bands[0]])
print numberOfPixelsInXDirection
numberOfPixelsInYDirection=len(best_data[bands[0]][0])
print numberOfPixelsInYDirection
pixels = numberOfPixelsInXDirection
if numberOfPixelsInYDirection > numberOfPixelsInXDirection:
pixels = numberOfPixelsInYDirection
if tile_count <1:
print "No tiles found for "+str(x)+", "+str(y)
return "None"
driver = gdal.GetDriverByName(file_format)
if driver is None:
print "No driver found for "+file_format
return "None"
#print f_name+'.tif'
raster = driver.Create(f_name+'.tif', pixels, pixels, numberOfBands, data_type, options=["BIGTIFF=YES", "INTERLEAVE=BAND"])
raster.SetGeoTransform(tile_metadata.transform)
raster.SetProjection(tile_metadata.projection)
index = 1
stats_file.close()
for band in bands:
stack_band = raster.GetRasterBand(index)
stack_band.SetNoDataValue(-999)
stack_band.WriteArray(best_data[band])
stack_band.ComputeStatistics(True)
index+=1
stack_band.FlushCache()
del stack_band
raster.FlushCache()
del raster
cache.set(c_name,f_name+".tif")
return f_name+".tif"
