def generate_raster_metadata(self): return { "X_INDEX": "{x:03d}".format(x=self.x), "Y_INDEX": "{y:04d}".format(y=self.y), "DATASET_TYPE": self.dataset_type.name, "ACQUISITION_DATE": "{acq_min} to {acq_max}".format( acq_min=format_date(self.acq_min), acq_max=format_date(self.acq_max) ), "SATELLITES": " ".join([s.name for s in self.satellites]), "PIXEL_QUALITY_FILTER": self.mask_pqa_apply and " ".join([mask.name for mask in self.mask_pqa_mask]) or "", "WATER_FILTER": self.mask_wofs_apply and " ".join([mask.name for mask in self.mask_wofs_mask]) or "", }
def output(self): from datacube.api.workflow import format_date from datacube.api.utils import get_satellite_string acq_min = format_date(self.acq_min) acq_max = format_date(self.acq_max) filename = os.path.join( self.output_directory, "TidalExtra_{x:03d}_{y:04d}_{acq_min}_{acq_max}.tif".format( x=self.x, y=self.y, acq_min=acq_min, acq_max=acq_max)) return luigi.LocalTarget(filename)
def get_tile_csv_filename(self): acq_min = format_date(self.acq_min) acq_max = format_date(self.acq_max) # TODO other distinguishing characteristics (e.g. dataset types) return os.path.join( self.output_directory, "tiles_{satellites}_{x_min:03d}_{x_max:03d}_{y_min:04d}_{y_max:04d}_{acq_min}_{acq_max}.csv".format( satellites=get_satellite_string(self.satellites), x_min=self.x, x_max=self.x, y_min=self.y, y_max=self.y, acq_min=acq_min, acq_max=acq_max ))
def output(self): from datacube.api.workflow import format_date from datacube.api.utils import get_satellite_string acq_min = format_date(self.acq_min) acq_max = format_date(self.acq_max) filename = os.path.join(self.output_directory, "TidalExtra_{x:03d}_{y:04d}_{acq_min}_{acq_max}.tif".format( x=self.x, y=self.y, acq_min=acq_min, acq_max=acq_max)) return luigi.LocalTarget(filename)
def output(self): from datacube.api.workflow import format_date from datacube.api.utils import get_satellite_string acq_min = format_date(self.acq_min) acq_max = format_date(self.acq_max) dataset_type_string = { DatasetType.ARG25: "NBAR", DatasetType.PQ25: "PQA", DatasetType.FC25: "FC", DatasetType.WATER: "WATER", DatasetType.NDVI: "NDVI", DatasetType.EVI: "EVI", DatasetType.NBR: "NBR", DatasetType.TCI: "TCI", DatasetType.DSM: "DSM" }[self.dataset_type] if self.mask_pqa_apply and self.mask_wofs_apply: dataset_type_string += "_WITH_PQA_WATER" elif self.mask_pqa_apply: dataset_type_string += "_WITH_PQA" elif self.mask_wofs_apply: dataset_type_string += + "_WITH_WATER" ext = { OutputFormat.GEOTIFF: "tif", OutputFormat.ENVI: "dat" }[self.output_format] filename = "{satellites}_{dataset_type}_STACK_{band}_{x:03d}_{y:04d}_{acq_min}_{acq_max}.{ext}".format( satellites=get_satellite_string(self.satellites), dataset_type=dataset_type_string, band=self.band, x=self.x, y=self.y, acq_min=acq_min, acq_max=acq_max, ext=ext) filename = os.path.join(self.output_directory, filename) return luigi.LocalTarget(filename)
def get_tile_csv_filename(self): acq_min = format_date(self.acq_min) acq_max = format_date(self.acq_max) # TODO other distinguishing characteristics (e.g. dataset types) return os.path.join( self.output_directory, "tiles_{satellites}_{x_min:03d}_{x_max:03d}_{y_min:04d}_{y_max:04d}_{acq_min}_{acq_max}.csv" .format(satellites=get_satellite_string(self.satellites), x_min=self.x, x_max=self.x, y_min=self.y, y_max=self.y, acq_min=acq_min, acq_max=acq_max))
def output(self): from datacube.api.workflow import format_date from datacube.api.utils import get_satellite_string acq_min = format_date(self.acq_min) acq_max = format_date(self.acq_max) dataset_type_string = { DatasetType.ARG25: "NBAR", DatasetType.PQ25: "PQA", DatasetType.FC25: "FC", DatasetType.WATER: "WATER", DatasetType.NDVI: "NDVI", DatasetType.EVI: "EVI", DatasetType.NBR: "NBR", DatasetType.TCI: "TCI", DatasetType.DSM: "DSM", }[self.dataset_type] if self.mask_pqa_apply and self.mask_wofs_apply: dataset_type_string += "_WITH_PQA_WATER" elif self.mask_pqa_apply: dataset_type_string += "_WITH_PQA" elif self.mask_wofs_apply: dataset_type_string += +"_WITH_WATER" ext = {OutputFormat.GEOTIFF: "tif", OutputFormat.ENVI: "dat"}[self.output_format] filename = "{satellites}_{dataset_type}_STACK_{band}_{x:03d}_{y:04d}_{acq_min}_{acq_max}.{ext}".format( satellites=get_satellite_string(self.satellites), dataset_type=dataset_type_string, band=self.band, x=self.x, y=self.y, acq_min=acq_min, acq_max=acq_max, ext=ext, ) filename = os.path.join(self.output_directory, filename) return luigi.LocalTarget(filename)
def generate_raster_metadata(self): return { "X_INDEX": "{x:03d}".format(x=self.x), "Y_INDEX": "{y:04d}".format(y=self.y), "DATASET_TYPE": "ARG25", "ACQUISITION_DATE": "{acq_min} to {acq_max}".format(acq_min=format_date(self.acq_min), acq_max=format_date(self.acq_max)), "SEASON": "Calendar year", "SATELLITES": " ".join([s.name for s in self.satellites]), "DESCRIPTION": "Intertidal offset fields" }
def generate_raster_metadata(self): return { "X_INDEX": "{x:03d}".format(x=self.x), "Y_INDEX": "{y:04d}".format(y=self.y), "DATASET_TYPE": "ARG25", "ACQUISITION_DATE": "{acq_min} to {acq_max}".format(acq_min=format_date(self.acq_min), acq_max=format_date(self.acq_max)), "SEASON": "Calendar year", "SATELLITES": " ".join([s.name for s in self.satellites]), "DESCRIPTION": "Tidal Image for each 10 percent of datasets" }
def generate_raster_metadata(self): return { "X_INDEX": "{x:03d}".format(x=self.x), "Y_INDEX": "{y:04d}".format(y=self.y), "DATASET_TYPE": "{dataset_type} STACK".format(dataset_type=self.dataset_type.name), "ACQUISITION_DATE": "{acq_min} to {acq_max}".format(acq_min=format_date(self.acq_min), acq_max=format_date(self.acq_max)), "SEASON": self.season.name, "SATELLITES": " ".join([s.name for s in self.satellites]), "PIXEL_QUALITY_FILTER": self.mask_pqa_apply and " ".join([mask.name for mask in self.mask_pqa_mask]) or "" }
def run(self): # TODO move the dicking around with bands stuff into utils? import gdal driver = raster = None metadata = None data_type = ndv = None tiles = self.get_tiles() _log.info("Total tiles found [%d]", len(tiles)) _log.info("Creating stack for band [%s]", self.band) relevant_tiles = [] for tile in tiles: dataset = self.dataset_type in tile.datasets and tile.datasets[self.dataset_type] or None if not dataset: _log.info("No applicable [%s] dataset for [%s]", self.dataset_type.name, tile.end_datetime) continue if self.band in [b.name for b in tile.datasets[self.dataset_type].bands]: relevant_tiles.append(tile) _log.info("Total tiles for band [%s] is [%d]", self.band, len(relevant_tiles)) for index, tile in enumerate(relevant_tiles, start=1): dataset = tile.datasets[self.dataset_type] assert dataset band = dataset.bands[self.band] assert band pqa = (self.mask_pqa_apply and DatasetType.PQ25 in tile.datasets) and tile.datasets[DatasetType.PQ25] or None wofs = (self.mask_wofs_apply and DatasetType.WATER in tile.datasets) and tile.datasets[DatasetType.WATER] or None if self.dataset_type not in tile.datasets: _log.debug("No [%s] dataset present for [%s] - skipping", self.dataset_type.name, tile.end_datetime) continue filename = self.output().path if not metadata: metadata = get_dataset_metadata(dataset) assert metadata if not data_type: data_type = get_dataset_datatype(dataset) assert data_type if not ndv: ndv = get_dataset_ndv(dataset) assert ndv if not driver: if self.output_format == OutputFormat.GEOTIFF: driver = gdal.GetDriverByName("GTiff") elif self.output_format == OutputFormat.ENVI: driver = gdal.GetDriverByName("ENVI") assert driver if not raster: if self.output_format == OutputFormat.GEOTIFF: raster = driver.Create(filename, metadata.shape[0], metadata.shape[1], len(tiles), data_type, options=["BIGTIFF=YES", "INTERLEAVE=BAND"]) elif self.output_format == OutputFormat.ENVI: raster = driver.Create(filename, metadata.shape[0], metadata.shape[1], len(tiles), data_type, options=["INTERLEAVE=BSQ"]) assert raster # NOTE: could do this without the metadata!! raster.SetGeoTransform(metadata.transform) raster.SetProjection(metadata.projection) raster.SetMetadata(self.generate_raster_metadata()) mask = None if pqa: mask = get_mask_pqa(pqa, self.mask_pqa_mask, mask=mask) if wofs: mask = get_mask_wofs(wofs, self.mask_wofs_mask, mask=mask) _log.info("Stacking [%s] band data from [%s] with PQA [%s] and PQA mask [%s] and WOFS [%s] and WOFS mask [%s] to [%s]", band.name, dataset.path, pqa and pqa.path or "", pqa and self.mask_pqa_mask or "", wofs and wofs.path or "", wofs and self.mask_wofs_mask or "", filename) data = get_dataset_data_masked(dataset, mask=mask, ndv=ndv) _log.debug("data is [%s]", data) stack_band = raster.GetRasterBand(index) stack_band.SetDescription(os.path.basename(dataset.path)) stack_band.SetNoDataValue(ndv) stack_band.WriteArray(data[band]) stack_band.ComputeStatistics(True) stack_band.SetMetadata({"ACQ_DATE": format_date(tile.end_datetime), "SATELLITE": dataset.satellite.name}) stack_band.FlushCache() del stack_band if raster: raster.FlushCache() raster = None del raster
def run(self): # TODO move the dicking around with bands stuff into utils? import gdal driver = raster = None metadata = None data_type = ndv = None tiles = self.get_tiles() _log.info("Total tiles found [%d]", len(tiles)) _log.info("Creating stack for band [%s]", self.band) relevant_tiles = [] for tile in tiles: dataset = self.dataset_type in tile.datasets and tile.datasets[ self.dataset_type] or None if not dataset: _log.info("No applicable [%s] dataset for [%s]", self.dataset_type.name, tile.end_datetime) continue if self.band in [ b.name for b in tile.datasets[self.dataset_type].bands ]: relevant_tiles.append(tile) _log.info("Total tiles for band [%s] is [%d]", self.band, len(relevant_tiles)) for index, tile in enumerate(relevant_tiles, start=1): dataset = tile.datasets[self.dataset_type] assert dataset band = dataset.bands[self.band] assert band pqa = (self.mask_pqa_apply and DatasetType.PQ25 in tile.datasets ) and tile.datasets[DatasetType.PQ25] or None wofs = (self.mask_wofs_apply and DatasetType.WATER in tile.datasets ) and tile.datasets[DatasetType.WATER] or None if self.dataset_type not in tile.datasets: _log.debug("No [%s] dataset present for [%s] - skipping", self.dataset_type.name, tile.end_datetime) continue filename = self.output().path if not metadata: metadata = get_dataset_metadata(dataset) assert metadata if not data_type: data_type = get_dataset_datatype(dataset) assert data_type if not ndv: ndv = get_dataset_ndv(dataset) assert ndv if not driver: if self.output_format == OutputFormat.GEOTIFF: driver = gdal.GetDriverByName("GTiff") elif self.output_format == OutputFormat.ENVI: driver = gdal.GetDriverByName("ENVI") assert driver if not raster: if self.output_format == OutputFormat.GEOTIFF: raster = driver.Create( filename, metadata.shape[0], metadata.shape[1], len(tiles), data_type, options=["BIGTIFF=YES", "INTERLEAVE=BAND"]) elif self.output_format == OutputFormat.ENVI: raster = driver.Create(filename, metadata.shape[0], metadata.shape[1], len(tiles), data_type, options=["INTERLEAVE=BSQ"]) assert raster # NOTE: could do this without the metadata!! raster.SetGeoTransform(metadata.transform) raster.SetProjection(metadata.projection) raster.SetMetadata(self.generate_raster_metadata()) mask = None if pqa: mask = get_mask_pqa(pqa, self.mask_pqa_mask, mask=mask) if wofs: mask = get_mask_wofs(wofs, self.mask_wofs_mask, mask=mask) _log.info( "Stacking [%s] band data from [%s] with PQA [%s] and PQA mask [%s] and WOFS [%s] and WOFS mask [%s] to [%s]", band.name, dataset.path, pqa and pqa.path or "", pqa and self.mask_pqa_mask or "", wofs and wofs.path or "", wofs and self.mask_wofs_mask or "", filename) data = get_dataset_data_masked(dataset, mask=mask, ndv=ndv) _log.debug("data is [%s]", data) stack_band = raster.GetRasterBand(index) stack_band.SetDescription(os.path.basename(dataset.path)) stack_band.SetNoDataValue(ndv) stack_band.WriteArray(data[band]) stack_band.ComputeStatistics(True) stack_band.SetMetadata({ "ACQ_DATE": format_date(tile.end_datetime), "SATELLITE": dataset.satellite.name }) stack_band.FlushCache() del stack_band if raster: raster.FlushCache() raster = None del raster
def run(self): _log.info("Creating stack for band [%s]", self.band.name) data_type = get_dataset_type_datatype(self.dataset_type) ndv = get_dataset_type_ndv(self.dataset_type) metadata = None driver = None raster = None acq_min, acq_max, criteria = build_season_date_criteria(self.acq_min, self.acq_max, self.season, seasons=SEASONS, extend=True) _log.info("\tacq %s to %s criteria is %s", acq_min, acq_max, criteria) dataset_types = [self.dataset_type] if self.mask_pqa_apply: dataset_types.append(DatasetType.PQ25) tiles = list_tiles_as_list(x=[self.x], y=[self.y], satellites=self.satellites, acq_min=acq_min, acq_max=acq_max, dataset_types=dataset_types, include=criteria) for index, tile in enumerate(tiles, start=1): dataset = tile.datasets[self.dataset_type] assert dataset # band = dataset.bands[self.band] # assert band band = self.band pqa = (self.mask_pqa_apply and DatasetType.PQ25 in tile.datasets) and tile.datasets[DatasetType.PQ25] or None if self.dataset_type not in tile.datasets: _log.debug("No [%s] dataset present for [%s] - skipping", self.dataset_type.name, tile.end_datetime) continue filename = self.output().path if not metadata: metadata = get_dataset_metadata(dataset) assert metadata if not driver: if self.output_format == OutputFormat.GEOTIFF: driver = gdal.GetDriverByName("GTiff") elif self.output_format == OutputFormat.ENVI: driver = gdal.GetDriverByName("ENVI") assert driver if not raster: if self.output_format == OutputFormat.GEOTIFF: raster = driver.Create(filename, metadata.shape[0], metadata.shape[1], len(tiles), data_type, options=["BIGTIFF=YES", "INTERLEAVE=BAND"]) elif self.output_format == OutputFormat.ENVI: raster = driver.Create(filename, metadata.shape[0], metadata.shape[1], len(tiles), data_type, options=["INTERLEAVE=BSQ"]) assert raster # NOTE: could do this without the metadata!! raster.SetGeoTransform(metadata.transform) raster.SetProjection(metadata.projection) raster.SetMetadata(self.generate_raster_metadata()) mask = None if pqa: mask = get_mask_pqa(pqa, self.mask_pqa_mask, mask=mask) _log.info("Stacking [%s] band data from [%s] with PQA [%s] and PQA mask [%s] to [%s]", band.name, dataset.path, pqa and pqa.path or "", pqa and self.mask_pqa_mask or "", filename) data = get_dataset_data_masked(dataset, mask=mask, ndv=ndv) _log.debug("data is [%s]", data) stack_band = raster.GetRasterBand(index) stack_band.SetDescription(os.path.basename(dataset.path)) stack_band.SetNoDataValue(ndv) stack_band.WriteArray(data[band]) stack_band.ComputeStatistics(True) stack_band.SetMetadata({"ACQ_DATE": format_date(tile.end_datetime), "SATELLITE": dataset.satellite.name}) stack_band.FlushCache() del stack_band if raster: raster.FlushCache() del raster raster = None
def go(self): # If we are applying a vector mask then calculate it not (once as it is the same for all tiles) mask_vector = None if self.mask_vector_apply: mask_vector = get_mask_vector_for_cell(self.x, self.y, self.mask_vector_file, self.mask_vector_layer, self.mask_vector_feature) # TODO move the dicking around with bands stuff into utils? import gdal if self.output_format == OutputFormat.GEOTIFF: driver = gdal.GetDriverByName("GTiff") elif self.output_format == OutputFormat.ENVI: driver = gdal.GetDriverByName("ENVI") assert driver tiles = self.get_tiles() _log.info("Total tiles found [%d]", len(tiles)) for band_name in self.bands: _log.info("Creating stack for band [%s]", band_name) relevant_tiles = [] for tile in tiles: dataset = self.dataset_type in tile.datasets and tile.datasets[self.dataset_type] or None if not dataset: _log.info("No applicable [%s] dataset for [%s]", self.dataset_type.name, tile.end_datetime) continue if band_name in [b.name for b in tile.datasets[self.dataset_type].bands]: relevant_tiles.append(tile) _log.info("Total tiles for band [%s] is [%d]", band_name, len(relevant_tiles)) filename = None raster = None metadata = None data_type = ndv = None for index, tile in enumerate(relevant_tiles, start=1): dataset = tile.datasets[self.dataset_type] assert dataset band = dataset.bands[band_name] assert band if self.list_only: _log.info("Would stack band [%s] from dataset [%s]", band.name, dataset.path) continue pqa = (self.mask_pqa_apply and DatasetType.PQ25 in tile.datasets) and tile.datasets[DatasetType.PQ25] or None wofs = (self.mask_wofs_apply and DatasetType.WATER in tile.datasets) and tile.datasets[DatasetType.WATER] or None if self.dataset_type not in tile.datasets: _log.debug("No [%s] dataset present for [%s] - skipping", self.dataset_type.name, tile.end_datetime) continue if not filename: filename = os.path.join(self.output_directory, get_dataset_band_stack_filename(satellites=self.satellites, dataset_type=self.dataset_type, band=band, x=self.x, y=self.y, acq_min=self.acq_min, acq_max=self.acq_max, season=self.season, output_format=self.output_format, mask_pqa_apply=self.mask_pqa_apply, mask_wofs_apply=self.mask_wofs_apply, mask_vector_apply=self.mask_vector_apply)) _log.info("Stacking [%s] band data from [%s] with PQA [%s] and PQA mask [%s] and WOFS [%s] and WOFS mask [%s] to band [%d] of [%s]", band.name, dataset.path, pqa and pqa.path or "", pqa and self.mask_pqa_mask or "", wofs and wofs.path or "", wofs and self.mask_wofs_mask or "", index, filename) if not metadata: metadata = get_dataset_metadata(dataset) assert metadata if not data_type: data_type = get_dataset_datatype(dataset) assert data_type if not ndv: ndv = get_dataset_ndv(dataset) assert ndv if not raster: if self.output_format == OutputFormat.GEOTIFF: raster = driver.Create(filename, metadata.shape[0], metadata.shape[1], len(relevant_tiles), data_type, options=["TILED=YES", "BIGTIFF=YES", "COMPRESS=LZW", "INTERLEAVE=BAND"]) elif self.output_format == OutputFormat.ENVI: raster = driver.Create(filename, metadata.shape[0], metadata.shape[1], len(relevant_tiles), data_type, options=["INTERLEAVE=BSQ"]) assert raster # NOTE: could do this without the metadata!! raster.SetGeoTransform(metadata.transform) raster.SetProjection(metadata.projection) raster.SetMetadata(self.generate_raster_metadata()) mask = mask_vector if pqa: mask = get_mask_pqa(pqa, self.mask_pqa_mask, mask=mask) if wofs: mask = get_mask_wofs(wofs, self.mask_wofs_mask, mask=mask) # _log.info("mask[3500,3500] is [%s]", mask[3500, 3500]) data = get_dataset_data_masked(dataset, mask=mask, ndv=ndv) _log.debug("data is [%s]", data) # _log.info("data[3500,3500] is [%s]", data[band][3500, 3500]) stack_band = raster.GetRasterBand(index) stack_band.SetDescription(os.path.basename(dataset.path)) stack_band.SetNoDataValue(ndv) stack_band.WriteArray(data[band]) stack_band.ComputeStatistics(True) stack_band.SetMetadata({"ACQ_DATE": format_date(tile.end_datetime), "SATELLITE": dataset.satellite.name}) stack_band.FlushCache() del stack_band if raster: raster.FlushCache() raster = None del raster
def run(self): _log.info("Creating stack for band [%s]", self.band.name) data_type = get_dataset_type_datatype(self.dataset_type) ndv = get_dataset_type_ndv(self.dataset_type) metadata = None driver = None raster = None acq_min, acq_max, criteria = build_season_date_criteria( self.acq_min, self.acq_max, self.season, seasons=SEASONS, extend=True) _log.info("\tacq %s to %s criteria is %s", acq_min, acq_max, criteria) dataset_types = [self.dataset_type] if self.mask_pqa_apply: dataset_types.append(DatasetType.PQ25) tiles = list_tiles_as_list(x=[self.x], y=[self.y], satellites=self.satellites, acq_min=acq_min, acq_max=acq_max, dataset_types=dataset_types, include=criteria) for index, tile in enumerate(tiles, start=1): dataset = tile.datasets[self.dataset_type] assert dataset # band = dataset.bands[self.band] # assert band band = self.band pqa = (self.mask_pqa_apply and DatasetType.PQ25 in tile.datasets ) and tile.datasets[DatasetType.PQ25] or None if self.dataset_type not in tile.datasets: _log.debug("No [%s] dataset present for [%s] - skipping", self.dataset_type.name, tile.end_datetime) continue filename = self.output().path if not metadata: metadata = get_dataset_metadata(dataset) assert metadata if not driver: if self.output_format == OutputFormat.GEOTIFF: driver = gdal.GetDriverByName("GTiff") elif self.output_format == OutputFormat.ENVI: driver = gdal.GetDriverByName("ENVI") assert driver if not raster: if self.output_format == OutputFormat.GEOTIFF: raster = driver.Create( filename, metadata.shape[0], metadata.shape[1], len(tiles), data_type, options=["BIGTIFF=YES", "INTERLEAVE=BAND"]) elif self.output_format == OutputFormat.ENVI: raster = driver.Create(filename, metadata.shape[0], metadata.shape[1], len(tiles), data_type, options=["INTERLEAVE=BSQ"]) assert raster # NOTE: could do this without the metadata!! raster.SetGeoTransform(metadata.transform) raster.SetProjection(metadata.projection) raster.SetMetadata(self.generate_raster_metadata()) mask = None if pqa: mask = get_mask_pqa(pqa, self.mask_pqa_mask, mask=mask) _log.info( "Stacking [%s] band data from [%s] with PQA [%s] and PQA mask [%s] to [%s]", band.name, dataset.path, pqa and pqa.path or "", pqa and self.mask_pqa_mask or "", filename) data = get_dataset_data_masked(dataset, mask=mask, ndv=ndv) _log.debug("data is [%s]", data) stack_band = raster.GetRasterBand(index) stack_band.SetDescription(os.path.basename(dataset.path)) stack_band.SetNoDataValue(ndv) stack_band.WriteArray(data[band]) stack_band.ComputeStatistics(True) stack_band.SetMetadata({ "ACQ_DATE": format_date(tile.end_datetime), "SATELLITE": dataset.satellite.name }) stack_band.FlushCache() del stack_band if raster: raster.FlushCache() del raster raster = None