def test_retrieve_data_ls5_mndwi(config=None): filename = "LS5_TM_MNDWI_{x:03d}_{y:04d}_{date}.{x_offset:04d}_{y_offset:04d}.{x_size:04d}x{y_size:04d}.tif".format(x=CELL_X, y=CELL_Y, date=DATE, x_offset=X_OFFSET, y_offset=Y_OFFSET, x_size=X_SIZE, y_size=Y_SIZE) tiles = list_tiles_as_list(x=[CELL_X], y=[CELL_Y], acq_min=ACQ_LS5, acq_max=ACQ_LS5, satellites=[Satellite.LS5], dataset_types=[MNDWI_DATASET_TYPE], config=config) assert len(tiles) == 1 dataset = tiles[0].datasets[MNDWI_DATASET_TYPE] data = get_dataset_data(dataset=dataset, x=X_OFFSET, y=Y_OFFSET, x_size=X_SIZE, y_size=Y_SIZE) assert(data) _log.info("data is [%s]\n%s", numpy.shape(data), data) ndv = get_dataset_ndv(dataset) assert(is_ndv(ndv, MNDWI_NDV)) data_type = get_dataset_datatype(dataset) assert(data_type == MNDWI_DATA_TYPE) metadata = generate_dataset_metadata(x=CELL_X, y=CELL_Y, acq_dt=ACQ_LS5, dataset=dataset, bands=None, mask_pqa_apply=False, mask_pqa_mask=None, mask_wofs_apply=False, mask_wofs_mask=None) raster_create_geotiff(filename, [data[b] for b in dataset.bands], CELL_GEO_TRANSFORM, CELL_PROJECTION, ndv, data_type, dataset_metadata=metadata, band_ids=[b.name for b in dataset.bands]) assert filecmp.cmp(filename, get_test_data_path(filename))
def run(self): print "****", self.output().path dataset = self.tile.datasets[self.dataset_type] metadata = get_dataset_metadata(dataset) mask = None # If doing PQA masking then get PQA mask if self.mask_pqa_apply and DatasetType.PQ25 in self.tile.datasets: mask = get_mask_pqa(self.tile.datasets[DatasetType.PQ25], self.mask_pqa_mask, mask=mask) # If doing WOFS masking then get WOFS mask if self.mask_wofs_apply and DatasetType.WATER in self.tile.datasets: mask = get_mask_wofs(self.tile.datasets[DatasetType.WATER], self.mask_wofs_mask, mask=mask) # TODO - no data value and data type ndv = get_dataset_ndv(dataset) data = get_dataset_data_masked(dataset, mask=mask, ndv=ndv) raster_create(self.output().path, [data[b] for b in dataset.bands], metadata.transform, metadata.projection, ndv, gdal.GDT_Int16, dataset_metadata=self.generate_raster_metadata(dataset), band_ids=[b.name for b in dataset.bands])
def retrieve_data(x, y, acq_dt, dataset, band_names, pqa, pqa_masks, wofs, wofs_masks, path, output_format, overwrite=False, data_type=None, ndv=None, mask=None): _log.info("Retrieving data from [%s] bands [%s] with pq [%s] and pq mask [%s] and wofs [%s] and wofs mask [%s] to [%s] file [%s]", dataset.path, band_names, pqa and pqa.path or "", pqa and pqa_masks or "", wofs and wofs.path or "", wofs and wofs_masks or "", output_format.name, path) if os.path.exists(path) and not overwrite: _log.error("Output file [%s] exists", path) raise Exception("Output file [%s] already exists" % path) metadata = get_dataset_metadata(dataset) # mask = None if pqa: mask = get_mask_pqa(pqa, pqa_masks, mask=mask) if wofs: mask = get_mask_wofs(wofs, wofs_masks, mask=mask) bands = [] for b in dataset.bands: if b.name in band_names: bands.append(b) ndv = ndv or get_dataset_ndv(dataset) data = get_dataset_data_masked(dataset, bands=bands, mask=mask, ndv=ndv) _log.debug("data is [%s]", data) data_type = data_type or get_dataset_datatype(dataset) dataset_info = generate_raster_metadata(x, y, acq_dt, dataset, bands, pqa is not None, pqa_masks, wofs is not None, wofs_masks) band_info = [b.name for b in bands] if output_format == OutputFormat.GEOTIFF: raster_create_geotiff(path, [data[b] for b in bands], metadata.transform, metadata.projection, ndv, data_type, dataset_metadata=dataset_info, band_ids=band_info) elif output_format == OutputFormat.ENVI: raster_create_envi(path, [data[b] for b in bands], metadata.transform, metadata.projection, ndv, data_type, dataset_metadata=dataset_info, band_ids=band_info)
def test_retrieve_data_ls5_arg_with_pqa_water_mask_dry(config=None): filename = "LS5_TM_NBAR_WITH_PQA_WATER_DRY_{x:03d}_{y:04d}_{date}.{x_offset:04d}_{y_offset:04d}.{x_size:04d}x{y_size:04d}.tif".format(x=CELL_X, y=CELL_Y, date=DATE, x_offset=X_OFFSET, y_offset=Y_OFFSET, x_size=X_SIZE, y_size=Y_SIZE) tiles = list_tiles_as_list(x=[CELL_X], y=[CELL_Y], acq_min=ACQ_LS5, acq_max=ACQ_LS5, satellites=[Satellite.LS5], dataset_types=[ARG_DATASET_TYPE, PQ_DATASET_TYPE, WOFS_DATASET_TYPE], config=config) assert len(tiles) == 1 tile = tiles[0] assert ARG_DATASET_TYPE in tile.datasets dataset = tile.datasets[ARG_DATASET_TYPE] assert PQ_DATASET_TYPE in tile.datasets pqa = tile.datasets[PQ_DATASET_TYPE] assert WOFS_DATASET_TYPE in tile.datasets wofs = tile.datasets[WOFS_DATASET_TYPE] mask = get_mask_pqa(pqa, x=X_OFFSET, y=Y_OFFSET, x_size=X_SIZE, y_size=Y_SIZE) mask = get_mask_wofs(wofs, wofs_masks=[WofsMask.DRY, WofsMask.NO_DATA, WofsMask.SATURATION_CONTIGUITY, WofsMask.SEA_WATER, WofsMask.TERRAIN_SHADOW, WofsMask.HIGH_SLOPE, WofsMask.CLOUD_SHADOW, WofsMask.CLOUD], x=X_OFFSET, y=Y_OFFSET, x_size=X_SIZE, y_size=Y_SIZE, mask=mask) data = get_dataset_data_masked(dataset=dataset, x=X_OFFSET, y=Y_OFFSET, x_size=X_SIZE, y_size=Y_SIZE, mask=mask) assert(data) _log.info("data is [%s]\n%s", numpy.shape(data), data) ndv = get_dataset_ndv(dataset) assert(is_ndv(ndv, ARG_NDV)) data_type = get_dataset_datatype(dataset) assert(data_type == ARG_DATA_TYPE) metadata = generate_dataset_metadata(x=CELL_X, y=CELL_Y, acq_dt=ACQ_LS5, dataset=dataset, bands=None, mask_pqa_apply=False, mask_pqa_mask=None, mask_wofs_apply=False, mask_wofs_mask=None) raster_create_geotiff(filename, [data[b] for b in dataset.bands], CELL_GEO_TRANSFORM, CELL_PROJECTION, ndv, data_type, dataset_metadata=metadata, band_ids=[b.name for b in dataset.bands]) assert filecmp.cmp(filename, get_test_data_path(filename))
def run(self): print "****", self.output().path dataset = self.tile.datasets[DatasetType.TCI] print "***", dataset.path transform = (self.x, 0.00025, 0.0, self.y+1, 0.0, -0.00025) srs = osr.SpatialReference() srs.ImportFromEPSG(4326) projection = srs.ExportToWkt() # metadata = get_dataset_metadata(dataset) mask = None # If doing PQA masking then get PQA mask if self.mask_pqa_apply and DatasetType.PQ25 in self.tile.datasets: mask = get_mask_pqa(self.tile.datasets[DatasetType.PQ25], self.mask_pqa_mask, mask=mask) # If doing WOFS masking then get WOFS mask if self.mask_wofs_apply and DatasetType.WATER in self.tile.datasets: mask = get_mask_wofs(self.tile.datasets[DatasetType.WATER], self.mask_wofs_mask, mask=mask) # TODO - no data value and data type ndv = get_dataset_ndv(dataset) data = get_dataset_data_masked(dataset, mask=mask, ndv=ndv) # Create ALL bands raster # raster_create(self.output().path, [data[b] for b in dataset.bands], # metadata.transform, metadata.projection, ndv, gdal.GDT_Float32, # dataset_metadata=self.generate_raster_metadata(dataset), # band_ids=[b.name for b in dataset.bands]) # Create just the WETNESS band raster raster_create(self.output().path, [data[TciBands.WETNESS]], transform, projection, ndv, gdal.GDT_Float32, dataset_metadata=self.generate_raster_metadata(dataset), band_ids=[TciBands.WETNESS.name])
def run(self): # TODO move the dicking around with bands stuff into utils? import gdal driver = raster = None metadata = None data_type = ndv = None tiles = self.get_tiles() _log.info("Total tiles found [%d]", len(tiles)) _log.info("Creating stack for band [%s]", self.band) relevant_tiles = [] for tile in tiles: dataset = self.dataset_type in tile.datasets and tile.datasets[self.dataset_type] or None if not dataset: _log.info("No applicable [%s] dataset for [%s]", self.dataset_type.name, tile.end_datetime) continue if self.band in [b.name for b in tile.datasets[self.dataset_type].bands]: relevant_tiles.append(tile) _log.info("Total tiles for band [%s] is [%d]", self.band, len(relevant_tiles)) for index, tile in enumerate(relevant_tiles, start=1): dataset = tile.datasets[self.dataset_type] assert dataset band = dataset.bands[self.band] assert band pqa = (self.mask_pqa_apply and DatasetType.PQ25 in tile.datasets) and tile.datasets[DatasetType.PQ25] or None wofs = (self.mask_wofs_apply and DatasetType.WATER in tile.datasets) and tile.datasets[DatasetType.WATER] or None if self.dataset_type not in tile.datasets: _log.debug("No [%s] dataset present for [%s] - skipping", self.dataset_type.name, tile.end_datetime) continue filename = self.output().path if not metadata: metadata = get_dataset_metadata(dataset) assert metadata if not data_type: data_type = get_dataset_datatype(dataset) assert data_type if not ndv: ndv = get_dataset_ndv(dataset) assert ndv if not driver: if self.output_format == OutputFormat.GEOTIFF: driver = gdal.GetDriverByName("GTiff") elif self.output_format == OutputFormat.ENVI: driver = gdal.GetDriverByName("ENVI") assert driver if not raster: if self.output_format == OutputFormat.GEOTIFF: raster = driver.Create(filename, metadata.shape[0], metadata.shape[1], len(tiles), data_type, options=["BIGTIFF=YES", "INTERLEAVE=BAND"]) elif self.output_format == OutputFormat.ENVI: raster = driver.Create(filename, metadata.shape[0], metadata.shape[1], len(tiles), data_type, options=["INTERLEAVE=BSQ"]) assert raster # NOTE: could do this without the metadata!! raster.SetGeoTransform(metadata.transform) raster.SetProjection(metadata.projection) raster.SetMetadata(self.generate_raster_metadata()) mask = None if pqa: mask = get_mask_pqa(pqa, self.mask_pqa_mask, mask=mask) if wofs: mask = get_mask_wofs(wofs, self.mask_wofs_mask, mask=mask) _log.info("Stacking [%s] band data from [%s] with PQA [%s] and PQA mask [%s] and WOFS [%s] and WOFS mask [%s] to [%s]", band.name, dataset.path, pqa and pqa.path or "", pqa and self.mask_pqa_mask or "", wofs and wofs.path or "", wofs and self.mask_wofs_mask or "", filename) data = get_dataset_data_masked(dataset, mask=mask, ndv=ndv) _log.debug("data is [%s]", data) stack_band = raster.GetRasterBand(index) stack_band.SetDescription(os.path.basename(dataset.path)) stack_band.SetNoDataValue(ndv) stack_band.WriteArray(data[band]) stack_band.ComputeStatistics(True) stack_band.SetMetadata({"ACQ_DATE": format_date(tile.end_datetime), "SATELLITE": dataset.satellite.name}) stack_band.FlushCache() del stack_band if raster: raster.FlushCache() raster = None del raster
def run(self): # TODO move the dicking around with bands stuff into utils? import gdal driver = raster = None metadata = None data_type = ndv = None tiles = self.get_tiles() _log.info("Total tiles found [%d]", len(tiles)) _log.info("Creating stack for band [%s]", self.band) relevant_tiles = [] for tile in tiles: dataset = self.dataset_type in tile.datasets and tile.datasets[ self.dataset_type] or None if not dataset: _log.info("No applicable [%s] dataset for [%s]", self.dataset_type.name, tile.end_datetime) continue if self.band in [ b.name for b in tile.datasets[self.dataset_type].bands ]: relevant_tiles.append(tile) _log.info("Total tiles for band [%s] is [%d]", self.band, len(relevant_tiles)) for index, tile in enumerate(relevant_tiles, start=1): dataset = tile.datasets[self.dataset_type] assert dataset band = dataset.bands[self.band] assert band pqa = (self.mask_pqa_apply and DatasetType.PQ25 in tile.datasets ) and tile.datasets[DatasetType.PQ25] or None wofs = (self.mask_wofs_apply and DatasetType.WATER in tile.datasets ) and tile.datasets[DatasetType.WATER] or None if self.dataset_type not in tile.datasets: _log.debug("No [%s] dataset present for [%s] - skipping", self.dataset_type.name, tile.end_datetime) continue filename = self.output().path if not metadata: metadata = get_dataset_metadata(dataset) assert metadata if not data_type: data_type = get_dataset_datatype(dataset) assert data_type if not ndv: ndv = get_dataset_ndv(dataset) assert ndv if not driver: if self.output_format == OutputFormat.GEOTIFF: driver = gdal.GetDriverByName("GTiff") elif self.output_format == OutputFormat.ENVI: driver = gdal.GetDriverByName("ENVI") assert driver if not raster: if self.output_format == OutputFormat.GEOTIFF: raster = driver.Create( filename, metadata.shape[0], metadata.shape[1], len(tiles), data_type, options=["BIGTIFF=YES", "INTERLEAVE=BAND"]) elif self.output_format == OutputFormat.ENVI: raster = driver.Create(filename, metadata.shape[0], metadata.shape[1], len(tiles), data_type, options=["INTERLEAVE=BSQ"]) assert raster # NOTE: could do this without the metadata!! raster.SetGeoTransform(metadata.transform) raster.SetProjection(metadata.projection) raster.SetMetadata(self.generate_raster_metadata()) mask = None if pqa: mask = get_mask_pqa(pqa, self.mask_pqa_mask, mask=mask) if wofs: mask = get_mask_wofs(wofs, self.mask_wofs_mask, mask=mask) _log.info( "Stacking [%s] band data from [%s] with PQA [%s] and PQA mask [%s] and WOFS [%s] and WOFS mask [%s] to [%s]", band.name, dataset.path, pqa and pqa.path or "", pqa and self.mask_pqa_mask or "", wofs and wofs.path or "", wofs and self.mask_wofs_mask or "", filename) data = get_dataset_data_masked(dataset, mask=mask, ndv=ndv) _log.debug("data is [%s]", data) stack_band = raster.GetRasterBand(index) stack_band.SetDescription(os.path.basename(dataset.path)) stack_band.SetNoDataValue(ndv) stack_band.WriteArray(data[band]) stack_band.ComputeStatistics(True) stack_band.SetMetadata({ "ACQ_DATE": format_date(tile.end_datetime), "SATELLITE": dataset.satellite.name }) stack_band.FlushCache() del stack_band if raster: raster.FlushCache() raster = None del raster
def go(self): # If we are applying a vector mask then calculate it not (once as it is the same for all tiles) mask_vector = None if self.mask_vector_apply: mask_vector = get_mask_vector_for_cell(self.x, self.y, self.mask_vector_file, self.mask_vector_layer, self.mask_vector_feature) # TODO move the dicking around with bands stuff into utils? import gdal if self.output_format == OutputFormat.GEOTIFF: driver = gdal.GetDriverByName("GTiff") elif self.output_format == OutputFormat.ENVI: driver = gdal.GetDriverByName("ENVI") assert driver tiles = self.get_tiles() _log.info("Total tiles found [%d]", len(tiles)) for band_name in self.bands: _log.info("Creating stack for band [%s]", band_name) relevant_tiles = [] for tile in tiles: dataset = self.dataset_type in tile.datasets and tile.datasets[self.dataset_type] or None if not dataset: _log.info("No applicable [%s] dataset for [%s]", self.dataset_type.name, tile.end_datetime) continue if band_name in [b.name for b in tile.datasets[self.dataset_type].bands]: relevant_tiles.append(tile) _log.info("Total tiles for band [%s] is [%d]", band_name, len(relevant_tiles)) filename = None raster = None metadata = None data_type = ndv = None for index, tile in enumerate(relevant_tiles, start=1): dataset = tile.datasets[self.dataset_type] assert dataset band = dataset.bands[band_name] assert band if self.list_only: _log.info("Would stack band [%s] from dataset [%s]", band.name, dataset.path) continue pqa = (self.mask_pqa_apply and DatasetType.PQ25 in tile.datasets) and tile.datasets[DatasetType.PQ25] or None wofs = (self.mask_wofs_apply and DatasetType.WATER in tile.datasets) and tile.datasets[DatasetType.WATER] or None if self.dataset_type not in tile.datasets: _log.debug("No [%s] dataset present for [%s] - skipping", self.dataset_type.name, tile.end_datetime) continue if not filename: filename = os.path.join(self.output_directory, get_dataset_band_stack_filename(satellites=self.satellites, dataset_type=self.dataset_type, band=band, x=self.x, y=self.y, acq_min=self.acq_min, acq_max=self.acq_max, season=self.season, output_format=self.output_format, mask_pqa_apply=self.mask_pqa_apply, mask_wofs_apply=self.mask_wofs_apply, mask_vector_apply=self.mask_vector_apply)) _log.info("Stacking [%s] band data from [%s] with PQA [%s] and PQA mask [%s] and WOFS [%s] and WOFS mask [%s] to band [%d] of [%s]", band.name, dataset.path, pqa and pqa.path or "", pqa and self.mask_pqa_mask or "", wofs and wofs.path or "", wofs and self.mask_wofs_mask or "", index, filename) if not metadata: metadata = get_dataset_metadata(dataset) assert metadata if not data_type: data_type = get_dataset_datatype(dataset) assert data_type if not ndv: ndv = get_dataset_ndv(dataset) assert ndv if not raster: if self.output_format == OutputFormat.GEOTIFF: raster = driver.Create(filename, metadata.shape[0], metadata.shape[1], len(relevant_tiles), data_type, options=["TILED=YES", "BIGTIFF=YES", "COMPRESS=LZW", "INTERLEAVE=BAND"]) elif self.output_format == OutputFormat.ENVI: raster = driver.Create(filename, metadata.shape[0], metadata.shape[1], len(relevant_tiles), data_type, options=["INTERLEAVE=BSQ"]) assert raster # NOTE: could do this without the metadata!! raster.SetGeoTransform(metadata.transform) raster.SetProjection(metadata.projection) raster.SetMetadata(self.generate_raster_metadata()) mask = mask_vector if pqa: mask = get_mask_pqa(pqa, self.mask_pqa_mask, mask=mask) if wofs: mask = get_mask_wofs(wofs, self.mask_wofs_mask, mask=mask) # _log.info("mask[3500,3500] is [%s]", mask[3500, 3500]) data = get_dataset_data_masked(dataset, mask=mask, ndv=ndv) _log.debug("data is [%s]", data) # _log.info("data[3500,3500] is [%s]", data[band][3500, 3500]) stack_band = raster.GetRasterBand(index) stack_band.SetDescription(os.path.basename(dataset.path)) stack_band.SetNoDataValue(ndv) stack_band.WriteArray(data[band]) stack_band.ComputeStatistics(True) stack_band.SetMetadata({"ACQ_DATE": format_date(tile.end_datetime), "SATELLITE": dataset.satellite.name}) stack_band.FlushCache() del stack_band if raster: raster.FlushCache() raster = None del raster