def go(self): # If we are applying a vector mask then calculate it (once as it is the same for all tiles) mask_vector = None if self.mask_vector_apply: mask_vector = get_mask_vector_for_cell(self.x, self.y, self.mask_vector_file, self.mask_vector_layer, self.mask_vector_feature) for tile in self.get_tiles(): pqa = (self.mask_pqa_apply and DatasetType.PQ25 in tile.datasets) and tile.datasets[DatasetType.PQ25] or None wofs = (self.mask_wofs_apply and DatasetType.WATER in tile.datasets) and tile.datasets[DatasetType.WATER] or None dataset = self.dataset_type in tile.datasets and tile.datasets[self.dataset_type] or None if not dataset: _log.info("No applicable [%s] dataset for [%s]", self.dataset_type.name, tile.end_datetime) continue if self.list_only: _log.info("Would retrieve dataset [%s]", tile.datasets[self.dataset_type].path) continue filename = os.path.join(self.output_directory, get_dataset_filename(dataset, output_format=self.output_format, mask_pqa_apply=self.mask_pqa_apply, mask_wofs_apply=self.mask_wofs_apply, mask_vector_apply=self.mask_vector_apply)) retrieve_data(tile.x, tile.y, tile.end_datetime, dataset, self.bands, pqa, self.mask_pqa_mask, wofs, self.mask_wofs_mask, filename, self.output_format, self.overwrite, mask=mask_vector)
def go(self): # If we are applying a vector mask then calculate it (once as it is the same for all tiles) mask = None if self.mask_vector_apply: mask = get_mask_vector_for_cell(self.x, self.y, self.mask_vector_file, self.mask_vector_layer, self.mask_vector_feature) for tile in self.get_tiles(): if self.list_only: _log.info("Would retrieve datasets [%s]", "\n".join([tile.datasets[t].path for t in intersection(self.dataset_types, [d for d in tile.datasets])])) continue pqa = (self.mask_pqa_apply and DatasetType.PQ25 in tile.datasets) and tile.datasets[DatasetType.PQ25] or None wofs = (self.mask_wofs_apply and DatasetType.WATER in tile.datasets) and tile.datasets[DatasetType.WATER] or None for dataset_type in self.dataset_types: if dataset_type not in tile.datasets: _log.debug("No [%s] dataset present for [%s] - skipping", dataset_type.name, tile.end_datetime) continue dataset = tile.datasets[dataset_type] filename = os.path.join(self.output_directory, get_dataset_filename(dataset, output_format=self.output_format, mask_pqa_apply=self.mask_pqa_apply, mask_wofs_apply=self.mask_wofs_apply, mask_vector_apply=self.mask_vector_apply)) retrieve_data(tile.x, tile.y, tile.end_datetime, dataset, pqa, self.mask_pqa_mask, wofs, self.mask_wofs_mask, filename, self.output_format, self.overwrite, mask=mask)
def go(self): # If we are applying a vector mask then calculate it not (once as it is the same for all tiles) mask_vector = None if self.mask_vector_apply: mask_vector = get_mask_vector_for_cell(self.x, self.y, self.mask_vector_file, self.mask_vector_layer, self.mask_vector_feature) # TODO move the dicking around with bands stuff into utils? import gdal if self.output_format == OutputFormat.GEOTIFF: driver = gdal.GetDriverByName("GTiff") elif self.output_format == OutputFormat.ENVI: driver = gdal.GetDriverByName("ENVI") assert driver tiles = self.get_tiles() _log.info("Total tiles found [%d]", len(tiles)) for band_name in self.bands: _log.info("Creating stack for band [%s]", band_name) relevant_tiles = [] for tile in tiles: dataset = self.dataset_type in tile.datasets and tile.datasets[self.dataset_type] or None if not dataset: _log.info("No applicable [%s] dataset for [%s]", self.dataset_type.name, tile.end_datetime) continue if band_name in [b.name for b in tile.datasets[self.dataset_type].bands]: relevant_tiles.append(tile) _log.info("Total tiles for band [%s] is [%d]", band_name, len(relevant_tiles)) filename = None raster = None metadata = None data_type = ndv = None for index, tile in enumerate(relevant_tiles, start=1): dataset = tile.datasets[self.dataset_type] assert dataset band = dataset.bands[band_name] assert band if self.list_only: _log.info("Would stack band [%s] from dataset [%s]", band.name, dataset.path) continue pqa = (self.mask_pqa_apply and DatasetType.PQ25 in tile.datasets) and tile.datasets[DatasetType.PQ25] or None wofs = (self.mask_wofs_apply and DatasetType.WATER in tile.datasets) and tile.datasets[DatasetType.WATER] or None if self.dataset_type not in tile.datasets: _log.debug("No [%s] dataset present for [%s] - skipping", self.dataset_type.name, tile.end_datetime) continue if not filename: filename = os.path.join(self.output_directory, get_dataset_band_stack_filename(satellites=self.satellites, dataset_type=self.dataset_type, band=band, x=self.x, y=self.y, acq_min=self.acq_min, acq_max=self.acq_max, season=self.season, output_format=self.output_format, mask_pqa_apply=self.mask_pqa_apply, mask_wofs_apply=self.mask_wofs_apply, mask_vector_apply=self.mask_vector_apply)) _log.info("Stacking [%s] band data from [%s] with PQA [%s] and PQA mask [%s] and WOFS [%s] and WOFS mask [%s] to band [%d] of [%s]", band.name, dataset.path, pqa and pqa.path or "", pqa and self.mask_pqa_mask or "", wofs and wofs.path or "", wofs and self.mask_wofs_mask or "", index, filename) if not metadata: metadata = get_dataset_metadata(dataset) assert metadata if not data_type: data_type = get_dataset_datatype(dataset) assert data_type if not ndv: ndv = get_dataset_ndv(dataset) assert ndv if not raster: if self.output_format == OutputFormat.GEOTIFF: raster = driver.Create(filename, metadata.shape[0], metadata.shape[1], len(relevant_tiles), data_type, options=["TILED=YES", "BIGTIFF=YES", "COMPRESS=LZW", "INTERLEAVE=BAND"]) elif self.output_format == OutputFormat.ENVI: raster = driver.Create(filename, metadata.shape[0], metadata.shape[1], len(relevant_tiles), data_type, options=["INTERLEAVE=BSQ"]) assert raster # NOTE: could do this without the metadata!! raster.SetGeoTransform(metadata.transform) raster.SetProjection(metadata.projection) raster.SetMetadata(self.generate_raster_metadata()) mask = mask_vector if pqa: mask = get_mask_pqa(pqa, self.mask_pqa_mask, mask=mask) if wofs: mask = get_mask_wofs(wofs, self.mask_wofs_mask, mask=mask) # _log.info("mask[3500,3500] is [%s]", mask[3500, 3500]) data = get_dataset_data_masked(dataset, mask=mask, ndv=ndv) _log.debug("data is [%s]", data) # _log.info("data[3500,3500] is [%s]", data[band][3500, 3500]) stack_band = raster.GetRasterBand(index) stack_band.SetDescription(os.path.basename(dataset.path)) stack_band.SetNoDataValue(ndv) stack_band.WriteArray(data[band]) stack_band.ComputeStatistics(True) stack_band.SetMetadata({"ACQ_DATE": format_date(tile.end_datetime), "SATELLITE": dataset.satellite.name}) stack_band.FlushCache() del stack_band if raster: raster.FlushCache() raster = None del raster