def run(self): self.parse_arguments() config = Config(os.path.expanduser("~/.datacube/config")) _log.debug(config.to_str()) # Clear stack files # TODO - filename consistency and safety and so on if self.stack_vrt: for satellite, dataset_type in itertools.product(self.satellites, self.dataset_types): path = os.path.join(self.output_directory, get_filename_file_list(satellite, dataset_type, self.x, self.y)) check_overwrite_remove_or_fail(path, self.overwrite) # TODO once WOFS is in the cube for tile in list_tiles(x=[self.x], y=[self.y], acq_min=self.acq_min, acq_max=self.acq_max, satellites=[satellite for satellite in self.satellites], dataset_types=intersection(self.dataset_types, dataset_type_database), database=config.get_db_database(), user=config.get_db_username(), password=config.get_db_password(), host=config.get_db_host(), port=config.get_db_port()): if self.list_only: _log.info("Would retrieve datasets [%s]", [tile.datasets[t].path for t in intersection(self.dataset_types, dataset_type_database)]) continue pqa = None # Apply PQA if specified if self.apply_pqa_filter: pqa = tile.datasets[DatasetType.PQ25] for dataset_type in intersection(self.dataset_types, dataset_type_database): retrieve_data(tile.datasets[dataset_type], pqa, self.pqa_mask, self.get_output_filename(tile.datasets[dataset_type]), tile.x, tile.y, self.overwrite, self.stack_vrt) nbar = tile.datasets[DatasetType.ARG25] self.generate_derived_nbar(intersection(self.dataset_types, dataset_type_derived_nbar), nbar, pqa, self.pqa_mask, self.overwrite) # Generate VRT stack if self.stack_vrt: for satellite, dataset_type in itertools.product(self.satellites, self.dataset_types): path = os.path.join(self.output_directory, get_filename_file_list(satellite, dataset_type, self.x, self.y)) if os.path.exists(path): for band in BANDS[dataset_type, satellite]: path_vrt = os.path.join(self.output_directory, get_filename_stack_vrt(satellite, dataset_type, self.x, self.y, band)) _log.info("Generating VRT file [%s] for band [%s]", path_vrt, band) # gdalbuildrt -separate -b <band> -input_file_list <input file> <vrt file> subprocess.call(["gdalbuildvrt", "-separate", "-b", str(band.value), "-input_file_list", path, path_vrt])
def go(self): # If we are applying a vector mask then calculate it (once as it is the same for all tiles) mask = None if self.mask_vector_apply: mask = get_mask_vector_for_cell(self.x, self.y, self.mask_vector_file, self.mask_vector_layer, self.mask_vector_feature) for tile in self.get_tiles(): if self.list_only: _log.info("Would retrieve datasets [%s]", "\n".join([tile.datasets[t].path for t in intersection(self.dataset_types, [d for d in tile.datasets])])) continue pqa = (self.mask_pqa_apply and DatasetType.PQ25 in tile.datasets) and tile.datasets[DatasetType.PQ25] or None wofs = (self.mask_wofs_apply and DatasetType.WATER in tile.datasets) and tile.datasets[DatasetType.WATER] or None for dataset_type in self.dataset_types: if dataset_type not in tile.datasets: _log.debug("No [%s] dataset present for [%s] - skipping", dataset_type.name, tile.end_datetime) continue dataset = tile.datasets[dataset_type] filename = os.path.join(self.output_directory, get_dataset_filename(dataset, output_format=self.output_format, mask_pqa_apply=self.mask_pqa_apply, mask_wofs_apply=self.mask_wofs_apply, mask_vector_apply=self.mask_vector_apply)) retrieve_data(tile.x, tile.y, tile.end_datetime, dataset, pqa, self.mask_pqa_mask, wofs, self.mask_wofs_mask, filename, self.output_format, self.overwrite, mask=mask)
def go(self): import numpy from datacube.api.query import list_cells_as_list, list_tiles_as_list from datacube.config import Config x_min, x_max, y_max, y_min = self.extract_bounds_from_vector() _log.debug("The bounds are [%s]", (x_min, x_max, y_min, y_max)) cells_vector = self.extract_cells_from_vector() _log.debug("Intersecting cells_vector are [%d] [%s]", len(cells_vector), cells_vector) config = Config() _log.debug(config.to_str()) x_list = range(x_min, x_max + 1) y_list = range(y_min, y_max + 1) _log.debug("x = [%s] y=[%s]", x_list, y_list) cells_db = list() for cell in list_cells_as_list(x=x_list, y=y_list, acq_min=self.acq_min, acq_max=self.acq_max, satellites=[satellite for satellite in self.satellites], dataset_types=[self.dataset_type]): cells_db.append((cell.x, cell.y)) _log.debug("Cells from DB are [%d] [%s]", len(cells_db), cells_db) cells = intersection(cells_vector, cells_db) _log.debug("Combined cells are [%d] [%s]", len(cells), cells) for (x, y) in cells: _log.info("Processing cell [%3d/%4d]", x, y) tiles = list_tiles_as_list(x=x_list, y=y_list, acq_min=self.acq_min, acq_max=self.acq_max, satellites=[satellite for satellite in self.satellites], dataset_types=[self.dataset_type]) _log.info("There are [%d] tiles", len(tiles)) if self.list_only: for tile in tiles: _log.info("Would process [%s]", tile.datasets[self.dataset_type].path) continue # Calculate the mask for the cell mask_aoi = self.get_mask_aoi_cell(x, y) pixel_count = 4000 * 4000 pixel_count_aoi = (mask_aoi == False).sum() _log.debug("mask_aoi is [%s]\n[%s]", numpy.shape(mask_aoi), mask_aoi) metadata = None with self.get_output_file() as csv_file: csv_writer = csv.writer(csv_file) import operator header = reduce(operator.add, [["DATE", "INSTRUMENT", "# PIXELS", "# PIXELS IN AOI"]] + [ ["%s - # DATA PIXELS" % band_name, "%s - # DATA PIXELS AFTER PQA" % band_name, "%s - # DATA PIXELS AFTER PQA WOFS" % band_name, "%s - # DATA PIXELS AFTER PQA WOFS AOI" % band_name, "%s - MIN" % band_name, "%s - MAX" % band_name, "%s - MEAN" % band_name] for band_name in self.bands]) csv_writer.writerow(header) for tile in tiles: _log.info("Processing tile [%s]", tile.datasets[self.dataset_type].path) if self.list_only: continue if not metadata: metadata = get_dataset_metadata(tile.datasets[self.dataset_type]) # Apply PQA if specified pqa = None mask_pqa = None if self.mask_pqa_apply and DatasetType.PQ25 in tile.datasets: pqa = tile.datasets[DatasetType.PQ25] mask_pqa = get_mask_pqa(pqa, self.mask_pqa_mask) _log.debug("mask_pqa is [%s]\n[%s]", numpy.shape(mask_pqa), mask_pqa) # Apply WOFS if specified wofs = None mask_wofs = None if self.mask_wofs_apply and DatasetType.WATER in tile.datasets: wofs = tile.datasets[DatasetType.WATER] mask_wofs = get_mask_wofs(wofs, self.mask_wofs_mask) _log.debug("mask_wofs is [%s]\n[%s]", numpy.shape(mask_wofs), mask_wofs) dataset = tile.datasets[self.dataset_type] bands = [] dataset_band_names = [b.name for b in dataset.bands] for b in self.bands: if b in dataset_band_names: bands.append(dataset.bands[b]) data = get_dataset_data(tile.datasets[self.dataset_type], bands=bands) _log.debug("data is [%s]\n[%s]", numpy.shape(data), data) pixel_count_data = dict() pixel_count_data_pqa = dict() pixel_count_data_pqa_wofs = dict() pixel_count_data_pqa_wofs_aoi = dict() mmin = dict() mmax = dict() mmean = dict() for band_name in self.bands: # Add "zeroed" entries for non-present bands - should only be if outputs for those bands have been explicitly requested if band_name not in dataset_band_names: pixel_count_data[band_name] = 0 pixel_count_data_pqa[band_name] = 0 pixel_count_data_pqa_wofs[band_name] = 0 pixel_count_data_pqa_wofs_aoi[band_name] = 0 mmin[band_name] = numpy.ma.masked mmax[band_name] = numpy.ma.masked mmean[band_name] = numpy.ma.masked continue band = dataset.bands[band_name] data[band] = numpy.ma.masked_equal(data[band], NDV) _log.debug("masked data is [%s] [%d]\n[%s]", numpy.shape(data), numpy.ma.count(data), data) pixel_count_data[band_name] = numpy.ma.count(data[band]) if pqa: data[band].mask = numpy.ma.mask_or(data[band].mask, mask_pqa) _log.debug("PQA masked data is [%s] [%d]\n[%s]", numpy.shape(data[band]), numpy.ma.count(data[band]), data[band]) pixel_count_data_pqa[band_name] = numpy.ma.count(data[band]) if wofs: data[band].mask = numpy.ma.mask_or(data[band].mask, mask_wofs) _log.debug("WOFS masked data is [%s] [%d]\n[%s]", numpy.shape(data[band]), numpy.ma.count(data[band]), data[band]) pixel_count_data_pqa_wofs[band_name] = numpy.ma.count(data[band]) data[band].mask = numpy.ma.mask_or(data[band].mask, mask_aoi) _log.debug("AOI masked data is [%s] [%d]\n[%s]", numpy.shape(data[band]), numpy.ma.count(data[band]), data[band]) pixel_count_data_pqa_wofs_aoi[band_name] = numpy.ma.count(data[band]) mmin[band_name] = numpy.ma.min(data[band]) mmax[band_name] = numpy.ma.max(data[band]) mmean[band_name] = numpy.ma.mean(data[band]) # Convert the mean to an int...taking into account masking.... if not numpy.ma.is_masked(mmean[band_name]): mmean[band_name] = mmean[band_name].astype(numpy.int16) pixel_count_data_pqa_wofs_aoi_all_bands = reduce(operator.add, pixel_count_data_pqa_wofs_aoi.itervalues()) if pixel_count_data_pqa_wofs_aoi_all_bands == 0 and not self.output_no_data: _log.info("Skipping dataset with no non-masked data values in ANY band") continue row = reduce( operator.add, [[tile.end_datetime, self.decode_satellite_as_instrument(tile.datasets[self.dataset_type].satellite), pixel_count, pixel_count_aoi]] + [[pixel_count_data[band_name], pixel_count_data_pqa[band_name], pixel_count_data_pqa_wofs[band_name], pixel_count_data_pqa_wofs_aoi[band_name], mmin[band_name], mmax[band_name], mmean[band_name]] for band_name in self.bands]) csv_writer.writerow(row)
def go(self): import numpy from datacube.api.query import list_cells_as_list, list_tiles_as_list from datacube.config import Config # Verify that all the requested satellites have the same band combinations dataset_bands = get_bands(self.dataset_type, self.satellites[0]) _log.info("dataset bands is [%s]", " ".join([b.name for b in dataset_bands])) for satellite in self.satellites: if dataset_bands != get_bands(self.dataset_type, satellite): _log.error("Satellites [%s] have differing bands", " ".join([satellite.name for satellite in self.satellites])) raise Exception("Satellites with different band combinations selected") bands = [] dataset_bands_list = list(dataset_bands) if not self.bands: bands = dataset_bands_list else: for b in self.bands: bands.append(dataset_bands_list[b - 1]) _log.info("Using bands [%s]", " ".join(band.name for band in bands)) x_min, x_max, y_max, y_min = self.extract_bounds_from_vector() _log.debug("The bounds are [%s]", (x_min, x_max, y_min, y_max)) cells_vector = self.extract_cells_from_vector() _log.debug("Intersecting cells_vector are [%d] [%s]", len(cells_vector), cells_vector) config = Config(os.path.expanduser("~/.datacube/config")) _log.debug(config.to_str()) x_list = range(x_min, x_max + 1) y_list = range(y_min, y_max + 1) _log.debug("x = [%s] y=[%s]", x_list, y_list) cells_db = list() for cell in list_cells_as_list(x=x_list, y=y_list, acq_min=self.acq_min, acq_max=self.acq_max, satellites=[satellite for satellite in self.satellites], dataset_types=[self.dataset_type]): cells_db.append((cell.x, cell.y)) _log.debug("Cells from DB are [%d] [%s]", len(cells_db), cells_db) cells = intersection(cells_vector, cells_db) _log.debug("Combined cells are [%d] [%s]", len(cells), cells) for (x, y) in cells: _log.info("Processing cell [%3d/%4d]", x, y) tiles = list_tiles_as_list(x=x_list, y=y_list, acq_min=self.acq_min, acq_max=self.acq_max, satellites=[satellite for satellite in self.satellites], dataset_types=[self.dataset_type]) _log.info("There are [%d] tiles", len(tiles)) if self.list_only: for tile in tiles: _log.info("Would process [%s]", tile.datasets[self.dataset_type].path) continue # Calculate the mask for the cell mask_aoi = self.get_mask_aoi_cell(x, y) pixel_count = 4000 * 4000 pixel_count_aoi = (mask_aoi == False).sum() _log.debug("mask_aoi is [%s]\n[%s]", numpy.shape(mask_aoi), mask_aoi) metadata = None with self.get_output_file() as csv_file: csv_writer = csv.writer(csv_file) import operator header = reduce(operator.add, [["DATE", "INSTRUMENT", "# PIXELS", "# PIXELS IN AOI"]] + [ ["%s - # DATA PIXELS" % b.name, "%s - # DATA PIXELS AFTER PQA" % b.name, "%s - # DATA PIXELS AFTER PQA WOFS" % b.name, "%s - # DATA PIXELS AFTER PQA WOFS AOI" % b.name, "%s - MIN" % b.name, "%s - MAX" % b.name, "%s - MEAN" % b.name] for b in bands]) csv_writer.writerow(header) for tile in tiles: _log.info("Processing tile [%s]", tile.datasets[self.dataset_type].path) if self.list_only: continue if not metadata: metadata = get_dataset_metadata(tile.datasets[self.dataset_type]) # Apply PQA if specified pqa = None mask_pqa = None if self.mask_pqa_apply and DatasetType.PQ25 in tile.datasets: pqa = tile.datasets[DatasetType.PQ25] mask_pqa = get_mask_pqa(pqa, self.mask_pqa_mask) _log.debug("mask_pqa is [%s]\n[%s]", numpy.shape(mask_pqa), mask_pqa) # Apply WOFS if specified wofs = None mask_wofs = None if self.mask_wofs_apply and DatasetType.WATER in tile.datasets: wofs = tile.datasets[DatasetType.WATER] mask_wofs = get_mask_wofs(wofs, self.mask_wofs_mask) _log.debug("mask_wofs is [%s]\n[%s]", numpy.shape(mask_wofs), mask_wofs) data = get_dataset_data(tile.datasets[self.dataset_type], bands=bands) _log.debug("data is [%s]\n[%s]", numpy.shape(data), data) pixel_count_data = dict() pixel_count_data_pqa = dict() pixel_count_data_pqa_wofs = dict() pixel_count_data_pqa_wofs_aoi = dict() mmin = dict() mmax = dict() mmean = dict() for band in bands: data[band] = numpy.ma.masked_equal(data[band], NDV) _log.debug("masked data is [%s] [%d]\n[%s]", numpy.shape(data), numpy.ma.count(data), data) pixel_count_data[band] = numpy.ma.count(data[band]) if pqa: data[band].mask = numpy.ma.mask_or(data[band].mask, mask_pqa) _log.debug("PQA masked data is [%s] [%d]\n[%s]", numpy.shape(data[band]), numpy.ma.count(data[band]), data[band]) pixel_count_data_pqa[band] = numpy.ma.count(data[band]) if wofs: data[band].mask = numpy.ma.mask_or(data[band].mask, mask_wofs) _log.debug("WOFS masked data is [%s] [%d]\n[%s]", numpy.shape(data[band]), numpy.ma.count(data[band]), data[band]) pixel_count_data_pqa_wofs[band] = numpy.ma.count(data[band]) data[band].mask = numpy.ma.mask_or(data[band].mask, mask_aoi) _log.debug("AOI masked data is [%s] [%d]\n[%s]", numpy.shape(data[band]), numpy.ma.count(data[band]), data[band]) pixel_count_data_pqa_wofs_aoi[band] = numpy.ma.count(data[band]) mmin[band] = numpy.ma.min(data[band]) mmax[band] = numpy.ma.max(data[band]) mmean[band] = numpy.ma.mean(data[band]) # Convert the mean to an int...which is actually trickier than you would expect due to masking.... if numpy.ma.count(mmean[band]) != 0: mmean[band] = mmean[band].astype(numpy.int16) # Should we output if no data values found? pixel_count_data_pqa_wofs_aoi_all_bands = reduce(operator.add, pixel_count_data_pqa_wofs_aoi.itervalues()) if pixel_count_data_pqa_wofs_aoi_all_bands == 0 and not self.output_no_data: _log.info("Skipping dataset with no non-masked data values in ANY band") continue row = reduce( operator.add, [[tile.end_datetime, self.decode_satellite_as_instrument(tile.datasets[self.dataset_type].satellite), pixel_count, pixel_count_aoi]] + [[pixel_count_data[band], pixel_count_data_pqa[band], pixel_count_data_pqa_wofs[band], pixel_count_data_pqa_wofs_aoi[band], mmin[band], mmax[band], mmean[band]] for band in bands]) csv_writer.writerow(row)