def run(self): print "****", self.output().path dataset = self.tile.datasets[self.dataset_type] metadata = get_dataset_metadata(dataset) mask = None # If doing PQA masking then get PQA mask if self.mask_pqa_apply and DatasetType.PQ25 in self.tile.datasets: mask = get_mask_pqa(self.tile.datasets[DatasetType.PQ25], self.mask_pqa_mask, mask=mask) # If doing WOFS masking then get WOFS mask if self.mask_wofs_apply and DatasetType.WATER in self.tile.datasets: mask = get_mask_wofs(self.tile.datasets[DatasetType.WATER], self.mask_wofs_mask, mask=mask) # TODO - no data value and data type ndv = get_dataset_ndv(dataset) data = get_dataset_data_masked(dataset, mask=mask, ndv=ndv) raster_create(self.output().path, [data[b] for b in dataset.bands], metadata.transform, metadata.projection, ndv, gdal.GDT_Int16, dataset_metadata=self.generate_raster_metadata(dataset), band_ids=[b.name for b in dataset.bands])
def run(self): ndv = NDV nbar = self.tile.datasets[DatasetType.ARG25] _log.info("Processing tile [%s]", nbar.path) # Apply PQA if specified pqa = None if self.mask_pqa_apply and DatasetType.PQ25 in self.tile.datasets: pqa = self.tile.datasets[DatasetType.PQ25] mask = None log_mem("Before get PQA mask") if pqa: mask = get_mask_pqa(pqa, self.mask_pqa_mask) data = get_dataset_data_masked(nbar, mask=mask, ndv=ndv) log_mem("After get data (masked)") metadata = get_dataset_metadata(nbar) data = calculate_tassel_cap_index(data, coefficients=TCI_COEFFICIENTS[nbar.satellite][TasselCapIndex.WETNESS]) raster_create(self.output().path, [data], metadata.transform, metadata.projection, numpy.nan, gdal.GDT_Float32)
def doit(self): # TODO this is working around the issue of it blowing up? # First create the wetness output for each tile for tile in self.get_tiles(): self.create_water_tile(tile) ######## bands = [] metadata = None for water_tile in self.input(): _log.debug("Processing %s", water_tile.path) if not metadata: metadata = raster_get_metadata(water_tile.path, 1) bands.append(raster_get_data(water_tile.path, 1)) data = numpy.array([band for band in bands]) _log.info("Creating summary raster %s", self.output().path) raster_create(self.output().path, [numpy.nanmin(data, axis=0), numpy.nanmax(data, axis=0), numpy.nanmean(data, axis=0)], metadata.transform, metadata.projection, numpy.nan, gdal.GDT_Float32)
def retrieve_data(dataset, pq, pq_masks, path, x, y, overwrite=False, stack=False): _log.info("Retrieving data from [%s] with pq [%s] and pq mask [%s] to [%s]", dataset.path, pq and pq.path or "", pq and pq_masks or "", path) if os.path.exists(path) and not overwrite: _log.error("Output file [%s] exists", path) raise Exception("Output file [%s] already exists" % path) data = None metadata = get_dataset_metadata(dataset) if pq: data = get_dataset_data_with_pq(dataset, pq, pq_masks=pq_masks) else: data = get_dataset_data(dataset) _log.debug("data is [%s]", data) raster_create(path, [data[b] for b in dataset.bands], metadata.transform, metadata.projection, NDV, gdal.GDT_Int16) # If we are creating a stack then also add to a file list file... if stack: path_file_list = os.path.join(os.path.dirname(path), get_filename_file_list(dataset.satellite, dataset.dataset_type, x, y)) _log.info("Also going to write file list to [%s]", path_file_list) with open(path_file_list, "ab") as f: print >>f, path
def doit(self): shape = (4000, 4000) masks = [PQ_MASK_CLEAR, PQ_MASK_SATURATION_OPTICAL, PQ_MASK_SATURATION_THERMAL, PQ_MASK_CONTIGUITY, PQ_MASK_LAND, PQ_MASK_CLOUD_ACCA, PQ_MASK_CLOUD_FMASK, PQ_MASK_CLOUD_SHADOW_ACCA, PQ_MASK_CLOUD_SHADOW_FMASK] observation_count = empty_array(shape=shape, dtype=numpy.int16, ndv=0) observation_count_clear = dict() for mask in masks: observation_count_clear[mask] = empty_array(shape=shape, dtype=numpy.int16, ndv=0) metadata = None for tile in self.get_tiles(): # Get the PQ mask pq = tile.datasets[DatasetType.PQ25] data = get_dataset_data(pq, [Pq25Bands.PQ])[Pq25Bands.PQ] # # Count any pixels that are no NDV - don't think we should actually have any but anyway # # Mask out any no data pixels - should actually be none but anyway pq = numpy.ma.masked_equal(data, NDV) # Count the data pixels - i.e. pixels that were NOT masked out observation_count += numpy.where(data.mask, 0, 1) # # Count and pixels that are not masked due to pixel quality # for mask in masks: # Apply the particular pixel mask pqm = numpy.ma.masked_where(numpy.bitwise_and(data, mask) != mask, data) # Count the pixels that were not masked out observation_count_clear[mask] += numpy.where(pqm.mask, 0, 1) if not metadata: metadata = get_dataset_metadata(pq) # Create the output datasets # Observation Count raster_create(self.output()[0].path, [observation_count] + [observation_count_clear[mask] for mask in masks], metadata.transform, metadata.projection, NDV, GDT_Int16)
def create_water_tile(self, tile): arg = tile.datasets[DatasetType.ARG25] pqa = tile.datasets[DatasetType.PQ25] _log.info("ARG tile [%s]", arg) _log.info("PQ tile [%s]", pqa) filename = os.path.basename(arg.path) filename = filename.replace("NBAR", "WETNESS") filename = filename.replace(".vrt", ".tif") filename = os.path.join(self.output_directory, filename) metadata = get_dataset_metadata(arg) data = get_dataset_data_with_pq(arg, Ls57Arg25Bands, pqa) # Calculate TCI Wetness tci = calculate_tassel_cap_index(data, coefficients=TCI_COEFFICIENTS[arg.satellite][TasselCapIndex.WETNESS]) _log.info("TCI shape is %s | min = %s | max = %s", numpy.shape(tci), tci.min(), tci.max()) raster_create(filename, [tci], metadata.transform, metadata.projection, numpy.nan, gdal.GDT_Float32)
def test_calculate_ndvi(): nbar = DatasetTile(satellite_id="LS5", type_id="ARG25", path="/data/tmp/cube/data/from.calum/LS5_TM_NBAR_150_-034_2004-01-13T23-22-17.088044.tif") metadata = get_dataset_metadata(nbar) band_data = calculate_ndvi(nbar) raster_create("/data/tmp/cube/data/unit_test/LS5_TM_NDVI_150_-034_2004-01-13T23-22-17.088044.tif", [band_data.filled(numpy.NaN)], metadata.transform, metadata.projection, numpy.NaN, gdal.GDT_Float32)
def test_apply_pq(): nbar = DatasetTile(satellite_id="LS5", type_id="ARG25", path="/data/tmp/cube/data/from.calum/LS5_TM_NBAR_150_-034_2004-01-13T23-22-17.088044.tif") pq = DatasetTile(satellite_id="LS5", type_id="PQ25", path="/data/tmp/cube/data/from.calum/LS5_TM_PQA_150_-034_2004-01-13T23-22-17.088044.tif") metadata = get_dataset_metadata(nbar) band_data = raster_get_band_data_with_pq(nbar, Ls57Arg25Bands, pq) raster_create("/data/tmp/cube/data/from.calum/LS5_TM_NBAR_PQD_150_-034_2004-01-13T23-22-17.088044.tif", [band_data[b].filled(-999) for b in Ls57Arg25Bands], metadata.transform, metadata.projection, -999, gdal.GDT_Int16)
def read_write_multi_images(self): newpat = '*'+'NDWI_'+ str(self.x) +'_'+ str(self.y).zfill(4) + '*' print 'reading input directory and searching pattern ', self.output_directory , newpat ndwi =None newimage=numpy.zeros((4000, 4000), dtype='int16') heights = ["tide_10","tide_20","tide_30","tide_40","tide_50","tide_60","tide_70","tide_80","tide_90","tide_100"] shape = (4000, 4000) band_desc= ["Relative DEM of average tidal offsets for each 10 percent of heights"] height_offset = dict() for height in heights: #height_offset[height] = numpy.zeros(shape, dtype=numpy.int16) height_offset[height] = 0 _log.info("searching pattern %s" , newpat) for fl in os.listdir(self.output_directory): fpath= self.output_directory + '/' + fl if fnmatch.fnmatch(fpath, newpat): _log.info("file matched %s" , fl) dataset = gdal.Open(fpath, GA_ReadOnly) if dataset is None: print 'No dataset found for ', fl _log.info("dataset problem %s" , fl) continue for height in heights: if height in fl: #get the first ndwi band from the source file band = dataset.GetRasterBand(1) ndwi = band.ReadAsArray(0,0, dataset.RasterXSize, dataset.RasterYSize) mask_nan = numpy.ma.masked_invalid(ndwi) fill = 0 mask_nan= numpy.ma.filled(mask_nan, fill) _log.info("shape of ndwi [%s] and data after nan masked %s" , mask_nan.shape, mask_nan) #ndwi>0 means water (0) else land(1) if fl.find("tide_100") > 0: if height.find("tide_100") != -1: newimage+=numpy.where(mask_nan >= 0, 0,1) else: newimage+=numpy.where(mask_nan >= 0, 0, 1) _log.info("shape of newimage [%s] and data %s" , newimage.shape, newimage) metadata_info = self.generate_raster_metadata() _log.info("new binary data %s" , newimage) srs = osr.SpatialReference() srs.ImportFromEPSG(4326) projection = srs.ExportToWkt() transform = (self.x, 0.00025, 0.0, self.y+1, 0.0, -0.00025) raster_create(self.output().path, [newimage], transform, projection, NDV, GDT_Int16, dataset_metadata=metadata_info, band_ids=band_desc)
def run(self): print "****", self.output().path dataset = self.tile.datasets[DatasetType.TCI] print "***", dataset.path transform = (self.x, 0.00025, 0.0, self.y+1, 0.0, -0.00025) srs = osr.SpatialReference() srs.ImportFromEPSG(4326) projection = srs.ExportToWkt() # metadata = get_dataset_metadata(dataset) mask = None # If doing PQA masking then get PQA mask if self.mask_pqa_apply and DatasetType.PQ25 in self.tile.datasets: mask = get_mask_pqa(self.tile.datasets[DatasetType.PQ25], self.mask_pqa_mask, mask=mask) # If doing WOFS masking then get WOFS mask if self.mask_wofs_apply and DatasetType.WATER in self.tile.datasets: mask = get_mask_wofs(self.tile.datasets[DatasetType.WATER], self.mask_wofs_mask, mask=mask) # TODO - no data value and data type ndv = get_dataset_ndv(dataset) data = get_dataset_data_masked(dataset, mask=mask, ndv=ndv) # Create ALL bands raster # raster_create(self.output().path, [data[b] for b in dataset.bands], # metadata.transform, metadata.projection, ndv, gdal.GDT_Float32, # dataset_metadata=self.generate_raster_metadata(dataset), # band_ids=[b.name for b in dataset.bands]) # Create just the WETNESS band raster raster_create(self.output().path, [data[TciBands.WETNESS]], transform, projection, ndv, gdal.GDT_Float32, dataset_metadata=self.generate_raster_metadata(dataset), band_ids=[TciBands.WETNESS.name])
def run(self): shape = (4000, 4000) extra = [ExtraBands.OBSERVATIONS, ExtraBands.LOW, ExtraBands.MEDIAN, ExtraBands.HIGH] band_desc = ["TOTAL_OBSERVATIONS","LOWEST TIDE","MEDIAN TIDE", "HIGHEST TIDE"] extra_fields = dict() for field in extra: extra_fields[field] = empty_array(shape=shape, dtype=numpy.int16, fill=0) metadata = None low_value = 0 high_value = 0 median_value = 0 observations = 0 cur = self.load_tidal_file() low=cur[0] high=cur[1] observations=cur[2] #import pdb; pdb.set_trace() low_value = int(float(low) * 1000) high_value = int(float(high) * 1000) median_value = (low_value + high_value)/2 #import pdb; pdb.set_trace() _log.info("\toffset file low %d high %d median %d lines %d ", low_value, high_value, median_value, observations) for layer in extra_fields: lv = layer.value if lv == 1: extra_fields[layer][:] = observations if lv == 2: extra_fields[layer][:] = low_value if lv == 3: extra_fields[layer][:] = median_value if lv == 4: extra_fields[layer][:] = high_value transform = (self.x, 0.00025, 0.0, self.y+1, 0.0, -0.00025) srs = osr.SpatialReference() srs.ImportFromEPSG(4326) projection = srs.ExportToWkt() # Create the output dataset metadata_info = self.generate_raster_metadata() raster_create(self.output().path, [extra_fields[field] for field in extra], transform, projection, NDV, GDT_Int16, dataset_metadata=metadata_info, band_ids=band_desc)
def generate_derived_nbar(self, dataset_types, nbar, pqa, pqa_masks, overwrite=False): for dataset_type in dataset_types: filename = self.get_output_filename_derived_nbar(nbar, dataset_type) _log.info("Generating data from [%s] with pq [%s] and pq mask [%s] to [%s]", nbar.path, pqa and pqa.path or "", pqa and pqa_masks or "", filename) metadata = get_dataset_metadata(nbar) data = None if pqa: data = get_dataset_data_with_pq(nbar, pqa, pq_masks=pqa_masks) else: data = get_dataset_data(nbar) _log.debug("data is [%s]", data) if dataset_type == DatasetType.NDVI: ndvi = calculate_ndvi(data[nbar.bands.RED], data[nbar.bands.NEAR_INFRARED]) raster_create(filename, [ndvi], metadata.transform, metadata.projection, NDV, gdal.GDT_Float32) elif dataset_type == DatasetType.EVI: evi = calculate_evi(data[nbar.bands.RED], data[nbar.bands.BLUE], data[nbar.bands.NEAR_INFRARED]) raster_create(filename, [evi], metadata.transform, metadata.projection, NDV, gdal.GDT_Float32) elif dataset_type == DatasetType.NBR: nbr = calculate_nbr(data[nbar.bands.NEAR_INFRARED], data[nbar.bands.SHORT_WAVE_INFRARED_2]) raster_create(filename, [nbr], metadata.transform, metadata.projection, NDV, gdal.GDT_Float32)
def read_write_multi_images(self): print 'reading input directory ', self.output_directory newpat = '*'+ str(self.x) +'_'+ str(self.y).zfill(4) + '*' ndwi =None newimage=numpy.zeros((4000, 4000), dtype='int16') heights = ["tide_10","tide_20","tide_30","tide_40","tide_50","tide_60","tide_70","tide_80","tide_90", "tide_100"] shape = (4000, 4000) band_desc= ["DEM of average tidal offsets for each 10 percent of heights"] height_offset = dict() for height in heights: height_offset[height] = 0 _log.info("searching pattern %s" , newpat) for fl in os.listdir(self.output_directory): fpath= self.output_directory + '/' + fl if fnmatch.fnmatch(fpath, newpat): _log.info("file matched %s" , fl) dataset = gdal.Open(fpath, GA_ReadOnly) if dataset is None: _log.info("dataset problem %s" , fl) continue for height in heights: if height in fl: band = dataset.GetRasterBand(1) ndwi = band.ReadAsArray(0,0, dataset.RasterXSize, dataset.RasterYSize) mask_nan = numpy.ma.masked_invalid(ndwi) fill = 0 mask_nan= numpy.ma.filled(mask_nan, fill) _log.info("shape of ndwi [%s] and data after nan masked %s" , mask_nan.shape, mask_nan) band=dataset.GetRasterBand(5) data=band.ReadAsArray(0,0, dataset.RasterXSize, dataset.RasterYSize) av=data.mean().astype(int) # set land and water depending on ndwi <0 and >0 respectively if fl.find("tide_100") > 0: if height.find("tide_100") != -1: height_offset[heights[len(heights)-1]]+=av newimage+=numpy.where(mask_nan >= 0, 0, 1) else: height_offset[height]+=av newimage+=numpy.where(mask_nan >= 0, 0, 1) _log.info("height_offset for %s is %s avrg %d" , height, height_offset[height] ,av) _log.info("last binary data %s" , newimage) for i in range(len(heights)): if height_offset[heights[i]] != 0: _log.info("height_offset for index %d is %d" , i, height_offset[heights[i]]) newimage[newimage==i+1] = height_offset[heights[i]] else: _log.info("no data for %d", height_offset[heights[i]]) _log.info("mod binary data -- %s" , newimage) newimage[newimage==0] = -6666 metadata_info = self.generate_raster_metadata() _log.info("new binary data %s" , newimage) srs = osr.SpatialReference() srs.ImportFromEPSG(4326) projection = srs.ExportToWkt() transform = (self.x, 0.00025, 0.0, self.y+1, 0.0, -0.00025) raster_create(self.output().path, [newimage], transform, projection, NDV, GDT_Int16, dataset_metadata=metadata_info, band_ids=band_desc)
def run(self): shape = (4000, 4000) extra = [ ExtraBands.OBSERVATIONS, ExtraBands.LOW, ExtraBands.MEDIAN, ExtraBands.HIGH ] band_desc = [ "TOTAL_OBSERVATIONS", "LOWEST TIDE", "MEDIAN TIDE", "HIGHEST TIDE" ] extra_fields = dict() for field in extra: extra_fields[field] = empty_array(shape=shape, dtype=numpy.int16, fill=0) metadata = None low_value = 0 high_value = 0 median_value = 0 observations = 0 cur = self.load_tidal_file() low = cur[0] high = cur[1] observations = cur[2] #import pdb; pdb.set_trace() low_value = int(float(low) * 1000) high_value = int(float(high) * 1000) median_value = (low_value + high_value) / 2 #import pdb; pdb.set_trace() _log.info("\toffset file low %d high %d median %d lines %d ", low_value, high_value, median_value, observations) for layer in extra_fields: lv = layer.value if lv == 1: extra_fields[layer][:] = observations if lv == 2: extra_fields[layer][:] = low_value if lv == 3: extra_fields[layer][:] = median_value if lv == 4: extra_fields[layer][:] = high_value transform = (self.x, 0.00025, 0.0, self.y + 1, 0.0, -0.00025) srs = osr.SpatialReference() srs.ImportFromEPSG(4326) projection = srs.ExportToWkt() # Create the output dataset metadata_info = self.generate_raster_metadata() raster_create(self.output().path, [extra_fields[field] for field in extra], transform, projection, NDV, GDT_Int16, dataset_metadata=metadata_info, band_ids=band_desc)
def read_write_multi_images(self): newpat = '*' + 'NDWI_' + str(self.x) + '_' + str(self.y).zfill(4) + '*' print 'reading input directory and searching pattern ', self.output_directory, newpat ndwi = None newimage = numpy.zeros((4000, 4000), dtype='int16') heights = [ "tide_10", "tide_20", "tide_30", "tide_40", "tide_50", "tide_60", "tide_70", "tide_80", "tide_90", "tide_100" ] shape = (4000, 4000) band_desc = [ "Relative DEM of average tidal offsets for each 10 percent of heights" ] height_offset = dict() for height in heights: #height_offset[height] = numpy.zeros(shape, dtype=numpy.int16) height_offset[height] = 0 _log.info("searching pattern %s", newpat) for fl in os.listdir(self.output_directory): fpath = self.output_directory + '/' + fl if fnmatch.fnmatch(fpath, newpat): _log.info("file matched %s", fl) dataset = gdal.Open(fpath, GA_ReadOnly) if dataset is None: print 'No dataset found for ', fl _log.info("dataset problem %s", fl) continue for height in heights: if height in fl: #get the first ndwi band from the source file band = dataset.GetRasterBand(1) ndwi = band.ReadAsArray(0, 0, dataset.RasterXSize, dataset.RasterYSize) mask_nan = numpy.ma.masked_invalid(ndwi) fill = 0 mask_nan = numpy.ma.filled(mask_nan, fill) _log.info( "shape of ndwi [%s] and data after nan masked %s", mask_nan.shape, mask_nan) #ndwi>0 means water (0) else land(1) if fl.find("tide_100") > 0: if height.find("tide_100") != -1: newimage += numpy.where(mask_nan >= 0, 0, 1) else: newimage += numpy.where(mask_nan >= 0, 0, 1) _log.info("shape of newimage [%s] and data %s", newimage.shape, newimage) metadata_info = self.generate_raster_metadata() _log.info("new binary data %s", newimage) srs = osr.SpatialReference() srs.ImportFromEPSG(4326) projection = srs.ExportToWkt() transform = (self.x, 0.00025, 0.0, self.y + 1, 0.0, -0.00025) raster_create(self.output().path, [newimage], transform, projection, NDV, GDT_Int16, dataset_metadata=metadata_info, band_ids=band_desc)
def run(self): shape = (4000, 4000) no_data_value = NDV best_pixel_fc = dict() for band in Fc25Bands: # best_pixel_fc[band] = empty_array(shape=shape, dtype=numpy.int16, ndv=INT16_MIN) best_pixel_fc[band] = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV) best_pixel_nbar = dict() for band in Ls57Arg25Bands: best_pixel_nbar[band] = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV) best_pixel_satellite = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV) best_pixel_date = empty_array(shape=shape, dtype=numpy.int32, ndv=NDV) current_satellite = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV) current_date = empty_array(shape=shape, dtype=numpy.int32, ndv=NDV) SATELLITE_DATA_VALUES = {Satellite.LS5: 5, Satellite.LS7: 7, Satellite.LS8: 8} metadata_nbar = None metadata_fc = None for tile in self.get_tiles(): pqa = tile.datasets[DatasetType.PQ25] nbar = tile.datasets[DatasetType.ARG25] fc = tile.datasets[DatasetType.FC25] wofs = DatasetType.WATER in tile.datasets and tile.datasets[DatasetType.WATER] or None _log.info("Processing [%s]", fc.path) data = dict() # Create an initial "no mask" mask mask = numpy.ma.make_mask_none((4000, 4000)) # _log.info("### mask is [%s]", mask[1000][1000]) # Add the PQA mask if we are doing PQA masking if self.mask_pqa_apply: mask = get_mask_pqa(pqa, pqa_masks=self.mask_pqa_mask, mask=mask) # _log.info("### mask PQA is [%s]", mask[1000][1000]) # Add the WOFS mask if we are doing WOFS masking if self.mask_wofs_apply and wofs: mask = get_mask_wofs(wofs, wofs_masks=self.mask_wofs_mask, mask=mask) # _log.info("### mask PQA is [%s]", mask[1000][1000]) # Get NBAR dataset data[DatasetType.ARG25] = get_dataset_data_masked(nbar, mask=mask) # _log.info("### NBAR/RED is [%s]", data[DatasetType.ARG25][Ls57Arg25Bands.RED][1000][1000]) # Get the NDVI dataset data[DatasetType.NDVI] = calculate_ndvi(data[DatasetType.ARG25][Ls57Arg25Bands.RED], data[DatasetType.ARG25][Ls57Arg25Bands.NEAR_INFRARED]) # _log.info("### NDVI is [%s]", data[DatasetType.NDVI][1000][1000]) # Add the NDVI value range mask (to the existing mask) mask = self.get_mask_range(data[DatasetType.NDVI], min_val=0.0, max_val=0.3, mask=mask) # _log.info("### mask NDVI is [%s]", mask[1000][1000]) # Get FC25 dataset data[DatasetType.FC25] = get_dataset_data_masked(fc, mask=mask) # _log.info("### FC/BS is [%s]", data[DatasetType.FC25][Fc25Bands.BARE_SOIL][1000][1000]) # Add the bare soil value range mask (to the existing mask) mask = self.get_mask_range(data[DatasetType.FC25][Fc25Bands.BARE_SOIL], min_val=0, max_val=8000, mask=mask) # _log.info("### mask BS is [%s]", mask[1000][1000]) # Apply the final mask to the FC25 bare soil data data_bare_soil = numpy.ma.MaskedArray(data=data[DatasetType.FC25][Fc25Bands.BARE_SOIL], mask=mask).filled(NDV) # _log.info("### bare soil is [%s]", data_bare_soil[1000][1000]) # Compare the bare soil value from this dataset to the current "best" value best_pixel_fc[Fc25Bands.BARE_SOIL] = numpy.fmax(best_pixel_fc[Fc25Bands.BARE_SOIL], data_bare_soil) # _log.info("### best pixel bare soil is [%s]", best_pixel_fc[Fc25Bands.BARE_SOIL][1000][1000]) # Now update the other best pixel datasets/bands to grab the pixels we just selected for band in Ls57Arg25Bands: best_pixel_nbar[band] = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BARE_SOIL], data_bare_soil, data[DatasetType.ARG25][band], best_pixel_nbar[band]) for band in [Fc25Bands.PHOTOSYNTHETIC_VEGETATION, Fc25Bands.NON_PHOTOSYNTHETIC_VEGETATION, Fc25Bands.UNMIXING_ERROR]: best_pixel_fc[band] = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BARE_SOIL], data_bare_soil, data[DatasetType.FC25][band], best_pixel_fc[band]) # And now the other "provenance" data # Satellite "provenance" data current_satellite.fill(SATELLITE_DATA_VALUES[fc.satellite]) best_pixel_satellite = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BARE_SOIL], data_bare_soil, current_satellite, best_pixel_satellite) # Date "provenance" data current_date.fill(date_to_integer(tile.end_datetime)) best_pixel_date = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BARE_SOIL], data_bare_soil, current_date, best_pixel_date) # Grab the metadata from the input datasets for use later when creating the output datasets if not metadata_nbar: metadata_nbar = get_dataset_metadata(nbar) if not metadata_fc: metadata_fc = get_dataset_metadata(fc) # Create the output datasets # FC composite raster_create(self.get_dataset_filename("FC"), [best_pixel_fc[b] for b in Fc25Bands], metadata_fc.transform, metadata_fc.projection, metadata_fc.bands[Fc25Bands.BARE_SOIL].no_data_value, metadata_fc.bands[Fc25Bands.BARE_SOIL].data_type) # NBAR composite raster_create(self.get_dataset_filename("NBAR"), [best_pixel_nbar[b] for b in Ls57Arg25Bands], metadata_nbar.transform, metadata_nbar.projection, metadata_nbar.bands[Ls57Arg25Bands.BLUE].no_data_value, metadata_nbar.bands[Ls57Arg25Bands.BLUE].data_type) # Satellite "provenance" composites raster_create(self.get_dataset_filename("SAT"), [best_pixel_satellite], metadata_nbar.transform, metadata_nbar.projection, no_data_value, gdal.GDT_Int16) # Date "provenance" composites raster_create(self.get_dataset_filename("DATE"), [best_pixel_date], metadata_nbar.transform, metadata_nbar.projection, no_data_value, gdal.GDT_Int32)
def doit(self): shape = (4000, 4000) no_data_value = NDV best_pixel_data = dict() # TODO if Satellite.LS8.value in self.satellites: bands = Ls8Arg25Bands else: bands = Ls57Arg25Bands for band in bands: best_pixel_data[band] = empty_array(shape=shape, dtype=numpy.int16, ndv=no_data_value) best_pixel_satellite = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV) # best_pixel_epoch = empty_array(shape=shape, dtype=numpy.int32, ndv=NDV) best_pixel_date = empty_array(shape=shape, dtype=numpy.int32, ndv=NDV) current_satellite = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV) # current_epoch = empty_array(shape=shape, dtype=numpy.int32, ndv=NDV) current_date = empty_array(shape=shape, dtype=numpy.int32, ndv=NDV) metadata = None SATELLITE_DATA_VALUES = {Satellite.LS5: 5, Satellite.LS7: 7, Satellite.LS8: 8} for tile in self.get_tiles(sort=SortType.DESC): # Get ARG25 dataset dataset = tile.datasets[DatasetType.ARG25] _log.info("Processing ARG tile [%s]", dataset.path) if not metadata: metadata = get_dataset_metadata(dataset) band_data = None if self.apply_pq_filter: band_data = get_dataset_data_with_pq(dataset, tile.datasets[DatasetType.PQ25]) else: band_data = get_dataset_data(dataset) # Create the provenance datasets # NOTE: need to do this BEFORE selecting the pixel since it is actually using the fact that the # selected pixel currently doesn't have a value # NOTE: band values are propagated "as a job lot" so can just check any band # TODO better way than just saying....RED....? band = bands.RED # Satellite current_satellite.fill(SATELLITE_DATA_VALUES[dataset.satellite]) best_pixel_satellite = numpy.where(best_pixel_data[band] == no_data_value, current_satellite, best_pixel_satellite) # # Epoch dataset # # current_epoch.fill(calendar.timegm(tile.end_datetime.timetuple())) # best_pixel_epoch = numpy.where(best_pixel_data[band] == no_data_value, current_epoch, best_pixel_epoch) # Date dataset (20150101) current_date.fill(tile.end_datetime.year * 10000 + tile.end_datetime.month * 100 + tile.end_datetime.day) best_pixel_date = numpy.where(best_pixel_data[band] == no_data_value, current_date, best_pixel_date) for band in bands: data = band_data[band] # _log.debug("data = \n%s", data) # Replace any NO DATA best pixels with data pixels # TODO should I explicitly do the AND data is not NO DATA VALUE? best_pixel_data[band] = numpy.where(best_pixel_data[band] == no_data_value, data, best_pixel_data[band]) # _log.debug("best pixel = \n%s", best_pixel_data[band]) still_no_data = numpy.any(numpy.array([best_pixel_data[b] for b in bands]) == no_data_value) # _log.debug("still no data pixels = %s", still_no_data) if not still_no_data: break # Now want to mask out values in the provenance datasets if we haven't actually got a value # TODO better way than just saying....RED....? band = bands.RED mask = numpy.ma.masked_equal(best_pixel_data[band], NDV).mask best_pixel_satellite = numpy.ma.array(best_pixel_satellite, mask=mask).filled(NDV) # best_pixel_epoch = numpy.ma.array(best_pixel_epoch, mask=mask).fill(NDV) best_pixel_date = numpy.ma.array(best_pixel_date, mask=mask).filled(NDV) # Composite NBAR dataset raster_create(self.get_output_path("NBAR"), [best_pixel_data[b] for b in bands], metadata.transform, metadata.projection, NDV, gdal.GDT_Int16) # Provenance (satellite) dataset raster_create(self.get_output_path("SAT"), [best_pixel_satellite], metadata.transform, metadata.projection, no_data_value, gdal.GDT_Int16) # # Provenance (epoch) dataset # # raster_create(self.get_output_path("EPOCH"), # [best_pixel_epoch], # metadata.transform, metadata.projection, no_data_value, # gdal.GDT_Int32) # Provenance (day of month) dataset raster_create(self.get_output_path("DATE"), [best_pixel_date], metadata.transform, metadata.projection, no_data_value, gdal.GDT_Int32)
def doit(self): _log.debug("Bare Soil Cell Task - doit()") shape = (4000, 4000) no_data_value = NDV best_pixel_fc = dict() for band in Fc25Bands: best_pixel_fc[band] = empty_array(shape=shape, dtype=numpy.int16, ndv=INT16_MIN) best_pixel_nbar = dict() for band in Ls57Arg25Bands: best_pixel_nbar[band] = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV) best_pixel_satellite = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV) best_pixel_year = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV) best_pixel_month = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV) best_pixel_epoch = empty_array(shape=shape, dtype=numpy.int32, ndv=NDV) current_satellite = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV) current_year = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV) current_month = empty_array(shape=shape, dtype=numpy.int16, ndv=NDV) current_epoch = empty_array(shape=shape, dtype=numpy.int32, ndv=NDV) SATELLITE_DATA_VALUES = {Satellite.LS5: 5, Satellite.LS7: 7, Satellite.LS8: 8} metadata_nbar = None metadata_fc = None for tile in self.get_tiles(): # Get the PQ mask pq = tile.datasets[DatasetType.PQ25] data_pq = get_dataset_data(pq, [Pq25Bands.PQ])[Pq25Bands.PQ] mask_pq = get_pq_mask(data_pq) # Get NBAR dataset nbar = tile.datasets[DatasetType.ARG25] _log.info("Processing NBAR tile [%s]", nbar.path) if not metadata_nbar: metadata_nbar = get_dataset_metadata(nbar) data_nbar = get_dataset_data_with_pq(nbar, Ls57Arg25Bands, tile.datasets[DatasetType.PQ25]) # Get the NDVI mask red = data_nbar[Ls57Arg25Bands.RED] nir = data_nbar[Ls57Arg25Bands.NEAR_INFRARED] ndvi_data = calculate_ndvi(red, nir) ndvi_data = numpy.ma.masked_equal(ndvi_data, NDV) ndvi_data = numpy.ma.masked_outside(ndvi_data, 0, 0.3, copy=False) mask_ndvi = ndvi_data.mask # Get FC25 dataset fc = tile.datasets[DatasetType.FC25] _log.info("Processing FC tile [%s]", fc.path) if not metadata_fc: metadata_fc = get_dataset_metadata(fc) _log.debug("metadata fc is %s", metadata_fc) data_fc = get_dataset_data(fc, Fc25Bands) data_bare_soil = data_fc[Fc25Bands.BS] data_bare_soil = numpy.ma.masked_equal(data_bare_soil, -999) data_bare_soil = numpy.ma.masked_outside(data_bare_soil, 0, 8000) data_bare_soil.mask = (data_bare_soil.mask | mask_pq | mask_ndvi) data_bare_soil = data_bare_soil.filled(NDV) # Compare the bare soil value from this dataset to the current "best" value best_pixel_fc[Fc25Bands.BS] = numpy.fmax(best_pixel_fc[Fc25Bands.BS], data_bare_soil) # Now update the other best pixel datasets/bands to grab the pixels we just selected for band in Ls57Arg25Bands: best_pixel_nbar[band] = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BS], data_bare_soil, data_nbar[band], best_pixel_nbar[band]) for band in [Fc25Bands.PV, Fc25Bands.NPV, Fc25Bands.ERROR]: best_pixel_fc[band] = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BS], data_bare_soil, data_fc[band], best_pixel_fc[band]) # And now the other "provenance" data current_satellite.fill(SATELLITE_DATA_VALUES[fc.satellite]) best_pixel_satellite = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BS], data_bare_soil, current_satellite, best_pixel_satellite) current_year.fill(tile.end_datetime_year) best_pixel_year = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BS], data_bare_soil, current_year, best_pixel_year) current_month.fill(tile.end_datetime_month) best_pixel_month = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BS], data_bare_soil, current_month, best_pixel_month) current_epoch.fill(calendar.timegm(tile.end_datetime.timetuple())) best_pixel_epoch = propagate_using_selected_pixel(best_pixel_fc[Fc25Bands.BS], data_bare_soil, current_epoch, best_pixel_epoch) # Create the output datasets # FC composite raster_create(self.get_dataset_filename("FC"), [best_pixel_fc[b] for b in Fc25Bands], metadata_fc.transform, metadata_fc.projection, metadata_fc.bands[Fc25Bands.BS].no_data_value, metadata_fc.bands[Fc25Bands.BS].data_type) # NBAR composite raster_create(self.get_dataset_filename("NBAR"), [best_pixel_nbar[b] for b in Ls57Arg25Bands], metadata_nbar.transform, metadata_nbar.projection, metadata_nbar.bands[Ls57Arg25Bands.BLUE].no_data_value, metadata_nbar.bands[Ls57Arg25Bands.BLUE].data_type) # "Provenance" composites raster_create(self.get_dataset_filename("SAT"), [best_pixel_satellite], metadata_nbar.transform, metadata_nbar.projection, no_data_value, gdal.GDT_Int16) raster_create(self.get_dataset_filename("YEAR"), [best_pixel_year], metadata_nbar.transform, metadata_nbar.projection, no_data_value, gdal.GDT_Int16) raster_create(self.get_dataset_filename("MONTH"), [best_pixel_month], metadata_nbar.transform, metadata_nbar.projection, no_data_value, gdal.GDT_Int16) raster_create(self.get_dataset_filename("EPOCH"), [best_pixel_epoch], metadata_nbar.transform, metadata_nbar.projection, no_data_value, gdal.GDT_Int32)