def test_reflectance_negative_elevation(): band = np.array([[0, 0, 0], [0, 2, 1], [2, 0, 1.00008]]).astype('float32') MR = 0.2 AR = -0.1 E = -90.0 with pytest.raises(ValueError): reflectance.reflectance(band, MR, AR, E)
def test_reflectance_wrong_shape(): band = np.array([[0, 0, 0], [0, 1, 1], [1, 0, 1]]).astype('float32') # wrong sun elevation shape with pytest.raises(ValueError): reflectance.reflectance(band, 0.2, -0.00000001, np.array([[1, 2, 3], [4, 5, 6]])) with pytest.raises(ValueError): reflectance.reflectance(band, 35.1, np.array([1, 3]), 90.0)
def _convert(arr: numpy.ndarray, band: str, metadata: Dict) -> numpy.ndarray: """Convert DN to TOA or Temp.""" if band in ["1", "2", "3", "4", "5", "6", "7", "8", "9"]: # OLI multi_reflect = metadata["RADIOMETRIC_RESCALING"].get( f"REFLECTANCE_MULT_BAND_{band}" ) add_reflect = metadata["RADIOMETRIC_RESCALING"].get( f"REFLECTANCE_ADD_BAND_{band}" ) sun_elev = metadata["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"] arr = 10000 * reflectance.reflectance( arr, multi_reflect, add_reflect, sun_elev, src_nodata=0 ) elif band in ["10", "11"]: # TIRS multi_rad = metadata["RADIOMETRIC_RESCALING"].get(f"RADIANCE_MULT_BAND_{band}") add_rad = metadata["RADIOMETRIC_RESCALING"].get(f"RADIANCE_ADD_BAND_{band}") k1 = metadata["TIRS_THERMAL_CONSTANTS"].get(f"K1_CONSTANT_BAND_{band}") k2 = metadata["TIRS_THERMAL_CONSTANTS"].get(f"K2_CONSTANT_BAND_{band}") arr = brightness_temp.brightness_temp(arr, multi_rad, add_rad, k1, k2) # TODO # elif band == "QA": return arr
def test_reflectance(): band = np.array([[0, 0, 0], [0, 1, 1], [1, 0, 1]]).astype('float32') MR = 0.2 AR = -0.1 E = 90.0 assert np.array_equal( reflectance.reflectance(band, MR, AR, E), np.array([[0., 0., 0.], [0., 0.1, 0.1], [0.1, 0., 0.1]]).astype(np.float32))
def landsat_min_max_worker(band, address, metadata, pmin=2, pmax=98, width=1024, height=1024): """Retrieve histogram percentage cut for a Landsat-8 scene. Attributes ---------- address : Landsat band AWS address band : Landsat band number metadata : Landsat metadata pmin : Histogram minimum cut (default: 2) pmax : Histogram maximum cut (default: 98) width : int, optional (default: 1024) Pixel width for the decimated read. height : int, optional (default: 1024) Pixel height for the decimated read. Returns ------- out : list, int returns a list of the min/max histogram cut values. """ if int(band) > 9: # TIRS multi_rad = metadata['RADIOMETRIC_RESCALING'].get( 'RADIANCE_MULT_BAND_{}'.format(band)) add_rad = metadata['RADIOMETRIC_RESCALING'].get( 'RADIANCE_ADD_BAND_{}'.format(band)) k1 = metadata['TIRS_THERMAL_CONSTANTS'].get( 'K1_CONSTANT_BAND_{}'.format(band)) k2 = metadata['TIRS_THERMAL_CONSTANTS'].get( 'K2_CONSTANT_BAND_{}'.format(band)) with rasterio.open('{}_B{}.TIF'.format(address, band)) as src: arr = src.read(indexes=1, out_shape=(height, width)).astype(src.profile['dtype']) arr = brightness_temp.brightness_temp(arr, multi_rad, add_rad, k1, k2) else: multi_reflect = metadata['RADIOMETRIC_RESCALING'].get( 'REFLECTANCE_MULT_BAND_{}'.format(band)) add_reflect = metadata['RADIOMETRIC_RESCALING'].get( 'REFLECTANCE_ADD_BAND_{}'.format(band)) sun_elev = metadata['IMAGE_ATTRIBUTES']['SUN_ELEVATION'] with rasterio.open('{}_B{}.TIF'.format(address, band)) as src: arr = src.read(indexes=1, out_shape=(height, width)).astype(src.profile['dtype']) arr = 10000 * reflectance.reflectance(arr, multi_reflect, add_reflect, sun_elev, src_nodata=0) return np.percentile(arr[arr > 0], (pmin, pmax)).astype(np.int).tolist()
def get_window(idx, window): band = bands[idx] multi_reflect = meta_data["RADIOMETRIC_RESCALING"].get( f"REFLECTANCE_MULT_BAND_{band}") add_reflect = meta_data["RADIOMETRIC_RESCALING"].get( f"REFLECTANCE_ADD_BAND_{band}") data = srcs[idx].read(window=window, boundless=True, indexes=(1)) return reflectance.reflectance(data, multi_reflect, add_reflect, sun_elev)
def worker(band): """Worker.""" address = f"{landsat_address}_B{band}.TIF" sun_elev = meta_data["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"] multi_reflect = meta_data["RADIOMETRIC_RESCALING"][ f"REFLECTANCE_MULT_BAND_{band}" ] add_reflect = meta_data["RADIOMETRIC_RESCALING"][f"REFLECTANCE_ADD_BAND_{band}"] band = get_area( address, bbox, max_img_size=max_img_size, bbox_crs=bbox_crs, out_crs=out_crs ) return reflectance(band, multi_reflect, add_reflect, sun_elev, src_nodata=0)
def worker(band, landsat_address, meta, ovr_size): """Worker.""" address = f"{landsat_address}_B{band}.TIF" sun_elev = meta["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"] multi_reflect = meta["RADIOMETRIC_RESCALING"][ f"REFLECTANCE_MULT_BAND_{band}"] add_reflect = meta["RADIOMETRIC_RESCALING"][f"REFLECTANCE_ADD_BAND_{band}"] matrix = get_overview(address, ovr_size) return reflectance(matrix, multi_reflect, add_reflect, sun_elev, src_nodata=0)
def worker(band, coordinates): """Worker.""" address = f"{landsat_address}_B{band}.TIF" sun_elev = meta_data["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"] multi_reflect = meta_data["RADIOMETRIC_RESCALING"][ f"REFLECTANCE_MULT_BAND_{band}" ] add_reflect = meta_data["RADIOMETRIC_RESCALING"][f"REFLECTANCE_ADD_BAND_{band}"] with rasterio.open(address) as band: lon_srs, lat_srs = warp.transform( "EPSG:4326", band.crs, [coordinates[0]], [coordinates[1]] ) point = list(band.sample([(lon_srs[0], lat_srs[0])]))[0] return reflectance(point, multi_reflect, add_reflect, sun_elev, src_nodata=0)[0]
def band_worker(band, landsat_address, meta, bounds=None): """ """ address = f'{landsat_address}_B{band}.TIF' sun_elev = meta['IMAGE_ATTRIBUTES']['SUN_ELEVATION'] multi_reflect = meta['RADIOMETRIC_RESCALING'][ f'REFLECTANCE_MULT_BAND_{band}'] add_reflect = meta['RADIOMETRIC_RESCALING'][f'REFLECTANCE_ADD_BAND_{band}'] ovrSize = 1024 with rasterio.open(address) as src: if bounds: w, s, e, n = bounds with WarpedVRT(src, dst_crs='EPSG:3857', resampling=Resampling.bilinear, src_nodata=0, dst_nodata=0) as vrt: window = vrt.window(w, s, e, n, precision=21) matrix = vrt.read(window=window, boundless=True, resampling=Resampling.bilinear, out_shape=(ovrSize, ovrSize), indexes=1).astype(src.profile['dtype']) else: matrix = src.read(indexes=1, out_shape=(ovrSize, ovrSize), resampling=Resampling.bilinear).astype( src.profile['dtype']) matrix = reflectance(matrix, multi_reflect, add_reflect, sun_elev, src_nodata=0) imgRange = np.percentile(matrix[matrix > 0], (2, 98)).tolist() matrix = np.where( matrix > 0, linear_rescale(matrix, in_range=imgRange, out_range=[1, 255]), 0).astype(np.uint8) return matrix
def percentiles(input, band, meta_url): meta = json.load(urllib.urlopen(meta_url)) with rasterio.Env(): with rasterio.open(input) as src: data = src.read(indexes=1, out_shape=(1024, 1024)) sun_elev = meta["L1_METADATA_FILE"]["IMAGE_ATTRIBUTES"][ "SUN_ELEVATION"] multi_reflect = meta[ "L1_METADATA_FILE"]["RADIOMETRIC_RESCALING"].get( "REFLECTANCE_MULT_BAND_{}".format(band)) add_reflect = meta[ "L1_METADATA_FILE"]["RADIOMETRIC_RESCALING"].get( "REFLECTANCE_ADD_BAND_{}".format(band)) data = 10000 * reflectance.reflectance( data, multi_reflect, add_reflect, sun_elev, src_nodata=0) return np.percentile(data[data > 0], (2, 5, 95, 98)).tolist()
def test_calculate_reflectance(test_data): tif_b, tif_output_single, mtl = test_data[0], test_data[5], test_data[-1] M = toa_utils._load_mtl_key(mtl, [ 'L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'REFLECTANCE_MULT_BAND_' ], 5) A = toa_utils._load_mtl_key( mtl, ['L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'REFLECTANCE_ADD_BAND_'], 5) E = toa_utils._load_mtl_key( mtl, ['L1_METADATA_FILE', 'IMAGE_ATTRIBUTES', 'SUN_ELEVATION']) assert (np.sin(np.radians(E)) <= 1) & (-1 <= np.sin(np.radians(E))) assert isinstance(M, float) toa = reflectance.reflectance(tif_b, M, A, E) toa_rescaled = toa_utils.rescale(toa, 55000.0, np.uint16, clip=False) assert toa_rescaled.dtype == np.uint16 # Note, the test data was created under a rescaling code, hence the fuzziness diff = toa_rescaled[310:315, 310:315] - tif_output_single[310:315, 310:315] assert diff.max() <= 1
def test_calculate_reflectance_uint8(test_data): tif_b, tif_output_single, mtl = test_data[0], test_data[5], test_data[-1] M = toa_utils._load_mtl_key(mtl, [ 'L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'REFLECTANCE_MULT_BAND_' ], 5) A = toa_utils._load_mtl_key( mtl, ['L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'REFLECTANCE_ADD_BAND_'], 5) E = toa_utils._load_mtl_key( mtl, ['L1_METADATA_FILE', 'IMAGE_ATTRIBUTES', 'SUN_ELEVATION']) assert (np.sin(np.radians(E)) <= 1) & (-1 <= np.sin(np.radians(E))) assert isinstance(M, float) toa = reflectance.reflectance(tif_b, M, A, E) toa_rescaled = toa_utils.rescale(toa, 215, np.uint8) scale = float(np.iinfo(np.uint16).max) / float(np.iinfo(np.uint8).max) tif_out_rescaled = np.clip((tif_output_single / scale), 0, np.iinfo(np.uint8).max).astype(np.uint8) assert toa_rescaled.dtype == np.uint8 assert np.min(tif_out_rescaled) == np.min(toa_rescaled)
def test_calculate_reflectance2(test_data): tif_b, tif_shape, mtl = test_data[0], test_data[4], test_data[-1] M = toa_utils._load_mtl_key(mtl, [ 'L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'REFLECTANCE_MULT_BAND_' ], 5) A = toa_utils._load_mtl_key( mtl, ['L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'REFLECTANCE_ADD_BAND_'], 5) date_collected = toa_utils._load_mtl_key( mtl, ['L1_METADATA_FILE', 'PRODUCT_METADATA', 'DATE_ACQUIRED']) time_collected_utc = toa_utils._load_mtl_key( mtl, ['L1_METADATA_FILE', 'PRODUCT_METADATA', 'SCENE_CENTER_TIME']) bounds = BoundingBox(*toa_utils._get_bounds_from_metadata( mtl['L1_METADATA_FILE']['PRODUCT_METADATA'])) E = sun_utils.sun_elevation(bounds, tif_shape, date_collected, time_collected_utc) toa = reflectance.reflectance(tif_b, M, A, E) toa_rescaled = toa_utils.rescale(toa, 55000.0, np.uint16) assert toa_rescaled.dtype == np.uint16 assert np.all(toa_rescaled) < 1.5 assert np.all(toa_rescaled) >= 0.0
def dn_to_toa(arr: numpy.ndarray, band: str, metadata: Dict) -> numpy.ndarray: """Convert DN to TOA or Temp. Args: arr (numpy.ndarray): Digital Number array values. band (str): Landsat 8 band's name. metadata (str): Landsat MTL metadata. Returns: numpy.ndarray: DN coverted to TOA or Temperature. """ if band in ["B1", "B2", "B3", "B4", "B5", "B6", "B7", "B8", "B9"]: # OLI multi_reflect = metadata["RADIOMETRIC_RESCALING"].get( f"REFLECTANCE_MULT_BAND_{band[1:]}") add_reflect = metadata["RADIOMETRIC_RESCALING"].get( f"REFLECTANCE_ADD_BAND_{band[1:]}") sun_elev = metadata["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"] arr = 10000 * reflectance.reflectance( arr, multi_reflect, add_reflect, sun_elev, src_nodata=0) arr = arr.astype("uint16") elif band in ["B10", "B11"]: # TIRS multi_rad = metadata["RADIOMETRIC_RESCALING"].get( f"RADIANCE_MULT_BAND_{band[1:]}") add_rad = metadata["RADIOMETRIC_RESCALING"].get( f"RADIANCE_ADD_BAND_{band[1:]}") k1 = metadata["TIRS_THERMAL_CONSTANTS"].get( f"K1_CONSTANT_BAND_{band[1:]}") k2 = metadata["TIRS_THERMAL_CONSTANTS"].get( f"K2_CONSTANT_BAND_{band[1:]}") arr = brightness_temp.brightness_temp(arr, multi_rad, add_rad, k1, k2) return arr
def tile(sceneid, tile_x, tile_y, tile_z, rgb=(4, 3, 2), tilesize=256, pan=False): """Create mercator tile from Landsat-8 data. Attributes ---------- sceneid : str Landsat sceneid. For scenes after May 2017, sceneid have to be LANDSAT_PRODUCT_ID. tile_x : int Mercator tile X index. tile_y : int Mercator tile Y index. tile_z : int Mercator tile ZOOM level. rgb : tuple, int, optional (default: (4, 3, 2)) Bands index for the RGB combination. tilesize : int, optional (default: 256) Output image size. pan : boolean, optional (default: False) If True, apply pan-sharpening. Returns ------- data : numpy ndarray mask: numpy array """ if not isinstance(rgb, tuple): rgb = tuple((rgb, )) scene_params = utils.landsat_parse_scene_id(sceneid) meta_data = utils.landsat_get_mtl(sceneid).get('L1_METADATA_FILE') landsat_address = '{}/{}'.format(LANDSAT_BUCKET, scene_params['key']) wgs_bounds = toa_utils._get_bounds_from_metadata( meta_data['PRODUCT_METADATA']) if not utils.tile_exists(wgs_bounds, tile_z, tile_x, tile_y): raise TileOutsideBounds( 'Tile {}/{}/{} is outside image bounds'.format( tile_z, tile_x, tile_y)) mercator_tile = mercantile.Tile(x=tile_x, y=tile_y, z=tile_z) tile_bounds = mercantile.xy_bounds(mercator_tile) ms_tile_size = int(tilesize / 2) if pan else tilesize addresses = ['{}_B{}.TIF'.format(landsat_address, band) for band in rgb] _tiler = partial(utils.tile_band_worker, bounds=tile_bounds, tilesize=ms_tile_size, nodata=0) with futures.ThreadPoolExecutor(max_workers=3) as executor: data, masks = zip(*list(executor.map(_tiler, addresses))) data = np.concatenate(data) mask = np.all(masks, axis=0).astype(np.uint8) * 255 if pan: pan_address = '{}_B8.TIF'.format(landsat_address) matrix_pan, mask = utils.tile_band_worker(pan_address, tile_bounds, tilesize, nodata=0) w, s, e, n = tile_bounds pan_transform = transform.from_bounds(w, s, e, n, tilesize, tilesize) vis_transform = pan_transform * Affine.scale(2.) data = pansharpen(data, vis_transform, matrix_pan, pan_transform, np.int16, 'EPSG:3857', 'EPSG:3857', 0.2, method='Brovey', src_nodata=0) sun_elev = meta_data['IMAGE_ATTRIBUTES']['SUN_ELEVATION'] for bdx, band in enumerate(rgb): if int(band) > 9: # TIRS multi_rad = meta_data['RADIOMETRIC_RESCALING'].get( 'RADIANCE_MULT_BAND_{}'.format(band)) add_rad = meta_data['RADIOMETRIC_RESCALING'].get( 'RADIANCE_ADD_BAND_{}'.format(band)) k1 = meta_data['TIRS_THERMAL_CONSTANTS'].get( 'K1_CONSTANT_BAND_{}'.format(band)) k2 = meta_data['TIRS_THERMAL_CONSTANTS'].get( 'K2_CONSTANT_BAND_{}'.format(band)) data[bdx] = brightness_temp.brightness_temp( data[bdx], multi_rad, add_rad, k1, k2) else: multi_reflect = meta_data['RADIOMETRIC_RESCALING'].get( 'REFLECTANCE_MULT_BAND_{}'.format(band)) add_reflect = meta_data['RADIOMETRIC_RESCALING'].get( 'REFLECTANCE_ADD_BAND_{}'.format(band)) data[bdx] = 10000 * reflectance.reflectance( data[bdx], multi_reflect, add_reflect, sun_elev) return data, mask
def apply(recipes, pixels, expand, source=None): data = pixels.data colormap = pixels.colormap if np.issubdtype(data.dtype, np.floating): dtype_min = np.finfo(data.dtype).min dtype_max = np.finfo(data.dtype).max else: dtype_min = np.iinfo(data.dtype).min dtype_max = np.iinfo(data.dtype).max if data.shape[0] == 1: if expand and colormap: # create a lookup table from the source's color map lut = make_colormap(colormap) # stash the mask mask = data.mask # apply the color map data = lut[data[0], :] # re-shape to match band-style data = np.ma.transpose(data, [2, 0, 1]) # re-apply the mask, merging it with pixels that were masked by the color map data.mask = data.mask | mask colormap = None if "landsat8" in recipes: LOG.info("Applying landsat 8 recipe") out = np.ma.empty(shape=(data.shape), dtype=np.float32) for bdx, source_band in enumerate((4, 3, 2)): sun_elev = source.meta["L1_METADATA_FILE"]["IMAGE_ATTRIBUTES"][ "SUN_ELEVATION" ] multi_reflect = source.meta["L1_METADATA_FILE"][ "RADIOMETRIC_RESCALING" ].get( "REFLECTANCE_MULT_BAND_{}".format(source_band) ) add_reflect = source.meta["L1_METADATA_FILE"]["RADIOMETRIC_RESCALING"].get( "REFLECTANCE_ADD_BAND_{}".format(source_band) ) min_val = source.meta.get("values", {}).get(str(source_band), {}).get( "min", dtype_min ) max_val = source.meta.get("values", {}).get(str(source_band), {}).get( "max", dtype_max ) band_data = 10000 * reflectance.reflectance( data[bdx], multi_reflect, add_reflect, sun_elev, src_nodata=0 ) # calculate local min/max as fallbacks if ( min_val == dtype_min and max_val == dtype_max and len(data.compressed()) > 0 ): local_min, local_max = np.percentile(band_data.compressed(), (2, 98)) min_val = max(min_val, local_min) max_val = min(max_val, local_max) out[bdx] = utils.linear_rescale( band_data, in_range=(min_val, max_val), out_range=(0.0, 1.0) ) data = out if "imagery" in recipes: LOG.info("Applying imagery recipe") if "rgb_bands" in recipes: data = np.ma.array( [data[i - 1] for i in recipes["rgb_bands"] if data.shape[0] >= i] ) elif "expr" in recipes: num_bands = data.shape[0] expressions = recipes["expr"].split(",") band_names = ["b" + str(i) for i in range(1, num_bands + 1)] local_dict = dict(zip(band_names, data.data)) old_mask = data.mask old_mask_shape = old_mask.shape new_mask_shape = (len(expressions), old_mask_shape[1], old_mask_shape[2]) new_mask = np.ndarray(new_mask_shape, dtype=np.bool) for (expression_index, expression) in enumerate(expressions): # identify bands used in expression band_indexes = [int(i) - 1 for i in re.findall(r"(?<=b)(\d{1,2})", expression)] old_band_masks = [old_mask[i] for i in band_indexes] new_mask[expression_index] = np.logical_and.reduce(old_band_masks) data = np.ma.array([ np.nan_to_num(ne.evaluate(expression.strip(), local_dict=local_dict)) for expression in expressions ], mask=new_mask) elif data.shape[0] > 3: # alpha(?) band (and beyond) present; drop it (them) # TODO use band 4 as an alpha channel if colorinterp == alpha instead # TODO re-order channels if BGR (whatever colorinterp says) data = data[0:3] if "linear_stretch" in recipes: # Added by Tushar Shukla # Add envi like property here to generate envi like browse tiles if recipes["linear_stretch"] == "global": data = utils.linear_rescale( data, in_range=(np.min(data), np.max(data)), out_range=(0.0, 1.0) ) elif recipes["linear_stretch"] == "per_band": out = np.ma.empty(shape=(data.shape), dtype=np.float32) for band in range(0, data.shape[0]): min_val = source.meta.get("values", {}).get(band, {}).get( "min", np.min(data[band]) ) max_val = source.meta.get("values", {}).get(band, {}).get( "max", np.max(data[band]) ) out[band] = utils.linear_rescale( data[band], in_range=(min_val, max_val), out_range=(0.0, 1.0) ) data = out else: # rescale after reducing and before increasing dimensionality if data.dtype != np.uint8: # rescale non-8-bit sources (assuming that they're raw sensor # data) and normalize to 0..1 out = np.ma.empty(shape=(data.shape), dtype=np.float32) for band in range(0, data.shape[0]): min_val = source.meta.get("values", {}).get(band, {}).get( "min", dtype_min ) max_val = source.meta.get("values", {}).get(band, {}).get( "max", dtype_max ) if ( min_val == dtype_min and max_val == dtype_max and len(data.compressed()) > 0 ): local_min, local_max = np.percentile( data[band].compressed(), (2, 98) ) min_val = max(min_val, local_min) max_val = min(max_val, local_max) out[band] = utils.linear_rescale( data[band], in_range=(min_val, max_val), out_range=(0.0, 1.0) ) data = out if not np.issubdtype(data.dtype, np.floating): # normalize to 0..1 based on the range of the source type (only # for int*s) data = data.astype(np.float32) / dtype_max if data.shape[0] == 1: # likely greyscale image; use the same band on all channels data = np.ma.array([data[0], data[0], data[0]]) return PixelCollection(data, pixels.bounds, None, colormap)
def tile(sceneid, tile_x, tile_y, tile_z, bands=("4", "3", "2"), tilesize=256, pan=False): """ Create mercator tile from Landsat-8 data. Attributes ---------- sceneid : str Landsat sceneid. For scenes after May 2017, sceneid have to be LANDSAT_PRODUCT_ID. tile_x : int Mercator tile X index. tile_y : int Mercator tile Y index. tile_z : int Mercator tile ZOOM level. bands : tuple, str, optional (default: ("4", "3", "2")) Bands index for the RGB combination. tilesize : int, optional (default: 256) Output image size. pan : boolean, optional (default: False) If True, apply pan-sharpening. Returns ------- data : numpy ndarray mask: numpy array """ if not isinstance(bands, tuple): bands = tuple((bands, )) for band in bands: if band not in LANDSAT_BANDS: raise InvalidBandName( "{} is not a valid Landsat band name".format(band)) scene_params = _landsat_parse_scene_id(sceneid) meta_data = _landsat_get_mtl(sceneid).get("L1_METADATA_FILE") landsat_address = "{}/{}".format(LANDSAT_BUCKET, scene_params["key"]) wgs_bounds = toa_utils._get_bounds_from_metadata( meta_data["PRODUCT_METADATA"]) if not utils.tile_exists(wgs_bounds, tile_z, tile_x, tile_y): raise TileOutsideBounds("Tile {}/{}/{} is outside image bounds".format( tile_z, tile_x, tile_y)) mercator_tile = mercantile.Tile(x=tile_x, y=tile_y, z=tile_z) tile_bounds = mercantile.xy_bounds(mercator_tile) ms_tile_size = int(tilesize / 2) if pan else tilesize addresses = ["{}_B{}.TIF".format(landsat_address, band) for band in bands] _tiler = partial(utils.tile_read, bounds=tile_bounds, tilesize=ms_tile_size, nodata=0) with futures.ThreadPoolExecutor(max_workers=MAX_THREADS) as executor: data, masks = zip(*list(executor.map(_tiler, addresses))) data = np.concatenate(data) mask = np.all(masks, axis=0).astype(np.uint8) * 255 if pan: pan_address = "{}_B8.TIF".format(landsat_address) matrix_pan, mask = utils.tile_read(pan_address, tile_bounds, tilesize, nodata=0) w, s, e, n = tile_bounds pan_transform = transform.from_bounds(w, s, e, n, tilesize, tilesize) vis_transform = pan_transform * Affine.scale(2.0) data = pansharpen( data, vis_transform, matrix_pan, pan_transform, np.int16, "EPSG:3857", "EPSG:3857", 0.2, method="Brovey", src_nodata=0, ) sun_elev = meta_data["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"] for bdx, band in enumerate(bands): if int(band) > 9: # TIRS multi_rad = meta_data["RADIOMETRIC_RESCALING"].get( "RADIANCE_MULT_BAND_{}".format(band)) add_rad = meta_data["RADIOMETRIC_RESCALING"].get( "RADIANCE_ADD_BAND_{}".format(band)) k1 = meta_data["TIRS_THERMAL_CONSTANTS"].get( "K1_CONSTANT_BAND_{}".format(band)) k2 = meta_data["TIRS_THERMAL_CONSTANTS"].get( "K2_CONSTANT_BAND_{}".format(band)) data[bdx] = brightness_temp.brightness_temp( data[bdx], multi_rad, add_rad, k1, k2) else: multi_reflect = meta_data["RADIOMETRIC_RESCALING"].get( "REFLECTANCE_MULT_BAND_{}".format(band)) add_reflect = meta_data["RADIOMETRIC_RESCALING"].get( "REFLECTANCE_ADD_BAND_{}".format(band)) data[bdx] = 10000 * reflectance.reflectance( data[bdx], multi_reflect, add_reflect, sun_elev) return data, mask
def tile(sceneid, tile_x, tile_y, tile_z, rgb=(4, 3, 2), r_bds=(0, 16000), g_bds=(0, 16000), b_bds=(0, 16000), tilesize=256, pan=False): """Create mercator tile from Landsat-8 data and encodes it in base64. Attributes ---------- sceneid : str Landsat sceneid. For scenes after May 2017, sceneid have to be LANDSAT_PRODUCT_ID. tile_x : int Mercator tile X index. tile_y : int Mercator tile Y index. tile_z : int Mercator tile ZOOM level. rgb : tuple, int, optional (default: (4, 3, 2)) Bands index for the RGB combination. r_bds : tuple, int, optional (default: (0, 16000)) First band (red) DN min and max values (DN * 10,000) used for the linear rescaling. g_bds : tuple, int, optional (default: (0, 16000)) Second band (green) DN min and max values (DN * 10,000) used for the linear rescaling. b_bds : tuple, int, optional (default: (0, 16000)) Third band (blue) DN min and max values (DN * 10,000) used for the linear rescaling. tilesize : int, optional (default: 256) Output image size. pan : boolean, optional (default: False) If True, apply pan-sharpening. Returns ------- out : numpy ndarray (type: uint8) """ scene_params = utils.landsat_parse_scene_id(sceneid) meta_data = utils.landsat_get_mtl(sceneid).get('L1_METADATA_FILE') landsat_address = '{}/{}'.format(LANDSAT_BUCKET, scene_params['key']) wgs_bounds = toa_utils._get_bounds_from_metadata( meta_data['PRODUCT_METADATA']) if not utils.tile_exists(wgs_bounds, tile_z, tile_x, tile_y): raise TileOutsideBounds('Tile {}/{}/{} is outside image bounds'.format( tile_z, tile_x, tile_y)) mercator_tile = mercantile.Tile(x=tile_x, y=tile_y, z=tile_z) tile_bounds = mercantile.xy_bounds(mercator_tile) # define a list of bands Min and Max Values (from input) histo_cuts = dict(zip(rgb, [r_bds, g_bds, b_bds])) ms_tile_size = int(tilesize / 2) if pan else tilesize addresses = ['{}_B{}.TIF'.format(landsat_address, band) for band in rgb] _tiler = partial(utils.tile_band_worker, bounds=tile_bounds, tilesize=ms_tile_size) with futures.ThreadPoolExecutor(max_workers=3) as executor: out = np.stack(list(executor.map(_tiler, addresses))) if pan: pan_address = '{}_B8.TIF'.format(landsat_address) matrix_pan = utils.tile_band_worker(pan_address, tile_bounds, tilesize) w, s, e, n = tile_bounds pan_transform = transform.from_bounds(w, s, e, n, tilesize, tilesize) vis_transform = pan_transform * Affine.scale(2.) out = pansharpen(out, vis_transform, matrix_pan, pan_transform, np.int16, 'EPSG:3857', 'EPSG:3857', 0.2, method='Brovey', src_nodata=0) sun_elev = meta_data['IMAGE_ATTRIBUTES']['SUN_ELEVATION'] for bdx, band in enumerate(rgb): multi_reflect = meta_data['RADIOMETRIC_RESCALING'].get( 'REFLECTANCE_MULT_BAND_{}'.format(band)) add_reflect = meta_data['RADIOMETRIC_RESCALING'].get( 'REFLECTANCE_ADD_BAND_{}'.format(band)) out[bdx] = 10000 * reflectance.reflectance( out[bdx], multi_reflect, add_reflect, sun_elev, src_nodata=0) out[bdx] = np.where( out[bdx] > 0, utils.linear_rescale(out[bdx], in_range=histo_cuts.get(band), out_range=[1, 255]), 0) return out.astype(np.uint8)
def dn2toa(self, platform, mtl_file=None, wavelengths=None): """This method converts digital numbers to top of atmosphere reflectance, like described here: https://www.usgs.gov/land-resources/nli/landsat/using-usgs-landsat-level-1-data-product :param platform: image platform, possible Platform.Landsat[5, 7, 8] or Platform.Sentinel2 (<enum 'Platform'>). :param mtl_file: path to Landsat MTL file that holds the band specific rescale factors (str). :param wavelengths: like ["Blue", "Green", "Red", "NIR", "SWIR1", "TIRS", "SWIR2"] for Landsat-5 (list of str). """ if platform in [ Platform.Landsat5, Platform.Landsat7, Platform.Landsat8, ]: if mtl_file is None: raise AttributeError( f"'mtl_file' has to be set if platform is {platform}.") else: # get rescale factors from mtl file mtl = toa_utils._load_mtl( str(mtl_file)) # no obvious reason not to call this metadata = mtl["L1_METADATA_FILE"] sun_elevation = metadata["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"] toa = [] for idx, b in enumerate( self._lookup_bands(platform, wavelengths)): if (platform == Platform.Landsat8 and b in ["10", "11"]) or (platform != Platform.Landsat8 and b.startswith("6")): if platform == Platform.Landsat8: thermal_conversion_constant1 = metadata[ "TIRS_THERMAL_CONSTANTS"][ f"K1_CONSTANT_BAND_{b}"] thermal_conversion_constant2 = metadata[ "TIRS_THERMAL_CONSTANTS"][ f"K2_CONSTANT_BAND_{b}"] else: thermal_conversion_constant1 = metadata[ "THERMAL_CONSTANTS"][f"K1_CONSTANT_BAND_{b}"] thermal_conversion_constant2 = metadata[ "THERMAL_CONSTANTS"][f"K2_CONSTANT_BAND_{b}"] multiplicative_rescaling_factors = metadata[ "RADIOMETRIC_RESCALING"][f"RADIANCE_MULT_BAND_{b}"] additive_rescaling_factors = metadata[ "RADIOMETRIC_RESCALING"][f"RADIANCE_ADD_BAND_{b}"] # rescale thermal bands toa.append( brightness_temp.brightness_temp( self.__arr[idx, :, :], ML=multiplicative_rescaling_factors, AL=additive_rescaling_factors, K1=thermal_conversion_constant1, K2=thermal_conversion_constant2, )) continue # rescale reflectance bands multiplicative_rescaling_factors = metadata[ "RADIOMETRIC_RESCALING"][f"REFLECTANCE_MULT_BAND_{b}"] additive_rescaling_factors = metadata[ "RADIOMETRIC_RESCALING"][f"REFLECTANCE_ADD_BAND_{b}"] toa.append( reflectance.reflectance( self.__arr[idx, :, :], MR=multiplicative_rescaling_factors, AR=additive_rescaling_factors, E=sun_elevation, )) self.__arr = np.array(np.stack(toa, axis=0)) elif platform == Platform.Sentinel2: self.__arr = self.__arr.astype(np.float32) / 10000.0 else: raise AttributeError( f"Cannot convert dn2toa. Platform {platform} not supported [Landsat-5, Landsat-7, Landsat-8, " f"Sentinel-2]. ") self.__update_dataset(self.dataset.crs, self.dataset.transform, nodata=self.dataset.nodata)
def worker(scene, bands): """Worker.""" try: scene_params = landsat_parse_scene_id(scene) meta_data = landsat_get_mtl(scene).get("L1_METADATA_FILE") landsat_address = f'{LANDSAT_BUCKET}/{scene_params["key"]}' bqa = f"{landsat_address}_BQA.TIF" with rasterio.open(bqa) as src: ovr = src.overviews(1) ovr_width = int(src.width / ovr[0]) ovr_height = int(src.height / ovr[0]) dst_affine, width, height = calculate_default_transform( src.crs, "epsg:3857", ovr_width, ovr_height, *src.bounds) meta = { "driver": "GTiff", "count": 3, "dtype": np.uint8, "nodata": 0, "height": height, "width": width, "compress": "DEFLATE", "crs": "epsg:3857", "transform": dst_affine, } outpath = f"/tmp/{scene}.tif" with rasterio.open(outpath, "w", **meta) as dataset: sun_elev = meta_data["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"] for idx, b in enumerate(bands): with rasterio.open(f"{landsat_address}_B{b}.TIF") as src: with WarpedVRT( src, dst_crs="EPSG:3857", resampling=Resampling.bilinear, src_nodata=0, dst_nodata=0, ) as vrt: matrix = vrt.read(indexes=1, out_shape=(height, width)) multi_reflect = meta_data["RADIOMETRIC_RESCALING"][ f"REFLECTANCE_MULT_BAND_{b}"] add_reflect = meta_data["RADIOMETRIC_RESCALING"][ f"REFLECTANCE_ADD_BAND_{b}"] matrix = (reflectance( matrix, multi_reflect, add_reflect, sun_elev, src_nodata=0) * 10000) minref = (meta_data["MIN_MAX_REFLECTANCE"] [f"REFLECTANCE_MINIMUM_BAND_{b}"] * 10000) maxref = (meta_data["MIN_MAX_REFLECTANCE"] [f"REFLECTANCE_MAXIMUM_BAND_{b}"] * 10000) matrix = np.where( matrix > 0, linear_rescale(matrix, in_range=[int(minref), int(maxref)], out_range=[1, 255]), 0, ).astype(np.uint8) mask = np.ma.masked_values(matrix, 0) s = np.ma.notmasked_contiguous(mask) matrix = matrix.ravel() for sl in s: matrix[sl.start:sl.start + 5] = 0 matrix[sl.stop - 5:sl.stop] = 0 matrix = matrix.reshape((height, width)) dataset.write(matrix, indexes=idx + 1) return outpath except: return None
def _landsat_stats( band, address_prefix, metadata, overview_level=None, max_size=1024, percentiles=(2, 98), dst_crs=CRS({"init": "EPSG:4326"}), histogram_bins=10, histogram_range=None, ): """ Retrieve landsat dataset statistics. Attributes ---------- band : str Landsat band number address_prefix : str A Landsat AWS S3 dataset prefix. metadata : dict Landsat metadata overview_level : int, optional Overview (decimation) level to fetch. max_size: int, optional Maximum size of dataset to retrieve (will be used to calculate the overview level to fetch). percentiles : tulple, optional Percentile or sequence of percentiles to compute, which must be between 0 and 100 inclusive (default: (2, 98)). dst_crs: CRS or dict Target coordinate reference system (default: EPSG:4326). histogram_bins: int, optional Defines the number of equal-width histogram bins (default: 10). histogram_range: tuple or list, optional The lower and upper range of the bins. If not provided, range is simply the min and max of the array. Returns ------- out : dict (percentiles), min, max, stdev, histogram for each band, e.g. { "4": { 'pc': [15, 121], 'min': 1, 'max': 162, 'std': 27.22067722127997, 'histogram': [ [102934, 135489, 20981, 13548, 11406, 8799, 7351, 5622, 2985, 662] [1., 17.1, 33.2, 49.3, 65.4, 81.5, 97.6, 113.7, 129.8, 145.9, 162.] ] } } """ src_path = "{}_B{}.TIF".format(address_prefix, band) with rasterio.open(src_path) as src: levels = src.overviews(1) width = src.width height = src.height bounds = transform_bounds(src.crs, dst_crs, *src.bounds, densify_pts=21) if len(levels): if overview_level: decim = levels[overview_level] else: # determine which zoom level to read for ii, decim in enumerate(levels): if max(width // decim, height // decim) < max_size: break else: decim = 1 warnings.warn("Dataset has no overviews, reading the full dataset", NoOverviewWarning) out_shape = (height // decim, width // decim) if band == "QA": nodata = 1 else: nodata = 0 vrt_params = dict(nodata=nodata, add_alpha=False, src_nodata=nodata, init_dest_nodata=False) with WarpedVRT(src, **vrt_params) as vrt: arr = vrt.read(out_shape=out_shape, indexes=[1], masked=True) if band in ["1", "2", "3", "4", "5", "6", "7", "8", "9"]: # OLI multi_reflect = metadata["RADIOMETRIC_RESCALING"].get( "REFLECTANCE_MULT_BAND_{}".format(band)) add_reflect = metadata["RADIOMETRIC_RESCALING"].get( "REFLECTANCE_ADD_BAND_{}".format(band)) sun_elev = metadata["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"] arr = 10000 * reflectance.reflectance( arr, multi_reflect, add_reflect, sun_elev, src_nodata=0) elif band in ["10", "11"]: # TIRS multi_rad = metadata["RADIOMETRIC_RESCALING"].get( "RADIANCE_MULT_BAND_{}".format(band)) add_rad = metadata["RADIOMETRIC_RESCALING"].get( "RADIANCE_ADD_BAND_{}".format(band)) k1 = metadata["TIRS_THERMAL_CONSTANTS"].get( "K1_CONSTANT_BAND_{}".format(band)) k2 = metadata["TIRS_THERMAL_CONSTANTS"].get( "K2_CONSTANT_BAND_{}".format(band)) arr = brightness_temp.brightness_temp(arr, multi_rad, add_rad, k1, k2) params = {} if histogram_bins: params.update(dict(bins=histogram_bins)) if histogram_range: params.update(dict(range=histogram_range)) stats = {band: utils._stats(arr, percentiles=percentiles, **params)} return { "bounds": { "value": bounds, "crs": dst_crs.to_string() if isinstance(dst_crs, CRS) else dst_crs, }, "statistics": stats, }
def test_reflectance_wrong_type(): band = np.array([[9931., 9872., 9939.], [0., 5000., 100.], [10000.1, 0., 100002.]]).astype('float32') with pytest.raises(TypeError): reflectance.reflectance(band, '45sldf', -0.1, 65.0)
def tile(sceneid, tile_x, tile_y, tile_z, bands=("4", "3", "2"), tilesize=256, pan=False, **kwargs): """ Create mercator tile from Landsat-8 data. Attributes ---------- sceneid : str Landsat sceneid. For scenes after May 2017, sceneid have to be LANDSAT_PRODUCT_ID. tile_x : int Mercator tile X index. tile_y : int Mercator tile Y index. tile_z : int Mercator tile ZOOM level. bands : tuple, str, optional (default: ("4", "3", "2")) Bands index for the RGB combination. tilesize : int, optional (default: 256) Output image size. pan : boolean, optional (default: False) If True, apply pan-sharpening. kwargs: dict, optional These will be passed to the 'rio_tiler.utils._tile_read' function. Returns ------- data : numpy ndarray mask: numpy array """ if not isinstance(bands, tuple): bands = tuple((bands, )) for band in bands: if band not in LANDSAT_BANDS: raise InvalidBandName( "{} is not a valid Landsat band name".format(band)) scene_params = _landsat_parse_scene_id(sceneid) meta_data = _landsat_get_mtl(sceneid).get("L1_METADATA_FILE") landsat_address = "{}/{}".format(LANDSAT_BUCKET, scene_params["key"]) wgs_bounds = toa_utils._get_bounds_from_metadata( meta_data["PRODUCT_METADATA"]) if not utils.tile_exists(wgs_bounds, tile_z, tile_x, tile_y): raise TileOutsideBounds("Tile {}/{}/{} is outside image bounds".format( tile_z, tile_x, tile_y)) mercator_tile = mercantile.Tile(x=tile_x, y=tile_y, z=tile_z) tile_bounds = mercantile.xy_bounds(mercator_tile) def _tiler(band): address = "{}_B{}.TIF".format(landsat_address, band) if band == "QA": nodata = 1 else: nodata = 0 return utils.tile_read(address, bounds=tile_bounds, tilesize=tilesize, nodata=nodata, **kwargs) with futures.ThreadPoolExecutor(max_workers=MAX_THREADS) as executor: data, masks = zip(*list(executor.map(_tiler, bands))) data = np.concatenate(data) mask = np.all(masks, axis=0).astype(np.uint8) * 255 if pan: pan_address = "{}_B8.TIF".format(landsat_address) matrix_pan, mask = utils.tile_read(pan_address, tile_bounds, tilesize, nodata=0) data = utils.pansharpening_brovey(data, matrix_pan, 0.2, matrix_pan.dtype) sun_elev = meta_data["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"] for bdx, band in enumerate(bands): if band in ["1", "2", "3", "4", "5", "6", "7", "8", "9"]: # OLI multi_reflect = meta_data["RADIOMETRIC_RESCALING"].get( "REFLECTANCE_MULT_BAND_{}".format(band)) add_reflect = meta_data["RADIOMETRIC_RESCALING"].get( "REFLECTANCE_ADD_BAND_{}".format(band)) data[bdx] = 10000 * reflectance.reflectance( data[bdx], multi_reflect, add_reflect, sun_elev) elif band in ["10", "11"]: # TIRS multi_rad = meta_data["RADIOMETRIC_RESCALING"].get( "RADIANCE_MULT_BAND_{}".format(band)) add_rad = meta_data["RADIOMETRIC_RESCALING"].get( "RADIANCE_ADD_BAND_{}".format(band)) k1 = meta_data["TIRS_THERMAL_CONSTANTS"].get( "K1_CONSTANT_BAND_{}".format(band)) k2 = meta_data["TIRS_THERMAL_CONSTANTS"].get( "K2_CONSTANT_BAND_{}".format(band)) data[bdx] = brightness_temp.brightness_temp( data[bdx], multi_rad, add_rad, k1, k2) return data, mask
def area(scene, bbox): """ """ max_width = 512 max_height = 512 scene_params = rputils.landsat_parse_scene_id(scene) meta_data = rputils.landsat_get_mtl(scene).get('L1_METADATA_FILE') sun_elev = meta_data['IMAGE_ATTRIBUTES']['SUN_ELEVATION'] multi_reflect = meta_data['RADIOMETRIC_RESCALING'][ 'REFLECTANCE_MULT_BAND_4'] add_reflect = meta_data['RADIOMETRIC_RESCALING']['REFLECTANCE_ADD_BAND_4'] band_address = scene_params["key"] + '_B4.TIF' s3 = boto3.resource('s3', 'us-west-2') if not os.path.exists('/tmp/' + scene + '_B4.TIF'): s3.Bucket('landsat-pds').download_file(band_address, '/tmp/' + scene + '_B4.TIF') with rio.open('/tmp/' + scene + '_B4.TIF') as band: crs_bounds = warp.transform_bounds('EPSG:4326', band.crs, *bbox) window = band.window(*crs_bounds) width = round(window.width) if window.width < max_width else max_width height = round( window.height) if window.height < max_height else max_height b4 = band.read(window=window, out_shape=(height, width), indexes=1, resampling=Resampling.bilinear, boundless=True) b4 = reflectance(b4, multi_reflect, add_reflect, sun_elev, src_nodata=0) multi_reflect = meta_data['RADIOMETRIC_RESCALING'][ 'REFLECTANCE_MULT_BAND_5'] add_reflect = meta_data['RADIOMETRIC_RESCALING']['REFLECTANCE_ADD_BAND_5'] band_address = scene_params["key"] + '_B5.TIF' if not os.path.exists('/tmp/' + scene + '_B5.TIF'): s3.Bucket('landsat-pds').download_file(band_address, '/tmp/' + scene + '_B5.TIF') with rio.open('/tmp/' + scene + '_B5.TIF') as band: crs_bounds = warp.transform_bounds('EPSG:4326', band.crs, *bbox) window = band.window(*crs_bounds) width = round(window.width) if window.width < max_width else max_width height = round( window.height) if window.height < max_height else max_height b5 = band.read(window=window, out_shape=(height, width), indexes=1, resampling=Resampling.bilinear, boundless=True) b5 = reflectance(b5, multi_reflect, add_reflect, sun_elev, src_nodata=0) ratio = np.where((b5 * b4) > 0, np.nan_to_num((b5 - b4) / (b5 + b4)), -9999) ratio = np.where( ratio > -9999, rputils.linear_rescale(ratio, in_range=[-1, 1], out_range=[1, 255]), 0).astype(np.uint8) cmap = list(np.array(rputils.get_colormap()).flatten()) img = Image.fromarray(ratio, 'P') img.putpalette(cmap) img = img.convert('RGB') sio = BytesIO() img.save(sio, 'jpeg', subsampling=0, quality=100) sio.seek(0) return base64.b64encode(sio.getvalue()).decode()
def point(scene, coord): """ """ try: scene_params = rputils.landsat_parse_scene_id(scene) meta_data = rputils.landsat_get_mtl(scene).get('L1_METADATA_FILE') sun_elev = meta_data['IMAGE_ATTRIBUTES']['SUN_ELEVATION'] multi_reflect = meta_data['RADIOMETRIC_RESCALING'][ 'REFLECTANCE_MULT_BAND_4'] add_reflect = meta_data['RADIOMETRIC_RESCALING'][ 'REFLECTANCE_ADD_BAND_4'] band_address = scene_params["key"] + '_B4.TIF' s3 = boto3.resource('s3', 'us-west-2') # if not os.path.exists('/tmp/'+scene+'_B4.TIF'): # s3.Bucket('landsat-pds').download_file(band_address, '/tmp/'+scene+'_B4.TIF') with rio.open('s3://landsat-pds/' + band_address) as band: lon_srs, lat_srs = warp.transform('EPSG:4326', band.crs, [coord[0]], [coord[1]]) b4 = list(band.sample([(lon_srs[0], lat_srs[0])]))[0] b4 = reflectance(b4, multi_reflect, add_reflect, sun_elev, src_nodata=0)[0] multi_reflect = meta_data['RADIOMETRIC_RESCALING'][ 'REFLECTANCE_MULT_BAND_5'] add_reflect = meta_data['RADIOMETRIC_RESCALING'][ 'REFLECTANCE_ADD_BAND_5'] band_address = scene_params["key"] + '_B5.TIF' #s3_object = s3.get_object(Bucket='landsat-pds', Key=band_address) #f = s3_object['Body'].read() # if not os.path.exists('/tmp/'+scene+'_B5.TIF'): # s3.Bucket('landsat-pds').download_file(band_address, '/tmp/'+scene+'_B5.TIF') with rio.open('s3://landsat-pds/' + band_address) as band: lon_srs, lat_srs = warp.transform('EPSG:4326', band.crs, [coord[0]], [coord[1]]) b5 = list(band.sample([(lon_srs[0], lat_srs[0])]))[0] b5 = reflectance(b5, multi_reflect, add_reflect, sun_elev, src_nodata=0)[0] ratio = np.nan_to_num((b5 - b4) / (b5 + b4)) if (b4 * b5) > 0 else 0. out = { 'scene': scene, 'ndvi': ratio, 'date': scene_params['date'], 'cloud': meta_data['IMAGE_ATTRIBUTES']['CLOUD_COVER'] } return out except: return {}
def apply(recipes, pixels, source=None): data = pixels.data dtype_min = np.iinfo(data.dtype).min dtype_max = np.iinfo(data.dtype).max if "landsat8" in recipes: LOG.info("Applying landsat 8 recipe") out = np.ma.empty(shape=(data.shape), dtype=np.float32) for bdx, source_band in enumerate((4, 3, 2)): sun_elev = source.meta["L1_METADATA_FILE"]["IMAGE_ATTRIBUTES"][ "SUN_ELEVATION" ] multi_reflect = source.meta["L1_METADATA_FILE"][ "RADIOMETRIC_RESCALING" ].get( "REFLECTANCE_MULT_BAND_{}".format(source_band) ) add_reflect = source.meta["L1_METADATA_FILE"]["RADIOMETRIC_RESCALING"].get( "REFLECTANCE_ADD_BAND_{}".format(source_band) ) min_val = source.meta.get("values", {}).get(str(source_band), {}).get( "min", dtype_min ) max_val = source.meta.get("values", {}).get(str(source_band), {}).get( "max", dtype_max ) band_data = 10000 * reflectance.reflectance( data[bdx], multi_reflect, add_reflect, sun_elev, src_nodata=0 ) # calculate local min/max as fallbacks if ( min_val == dtype_min and max_val == dtype_max and len(data.compressed()) > 0 ): local_min, local_max = np.percentile(band_data.compressed(), (2, 98)) min_val = max(min_val, local_min) max_val = min(max_val, local_max) out[bdx] = np.ma.where( band_data > 0, utils.linear_rescale( band_data, in_range=[min_val, max_val], out_range=[0, 1] ), 0, ) data = out if "imagery" in recipes: LOG.info("Applying imagery recipe") if "rgb_bands" in recipes: data = np.ma.array([data[i - 1] for i in recipes["rgb_bands"]]) if data.shape[0] > 3: # alpha band (and beyond) present; drop it (them) # TODO use band 4 as a mask instead data = data[0:3] if "linear_stretch" in recipes: if recipes["linear_stretch"] == "global": data = utils.linear_rescale( data, in_range=(np.min(data), np.max(data)), out_range=(dtype_min, dtype_max), ) elif recipes["linear_stretch"] == "per_band": for band in xrange(0, data.shape[0]): min_val = source.meta.get("values", {}).get(band, {}).get( "min", np.min(data[band]) ) max_val = source.meta.get("values", {}).get(band, {}).get( "max", np.max(data[band]) ) data[band] = np.ma.where( data[band] > 0, utils.linear_rescale( data[band], in_range=(min_val, max_val), out_range=(dtype_min, dtype_max), ), 0, ) else: # rescale after reducing and before increasing dimensionality if data.dtype != np.uint8 and not np.issubdtype(data.dtype, float): # rescale non-8-bit sources (assuming that they're raw sensor data) for band in xrange(0, data.shape[0]): min_val = source.meta.get("values", {}).get(band, {}).get( "min", dtype_min ) max_val = source.meta.get("values", {}).get(band, {}).get( "max", dtype_max ) if ( min_val == dtype_min and max_val == dtype_max and len(data.compressed()) > 0 ): local_min, local_max = np.percentile( data[band].compressed(), (2, 98) ) min_val = max(min_val, local_min) max_val = min(max_val, local_max) data[band] = np.ma.where( data[band] > 0, utils.linear_rescale( data[band], in_range=(min_val, max_val), out_range=(dtype_min, dtype_max), ), 0, ) if data.shape[0] == 1: # likely greyscale image; use the same band on all channels data = np.ma.array([data[0], data[0], data[0]]) # normalize to 0..1 based on the range of the source type (only # for int*s) if not np.issubdtype(data.dtype, float): data = data.astype(np.float32) / np.iinfo(data.dtype).max return PixelCollection(data, pixels.bounds)
def landsat8_tile(sceneid, tile_x, tile_y, tile_z, bands=("4", "3", "2"), tilesize=256, pan=False, percents="", **kwargs): """ Create mercator tile from Landsat-8 data. Attributes ---------- sceneid : str Landsat sceneid. For scenes after May 2017, sceneid have to be LANDSAT_PRODUCT_ID. tile_x : int Mercator tile X index. tile_y : int Mercator tile Y index. tile_z : int Mercator tile ZOOM level. bands : tuple, str, optional (default: ("4", "3", "2")) Bands index for the RGB combination. tilesize : int, optional (default: 256) Output image size. pan : boolean, optional (default: False) If True, apply pan-sharpening. kwargs: dict, optional These will be passed to the 'rio_tiler.utils._tile_read' function. Returns ------- data : numpy ndarray mask: numpy array """ if not isinstance(bands, tuple): bands = tuple((bands, )) for band in bands: if band not in LANDSAT_BANDS: raise InvalidBandName( "{} is not a valid Landsat band name".format(band)) scene_params = landsat8._landsat_parse_scene_id(sceneid) meta_data = landsat8._landsat_get_mtl(sceneid).get("L1_METADATA_FILE") landsat_address = "{}/{}".format(LANDSAT_BUCKET, scene_params["key"]) wgs_bounds = toa_utils._get_bounds_from_metadata( meta_data["PRODUCT_METADATA"]) addresses = ["{}_B{}.TIF".format(landsat_address, band) for band in bands] values = [] percents = percents.split(',') i = 0 for address in addresses: with rasterio.open(address) as src: if int(percents[i]) != 0 and int(percents[i + 1]) != 100: overviews = src.overviews(1) if len(overviews) > 0: d = src.read( out_shape=(1, int(src.height / overviews[len(overviews) - 1]), int(src.width / overviews[len(overviews) - 1]))) else: d = src.read() dflatten = numpy.array(d.flatten()) p_start, p_end = numpy.percentile(dflatten[dflatten > 0], (int(percents[i]), (int(percents[i + 1])))) values.append([p_start, p_end]) else: values.append([None, None]) i += 2 if not utils.tile_exists(wgs_bounds, tile_z, tile_x, tile_y): # raise TileOutsideBounds( # "Tile {}/{}/{} is outside image bounds".format(tile_z, tile_x, tile_y) # ) return None, None mercator_tile = mercantile.Tile(x=tile_x, y=tile_y, z=tile_z) tile_bounds = mercantile.xy_bounds(mercator_tile) def _tiler(band): address = "{}_B{}.TIF".format(landsat_address, band) if band == "QA": nodata = 1 else: nodata = 0 return utils.tile_read(address, bounds=tile_bounds, tilesize=tilesize, nodata=nodata, **kwargs) with futures.ThreadPoolExecutor(max_workers=MAX_THREADS) as executor: data, masks = zip(*list(executor.map(_tiler, bands))) mask = numpy.all(masks, axis=0).astype(numpy.uint8) * 255 new_data = list(data) has_modification = False for ds in range(0, len(new_data)): if values[ds][0] is not None and values[ds][1] is not None: has_modification = True new_data[ds] = rescale_intensity(new_data[ds], in_range=(values[ds][0], values[ds][1]), out_range=(0, 255)) if has_modification == True: data = numpy.array(new_data).astype(numpy.uint8) data = numpy.concatenate(data) if pan: pan_address = "{}_B8.TIF".format(landsat_address) matrix_pan, mask = utils.tile_read(pan_address, tile_bounds, tilesize, nodata=0) data = utils.pansharpening_brovey(data, matrix_pan, 0.2, matrix_pan.dtype) sun_elev = meta_data["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"] for bdx, band in enumerate(bands): if band in ["1", "2", "3", "4", "5", "6", "7", "8", "9"]: # OLI multi_reflect = meta_data["RADIOMETRIC_RESCALING"].get( "REFLECTANCE_MULT_BAND_{}".format(band)) add_reflect = meta_data["RADIOMETRIC_RESCALING"].get( "REFLECTANCE_ADD_BAND_{}".format(band)) data[bdx] = 10000 * reflectance.reflectance( data[bdx], multi_reflect, add_reflect, sun_elev) elif band in ["10", "11"]: # TIRS multi_rad = meta_data["RADIOMETRIC_RESCALING"].get( "RADIANCE_MULT_BAND_{}".format(band)) add_rad = meta_data["RADIOMETRIC_RESCALING"].get( "RADIANCE_ADD_BAND_{}".format(band)) k1 = meta_data["TIRS_THERMAL_CONSTANTS"].get( "K1_CONSTANT_BAND_{}".format(band)) k2 = meta_data["TIRS_THERMAL_CONSTANTS"].get( "K2_CONSTANT_BAND_{}".format(band)) data[bdx] = brightness_temp.brightness_temp( data[bdx], multi_rad, add_rad, k1, k2) return data, mask