def test_reflectance_negative_elevation():
band = np.array([[0, 0, 0], [0, 2, 1], [2, 0, 1.00008]]).astype('float32')
MR = 0.2
AR = -0.1
E = -90.0
with pytest.raises(ValueError):
reflectance.reflectance(band, MR, AR, E)
def test_reflectance_wrong_shape():
band = np.array([[0, 0, 0], [0, 1, 1], [1, 0, 1]]).astype('float32')
# wrong sun elevation shape
with pytest.raises(ValueError):
reflectance.reflectance(band, 0.2, -0.00000001,
np.array([[1, 2, 3], [4, 5, 6]]))
with pytest.raises(ValueError):
reflectance.reflectance(band, 35.1, np.array([1, 3]), 90.0)
def _convert(arr: numpy.ndarray, band: str, metadata: Dict) -> numpy.ndarray:
"""Convert DN to TOA or Temp."""
if band in ["1", "2", "3", "4", "5", "6", "7", "8", "9"]: # OLI
multi_reflect = metadata["RADIOMETRIC_RESCALING"].get(
f"REFLECTANCE_MULT_BAND_{band}"
)
add_reflect = metadata["RADIOMETRIC_RESCALING"].get(
f"REFLECTANCE_ADD_BAND_{band}"
)
sun_elev = metadata["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"]
arr = 10000 * reflectance.reflectance(
arr, multi_reflect, add_reflect, sun_elev, src_nodata=0
)
elif band in ["10", "11"]: # TIRS
multi_rad = metadata["RADIOMETRIC_RESCALING"].get(f"RADIANCE_MULT_BAND_{band}")
add_rad = metadata["RADIOMETRIC_RESCALING"].get(f"RADIANCE_ADD_BAND_{band}")
k1 = metadata["TIRS_THERMAL_CONSTANTS"].get(f"K1_CONSTANT_BAND_{band}")
k2 = metadata["TIRS_THERMAL_CONSTANTS"].get(f"K2_CONSTANT_BAND_{band}")
arr = brightness_temp.brightness_temp(arr, multi_rad, add_rad, k1, k2)
# TODO
# elif band == "QA":
return arr
def test_reflectance():
band = np.array([[0, 0, 0], [0, 1, 1], [1, 0, 1]]).astype('float32')
MR = 0.2
AR = -0.1
E = 90.0
assert np.array_equal(
reflectance.reflectance(band, MR, AR, E),
np.array([[0., 0., 0.], [0., 0.1, 0.1], [0.1, 0.,
0.1]]).astype(np.float32))
def landsat_min_max_worker(band, address, metadata, pmin=2, pmax=98,
width=1024, height=1024):
"""Retrieve histogram percentage cut for a Landsat-8 scene.
Attributes
----------
address : Landsat band AWS address
band : Landsat band number
metadata : Landsat metadata
pmin : Histogram minimum cut (default: 2)
pmax : Histogram maximum cut (default: 98)
width : int, optional (default: 1024)
Pixel width for the decimated read.
height : int, optional (default: 1024)
Pixel height for the decimated read.
Returns
-------
out : list, int
returns a list of the min/max histogram cut values.
"""
if int(band) > 9: # TIRS
multi_rad = metadata['RADIOMETRIC_RESCALING'].get(
'RADIANCE_MULT_BAND_{}'.format(band))
add_rad = metadata['RADIOMETRIC_RESCALING'].get(
'RADIANCE_ADD_BAND_{}'.format(band))
k1 = metadata['TIRS_THERMAL_CONSTANTS'].get(
'K1_CONSTANT_BAND_{}'.format(band))
k2 = metadata['TIRS_THERMAL_CONSTANTS'].get(
'K2_CONSTANT_BAND_{}'.format(band))
with rasterio.open('{}_B{}.TIF'.format(address, band)) as src:
arr = src.read(indexes=1,
out_shape=(height, width)).astype(src.profile['dtype'])
arr = brightness_temp.brightness_temp(arr, multi_rad, add_rad, k1, k2)
else:
multi_reflect = metadata['RADIOMETRIC_RESCALING'].get(
'REFLECTANCE_MULT_BAND_{}'.format(band))
add_reflect = metadata['RADIOMETRIC_RESCALING'].get(
'REFLECTANCE_ADD_BAND_{}'.format(band))
sun_elev = metadata['IMAGE_ATTRIBUTES']['SUN_ELEVATION']
with rasterio.open('{}_B{}.TIF'.format(address, band)) as src:
arr = src.read(indexes=1,
out_shape=(height, width)).astype(src.profile['dtype'])
arr = 10000 * reflectance.reflectance(arr, multi_reflect, add_reflect,
sun_elev, src_nodata=0)
return np.percentile(arr[arr > 0], (pmin, pmax)).astype(np.int).tolist()
def get_window(idx, window):
band = bands[idx]
multi_reflect = meta_data["RADIOMETRIC_RESCALING"].get(
f"REFLECTANCE_MULT_BAND_{band}")
add_reflect = meta_data["RADIOMETRIC_RESCALING"].get(
f"REFLECTANCE_ADD_BAND_{band}")
data = srcs[idx].read(window=window,
boundless=True,
indexes=(1))
return reflectance.reflectance(data, multi_reflect,
add_reflect, sun_elev)
def worker(band):
"""Worker."""
address = f"{landsat_address}_B{band}.TIF"
sun_elev = meta_data["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"]
multi_reflect = meta_data["RADIOMETRIC_RESCALING"][
f"REFLECTANCE_MULT_BAND_{band}"
]
add_reflect = meta_data["RADIOMETRIC_RESCALING"][f"REFLECTANCE_ADD_BAND_{band}"]
band = get_area(
address, bbox, max_img_size=max_img_size, bbox_crs=bbox_crs, out_crs=out_crs
)
return reflectance(band, multi_reflect, add_reflect, sun_elev, src_nodata=0)
def worker(band, landsat_address, meta, ovr_size):
"""Worker."""
address = f"{landsat_address}_B{band}.TIF"
sun_elev = meta["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"]
multi_reflect = meta["RADIOMETRIC_RESCALING"][
f"REFLECTANCE_MULT_BAND_{band}"]
add_reflect = meta["RADIOMETRIC_RESCALING"][f"REFLECTANCE_ADD_BAND_{band}"]
matrix = get_overview(address, ovr_size)
return reflectance(matrix,
multi_reflect,
add_reflect,
sun_elev,
src_nodata=0)
def worker(band, coordinates):
"""Worker."""
address = f"{landsat_address}_B{band}.TIF"
sun_elev = meta_data["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"]
multi_reflect = meta_data["RADIOMETRIC_RESCALING"][
f"REFLECTANCE_MULT_BAND_{band}"
]
add_reflect = meta_data["RADIOMETRIC_RESCALING"][f"REFLECTANCE_ADD_BAND_{band}"]
with rasterio.open(address) as band:
lon_srs, lat_srs = warp.transform(
"EPSG:4326", band.crs, [coordinates[0]], [coordinates[1]]
)
point = list(band.sample([(lon_srs[0], lat_srs[0])]))[0]
return reflectance(point, multi_reflect, add_reflect, sun_elev, src_nodata=0)[0]
def band_worker(band, landsat_address, meta, bounds=None):
"""
"""
address = f'{landsat_address}_B{band}.TIF'
sun_elev = meta['IMAGE_ATTRIBUTES']['SUN_ELEVATION']
multi_reflect = meta['RADIOMETRIC_RESCALING'][
f'REFLECTANCE_MULT_BAND_{band}']
add_reflect = meta['RADIOMETRIC_RESCALING'][f'REFLECTANCE_ADD_BAND_{band}']
ovrSize = 1024
with rasterio.open(address) as src:
if bounds:
w, s, e, n = bounds
with WarpedVRT(src,
dst_crs='EPSG:3857',
resampling=Resampling.bilinear,
src_nodata=0,
dst_nodata=0) as vrt:
window = vrt.window(w, s, e, n, precision=21)
matrix = vrt.read(window=window,
boundless=True,
resampling=Resampling.bilinear,
out_shape=(ovrSize, ovrSize),
indexes=1).astype(src.profile['dtype'])
else:
matrix = src.read(indexes=1,
out_shape=(ovrSize, ovrSize),
resampling=Resampling.bilinear).astype(
src.profile['dtype'])
matrix = reflectance(matrix,
multi_reflect,
add_reflect,
sun_elev,
src_nodata=0)
imgRange = np.percentile(matrix[matrix > 0], (2, 98)).tolist()
matrix = np.where(
matrix > 0,
linear_rescale(matrix, in_range=imgRange, out_range=[1, 255]),
0).astype(np.uint8)
return matrix
def percentiles(input, band, meta_url):
meta = json.load(urllib.urlopen(meta_url))
with rasterio.Env():
with rasterio.open(input) as src:
data = src.read(indexes=1, out_shape=(1024, 1024))
sun_elev = meta["L1_METADATA_FILE"]["IMAGE_ATTRIBUTES"][
"SUN_ELEVATION"]
multi_reflect = meta[
"L1_METADATA_FILE"]["RADIOMETRIC_RESCALING"].get(
"REFLECTANCE_MULT_BAND_{}".format(band))
add_reflect = meta[
"L1_METADATA_FILE"]["RADIOMETRIC_RESCALING"].get(
"REFLECTANCE_ADD_BAND_{}".format(band))
data = 10000 * reflectance.reflectance(
data, multi_reflect, add_reflect, sun_elev, src_nodata=0)
return np.percentile(data[data > 0], (2, 5, 95, 98)).tolist()
def test_calculate_reflectance(test_data):
tif_b, tif_output_single, mtl = test_data[0], test_data[5], test_data[-1]
M = toa_utils._load_mtl_key(mtl, [
'L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'REFLECTANCE_MULT_BAND_'
], 5)
A = toa_utils._load_mtl_key(
mtl,
['L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'REFLECTANCE_ADD_BAND_'],
5)
E = toa_utils._load_mtl_key(
mtl, ['L1_METADATA_FILE', 'IMAGE_ATTRIBUTES', 'SUN_ELEVATION'])
assert (np.sin(np.radians(E)) <= 1) & (-1 <= np.sin(np.radians(E)))
assert isinstance(M, float)
toa = reflectance.reflectance(tif_b, M, A, E)
toa_rescaled = toa_utils.rescale(toa, 55000.0, np.uint16, clip=False)
assert toa_rescaled.dtype == np.uint16
# Note, the test data was created under a rescaling code, hence the fuzziness
diff = toa_rescaled[310:315, 310:315] - tif_output_single[310:315, 310:315]
assert diff.max() <= 1
def test_calculate_reflectance_uint8(test_data):
tif_b, tif_output_single, mtl = test_data[0], test_data[5], test_data[-1]
M = toa_utils._load_mtl_key(mtl, [
'L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'REFLECTANCE_MULT_BAND_'
], 5)
A = toa_utils._load_mtl_key(
mtl,
['L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'REFLECTANCE_ADD_BAND_'],
5)
E = toa_utils._load_mtl_key(
mtl, ['L1_METADATA_FILE', 'IMAGE_ATTRIBUTES', 'SUN_ELEVATION'])
assert (np.sin(np.radians(E)) <= 1) & (-1 <= np.sin(np.radians(E)))
assert isinstance(M, float)
toa = reflectance.reflectance(tif_b, M, A, E)
toa_rescaled = toa_utils.rescale(toa, 215, np.uint8)
scale = float(np.iinfo(np.uint16).max) / float(np.iinfo(np.uint8).max)
tif_out_rescaled = np.clip((tif_output_single / scale), 0,
np.iinfo(np.uint8).max).astype(np.uint8)
assert toa_rescaled.dtype == np.uint8
assert np.min(tif_out_rescaled) == np.min(toa_rescaled)
def test_calculate_reflectance2(test_data):
tif_b, tif_shape, mtl = test_data[0], test_data[4], test_data[-1]
M = toa_utils._load_mtl_key(mtl, [
'L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'REFLECTANCE_MULT_BAND_'
], 5)
A = toa_utils._load_mtl_key(
mtl,
['L1_METADATA_FILE', 'RADIOMETRIC_RESCALING', 'REFLECTANCE_ADD_BAND_'],
5)
date_collected = toa_utils._load_mtl_key(
mtl, ['L1_METADATA_FILE', 'PRODUCT_METADATA', 'DATE_ACQUIRED'])
time_collected_utc = toa_utils._load_mtl_key(
mtl, ['L1_METADATA_FILE', 'PRODUCT_METADATA', 'SCENE_CENTER_TIME'])
bounds = BoundingBox(*toa_utils._get_bounds_from_metadata(
mtl['L1_METADATA_FILE']['PRODUCT_METADATA']))
E = sun_utils.sun_elevation(bounds, tif_shape, date_collected,
time_collected_utc)
toa = reflectance.reflectance(tif_b, M, A, E)
toa_rescaled = toa_utils.rescale(toa, 55000.0, np.uint16)
assert toa_rescaled.dtype == np.uint16
assert np.all(toa_rescaled) < 1.5
assert np.all(toa_rescaled) >= 0.0
def dn_to_toa(arr: numpy.ndarray, band: str, metadata: Dict) -> numpy.ndarray:
"""Convert DN to TOA or Temp.
Args:
arr (numpy.ndarray): Digital Number array values.
band (str): Landsat 8 band's name.
metadata (str): Landsat MTL metadata.
Returns:
numpy.ndarray: DN coverted to TOA or Temperature.
"""
if band in ["B1", "B2", "B3", "B4", "B5", "B6", "B7", "B8", "B9"]: # OLI
multi_reflect = metadata["RADIOMETRIC_RESCALING"].get(
f"REFLECTANCE_MULT_BAND_{band[1:]}")
add_reflect = metadata["RADIOMETRIC_RESCALING"].get(
f"REFLECTANCE_ADD_BAND_{band[1:]}")
sun_elev = metadata["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"]
arr = 10000 * reflectance.reflectance(
arr, multi_reflect, add_reflect, sun_elev, src_nodata=0)
arr = arr.astype("uint16")
elif band in ["B10", "B11"]: # TIRS
multi_rad = metadata["RADIOMETRIC_RESCALING"].get(
f"RADIANCE_MULT_BAND_{band[1:]}")
add_rad = metadata["RADIOMETRIC_RESCALING"].get(
f"RADIANCE_ADD_BAND_{band[1:]}")
k1 = metadata["TIRS_THERMAL_CONSTANTS"].get(
f"K1_CONSTANT_BAND_{band[1:]}")
k2 = metadata["TIRS_THERMAL_CONSTANTS"].get(
f"K2_CONSTANT_BAND_{band[1:]}")
arr = brightness_temp.brightness_temp(arr, multi_rad, add_rad, k1, k2)
return arr
def tile(sceneid, tile_x, tile_y, tile_z, rgb=(4, 3, 2), tilesize=256, pan=False):
"""Create mercator tile from Landsat-8 data.
Attributes
----------
sceneid : str
Landsat sceneid. For scenes after May 2017,
sceneid have to be LANDSAT_PRODUCT_ID.
tile_x : int
Mercator tile X index.
tile_y : int
Mercator tile Y index.
tile_z : int
Mercator tile ZOOM level.
rgb : tuple, int, optional (default: (4, 3, 2))
Bands index for the RGB combination.
tilesize : int, optional (default: 256)
Output image size.
pan : boolean, optional (default: False)
If True, apply pan-sharpening.
Returns
-------
data : numpy ndarray
mask: numpy array
"""
if not isinstance(rgb, tuple):
rgb = tuple((rgb, ))
scene_params = utils.landsat_parse_scene_id(sceneid)
meta_data = utils.landsat_get_mtl(sceneid).get('L1_METADATA_FILE')
landsat_address = '{}/{}'.format(LANDSAT_BUCKET, scene_params['key'])
wgs_bounds = toa_utils._get_bounds_from_metadata(
meta_data['PRODUCT_METADATA'])
if not utils.tile_exists(wgs_bounds, tile_z, tile_x, tile_y):
raise TileOutsideBounds(
'Tile {}/{}/{} is outside image bounds'.format(
tile_z, tile_x, tile_y))
mercator_tile = mercantile.Tile(x=tile_x, y=tile_y, z=tile_z)
tile_bounds = mercantile.xy_bounds(mercator_tile)
ms_tile_size = int(tilesize / 2) if pan else tilesize
addresses = ['{}_B{}.TIF'.format(landsat_address, band) for band in rgb]
_tiler = partial(utils.tile_band_worker, bounds=tile_bounds, tilesize=ms_tile_size, nodata=0)
with futures.ThreadPoolExecutor(max_workers=3) as executor:
data, masks = zip(*list(executor.map(_tiler, addresses)))
data = np.concatenate(data)
mask = np.all(masks, axis=0).astype(np.uint8) * 255
if pan:
pan_address = '{}_B8.TIF'.format(landsat_address)
matrix_pan, mask = utils.tile_band_worker(pan_address, tile_bounds, tilesize, nodata=0)
w, s, e, n = tile_bounds
pan_transform = transform.from_bounds(w, s, e, n, tilesize, tilesize)
vis_transform = pan_transform * Affine.scale(2.)
data = pansharpen(data, vis_transform, matrix_pan, pan_transform,
np.int16, 'EPSG:3857', 'EPSG:3857', 0.2,
method='Brovey', src_nodata=0)
sun_elev = meta_data['IMAGE_ATTRIBUTES']['SUN_ELEVATION']
for bdx, band in enumerate(rgb):
if int(band) > 9: # TIRS
multi_rad = meta_data['RADIOMETRIC_RESCALING'].get(
'RADIANCE_MULT_BAND_{}'.format(band))
add_rad = meta_data['RADIOMETRIC_RESCALING'].get(
'RADIANCE_ADD_BAND_{}'.format(band))
k1 = meta_data['TIRS_THERMAL_CONSTANTS'].get(
'K1_CONSTANT_BAND_{}'.format(band))
k2 = meta_data['TIRS_THERMAL_CONSTANTS'].get(
'K2_CONSTANT_BAND_{}'.format(band))
data[bdx] = brightness_temp.brightness_temp(
data[bdx], multi_rad, add_rad, k1, k2)
else:
multi_reflect = meta_data['RADIOMETRIC_RESCALING'].get(
'REFLECTANCE_MULT_BAND_{}'.format(band))
add_reflect = meta_data['RADIOMETRIC_RESCALING'].get(
'REFLECTANCE_ADD_BAND_{}'.format(band))
data[bdx] = 10000 * reflectance.reflectance(
data[bdx], multi_reflect, add_reflect, sun_elev)
return data, mask
def apply(recipes, pixels, expand, source=None):
data = pixels.data
colormap = pixels.colormap
if np.issubdtype(data.dtype, np.floating):
dtype_min = np.finfo(data.dtype).min
dtype_max = np.finfo(data.dtype).max
else:
dtype_min = np.iinfo(data.dtype).min
dtype_max = np.iinfo(data.dtype).max
if data.shape[0] == 1:
if expand and colormap:
# create a lookup table from the source's color map
lut = make_colormap(colormap)
# stash the mask
mask = data.mask
# apply the color map
data = lut[data[0], :]
# re-shape to match band-style
data = np.ma.transpose(data, [2, 0, 1])
# re-apply the mask, merging it with pixels that were masked by the color map
data.mask = data.mask | mask
colormap = None
if "landsat8" in recipes:
LOG.info("Applying landsat 8 recipe")
out = np.ma.empty(shape=(data.shape), dtype=np.float32)
for bdx, source_band in enumerate((4, 3, 2)):
sun_elev = source.meta["L1_METADATA_FILE"]["IMAGE_ATTRIBUTES"][
"SUN_ELEVATION"
]
multi_reflect = source.meta["L1_METADATA_FILE"][
"RADIOMETRIC_RESCALING"
].get(
"REFLECTANCE_MULT_BAND_{}".format(source_band)
)
add_reflect = source.meta["L1_METADATA_FILE"]["RADIOMETRIC_RESCALING"].get(
"REFLECTANCE_ADD_BAND_{}".format(source_band)
)
min_val = source.meta.get("values", {}).get(str(source_band), {}).get(
"min", dtype_min
)
max_val = source.meta.get("values", {}).get(str(source_band), {}).get(
"max", dtype_max
)
band_data = 10000 * reflectance.reflectance(
data[bdx], multi_reflect, add_reflect, sun_elev, src_nodata=0
)
# calculate local min/max as fallbacks
if (
min_val == dtype_min
and max_val == dtype_max
and len(data.compressed()) > 0
):
local_min, local_max = np.percentile(band_data.compressed(), (2, 98))
min_val = max(min_val, local_min)
max_val = min(max_val, local_max)
out[bdx] = utils.linear_rescale(
band_data, in_range=(min_val, max_val), out_range=(0.0, 1.0)
)
data = out
if "imagery" in recipes:
LOG.info("Applying imagery recipe")
if "rgb_bands" in recipes:
data = np.ma.array(
[data[i - 1] for i in recipes["rgb_bands"] if data.shape[0] >= i]
)
elif "expr" in recipes:
num_bands = data.shape[0]
expressions = recipes["expr"].split(",")
band_names = ["b" + str(i) for i in range(1, num_bands + 1)]
local_dict = dict(zip(band_names, data.data))
old_mask = data.mask
old_mask_shape = old_mask.shape
new_mask_shape = (len(expressions), old_mask_shape[1], old_mask_shape[2])
new_mask = np.ndarray(new_mask_shape, dtype=np.bool)
for (expression_index, expression) in enumerate(expressions):
# identify bands used in expression
band_indexes = [int(i) - 1 for i in re.findall(r"(?<=b)(\d{1,2})", expression)]
old_band_masks = [old_mask[i] for i in band_indexes]
new_mask[expression_index] = np.logical_and.reduce(old_band_masks)
data = np.ma.array([
np.nan_to_num(ne.evaluate(expression.strip(), local_dict=local_dict))
for expression in expressions
], mask=new_mask)
elif data.shape[0] > 3:
# alpha(?) band (and beyond) present; drop it (them)
# TODO use band 4 as an alpha channel if colorinterp == alpha instead
# TODO re-order channels if BGR (whatever colorinterp says)
data = data[0:3]
if "linear_stretch" in recipes:
# Added by Tushar Shukla
# Add envi like property here to generate envi like browse tiles
if recipes["linear_stretch"] == "global":
data = utils.linear_rescale(
data, in_range=(np.min(data), np.max(data)), out_range=(0.0, 1.0)
)
elif recipes["linear_stretch"] == "per_band":
out = np.ma.empty(shape=(data.shape), dtype=np.float32)
for band in range(0, data.shape[0]):
min_val = source.meta.get("values", {}).get(band, {}).get(
"min", np.min(data[band])
)
max_val = source.meta.get("values", {}).get(band, {}).get(
"max", np.max(data[band])
)
out[band] = utils.linear_rescale(
data[band], in_range=(min_val, max_val), out_range=(0.0, 1.0)
)
data = out
else:
# rescale after reducing and before increasing dimensionality
if data.dtype != np.uint8:
# rescale non-8-bit sources (assuming that they're raw sensor
# data) and normalize to 0..1
out = np.ma.empty(shape=(data.shape), dtype=np.float32)
for band in range(0, data.shape[0]):
min_val = source.meta.get("values", {}).get(band, {}).get(
"min", dtype_min
)
max_val = source.meta.get("values", {}).get(band, {}).get(
"max", dtype_max
)
if (
min_val == dtype_min
and max_val == dtype_max
and len(data.compressed()) > 0
):
local_min, local_max = np.percentile(
data[band].compressed(), (2, 98)
)
min_val = max(min_val, local_min)
max_val = min(max_val, local_max)
out[band] = utils.linear_rescale(
data[band], in_range=(min_val, max_val), out_range=(0.0, 1.0)
)
data = out
if not np.issubdtype(data.dtype, np.floating):
# normalize to 0..1 based on the range of the source type (only
# for int*s)
data = data.astype(np.float32) / dtype_max
if data.shape[0] == 1:
# likely greyscale image; use the same band on all channels
data = np.ma.array([data[0], data[0], data[0]])
return PixelCollection(data, pixels.bounds, None, colormap)
def tile(sceneid,
tile_x,
tile_y,
tile_z,
bands=("4", "3", "2"),
tilesize=256,
pan=False):
"""
Create mercator tile from Landsat-8 data.
Attributes
----------
sceneid : str
Landsat sceneid. For scenes after May 2017,
sceneid have to be LANDSAT_PRODUCT_ID.
tile_x : int
Mercator tile X index.
tile_y : int
Mercator tile Y index.
tile_z : int
Mercator tile ZOOM level.
bands : tuple, str, optional (default: ("4", "3", "2"))
Bands index for the RGB combination.
tilesize : int, optional (default: 256)
Output image size.
pan : boolean, optional (default: False)
If True, apply pan-sharpening.
Returns
-------
data : numpy ndarray
mask: numpy array
"""
if not isinstance(bands, tuple):
bands = tuple((bands, ))
for band in bands:
if band not in LANDSAT_BANDS:
raise InvalidBandName(
"{} is not a valid Landsat band name".format(band))
scene_params = _landsat_parse_scene_id(sceneid)
meta_data = _landsat_get_mtl(sceneid).get("L1_METADATA_FILE")
landsat_address = "{}/{}".format(LANDSAT_BUCKET, scene_params["key"])
wgs_bounds = toa_utils._get_bounds_from_metadata(
meta_data["PRODUCT_METADATA"])
if not utils.tile_exists(wgs_bounds, tile_z, tile_x, tile_y):
raise TileOutsideBounds("Tile {}/{}/{} is outside image bounds".format(
tile_z, tile_x, tile_y))
mercator_tile = mercantile.Tile(x=tile_x, y=tile_y, z=tile_z)
tile_bounds = mercantile.xy_bounds(mercator_tile)
ms_tile_size = int(tilesize / 2) if pan else tilesize
addresses = ["{}_B{}.TIF".format(landsat_address, band) for band in bands]
_tiler = partial(utils.tile_read,
bounds=tile_bounds,
tilesize=ms_tile_size,
nodata=0)
with futures.ThreadPoolExecutor(max_workers=MAX_THREADS) as executor:
data, masks = zip(*list(executor.map(_tiler, addresses)))
data = np.concatenate(data)
mask = np.all(masks, axis=0).astype(np.uint8) * 255
if pan:
pan_address = "{}_B8.TIF".format(landsat_address)
matrix_pan, mask = utils.tile_read(pan_address,
tile_bounds,
tilesize,
nodata=0)
w, s, e, n = tile_bounds
pan_transform = transform.from_bounds(w, s, e, n, tilesize,
tilesize)
vis_transform = pan_transform * Affine.scale(2.0)
data = pansharpen(
data,
vis_transform,
matrix_pan,
pan_transform,
np.int16,
"EPSG:3857",
"EPSG:3857",
0.2,
method="Brovey",
src_nodata=0,
)
sun_elev = meta_data["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"]
for bdx, band in enumerate(bands):
if int(band) > 9: # TIRS
multi_rad = meta_data["RADIOMETRIC_RESCALING"].get(
"RADIANCE_MULT_BAND_{}".format(band))
add_rad = meta_data["RADIOMETRIC_RESCALING"].get(
"RADIANCE_ADD_BAND_{}".format(band))
k1 = meta_data["TIRS_THERMAL_CONSTANTS"].get(
"K1_CONSTANT_BAND_{}".format(band))
k2 = meta_data["TIRS_THERMAL_CONSTANTS"].get(
"K2_CONSTANT_BAND_{}".format(band))
data[bdx] = brightness_temp.brightness_temp(
data[bdx], multi_rad, add_rad, k1, k2)
else:
multi_reflect = meta_data["RADIOMETRIC_RESCALING"].get(
"REFLECTANCE_MULT_BAND_{}".format(band))
add_reflect = meta_data["RADIOMETRIC_RESCALING"].get(
"REFLECTANCE_ADD_BAND_{}".format(band))
data[bdx] = 10000 * reflectance.reflectance(
data[bdx], multi_reflect, add_reflect, sun_elev)
return data, mask
def tile(sceneid,
tile_x,
tile_y,
tile_z,
rgb=(4, 3, 2),
r_bds=(0, 16000),
g_bds=(0, 16000),
b_bds=(0, 16000),
tilesize=256,
pan=False):
"""Create mercator tile from Landsat-8 data and encodes it in base64.
Attributes
----------
sceneid : str
Landsat sceneid. For scenes after May 2017,
sceneid have to be LANDSAT_PRODUCT_ID.
tile_x : int
Mercator tile X index.
tile_y : int
Mercator tile Y index.
tile_z : int
Mercator tile ZOOM level.
rgb : tuple, int, optional (default: (4, 3, 2))
Bands index for the RGB combination.
r_bds : tuple, int, optional (default: (0, 16000))
First band (red) DN min and max values (DN * 10,000)
used for the linear rescaling.
g_bds : tuple, int, optional (default: (0, 16000))
Second band (green) DN min and max values (DN * 10,000)
used for the linear rescaling.
b_bds : tuple, int, optional (default: (0, 16000))
Third band (blue) DN min and max values (DN * 10,000)
used for the linear rescaling.
tilesize : int, optional (default: 256)
Output image size.
pan : boolean, optional (default: False)
If True, apply pan-sharpening.
Returns
-------
out : numpy ndarray (type: uint8)
"""
scene_params = utils.landsat_parse_scene_id(sceneid)
meta_data = utils.landsat_get_mtl(sceneid).get('L1_METADATA_FILE')
landsat_address = '{}/{}'.format(LANDSAT_BUCKET, scene_params['key'])
wgs_bounds = toa_utils._get_bounds_from_metadata(
meta_data['PRODUCT_METADATA'])
if not utils.tile_exists(wgs_bounds, tile_z, tile_x, tile_y):
raise TileOutsideBounds('Tile {}/{}/{} is outside image bounds'.format(
tile_z, tile_x, tile_y))
mercator_tile = mercantile.Tile(x=tile_x, y=tile_y, z=tile_z)
tile_bounds = mercantile.xy_bounds(mercator_tile)
# define a list of bands Min and Max Values (from input)
histo_cuts = dict(zip(rgb, [r_bds, g_bds, b_bds]))
ms_tile_size = int(tilesize / 2) if pan else tilesize
addresses = ['{}_B{}.TIF'.format(landsat_address, band) for band in rgb]
_tiler = partial(utils.tile_band_worker,
bounds=tile_bounds,
tilesize=ms_tile_size)
with futures.ThreadPoolExecutor(max_workers=3) as executor:
out = np.stack(list(executor.map(_tiler, addresses)))
if pan:
pan_address = '{}_B8.TIF'.format(landsat_address)
matrix_pan = utils.tile_band_worker(pan_address, tile_bounds,
tilesize)
w, s, e, n = tile_bounds
pan_transform = transform.from_bounds(w, s, e, n, tilesize,
tilesize)
vis_transform = pan_transform * Affine.scale(2.)
out = pansharpen(out,
vis_transform,
matrix_pan,
pan_transform,
np.int16,
'EPSG:3857',
'EPSG:3857',
0.2,
method='Brovey',
src_nodata=0)
sun_elev = meta_data['IMAGE_ATTRIBUTES']['SUN_ELEVATION']
for bdx, band in enumerate(rgb):
multi_reflect = meta_data['RADIOMETRIC_RESCALING'].get(
'REFLECTANCE_MULT_BAND_{}'.format(band))
add_reflect = meta_data['RADIOMETRIC_RESCALING'].get(
'REFLECTANCE_ADD_BAND_{}'.format(band))
out[bdx] = 10000 * reflectance.reflectance(
out[bdx], multi_reflect, add_reflect, sun_elev, src_nodata=0)
out[bdx] = np.where(
out[bdx] > 0,
utils.linear_rescale(out[bdx],
in_range=histo_cuts.get(band),
out_range=[1, 255]), 0)
return out.astype(np.uint8)
def dn2toa(self, platform, mtl_file=None, wavelengths=None):
"""This method converts digital numbers to top of atmosphere reflectance, like described here:
https://www.usgs.gov/land-resources/nli/landsat/using-usgs-landsat-level-1-data-product
:param platform: image platform, possible Platform.Landsat[5, 7, 8] or Platform.Sentinel2 (<enum 'Platform'>).
:param mtl_file: path to Landsat MTL file that holds the band specific rescale factors (str).
:param wavelengths: like ["Blue", "Green", "Red", "NIR", "SWIR1", "TIRS", "SWIR2"] for Landsat-5 (list of str).
"""
if platform in [
Platform.Landsat5,
Platform.Landsat7,
Platform.Landsat8,
]:
if mtl_file is None:
raise AttributeError(
f"'mtl_file' has to be set if platform is {platform}.")
else:
# get rescale factors from mtl file
mtl = toa_utils._load_mtl(
str(mtl_file)) # no obvious reason not to call this
metadata = mtl["L1_METADATA_FILE"]
sun_elevation = metadata["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"]
toa = []
for idx, b in enumerate(
self._lookup_bands(platform, wavelengths)):
if (platform == Platform.Landsat8 and b
in ["10", "11"]) or (platform != Platform.Landsat8
and b.startswith("6")):
if platform == Platform.Landsat8:
thermal_conversion_constant1 = metadata[
"TIRS_THERMAL_CONSTANTS"][
f"K1_CONSTANT_BAND_{b}"]
thermal_conversion_constant2 = metadata[
"TIRS_THERMAL_CONSTANTS"][
f"K2_CONSTANT_BAND_{b}"]
else:
thermal_conversion_constant1 = metadata[
"THERMAL_CONSTANTS"][f"K1_CONSTANT_BAND_{b}"]
thermal_conversion_constant2 = metadata[
"THERMAL_CONSTANTS"][f"K2_CONSTANT_BAND_{b}"]
multiplicative_rescaling_factors = metadata[
"RADIOMETRIC_RESCALING"][f"RADIANCE_MULT_BAND_{b}"]
additive_rescaling_factors = metadata[
"RADIOMETRIC_RESCALING"][f"RADIANCE_ADD_BAND_{b}"]
# rescale thermal bands
toa.append(
brightness_temp.brightness_temp(
self.__arr[idx, :, :],
ML=multiplicative_rescaling_factors,
AL=additive_rescaling_factors,
K1=thermal_conversion_constant1,
K2=thermal_conversion_constant2,
))
continue
# rescale reflectance bands
multiplicative_rescaling_factors = metadata[
"RADIOMETRIC_RESCALING"][f"REFLECTANCE_MULT_BAND_{b}"]
additive_rescaling_factors = metadata[
"RADIOMETRIC_RESCALING"][f"REFLECTANCE_ADD_BAND_{b}"]
toa.append(
reflectance.reflectance(
self.__arr[idx, :, :],
MR=multiplicative_rescaling_factors,
AR=additive_rescaling_factors,
E=sun_elevation,
))
self.__arr = np.array(np.stack(toa, axis=0))
elif platform == Platform.Sentinel2:
self.__arr = self.__arr.astype(np.float32) / 10000.0
else:
raise AttributeError(
f"Cannot convert dn2toa. Platform {platform} not supported [Landsat-5, Landsat-7, Landsat-8, "
f"Sentinel-2]. ")
self.__update_dataset(self.dataset.crs,
self.dataset.transform,
nodata=self.dataset.nodata)
def worker(scene, bands):
"""Worker."""
try:
scene_params = landsat_parse_scene_id(scene)
meta_data = landsat_get_mtl(scene).get("L1_METADATA_FILE")
landsat_address = f'{LANDSAT_BUCKET}/{scene_params["key"]}'
bqa = f"{landsat_address}_BQA.TIF"
with rasterio.open(bqa) as src:
ovr = src.overviews(1)
ovr_width = int(src.width / ovr[0])
ovr_height = int(src.height / ovr[0])
dst_affine, width, height = calculate_default_transform(
src.crs, "epsg:3857", ovr_width, ovr_height, *src.bounds)
meta = {
"driver": "GTiff",
"count": 3,
"dtype": np.uint8,
"nodata": 0,
"height": height,
"width": width,
"compress": "DEFLATE",
"crs": "epsg:3857",
"transform": dst_affine,
}
outpath = f"/tmp/{scene}.tif"
with rasterio.open(outpath, "w", **meta) as dataset:
sun_elev = meta_data["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"]
for idx, b in enumerate(bands):
with rasterio.open(f"{landsat_address}_B{b}.TIF") as src:
with WarpedVRT(
src,
dst_crs="EPSG:3857",
resampling=Resampling.bilinear,
src_nodata=0,
dst_nodata=0,
) as vrt:
matrix = vrt.read(indexes=1, out_shape=(height, width))
multi_reflect = meta_data["RADIOMETRIC_RESCALING"][
f"REFLECTANCE_MULT_BAND_{b}"]
add_reflect = meta_data["RADIOMETRIC_RESCALING"][
f"REFLECTANCE_ADD_BAND_{b}"]
matrix = (reflectance(
matrix, multi_reflect, add_reflect, sun_elev, src_nodata=0)
* 10000)
minref = (meta_data["MIN_MAX_REFLECTANCE"]
[f"REFLECTANCE_MINIMUM_BAND_{b}"] * 10000)
maxref = (meta_data["MIN_MAX_REFLECTANCE"]
[f"REFLECTANCE_MAXIMUM_BAND_{b}"] * 10000)
matrix = np.where(
matrix > 0,
linear_rescale(matrix,
in_range=[int(minref),
int(maxref)],
out_range=[1, 255]),
0,
).astype(np.uint8)
mask = np.ma.masked_values(matrix, 0)
s = np.ma.notmasked_contiguous(mask)
matrix = matrix.ravel()
for sl in s:
matrix[sl.start:sl.start + 5] = 0
matrix[sl.stop - 5:sl.stop] = 0
matrix = matrix.reshape((height, width))
dataset.write(matrix, indexes=idx + 1)
return outpath
except:
return None
def _landsat_stats(
band,
address_prefix,
metadata,
overview_level=None,
max_size=1024,
percentiles=(2, 98),
dst_crs=CRS({"init": "EPSG:4326"}),
histogram_bins=10,
histogram_range=None,
):
"""
Retrieve landsat dataset statistics.
Attributes
----------
band : str
Landsat band number
address_prefix : str
A Landsat AWS S3 dataset prefix.
metadata : dict
Landsat metadata
overview_level : int, optional
Overview (decimation) level to fetch.
max_size: int, optional
Maximum size of dataset to retrieve
(will be used to calculate the overview level to fetch).
percentiles : tulple, optional
Percentile or sequence of percentiles to compute,
which must be between 0 and 100 inclusive (default: (2, 98)).
dst_crs: CRS or dict
Target coordinate reference system (default: EPSG:4326).
histogram_bins: int, optional
Defines the number of equal-width histogram bins (default: 10).
histogram_range: tuple or list, optional
The lower and upper range of the bins. If not provided, range is simply
the min and max of the array.
Returns
-------
out : dict
(percentiles), min, max, stdev, histogram for each band,
e.g.
{
"4": {
'pc': [15, 121],
'min': 1,
'max': 162,
'std': 27.22067722127997,
'histogram': [
[102934, 135489, 20981, 13548, 11406, 8799, 7351, 5622, 2985, 662]
[1., 17.1, 33.2, 49.3, 65.4, 81.5, 97.6, 113.7, 129.8, 145.9, 162.]
]
}
}
"""
src_path = "{}_B{}.TIF".format(address_prefix, band)
with rasterio.open(src_path) as src:
levels = src.overviews(1)
width = src.width
height = src.height
bounds = transform_bounds(src.crs,
dst_crs,
*src.bounds,
densify_pts=21)
if len(levels):
if overview_level:
decim = levels[overview_level]
else:
# determine which zoom level to read
for ii, decim in enumerate(levels):
if max(width // decim, height // decim) < max_size:
break
else:
decim = 1
warnings.warn("Dataset has no overviews, reading the full dataset",
NoOverviewWarning)
out_shape = (height // decim, width // decim)
if band == "QA":
nodata = 1
else:
nodata = 0
vrt_params = dict(nodata=nodata,
add_alpha=False,
src_nodata=nodata,
init_dest_nodata=False)
with WarpedVRT(src, **vrt_params) as vrt:
arr = vrt.read(out_shape=out_shape, indexes=[1], masked=True)
if band in ["1", "2", "3", "4", "5", "6", "7", "8", "9"]: # OLI
multi_reflect = metadata["RADIOMETRIC_RESCALING"].get(
"REFLECTANCE_MULT_BAND_{}".format(band))
add_reflect = metadata["RADIOMETRIC_RESCALING"].get(
"REFLECTANCE_ADD_BAND_{}".format(band))
sun_elev = metadata["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"]
arr = 10000 * reflectance.reflectance(
arr, multi_reflect, add_reflect, sun_elev, src_nodata=0)
elif band in ["10", "11"]: # TIRS
multi_rad = metadata["RADIOMETRIC_RESCALING"].get(
"RADIANCE_MULT_BAND_{}".format(band))
add_rad = metadata["RADIOMETRIC_RESCALING"].get(
"RADIANCE_ADD_BAND_{}".format(band))
k1 = metadata["TIRS_THERMAL_CONSTANTS"].get(
"K1_CONSTANT_BAND_{}".format(band))
k2 = metadata["TIRS_THERMAL_CONSTANTS"].get(
"K2_CONSTANT_BAND_{}".format(band))
arr = brightness_temp.brightness_temp(arr, multi_rad, add_rad, k1, k2)
params = {}
if histogram_bins:
params.update(dict(bins=histogram_bins))
if histogram_range:
params.update(dict(range=histogram_range))
stats = {band: utils._stats(arr, percentiles=percentiles, **params)}
return {
"bounds": {
"value": bounds,
"crs":
dst_crs.to_string() if isinstance(dst_crs, CRS) else dst_crs,
},
"statistics": stats,
}
def test_reflectance_wrong_type():
band = np.array([[9931., 9872., 9939.], [0., 5000., 100.],
[10000.1, 0., 100002.]]).astype('float32')
with pytest.raises(TypeError):
reflectance.reflectance(band, '45sldf', -0.1, 65.0)
def tile(sceneid,
tile_x,
tile_y,
tile_z,
bands=("4", "3", "2"),
tilesize=256,
pan=False,
**kwargs):
"""
Create mercator tile from Landsat-8 data.
Attributes
----------
sceneid : str
Landsat sceneid. For scenes after May 2017,
sceneid have to be LANDSAT_PRODUCT_ID.
tile_x : int
Mercator tile X index.
tile_y : int
Mercator tile Y index.
tile_z : int
Mercator tile ZOOM level.
bands : tuple, str, optional (default: ("4", "3", "2"))
Bands index for the RGB combination.
tilesize : int, optional (default: 256)
Output image size.
pan : boolean, optional (default: False)
If True, apply pan-sharpening.
kwargs: dict, optional
These will be passed to the 'rio_tiler.utils._tile_read' function.
Returns
-------
data : numpy ndarray
mask: numpy array
"""
if not isinstance(bands, tuple):
bands = tuple((bands, ))
for band in bands:
if band not in LANDSAT_BANDS:
raise InvalidBandName(
"{} is not a valid Landsat band name".format(band))
scene_params = _landsat_parse_scene_id(sceneid)
meta_data = _landsat_get_mtl(sceneid).get("L1_METADATA_FILE")
landsat_address = "{}/{}".format(LANDSAT_BUCKET, scene_params["key"])
wgs_bounds = toa_utils._get_bounds_from_metadata(
meta_data["PRODUCT_METADATA"])
if not utils.tile_exists(wgs_bounds, tile_z, tile_x, tile_y):
raise TileOutsideBounds("Tile {}/{}/{} is outside image bounds".format(
tile_z, tile_x, tile_y))
mercator_tile = mercantile.Tile(x=tile_x, y=tile_y, z=tile_z)
tile_bounds = mercantile.xy_bounds(mercator_tile)
def _tiler(band):
address = "{}_B{}.TIF".format(landsat_address, band)
if band == "QA":
nodata = 1
else:
nodata = 0
return utils.tile_read(address,
bounds=tile_bounds,
tilesize=tilesize,
nodata=nodata,
**kwargs)
with futures.ThreadPoolExecutor(max_workers=MAX_THREADS) as executor:
data, masks = zip(*list(executor.map(_tiler, bands)))
data = np.concatenate(data)
mask = np.all(masks, axis=0).astype(np.uint8) * 255
if pan:
pan_address = "{}_B8.TIF".format(landsat_address)
matrix_pan, mask = utils.tile_read(pan_address,
tile_bounds,
tilesize,
nodata=0)
data = utils.pansharpening_brovey(data, matrix_pan, 0.2,
matrix_pan.dtype)
sun_elev = meta_data["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"]
for bdx, band in enumerate(bands):
if band in ["1", "2", "3", "4", "5", "6", "7", "8", "9"]: # OLI
multi_reflect = meta_data["RADIOMETRIC_RESCALING"].get(
"REFLECTANCE_MULT_BAND_{}".format(band))
add_reflect = meta_data["RADIOMETRIC_RESCALING"].get(
"REFLECTANCE_ADD_BAND_{}".format(band))
data[bdx] = 10000 * reflectance.reflectance(
data[bdx], multi_reflect, add_reflect, sun_elev)
elif band in ["10", "11"]: # TIRS
multi_rad = meta_data["RADIOMETRIC_RESCALING"].get(
"RADIANCE_MULT_BAND_{}".format(band))
add_rad = meta_data["RADIOMETRIC_RESCALING"].get(
"RADIANCE_ADD_BAND_{}".format(band))
k1 = meta_data["TIRS_THERMAL_CONSTANTS"].get(
"K1_CONSTANT_BAND_{}".format(band))
k2 = meta_data["TIRS_THERMAL_CONSTANTS"].get(
"K2_CONSTANT_BAND_{}".format(band))
data[bdx] = brightness_temp.brightness_temp(
data[bdx], multi_rad, add_rad, k1, k2)
return data, mask
def area(scene, bbox):
"""
"""
max_width = 512
max_height = 512
scene_params = rputils.landsat_parse_scene_id(scene)
meta_data = rputils.landsat_get_mtl(scene).get('L1_METADATA_FILE')
sun_elev = meta_data['IMAGE_ATTRIBUTES']['SUN_ELEVATION']
multi_reflect = meta_data['RADIOMETRIC_RESCALING'][
'REFLECTANCE_MULT_BAND_4']
add_reflect = meta_data['RADIOMETRIC_RESCALING']['REFLECTANCE_ADD_BAND_4']
band_address = scene_params["key"] + '_B4.TIF'
s3 = boto3.resource('s3', 'us-west-2')
if not os.path.exists('/tmp/' + scene + '_B4.TIF'):
s3.Bucket('landsat-pds').download_file(band_address,
'/tmp/' + scene + '_B4.TIF')
with rio.open('/tmp/' + scene + '_B4.TIF') as band:
crs_bounds = warp.transform_bounds('EPSG:4326', band.crs, *bbox)
window = band.window(*crs_bounds)
width = round(window.width) if window.width < max_width else max_width
height = round(
window.height) if window.height < max_height else max_height
b4 = band.read(window=window,
out_shape=(height, width),
indexes=1,
resampling=Resampling.bilinear,
boundless=True)
b4 = reflectance(b4,
multi_reflect,
add_reflect,
sun_elev,
src_nodata=0)
multi_reflect = meta_data['RADIOMETRIC_RESCALING'][
'REFLECTANCE_MULT_BAND_5']
add_reflect = meta_data['RADIOMETRIC_RESCALING']['REFLECTANCE_ADD_BAND_5']
band_address = scene_params["key"] + '_B5.TIF'
if not os.path.exists('/tmp/' + scene + '_B5.TIF'):
s3.Bucket('landsat-pds').download_file(band_address,
'/tmp/' + scene + '_B5.TIF')
with rio.open('/tmp/' + scene + '_B5.TIF') as band:
crs_bounds = warp.transform_bounds('EPSG:4326', band.crs, *bbox)
window = band.window(*crs_bounds)
width = round(window.width) if window.width < max_width else max_width
height = round(
window.height) if window.height < max_height else max_height
b5 = band.read(window=window,
out_shape=(height, width),
indexes=1,
resampling=Resampling.bilinear,
boundless=True)
b5 = reflectance(b5,
multi_reflect,
add_reflect,
sun_elev,
src_nodata=0)
ratio = np.where((b5 * b4) > 0, np.nan_to_num((b5 - b4) / (b5 + b4)),
-9999)
ratio = np.where(
ratio > -9999,
rputils.linear_rescale(ratio, in_range=[-1, 1], out_range=[1, 255]),
0).astype(np.uint8)
cmap = list(np.array(rputils.get_colormap()).flatten())
img = Image.fromarray(ratio, 'P')
img.putpalette(cmap)
img = img.convert('RGB')
sio = BytesIO()
img.save(sio, 'jpeg', subsampling=0, quality=100)
sio.seek(0)
return base64.b64encode(sio.getvalue()).decode()
def point(scene, coord):
"""
"""
try:
scene_params = rputils.landsat_parse_scene_id(scene)
meta_data = rputils.landsat_get_mtl(scene).get('L1_METADATA_FILE')
sun_elev = meta_data['IMAGE_ATTRIBUTES']['SUN_ELEVATION']
multi_reflect = meta_data['RADIOMETRIC_RESCALING'][
'REFLECTANCE_MULT_BAND_4']
add_reflect = meta_data['RADIOMETRIC_RESCALING'][
'REFLECTANCE_ADD_BAND_4']
band_address = scene_params["key"] + '_B4.TIF'
s3 = boto3.resource('s3', 'us-west-2')
# if not os.path.exists('/tmp/'+scene+'_B4.TIF'):
# s3.Bucket('landsat-pds').download_file(band_address, '/tmp/'+scene+'_B4.TIF')
with rio.open('s3://landsat-pds/' + band_address) as band:
lon_srs, lat_srs = warp.transform('EPSG:4326', band.crs,
[coord[0]], [coord[1]])
b4 = list(band.sample([(lon_srs[0], lat_srs[0])]))[0]
b4 = reflectance(b4,
multi_reflect,
add_reflect,
sun_elev,
src_nodata=0)[0]
multi_reflect = meta_data['RADIOMETRIC_RESCALING'][
'REFLECTANCE_MULT_BAND_5']
add_reflect = meta_data['RADIOMETRIC_RESCALING'][
'REFLECTANCE_ADD_BAND_5']
band_address = scene_params["key"] + '_B5.TIF'
#s3_object = s3.get_object(Bucket='landsat-pds', Key=band_address)
#f = s3_object['Body'].read()
# if not os.path.exists('/tmp/'+scene+'_B5.TIF'):
# s3.Bucket('landsat-pds').download_file(band_address, '/tmp/'+scene+'_B5.TIF')
with rio.open('s3://landsat-pds/' + band_address) as band:
lon_srs, lat_srs = warp.transform('EPSG:4326', band.crs,
[coord[0]], [coord[1]])
b5 = list(band.sample([(lon_srs[0], lat_srs[0])]))[0]
b5 = reflectance(b5,
multi_reflect,
add_reflect,
sun_elev,
src_nodata=0)[0]
ratio = np.nan_to_num((b5 - b4) / (b5 + b4)) if (b4 * b5) > 0 else 0.
out = {
'scene': scene,
'ndvi': ratio,
'date': scene_params['date'],
'cloud': meta_data['IMAGE_ATTRIBUTES']['CLOUD_COVER']
}
return out
except:
return {}
def apply(recipes, pixels, source=None):
data = pixels.data
dtype_min = np.iinfo(data.dtype).min
dtype_max = np.iinfo(data.dtype).max
if "landsat8" in recipes:
LOG.info("Applying landsat 8 recipe")
out = np.ma.empty(shape=(data.shape), dtype=np.float32)
for bdx, source_band in enumerate((4, 3, 2)):
sun_elev = source.meta["L1_METADATA_FILE"]["IMAGE_ATTRIBUTES"][
"SUN_ELEVATION"
]
multi_reflect = source.meta["L1_METADATA_FILE"][
"RADIOMETRIC_RESCALING"
].get(
"REFLECTANCE_MULT_BAND_{}".format(source_band)
)
add_reflect = source.meta["L1_METADATA_FILE"]["RADIOMETRIC_RESCALING"].get(
"REFLECTANCE_ADD_BAND_{}".format(source_band)
)
min_val = source.meta.get("values", {}).get(str(source_band), {}).get(
"min", dtype_min
)
max_val = source.meta.get("values", {}).get(str(source_band), {}).get(
"max", dtype_max
)
band_data = 10000 * reflectance.reflectance(
data[bdx], multi_reflect, add_reflect, sun_elev, src_nodata=0
)
# calculate local min/max as fallbacks
if (
min_val == dtype_min
and max_val == dtype_max
and len(data.compressed()) > 0
):
local_min, local_max = np.percentile(band_data.compressed(), (2, 98))
min_val = max(min_val, local_min)
max_val = min(max_val, local_max)
out[bdx] = np.ma.where(
band_data > 0,
utils.linear_rescale(
band_data, in_range=[min_val, max_val], out_range=[0, 1]
),
0,
)
data = out
if "imagery" in recipes:
LOG.info("Applying imagery recipe")
if "rgb_bands" in recipes:
data = np.ma.array([data[i - 1] for i in recipes["rgb_bands"]])
if data.shape[0] > 3:
# alpha band (and beyond) present; drop it (them)
# TODO use band 4 as a mask instead
data = data[0:3]
if "linear_stretch" in recipes:
if recipes["linear_stretch"] == "global":
data = utils.linear_rescale(
data,
in_range=(np.min(data), np.max(data)),
out_range=(dtype_min, dtype_max),
)
elif recipes["linear_stretch"] == "per_band":
for band in xrange(0, data.shape[0]):
min_val = source.meta.get("values", {}).get(band, {}).get(
"min", np.min(data[band])
)
max_val = source.meta.get("values", {}).get(band, {}).get(
"max", np.max(data[band])
)
data[band] = np.ma.where(
data[band] > 0,
utils.linear_rescale(
data[band],
in_range=(min_val, max_val),
out_range=(dtype_min, dtype_max),
),
0,
)
else:
# rescale after reducing and before increasing dimensionality
if data.dtype != np.uint8 and not np.issubdtype(data.dtype, float):
# rescale non-8-bit sources (assuming that they're raw sensor data)
for band in xrange(0, data.shape[0]):
min_val = source.meta.get("values", {}).get(band, {}).get(
"min", dtype_min
)
max_val = source.meta.get("values", {}).get(band, {}).get(
"max", dtype_max
)
if (
min_val == dtype_min
and max_val == dtype_max
and len(data.compressed()) > 0
):
local_min, local_max = np.percentile(
data[band].compressed(), (2, 98)
)
min_val = max(min_val, local_min)
max_val = min(max_val, local_max)
data[band] = np.ma.where(
data[band] > 0,
utils.linear_rescale(
data[band],
in_range=(min_val, max_val),
out_range=(dtype_min, dtype_max),
),
0,
)
if data.shape[0] == 1:
# likely greyscale image; use the same band on all channels
data = np.ma.array([data[0], data[0], data[0]])
# normalize to 0..1 based on the range of the source type (only
# for int*s)
if not np.issubdtype(data.dtype, float):
data = data.astype(np.float32) / np.iinfo(data.dtype).max
return PixelCollection(data, pixels.bounds)
def landsat8_tile(sceneid,
tile_x,
tile_y,
tile_z,
bands=("4", "3", "2"),
tilesize=256,
pan=False,
percents="",
**kwargs):
"""
Create mercator tile from Landsat-8 data.
Attributes
----------
sceneid : str
Landsat sceneid. For scenes after May 2017,
sceneid have to be LANDSAT_PRODUCT_ID.
tile_x : int
Mercator tile X index.
tile_y : int
Mercator tile Y index.
tile_z : int
Mercator tile ZOOM level.
bands : tuple, str, optional (default: ("4", "3", "2"))
Bands index for the RGB combination.
tilesize : int, optional (default: 256)
Output image size.
pan : boolean, optional (default: False)
If True, apply pan-sharpening.
kwargs: dict, optional
These will be passed to the 'rio_tiler.utils._tile_read' function.
Returns
-------
data : numpy ndarray
mask: numpy array
"""
if not isinstance(bands, tuple):
bands = tuple((bands, ))
for band in bands:
if band not in LANDSAT_BANDS:
raise InvalidBandName(
"{} is not a valid Landsat band name".format(band))
scene_params = landsat8._landsat_parse_scene_id(sceneid)
meta_data = landsat8._landsat_get_mtl(sceneid).get("L1_METADATA_FILE")
landsat_address = "{}/{}".format(LANDSAT_BUCKET, scene_params["key"])
wgs_bounds = toa_utils._get_bounds_from_metadata(
meta_data["PRODUCT_METADATA"])
addresses = ["{}_B{}.TIF".format(landsat_address, band) for band in bands]
values = []
percents = percents.split(',')
i = 0
for address in addresses:
with rasterio.open(address) as src:
if int(percents[i]) != 0 and int(percents[i + 1]) != 100:
overviews = src.overviews(1)
if len(overviews) > 0:
d = src.read(
out_shape=(1,
int(src.height /
overviews[len(overviews) - 1]),
int(src.width /
overviews[len(overviews) - 1])))
else:
d = src.read()
dflatten = numpy.array(d.flatten())
p_start, p_end = numpy.percentile(dflatten[dflatten > 0],
(int(percents[i]),
(int(percents[i + 1]))))
values.append([p_start, p_end])
else:
values.append([None, None])
i += 2
if not utils.tile_exists(wgs_bounds, tile_z, tile_x, tile_y):
# raise TileOutsideBounds(
# "Tile {}/{}/{} is outside image bounds".format(tile_z, tile_x, tile_y)
# )
return None, None
mercator_tile = mercantile.Tile(x=tile_x, y=tile_y, z=tile_z)
tile_bounds = mercantile.xy_bounds(mercator_tile)
def _tiler(band):
address = "{}_B{}.TIF".format(landsat_address, band)
if band == "QA":
nodata = 1
else:
nodata = 0
return utils.tile_read(address,
bounds=tile_bounds,
tilesize=tilesize,
nodata=nodata,
**kwargs)
with futures.ThreadPoolExecutor(max_workers=MAX_THREADS) as executor:
data, masks = zip(*list(executor.map(_tiler, bands)))
mask = numpy.all(masks, axis=0).astype(numpy.uint8) * 255
new_data = list(data)
has_modification = False
for ds in range(0, len(new_data)):
if values[ds][0] is not None and values[ds][1] is not None:
has_modification = True
new_data[ds] = rescale_intensity(new_data[ds],
in_range=(values[ds][0],
values[ds][1]),
out_range=(0, 255))
if has_modification == True:
data = numpy.array(new_data).astype(numpy.uint8)
data = numpy.concatenate(data)
if pan:
pan_address = "{}_B8.TIF".format(landsat_address)
matrix_pan, mask = utils.tile_read(pan_address,
tile_bounds,
tilesize,
nodata=0)
data = utils.pansharpening_brovey(data, matrix_pan, 0.2,
matrix_pan.dtype)
sun_elev = meta_data["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"]
for bdx, band in enumerate(bands):
if band in ["1", "2", "3", "4", "5", "6", "7", "8", "9"]: # OLI
multi_reflect = meta_data["RADIOMETRIC_RESCALING"].get(
"REFLECTANCE_MULT_BAND_{}".format(band))
add_reflect = meta_data["RADIOMETRIC_RESCALING"].get(
"REFLECTANCE_ADD_BAND_{}".format(band))
data[bdx] = 10000 * reflectance.reflectance(
data[bdx], multi_reflect, add_reflect, sun_elev)
elif band in ["10", "11"]: # TIRS
multi_rad = meta_data["RADIOMETRIC_RESCALING"].get(
"RADIANCE_MULT_BAND_{}".format(band))
add_rad = meta_data["RADIOMETRIC_RESCALING"].get(
"RADIANCE_ADD_BAND_{}".format(band))
k1 = meta_data["TIRS_THERMAL_CONSTANTS"].get(
"K1_CONSTANT_BAND_{}".format(band))
k2 = meta_data["TIRS_THERMAL_CONSTANTS"].get(
"K2_CONSTANT_BAND_{}".format(band))
data[bdx] = brightness_temp.brightness_temp(
data[bdx], multi_rad, add_rad, k1, k2)
return data, mask
