def rescale_tile(tile, mask, rescale=None):
if rescale:
try:
rescale_arr = list(map(float, rescale.split(",")))
except ValueError:
raise exceptions.ValidationError("Invalid rescale value")
rescale_arr = list(_chunks(rescale_arr, 2))
if len(rescale_arr) != tile.shape[0]:
rescale_arr = ((rescale_arr[0]), ) * tile.shape[0]
for bdx in range(tile.shape[0]):
if mask is not None:
tile[bdx] = np.where(
mask,
linear_rescale(tile[bdx],
in_range=rescale_arr[bdx],
out_range=[0, 255]),
0,
)
else:
tile[bdx] = linear_rescale(tile[bdx],
in_range=rescale_arr[bdx],
out_range=[0, 255])
tile = tile.astype(np.uint8)
return tile, mask
def post_process_tile(
tile: numpy.ndarray,
mask: numpy.ndarray,
rescale: str = None,
color_formula: str = None,
) -> Tuple[numpy.ndarray, numpy.ndarray]:
"""Tile data post processing."""
if rescale:
rescale_arr = (tuple(map(float, rescale.split(","))), ) * tile.shape[0]
for bdx in range(tile.shape[0]):
tile[bdx] = numpy.where(
mask,
linear_rescale(tile[bdx],
in_range=rescale_arr[bdx],
out_range=[0, 255]),
0,
)
tile = tile.astype(numpy.uint8)
if color_formula:
if issubclass(tile.dtype.type, numpy.floating):
tile = tile.astype(numpy.int16)
# make sure one last time we don't have
# negative value before applying color formula
tile[tile < 0] = 0
for ops in parse_operations(color_formula):
tile = scale_dtype(ops(to_math_type(tile)), numpy.uint8)
return tile
def tile(scene, tile_z, tile_x, tile_y, tileformat):
"""Handle tile requests
"""
if tileformat == 'jpg':
tileformat = 'jpeg'
query_args = APP.current_request.query_params
query_args = query_args if isinstance(query_args, dict) else {}
bands = query_args.get('rgb', '4,3,2')
bands = tuple(re.findall(r'\d+', bands))
histoCut = query_args.get('histo', ';'.join(['0,16000'] * len(bands)))
histoCut = re.findall(r'\d+,\d+', histoCut)
histoCut = list(map(lambda x: list(map(int, x.split(','))), histoCut))
if len(bands) != len(histoCut):
raise LandsatTilerError('The number of bands doesn\'t match the number of histogramm values')
tilesize = query_args.get('tile', 256)
tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize
pan = True if query_args.get('pan') else False
tile, mask = landsat8.tile(scene, tile_x, tile_y, tile_z, bands, pan=pan, tilesize=tilesize)
rtile = np.zeros((len(bands), tilesize, tilesize), dtype=np.uint8)
for bdx in range(len(bands)):
rtile[bdx] = np.where(mask, linear_rescale(tile[bdx], in_range=histoCut[bdx], out_range=[0, 255]), 0)
img = array_to_img(rtile, mask=mask)
str_img = b64_encode_img(img, tileformat)
return ('OK', f'image/{tileformat}', str_img)
def ratio(scene, tile_z, tile_x, tile_y, tileformat):
"""Handle processing requests
"""
if tileformat == 'jpg':
tileformat = 'jpeg'
query_args = APP.current_request.query_params
query_args = query_args if isinstance(query_args, dict) else {}
ratio_value = query_args['ratio']
range_value = query_args.get('range', [-1, 1])
tilesize = query_args.get('tile', 256)
tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize
tile, mask = expression(scene,
tile_x,
tile_y,
tile_z,
ratio_value,
tilesize=tilesize)
if len(tile.shape) == 2:
tile = np.expand_dims(tile, axis=0)
rtile = np.where(
mask, linear_rescale(tile, in_range=range_value, out_range=[0, 255]),
0).astype(np.uint8)
img = array_to_img(rtile,
color_map=get_colormap(name='cfastie'),
mask=mask)
str_img = b64_encode_img(img, tileformat)
return ('OK', f'image/{tileformat}', str_img)
def gdal(
input,
tile,
ext="png",
bidx=(1, 2, 3),
colormap=None,
scale=None,
save=None,
):
"""Handle tile requests."""
tile_z, tile_x, tile_y = list(map(int, tile.split('-')))
tile, mask = main.tile(
input, tile_x, tile_y, tile_z, indexes=bidx, tilesize=512
)
if scale:
nbands = tile.shape[0]
for bdx in range(nbands):
tile[bdx] = numpy.where(
mask,
utils.linear_rescale(tile[bdx], in_range=scale[bdx], out_range=[0, 255]),
0,
)
tile = tile.astype(numpy.uint8)
if colormap:
colormap = numpy.array(list(chunks(utils.get_colormap(name=colormap), 3)))
buffer = array_to_img_rasterio(tile, mask, img_format=ext, color_map=colormap)
if save:
with open(f"{tile_x}-{tile_y}-{tile_z}_gdal.{ext}", "wb") as f:
f.write(buffer)
def _postprocess(
tile: numpy.ndarray,
mask: numpy.ndarray,
tilesize: int,
rescale: str = None,
color_formula: str = None,
) -> Tuple[numpy.ndarray, numpy.ndarray]:
if tile is None:
# Return empty tile
tile = numpy.zeros((1, tilesize, tilesize), dtype=numpy.uint8)
mask = numpy.zeros((tilesize, tilesize), dtype=numpy.uint8)
else:
if rescale:
rescale_arr = (tuple(map(float,
rescale.split(","))), ) * tile.shape[0]
for bdx in range(tile.shape[0]):
tile[bdx] = numpy.where(
mask,
linear_rescale(tile[bdx],
in_range=rescale_arr[bdx],
out_range=[0, 255]),
0,
)
tile = tile.astype(numpy.uint8)
if color_formula:
# make sure one last time we don't have
# negative value before applying color formula
tile[tile < 0] = 0
for ops in parse_operations(color_formula):
tile = scale_dtype(ops(to_math_type(tile)), numpy.uint8)
return tile, mask
def _postprocess(
tile: numpy.ndarray,
mask: numpy.ndarray,
rescale: str = None,
color_formula: str = None,
) -> Tuple[numpy.ndarray, numpy.ndarray]:
"""Post-process tile data."""
if rescale:
rescale_arr = list(map(float, rescale.split(",")))
rescale_arr = list(_chunks(rescale_arr, 2))
if len(rescale_arr) != tile.shape[0]:
rescale_arr = ((rescale_arr[0]),) * tile.shape[0]
for bdx in range(tile.shape[0]):
tile[bdx] = numpy.where(
mask,
linear_rescale(
tile[bdx], in_range=rescale_arr[bdx], out_range=[0, 255]
),
0,
)
tile = tile.astype(numpy.uint8)
if color_formula:
# make sure one last time we don't have
# negative value before applying color formula
tile[tile < 0] = 0
for ops in parse_operations(color_formula):
tile = scale_dtype(ops(to_math_type(tile)), numpy.uint8)
return tile, mask
def pil(
input,
tile,
ext="png",
bidx=(1, 2, 3),
colormap=None,
scale=None,
save=None,
):
"""Handle tile requests."""
tile_z, tile_x, tile_y = list(map(int, tile.split('-')))
tile, mask = main.tile(
input, tile_x, tile_y, tile_z, indexes=bidx, tilesize=512
)
if scale:
nbands = tile.shape[0]
for bdx in range(nbands):
tile[bdx] = numpy.where(
mask,
utils.linear_rescale(tile[bdx], in_range=scale[bdx], out_range=[0, 255]),
0,
)
tile = tile.astype(numpy.uint8)
if colormap:
colormap = utils.get_colormap(name=colormap)
img = utils.array_to_img(tile, mask=mask, color_map=colormap)
img_format = "JPEG2000" if ext == "jp2" else ext
buffer = img_to_buffer(img, img_format)
if save:
with open(f"{tile_x}-{tile_y}-{tile_z}_pil.{ext}", "wb") as f:
f.write(buffer)
def test_linear_rescale_valid():
"""Should work as expected (read data band)."""
data = np.zeros((1, 1), dtype=np.int16) + 1000
expected_value = np.zeros((1, 1), dtype=np.int16) + 25.5
assert (
utils.linear_rescale(data, in_range=(0, 10000), out_range=(0, 255))
== expected_value
)
def area(
scene,
bbox,
expression,
expression_range=[-1, 1],
bbox_crs="epsg:4326",
out_crs="epsg:3857",
):
"""Area handler."""
max_img_size = 512
bands = tuple(set(re.findall(r"b(?P<bands>[0-9]{1,2})", expression)))
scene_params = sentinel_parse_scene_id(scene)
sentinel_address = f'{SENTINEL_BUCKET}/{scene_params["key"]}'
scene_info = sentinel2_get_info(os.path.basename(SENTINEL_BUCKET),
scene_params["key"],
request_pays=True)
addresses = [f"{sentinel_address}/B{band}.jp2" for band in bands]
_worker = partial(
get_area,
bbox=bbox,
max_img_size=max_img_size,
bbox_crs=bbox_crs,
out_crs=out_crs,
)
with futures.ThreadPoolExecutor(max_workers=3) as executor:
data = np.concatenate(list(executor.map(_worker, addresses)))
if not np.any(data):
raise Exception("No valid data in array")
mask = np.all(data != 0, axis=0).astype(np.uint8) * 255
ctx = {}
for bdx, b in enumerate(bands):
ctx["b{}".format(b)] = data[bdx]
ratio = np.nan_to_num(ne.evaluate(expression, local_dict=ctx))
ratio = np.where(
mask,
linear_rescale(ratio, in_range=expression_range, out_range=[0, 255]),
0).astype(np.uint8)
img = array_to_img(ratio, mask, get_colormap(name="cfastie"))
ndvi = b64_encode_img(img, "jpeg")
date = (scene_params["acquisitionYear"] + "-" +
scene_params["acquisitionMonth"] + "-" +
scene_params["acquisitionDay"])
return {
"ndvi": ndvi,
"date": date,
"sat": scene_info["sat"],
"scene": scene,
"cloud": scene_info["cloud_coverage"],
}
def process(arr, skip_rescale=False, skip_alpha=False, skip_type=False):
if not skip_rescale and rescale is not None:
arr = linear_rescale(arr, in_range=rescale)
if not skip_alpha and not with_alpha:
arr[mask==0] = jpg_background
if not skip_type and rgb and arr.dtype != np.uint8:
arr = arr.astype(np.uint8)
return arr
def area(
scene,
bbox,
expression,
expression_range=[-1, 1],
bbox_crs="epsg:4326",
out_crs="epsg:3857",
):
"""Area handler."""
max_img_size = 512
bands = tuple(set(re.findall(r"b(?P<bands>[0-9]{1,2})", expression)))
scene_params = landsat_parse_scene_id(scene)
meta_data = landsat_get_mtl(scene).get("L1_METADATA_FILE")
landsat_address = f'{LANDSAT_BUCKET}/{scene_params["key"]}'
def worker(band):
"""Worker."""
address = f"{landsat_address}_B{band}.TIF"
sun_elev = meta_data["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"]
multi_reflect = meta_data["RADIOMETRIC_RESCALING"][
f"REFLECTANCE_MULT_BAND_{band}"
]
add_reflect = meta_data["RADIOMETRIC_RESCALING"][f"REFLECTANCE_ADD_BAND_{band}"]
band = get_area(
address, bbox, max_img_size=max_img_size, bbox_crs=bbox_crs, out_crs=out_crs
)
return reflectance(band, multi_reflect, add_reflect, sun_elev, src_nodata=0)
with futures.ThreadPoolExecutor(max_workers=3) as executor:
data = np.concatenate(list(executor.map(worker, bands)))
if not np.any(data):
raise Exception("No valid data in array")
mask = np.all(data != 0, axis=0).astype(np.uint8) * 255
ctx = {}
for bdx, b in enumerate(bands):
ctx["b{}".format(b)] = data[bdx]
ratio = np.nan_to_num(ne.evaluate(expression, local_dict=ctx))
ratio = np.where(
mask, linear_rescale(ratio, in_range=expression_range, out_range=[0, 255]), 0
).astype(np.uint8)
img = array_to_img(ratio, mask, get_colormap(name="cfastie"))
ndvi = b64_encode_img(img, "jpeg")
return {
"ndvi": ndvi,
"date": scene_params["date"],
"scene": scene,
"cloud": meta_data["IMAGE_ATTRIBUTES"]["CLOUD_COVER"],
}
def tile(tile_z, tile_x, tile_y, tileformat):
"""Handle tile requests."""
if tileformat == 'jpg':
tileformat = 'jpeg'
query_args = APP.current_request.query_params
query_args = query_args if isinstance(query_args, dict) else {}
address = query_args['url']
## Read Stac File
with urllib.request.urlopen(address) as url:
data = json.loads(url.read().decode())
asset_key = query_args.get('asset_key', 'raster')
raster_address = data['assets'][asset_key]['href']
indexes = query_args.get('indexes')
if indexes:
indexes = tuple(int(s) for s in re.findall(r'\d+', indexes))
tilesize = query_args.get('tile', 512)
tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize
nodata = query_args.get('nodata')
if nodata is not None:
nodata = int(nodata)
raster_address = data['assets'][asset_key]['href']
tile, mask = main.tile(raster_address,
tile_x,
tile_y,
tile_z,
indexes=indexes,
tilesize=tilesize,
nodata=nodata)
linearStretch = query_args.get('linearStretch')
if linearStretch is not None:
if util.strtobool(linearStretch):
tile = linear_rescale(tile, in_range=(np.min(tile), np.max(tile)))
img = array_to_img(tile, mask=mask)
str_img = b64_encode_img(img, tileformat)
return ('OK', f'image/{tileformat}', str_img)
def tile(
tile_z,
tile_x,
tile_y,
tileformat,
indexes=1,
shadder=None,
colormap=None,
range=None,
):
"""Handle tile requests"""
if tileformat == "jpg":
tileformat = "jpeg"
if colormap and shadder:
return ("NOK", "text/plain",
"Cannot pass shadder and colormap options")
address = f"s3://elevation-tiles-prod/geotiff/{tile_z}/{tile_x}/{tile_y}.tif"
data, mask = main.tile(address,
tile_x,
tile_y,
tile_z,
indexes=indexes,
tilesize=512)
if shadder:
tileformat = "png"
if shadder == "mapbox":
tile = encoders.data_to_rgb(data[0], -10000, 1)
elif shadder == "mapzen":
tile = mapzen_elevation_rgb.data_to_rgb(data)
else:
return ("NOK", "text/plain", f"Invalid shadder mode: {shadder}")
else:
if not range:
range = "0,10000"
histoCut = list(map(int, range.split(',')))
tile = numpy.where(
mask, linear_rescale(data, in_range=histoCut, out_range=[0, 255]),
0)
options = dict(mask=mask)
if colormap:
options.update(color_map=get_colormap(name=colormap))
img = array_to_img(tile, **options)
return ("OK", f"image/{tileformat}", img_to_buffer(img, tileformat))
def band_worker(band, landsat_address, meta, bounds=None):
"""
"""
address = f'{landsat_address}_B{band}.TIF'
sun_elev = meta['IMAGE_ATTRIBUTES']['SUN_ELEVATION']
multi_reflect = meta['RADIOMETRIC_RESCALING'][
f'REFLECTANCE_MULT_BAND_{band}']
add_reflect = meta['RADIOMETRIC_RESCALING'][f'REFLECTANCE_ADD_BAND_{band}']
ovrSize = 1024
with rasterio.open(address) as src:
if bounds:
w, s, e, n = bounds
with WarpedVRT(src,
dst_crs='EPSG:3857',
resampling=Resampling.bilinear,
src_nodata=0,
dst_nodata=0) as vrt:
window = vrt.window(w, s, e, n, precision=21)
matrix = vrt.read(window=window,
boundless=True,
resampling=Resampling.bilinear,
out_shape=(ovrSize, ovrSize),
indexes=1).astype(src.profile['dtype'])
else:
matrix = src.read(indexes=1,
out_shape=(ovrSize, ovrSize),
resampling=Resampling.bilinear).astype(
src.profile['dtype'])
matrix = reflectance(matrix,
multi_reflect,
add_reflect,
sun_elev,
src_nodata=0)
imgRange = np.percentile(matrix[matrix > 0], (2, 98)).tolist()
matrix = np.where(
matrix > 0,
linear_rescale(matrix, in_range=imgRange, out_range=[1, 255]),
0).astype(np.uint8)
return matrix
def landsat_ratio(scene, tile_z, tile_x, tile_y, tileformat):
"""
Handle processing requests
"""
query_args = LANDSAT_APP.current_request.query_params
query_args = query_args if isinstance(query_args, dict) else {}
ratio_value = query_args.get('ratio', 'ndvi')
if ratio_value not in RATIOS.keys():
raise LandsatTilerError('Invalid ratio: {}'.format(ratio_value))
equation = RATIOS[ratio_value]['eq']
band_names = list(set(re.findall('b[0-9]{1,2}', equation)))
bands = tuple(map(lambda x: x.strip('b'), band_names))
tilesize = query_args.get('tile', 256)
tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize
tile = landsat8.tile(scene,
tile_x,
tile_y,
tile_z,
bands,
tilesize=tilesize)
for bdx, b in enumerate(band_names):
globals()[b] = tile[bdx]
tile = np.where(
reduce(lambda x, y: x * y, [globals()[i] for i in band_names]) > 0,
np.nan_to_num(ne.evaluate(equation)), -9999)
range_val = equation = RATIOS[ratio_value]['rg']
rtile = np.where(
tile != -9999,
linear_rescale(tile, in_range=range_val, out_range=[1, 255]),
0).astype(np.uint8)
tile = array_to_img(rtile,
tileformat,
color_map=get_colormap(name='cfastie'))
if tileformat == 'jpg':
tileformat = 'jpeg'
return ('OK', f'image/{tileformat}', tile)
def tile(scene, tile_z, tile_x, tile_y, tileformat):
"""Handle tile requests."""
if tileformat == "jpg":
tileformat = "jpeg"
query_args = APP.current_request.query_params
query_args = query_args if isinstance(query_args, dict) else {}
bands = query_args.get("rgb", "04,03,02")
bands = tuple(re.findall(r"[0-9A]{2}", bands))
histoCut = query_args.get("histo", "-".join(["0,16000"] * len(bands)))
histoCut = re.findall(r"\d+,\d+", histoCut)
histoCut = list(map(lambda x: list(map(int, x.split(","))), histoCut))
if len(bands) != len(histoCut):
raise SentinelTilerError(
"The number of bands doesn't match the number of histogramm values"
)
tilesize = query_args.get("tile", 256)
tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize
tile, mask = sentinel2.tile(scene,
tile_x,
tile_y,
tile_z,
bands,
tilesize=tilesize)
rtile = np.zeros((len(bands), tilesize, tilesize), dtype=np.uint8)
for bdx in range(len(bands)):
rtile[bdx] = np.where(
mask,
linear_rescale(tile[bdx],
in_range=histoCut[bdx],
out_range=[0, 255]),
0,
)
img = array_to_img(rtile, mask=mask)
str_img = b64_encode_img(img, tileformat)
return ("OK", f"image/{tileformat}", str_img)
def tile(scene, tile_z, tile_x, tile_y, tileformat):
"""Handle tile requests
"""
query_args = APP.current_request.query_params
query_args = query_args if isinstance(query_args, dict) else {}
bands = query_args.get('rgb', '04,03,02')
bands = tuple(re.findall(r'[0-9A]{2}', bands))
histoCut = query_args.get('histo', '-'.join(['0,16000'] * len(bands)))
histoCut = re.findall(r'\d+,\d+', histoCut)
histoCut = list(map(lambda x: list(map(int, x.split(','))), histoCut))
if len(bands) != len(histoCut):
raise SentinelTilerError(
'The number of bands doesn\'t match the number of histogramm values'
)
tilesize = query_args.get('tile', 256)
tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize
tile, mask = sentinel2.tile(scene,
tile_x,
tile_y,
tile_z,
bands,
tilesize=tilesize)
rtile = np.zeros((len(bands), tilesize, tilesize), dtype=np.uint8)
for bdx in range(len(bands)):
rtile[bdx] = np.where(
mask,
linear_rescale(tile[bdx],
in_range=histoCut[bdx],
out_range=[0, 255]), 0)
tile = array_to_img(rtile, tileformat, mask=mask)
if tileformat == 'jpg':
tileformat = 'jpeg'
return ('OK', f'image/{tileformat}', tile)
def _get_tile(self, z, x, y, tileformat, color_ops=None):
if tileformat == "jpg":
tileformat = "jpeg"
if not self.raster.tile_exists(z, x, y):
raise web.HTTPError(404)
data, mask = self.raster.read_tile(z, x, y)
if len(data.shape) == 2:
data = numpy.expand_dims(data, axis=0)
if self.scale:
nbands = data.shape[0]
scale = self.scale
if len(scale) != nbands:
scale = scale * nbands
for bdx in range(nbands):
data[bdx] = numpy.where(
mask,
linear_rescale(data[bdx],
in_range=scale[bdx],
out_range=[0, 255]),
0,
)
data = data.astype(numpy.uint8)
if color_ops:
data = self._apply_color_operations(data, color_ops)
img = array_to_img(data, mask=mask, color_map=self.colormap)
params = TileProfiles.get(tileformat)
if tileformat == "jpeg":
img = img.convert("RGB")
sio = BytesIO()
img.save(sio, tileformat.upper(), **params)
sio.seek(0)
return sio
def _get_tile(self, z, x, y, tileformat, color_ops=None):
if tileformat == "jpg":
tileformat = "jpeg"
if not self.raster.tile_exists(z, x, y):
raise web.HTTPError(404)
data, mask = self.raster.read_tile(z, x, y)
if len(data.shape) == 2:
data = numpy.expand_dims(data, axis=0)
if self.scale:
nbands = data.shape[0]
scale = self.scale
if len(scale) != nbands:
scale = scale * nbands
for bdx in range(nbands):
data[bdx] = numpy.where(
mask,
linear_rescale(data[bdx],
in_range=scale[bdx],
out_range=[0, 255]),
0,
)
data = data.astype(numpy.uint8)
if color_ops:
data = self._apply_color_operations(data, color_ops)
options = img_profiles.get(tileformat, {})
return BytesIO(
array_to_image(data,
mask=mask,
color_map=self.colormap,
img_format=tileformat,
**options))
def _postprocess_tile(tile, mask, rescale=None, color_ops=None):
"""Tile data post-processing."""
if rescale:
rescale = (tuple(map(float, rescale.split(","))), ) * tile.shape[0]
for bdx in range(tile.shape[0]):
tile[bdx] = numpy.where(
mask,
linear_rescale(tile[bdx],
in_range=rescale[bdx],
out_range=[0, 255]),
0,
)
tile = tile.astype(numpy.uint8)
if color_ops:
# make sure one last time we don't have
# negative value before applying color formula
tile[tile < 0] = 0
for ops in parse_operations(color_ops):
tile = scale_dtype(ops(to_math_type(tile)), numpy.uint8)
return tile, mask
def tile(tile_z, tile_x, tile_y, tileformat):
"""Handle tile requests."""
if tileformat == 'jpg':
tileformat = 'jpeg'
query_args = APP.current_request.query_params
query_args = query_args if isinstance(query_args, dict) else {}
address = query_args['url']
indexes = query_args.get('indexes')
if indexes:
indexes = tuple(int(s) for s in re.findall(r'\d+', indexes))
tilesize = query_args.get('tile', 512)
tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize
nodata = query_args.get('nodata')
if nodata is not None:
nodata = int(nodata)
tile, mask = main.tile(address,
tile_x,
tile_y,
tile_z,
indexes=indexes,
tilesize=tilesize,
nodata=nodata)
linearStretch = query_args.get('linearStretch')
if linearStretch is not None:
if util.strtobool(linearStretch):
tile = linear_rescale(tile, in_range=(np.min(tile), np.max(tile)))
img = array_to_img(tile, mask=mask)
str_img = b64_encode_img(img, tileformat)
return ('OK', f'image/{tileformat}', str_img)
def _postprocess(tile, mask, tilesize, rescale=None, color_formula=None):
if tile is None:
# Return empty tile
tile = numpy.zeros((1, tilesize, tilesize), dtype=numpy.uint8)
mask = numpy.zeros((tilesize, tilesize), dtype=numpy.uint8)
else:
if rescale:
rescale = (tuple(map(float, rescale.split(","))), ) * tile.shape[0]
for bdx in range(tile.shape[0]):
tile[bdx] = numpy.where(
mask,
linear_rescale(tile[bdx],
in_range=rescale[bdx],
out_range=[0, 255]),
0,
)
tile = tile.astype(numpy.uint8)
if color_formula:
for ops in parse_operations(color_formula):
tile = scale_dtype(ops(to_math_type(tile)), numpy.uint8)
return tile, mask
def apply(recipes, pixels, source=None):
data = pixels.data
dtype_min = np.iinfo(data.dtype).min
dtype_max = np.iinfo(data.dtype).max
if "landsat8" in recipes:
LOG.info("Applying landsat 8 recipe")
out = np.ma.empty(shape=(data.shape), dtype=np.float32)
for bdx, source_band in enumerate((4, 3, 2)):
sun_elev = source.meta["L1_METADATA_FILE"]["IMAGE_ATTRIBUTES"][
"SUN_ELEVATION"
]
multi_reflect = source.meta["L1_METADATA_FILE"][
"RADIOMETRIC_RESCALING"
].get(
"REFLECTANCE_MULT_BAND_{}".format(source_band)
)
add_reflect = source.meta["L1_METADATA_FILE"]["RADIOMETRIC_RESCALING"].get(
"REFLECTANCE_ADD_BAND_{}".format(source_band)
)
min_val = source.meta.get("values", {}).get(str(source_band), {}).get(
"min", dtype_min
)
max_val = source.meta.get("values", {}).get(str(source_band), {}).get(
"max", dtype_max
)
band_data = 10000 * reflectance.reflectance(
data[bdx], multi_reflect, add_reflect, sun_elev, src_nodata=0
)
# calculate local min/max as fallbacks
if (
min_val == dtype_min
and max_val == dtype_max
and len(data.compressed()) > 0
):
local_min, local_max = np.percentile(band_data.compressed(), (2, 98))
min_val = max(min_val, local_min)
max_val = min(max_val, local_max)
out[bdx] = np.ma.where(
band_data > 0,
utils.linear_rescale(
band_data, in_range=[min_val, max_val], out_range=[0, 1]
),
0,
)
data = out
if "imagery" in recipes:
LOG.info("Applying imagery recipe")
if "rgb_bands" in recipes:
data = np.ma.array([data[i - 1] for i in recipes["rgb_bands"]])
if data.shape[0] > 3:
# alpha band (and beyond) present; drop it (them)
# TODO use band 4 as a mask instead
data = data[0:3]
if "linear_stretch" in recipes:
if recipes["linear_stretch"] == "global":
data = utils.linear_rescale(
data,
in_range=(np.min(data), np.max(data)),
out_range=(dtype_min, dtype_max),
)
elif recipes["linear_stretch"] == "per_band":
for band in xrange(0, data.shape[0]):
min_val = source.meta.get("values", {}).get(band, {}).get(
"min", np.min(data[band])
)
max_val = source.meta.get("values", {}).get(band, {}).get(
"max", np.max(data[band])
)
data[band] = np.ma.where(
data[band] > 0,
utils.linear_rescale(
data[band],
in_range=(min_val, max_val),
out_range=(dtype_min, dtype_max),
),
0,
)
else:
# rescale after reducing and before increasing dimensionality
if data.dtype != np.uint8 and not np.issubdtype(data.dtype, float):
# rescale non-8-bit sources (assuming that they're raw sensor data)
for band in xrange(0, data.shape[0]):
min_val = source.meta.get("values", {}).get(band, {}).get(
"min", dtype_min
)
max_val = source.meta.get("values", {}).get(band, {}).get(
"max", dtype_max
)
if (
min_val == dtype_min
and max_val == dtype_max
and len(data.compressed()) > 0
):
local_min, local_max = np.percentile(
data[band].compressed(), (2, 98)
)
min_val = max(min_val, local_min)
max_val = min(max_val, local_max)
data[band] = np.ma.where(
data[band] > 0,
utils.linear_rescale(
data[band],
in_range=(min_val, max_val),
out_range=(dtype_min, dtype_max),
),
0,
)
if data.shape[0] == 1:
# likely greyscale image; use the same band on all channels
data = np.ma.array([data[0], data[0], data[0]])
# normalize to 0..1 based on the range of the source type (only
# for int*s)
if not np.issubdtype(data.dtype, float):
data = data.astype(np.float32) / np.iinfo(data.dtype).max
return PixelCollection(data, pixels.bounds)
def create(
scene,
bands=None,
expression=None,
expression_range=[-1, 1],
img_format="jpeg",
ovrSize=512,
):
"""Handler."""
if img_format not in ["png", "jpeg"]:
raise UserWarning(f"Invalid {img_format} extension")
if not expression and not bands:
raise Exception("Expression or Bands must be provided")
if bands:
nb_bands = len(bands)
if nb_bands != 3:
raise Exception("RGB combination only")
if expression:
bands = tuple(set(re.findall(r"b(?P<bands>[0-9]{1,2})", expression)))
rgb = expression.split(",")
nb_bands = len(rgb)
scene_params = landsat_parse_scene_id(scene)
meta_data = landsat_get_mtl(scene).get("L1_METADATA_FILE")
landsat_address = f'{LANDSAT_BUCKET}/{scene_params["key"]}'
_worker = partial(worker,
landsat_address=landsat_address,
meta=meta_data,
ovr_size=ovrSize)
with futures.ThreadPoolExecutor(max_workers=3) as executor:
data = np.concatenate(list(executor.map(_worker, bands)))
mask = np.all(data != 0, axis=0).astype(np.uint8) * 255
if expression:
ctx = {}
for bdx, b in enumerate(bands):
ctx["b{}".format(b)] = data[bdx]
data = np.array([
np.nan_to_num(ne.evaluate(bloc.strip(), local_dict=ctx))
for bloc in rgb
])
for band in range(data.shape[0]):
imgRange = (expression_range if expression else np.percentile(
data[band][mask > 0], (2, 98)).tolist())
data[band] = np.where(
mask,
linear_rescale(data[band],
in_range=imgRange,
out_range=[0, 255]),
0,
)
data = data.squeeze()
colormap = None if len(data.shape) >= 3 else get_colormap(name="cfastie")
img = array_to_img(data, mask, colormap)
return b64_encode_img(img, img_format)
def worker(scene, bands):
"""Worker."""
try:
scene_params = landsat_parse_scene_id(scene)
meta_data = landsat_get_mtl(scene).get("L1_METADATA_FILE")
landsat_address = f'{LANDSAT_BUCKET}/{scene_params["key"]}'
bqa = f"{landsat_address}_BQA.TIF"
with rasterio.open(bqa) as src:
ovr = src.overviews(1)
ovr_width = int(src.width / ovr[0])
ovr_height = int(src.height / ovr[0])
dst_affine, width, height = calculate_default_transform(
src.crs, "epsg:3857", ovr_width, ovr_height, *src.bounds)
meta = {
"driver": "GTiff",
"count": 3,
"dtype": np.uint8,
"nodata": 0,
"height": height,
"width": width,
"compress": "DEFLATE",
"crs": "epsg:3857",
"transform": dst_affine,
}
outpath = f"/tmp/{scene}.tif"
with rasterio.open(outpath, "w", **meta) as dataset:
sun_elev = meta_data["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"]
for idx, b in enumerate(bands):
with rasterio.open(f"{landsat_address}_B{b}.TIF") as src:
with WarpedVRT(
src,
dst_crs="EPSG:3857",
resampling=Resampling.bilinear,
src_nodata=0,
dst_nodata=0,
) as vrt:
matrix = vrt.read(indexes=1, out_shape=(height, width))
multi_reflect = meta_data["RADIOMETRIC_RESCALING"][
f"REFLECTANCE_MULT_BAND_{b}"]
add_reflect = meta_data["RADIOMETRIC_RESCALING"][
f"REFLECTANCE_ADD_BAND_{b}"]
matrix = (reflectance(
matrix, multi_reflect, add_reflect, sun_elev, src_nodata=0)
* 10000)
minref = (meta_data["MIN_MAX_REFLECTANCE"]
[f"REFLECTANCE_MINIMUM_BAND_{b}"] * 10000)
maxref = (meta_data["MIN_MAX_REFLECTANCE"]
[f"REFLECTANCE_MAXIMUM_BAND_{b}"] * 10000)
matrix = np.where(
matrix > 0,
linear_rescale(matrix,
in_range=[int(minref),
int(maxref)],
out_range=[1, 255]),
0,
).astype(np.uint8)
mask = np.ma.masked_values(matrix, 0)
s = np.ma.notmasked_contiguous(mask)
matrix = matrix.ravel()
for sl in s:
matrix[sl.start:sl.start + 5] = 0
matrix[sl.stop - 5:sl.stop] = 0
matrix = matrix.reshape((height, width))
dataset.write(matrix, indexes=idx + 1)
return outpath
except:
return None
def tile(scene, tile_z, tile_x, tile_y, tileformat):
if tileformat == 'jpg':
tileformat = 'jpeg'
query_args = APP.current_request.query_params
query_args = query_args if isinstance(query_args, dict) else {}
bands = query_args.get('rgb', '4,3,2')
label_id = -1
if bands == '12,12,12':
label_id = 0
bands = '4,3,2'
elif bands == '13,13,13':
label_id = 1
bands = '4,3,2'
elif bands == '14,14,14':
label_id = 2
bands = '4,3,2'
elif bands == '15,15,15':
label_id = 3
bands = '4,3,2'
bands = tuple(re.findall(r'\d+', bands))
histoCut = query_args.get('histo', ';'.join(['0,16000'] * len(bands)))
histoCut = re.findall(r'\d+,\d+', histoCut)
histoCut = list(map(lambda x: list(map(int, x.split(','))), histoCut))
if len(bands) != len(histoCut):
raise LandsatTilerError(
'The number of bands doesn\'t match the number of histogramm values'
)
tilesize = query_args.get('tile', 256)
tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize
pan = True if query_args.get('pan') else False
tile, mask = landsat8.tile(scene,
tile_x,
tile_y,
tile_z,
bands,
pan=pan,
tilesize=tilesize)
rtile = np.zeros((len(bands), tilesize, tilesize), dtype=np.uint8)
for bdx in range(len(bands)):
rtile[bdx] = np.where(
mask,
linear_rescale(tile[bdx],
in_range=histoCut[bdx],
out_range=[0, 255]), 0)
img = array_to_img(rtile, mask=mask)
str_img = b64_encode_img(img, tileformat)
if label_id != -1:
data = {
'label_id': label_id,
'img': str_img,
}
r = requests.post(f"http://kursach.ngrok.io/getmask/", json=data)
str_img = r.json()['mask']
img = Image.open(io.BytesIO(base64.decodebytes(eval(f"b'{str_img}'"))))
img_np = np.asarray(img).T
img = array_to_img(img_np, mask=mask)
str_img = b64_encode_img(img, tileformat)
return ('OK', f'image/{tileformat}', str_img)
if __name__ == '__main__':
tile, mask = main.tile("/tmp/SRTM_cogeo.tif",
62,
44,
7,
indexes=(1),
tilesize=512)
scale = [(0, 1000)]
nbands = tile.shape[0]
for bdx in range(nbands):
tile[bdx] = numpy.where(
mask,
utils.linear_rescale(tile[bdx],
in_range=scale[bdx],
out_range=[0, 255]),
0,
)
tile = tile.astype(numpy.uint8)
pil_cmap = utils.get_colormap(name="schwarzwald")
gdal_cmap = numpy.array(
list(chunks(utils.get_colormap(name="schwarzwald"), 3)))
def _ti(ty, img_format, number, repeat):
t = timeit.Timer(f"array_to_img_{ty}('{img_format}')",
setup=f"from __main__ import array_to_img_{ty}")
times = t.repeat(repeat=repeat, number=number)
return min(times), max(times), sum(times)
def apply(recipes, pixels, expand, source=None):
data = pixels.data
colormap = pixels.colormap
if np.issubdtype(data.dtype, np.floating):
dtype_min = np.finfo(data.dtype).min
dtype_max = np.finfo(data.dtype).max
else:
dtype_min = np.iinfo(data.dtype).min
dtype_max = np.iinfo(data.dtype).max
if data.shape[0] == 1:
if expand and colormap:
# create a lookup table from the source's color map
lut = make_colormap(colormap)
# stash the mask
mask = data.mask
# apply the color map
data = lut[data[0], :]
# re-shape to match band-style
data = np.ma.transpose(data, [2, 0, 1])
# re-apply the mask, merging it with pixels that were masked by the color map
data.mask = data.mask | mask
colormap = None
if "landsat8" in recipes:
LOG.info("Applying landsat 8 recipe")
out = np.ma.empty(shape=(data.shape), dtype=np.float32)
for bdx, source_band in enumerate((4, 3, 2)):
sun_elev = source.meta["L1_METADATA_FILE"]["IMAGE_ATTRIBUTES"][
"SUN_ELEVATION"
]
multi_reflect = source.meta["L1_METADATA_FILE"][
"RADIOMETRIC_RESCALING"
].get(
"REFLECTANCE_MULT_BAND_{}".format(source_band)
)
add_reflect = source.meta["L1_METADATA_FILE"]["RADIOMETRIC_RESCALING"].get(
"REFLECTANCE_ADD_BAND_{}".format(source_band)
)
min_val = source.meta.get("values", {}).get(str(source_band), {}).get(
"min", dtype_min
)
max_val = source.meta.get("values", {}).get(str(source_band), {}).get(
"max", dtype_max
)
band_data = 10000 * reflectance.reflectance(
data[bdx], multi_reflect, add_reflect, sun_elev, src_nodata=0
)
# calculate local min/max as fallbacks
if (
min_val == dtype_min
and max_val == dtype_max
and len(data.compressed()) > 0
):
local_min, local_max = np.percentile(band_data.compressed(), (2, 98))
min_val = max(min_val, local_min)
max_val = min(max_val, local_max)
out[bdx] = utils.linear_rescale(
band_data, in_range=(min_val, max_val), out_range=(0.0, 1.0)
)
data = out
if "imagery" in recipes:
LOG.info("Applying imagery recipe")
if "rgb_bands" in recipes:
data = np.ma.array(
[data[i - 1] for i in recipes["rgb_bands"] if data.shape[0] >= i]
)
elif "expr" in recipes:
num_bands = data.shape[0]
expressions = recipes["expr"].split(",")
band_names = ["b" + str(i) for i in range(1, num_bands + 1)]
local_dict = dict(zip(band_names, data.data))
old_mask = data.mask
old_mask_shape = old_mask.shape
new_mask_shape = (len(expressions), old_mask_shape[1], old_mask_shape[2])
new_mask = np.ndarray(new_mask_shape, dtype=np.bool)
for (expression_index, expression) in enumerate(expressions):
# identify bands used in expression
band_indexes = [int(i) - 1 for i in re.findall(r"(?<=b)(\d{1,2})", expression)]
old_band_masks = [old_mask[i] for i in band_indexes]
new_mask[expression_index] = np.logical_and.reduce(old_band_masks)
data = np.ma.array([
np.nan_to_num(ne.evaluate(expression.strip(), local_dict=local_dict))
for expression in expressions
], mask=new_mask)
elif data.shape[0] > 3:
# alpha(?) band (and beyond) present; drop it (them)
# TODO use band 4 as an alpha channel if colorinterp == alpha instead
# TODO re-order channels if BGR (whatever colorinterp says)
data = data[0:3]
if "linear_stretch" in recipes:
# Added by Tushar Shukla
# Add envi like property here to generate envi like browse tiles
if recipes["linear_stretch"] == "global":
data = utils.linear_rescale(
data, in_range=(np.min(data), np.max(data)), out_range=(0.0, 1.0)
)
elif recipes["linear_stretch"] == "per_band":
out = np.ma.empty(shape=(data.shape), dtype=np.float32)
for band in range(0, data.shape[0]):
min_val = source.meta.get("values", {}).get(band, {}).get(
"min", np.min(data[band])
)
max_val = source.meta.get("values", {}).get(band, {}).get(
"max", np.max(data[band])
)
out[band] = utils.linear_rescale(
data[band], in_range=(min_val, max_val), out_range=(0.0, 1.0)
)
data = out
else:
# rescale after reducing and before increasing dimensionality
if data.dtype != np.uint8:
# rescale non-8-bit sources (assuming that they're raw sensor
# data) and normalize to 0..1
out = np.ma.empty(shape=(data.shape), dtype=np.float32)
for band in range(0, data.shape[0]):
min_val = source.meta.get("values", {}).get(band, {}).get(
"min", dtype_min
)
max_val = source.meta.get("values", {}).get(band, {}).get(
"max", dtype_max
)
if (
min_val == dtype_min
and max_val == dtype_max
and len(data.compressed()) > 0
):
local_min, local_max = np.percentile(
data[band].compressed(), (2, 98)
)
min_val = max(min_val, local_min)
max_val = min(max_val, local_max)
out[band] = utils.linear_rescale(
data[band], in_range=(min_val, max_val), out_range=(0.0, 1.0)
)
data = out
if not np.issubdtype(data.dtype, np.floating):
# normalize to 0..1 based on the range of the source type (only
# for int*s)
data = data.astype(np.float32) / dtype_max
if data.shape[0] == 1:
# likely greyscale image; use the same band on all channels
data = np.ma.array([data[0], data[0], data[0]])
return PixelCollection(data, pixels.bounds, None, colormap)
def tile(sceneid,
tile_x,
tile_y,
tile_z,
rgb=(4, 3, 2),
r_bds=(0, 16000),
g_bds=(0, 16000),
b_bds=(0, 16000),
tilesize=256,
pan=False):
"""Create mercator tile from Landsat-8 data and encodes it in base64.
Attributes
----------
sceneid : str
Landsat sceneid. For scenes after May 2017,
sceneid have to be LANDSAT_PRODUCT_ID.
tile_x : int
Mercator tile X index.
tile_y : int
Mercator tile Y index.
tile_z : int
Mercator tile ZOOM level.
rgb : tuple, int, optional (default: (4, 3, 2))
Bands index for the RGB combination.
r_bds : tuple, int, optional (default: (0, 16000))
First band (red) DN min and max values (DN * 10,000)
used for the linear rescaling.
g_bds : tuple, int, optional (default: (0, 16000))
Second band (green) DN min and max values (DN * 10,000)
used for the linear rescaling.
b_bds : tuple, int, optional (default: (0, 16000))
Third band (blue) DN min and max values (DN * 10,000)
used for the linear rescaling.
tilesize : int, optional (default: 256)
Output image size.
pan : boolean, optional (default: False)
If True, apply pan-sharpening.
Returns
-------
out : numpy ndarray (type: uint8)
"""
scene_params = utils.landsat_parse_scene_id(sceneid)
meta_data = utils.landsat_get_mtl(sceneid).get('L1_METADATA_FILE')
landsat_address = '{}/{}'.format(LANDSAT_BUCKET, scene_params['key'])
wgs_bounds = toa_utils._get_bounds_from_metadata(
meta_data['PRODUCT_METADATA'])
if not utils.tile_exists(wgs_bounds, tile_z, tile_x, tile_y):
raise TileOutsideBounds('Tile {}/{}/{} is outside image bounds'.format(
tile_z, tile_x, tile_y))
mercator_tile = mercantile.Tile(x=tile_x, y=tile_y, z=tile_z)
tile_bounds = mercantile.xy_bounds(mercator_tile)
# define a list of bands Min and Max Values (from input)
histo_cuts = dict(zip(rgb, [r_bds, g_bds, b_bds]))
ms_tile_size = int(tilesize / 2) if pan else tilesize
addresses = ['{}_B{}.TIF'.format(landsat_address, band) for band in rgb]
_tiler = partial(utils.tile_band_worker,
bounds=tile_bounds,
tilesize=ms_tile_size)
with futures.ThreadPoolExecutor(max_workers=3) as executor:
out = np.stack(list(executor.map(_tiler, addresses)))
if pan:
pan_address = '{}_B8.TIF'.format(landsat_address)
matrix_pan = utils.tile_band_worker(pan_address, tile_bounds,
tilesize)
w, s, e, n = tile_bounds
pan_transform = transform.from_bounds(w, s, e, n, tilesize,
tilesize)
vis_transform = pan_transform * Affine.scale(2.)
out = pansharpen(out,
vis_transform,
matrix_pan,
pan_transform,
np.int16,
'EPSG:3857',
'EPSG:3857',
0.2,
method='Brovey',
src_nodata=0)
sun_elev = meta_data['IMAGE_ATTRIBUTES']['SUN_ELEVATION']
for bdx, band in enumerate(rgb):
multi_reflect = meta_data['RADIOMETRIC_RESCALING'].get(
'REFLECTANCE_MULT_BAND_{}'.format(band))
add_reflect = meta_data['RADIOMETRIC_RESCALING'].get(
'REFLECTANCE_ADD_BAND_{}'.format(band))
out[bdx] = 10000 * reflectance.reflectance(
out[bdx], multi_reflect, add_reflect, sun_elev, src_nodata=0)
out[bdx] = np.where(
out[bdx] > 0,
utils.linear_rescale(out[bdx],
in_range=histo_cuts.get(band),
out_range=[1, 255]), 0)
return out.astype(np.uint8)