def rescale_tile(tile, mask, rescale=None): if rescale: try: rescale_arr = list(map(float, rescale.split(","))) except ValueError: raise exceptions.ValidationError("Invalid rescale value") rescale_arr = list(_chunks(rescale_arr, 2)) if len(rescale_arr) != tile.shape[0]: rescale_arr = ((rescale_arr[0]), ) * tile.shape[0] for bdx in range(tile.shape[0]): if mask is not None: tile[bdx] = np.where( mask, linear_rescale(tile[bdx], in_range=rescale_arr[bdx], out_range=[0, 255]), 0, ) else: tile[bdx] = linear_rescale(tile[bdx], in_range=rescale_arr[bdx], out_range=[0, 255]) tile = tile.astype(np.uint8) return tile, mask
def post_process_tile( tile: numpy.ndarray, mask: numpy.ndarray, rescale: str = None, color_formula: str = None, ) -> Tuple[numpy.ndarray, numpy.ndarray]: """Tile data post processing.""" if rescale: rescale_arr = (tuple(map(float, rescale.split(","))), ) * tile.shape[0] for bdx in range(tile.shape[0]): tile[bdx] = numpy.where( mask, linear_rescale(tile[bdx], in_range=rescale_arr[bdx], out_range=[0, 255]), 0, ) tile = tile.astype(numpy.uint8) if color_formula: if issubclass(tile.dtype.type, numpy.floating): tile = tile.astype(numpy.int16) # make sure one last time we don't have # negative value before applying color formula tile[tile < 0] = 0 for ops in parse_operations(color_formula): tile = scale_dtype(ops(to_math_type(tile)), numpy.uint8) return tile
def tile(scene, tile_z, tile_x, tile_y, tileformat): """Handle tile requests """ if tileformat == 'jpg': tileformat = 'jpeg' query_args = APP.current_request.query_params query_args = query_args if isinstance(query_args, dict) else {} bands = query_args.get('rgb', '4,3,2') bands = tuple(re.findall(r'\d+', bands)) histoCut = query_args.get('histo', ';'.join(['0,16000'] * len(bands))) histoCut = re.findall(r'\d+,\d+', histoCut) histoCut = list(map(lambda x: list(map(int, x.split(','))), histoCut)) if len(bands) != len(histoCut): raise LandsatTilerError('The number of bands doesn\'t match the number of histogramm values') tilesize = query_args.get('tile', 256) tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize pan = True if query_args.get('pan') else False tile, mask = landsat8.tile(scene, tile_x, tile_y, tile_z, bands, pan=pan, tilesize=tilesize) rtile = np.zeros((len(bands), tilesize, tilesize), dtype=np.uint8) for bdx in range(len(bands)): rtile[bdx] = np.where(mask, linear_rescale(tile[bdx], in_range=histoCut[bdx], out_range=[0, 255]), 0) img = array_to_img(rtile, mask=mask) str_img = b64_encode_img(img, tileformat) return ('OK', f'image/{tileformat}', str_img)
def ratio(scene, tile_z, tile_x, tile_y, tileformat): """Handle processing requests """ if tileformat == 'jpg': tileformat = 'jpeg' query_args = APP.current_request.query_params query_args = query_args if isinstance(query_args, dict) else {} ratio_value = query_args['ratio'] range_value = query_args.get('range', [-1, 1]) tilesize = query_args.get('tile', 256) tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize tile, mask = expression(scene, tile_x, tile_y, tile_z, ratio_value, tilesize=tilesize) if len(tile.shape) == 2: tile = np.expand_dims(tile, axis=0) rtile = np.where( mask, linear_rescale(tile, in_range=range_value, out_range=[0, 255]), 0).astype(np.uint8) img = array_to_img(rtile, color_map=get_colormap(name='cfastie'), mask=mask) str_img = b64_encode_img(img, tileformat) return ('OK', f'image/{tileformat}', str_img)
def gdal( input, tile, ext="png", bidx=(1, 2, 3), colormap=None, scale=None, save=None, ): """Handle tile requests.""" tile_z, tile_x, tile_y = list(map(int, tile.split('-'))) tile, mask = main.tile( input, tile_x, tile_y, tile_z, indexes=bidx, tilesize=512 ) if scale: nbands = tile.shape[0] for bdx in range(nbands): tile[bdx] = numpy.where( mask, utils.linear_rescale(tile[bdx], in_range=scale[bdx], out_range=[0, 255]), 0, ) tile = tile.astype(numpy.uint8) if colormap: colormap = numpy.array(list(chunks(utils.get_colormap(name=colormap), 3))) buffer = array_to_img_rasterio(tile, mask, img_format=ext, color_map=colormap) if save: with open(f"{tile_x}-{tile_y}-{tile_z}_gdal.{ext}", "wb") as f: f.write(buffer)
def _postprocess( tile: numpy.ndarray, mask: numpy.ndarray, tilesize: int, rescale: str = None, color_formula: str = None, ) -> Tuple[numpy.ndarray, numpy.ndarray]: if tile is None: # Return empty tile tile = numpy.zeros((1, tilesize, tilesize), dtype=numpy.uint8) mask = numpy.zeros((tilesize, tilesize), dtype=numpy.uint8) else: if rescale: rescale_arr = (tuple(map(float, rescale.split(","))), ) * tile.shape[0] for bdx in range(tile.shape[0]): tile[bdx] = numpy.where( mask, linear_rescale(tile[bdx], in_range=rescale_arr[bdx], out_range=[0, 255]), 0, ) tile = tile.astype(numpy.uint8) if color_formula: # make sure one last time we don't have # negative value before applying color formula tile[tile < 0] = 0 for ops in parse_operations(color_formula): tile = scale_dtype(ops(to_math_type(tile)), numpy.uint8) return tile, mask
def _postprocess( tile: numpy.ndarray, mask: numpy.ndarray, rescale: str = None, color_formula: str = None, ) -> Tuple[numpy.ndarray, numpy.ndarray]: """Post-process tile data.""" if rescale: rescale_arr = list(map(float, rescale.split(","))) rescale_arr = list(_chunks(rescale_arr, 2)) if len(rescale_arr) != tile.shape[0]: rescale_arr = ((rescale_arr[0]),) * tile.shape[0] for bdx in range(tile.shape[0]): tile[bdx] = numpy.where( mask, linear_rescale( tile[bdx], in_range=rescale_arr[bdx], out_range=[0, 255] ), 0, ) tile = tile.astype(numpy.uint8) if color_formula: # make sure one last time we don't have # negative value before applying color formula tile[tile < 0] = 0 for ops in parse_operations(color_formula): tile = scale_dtype(ops(to_math_type(tile)), numpy.uint8) return tile, mask
def pil( input, tile, ext="png", bidx=(1, 2, 3), colormap=None, scale=None, save=None, ): """Handle tile requests.""" tile_z, tile_x, tile_y = list(map(int, tile.split('-'))) tile, mask = main.tile( input, tile_x, tile_y, tile_z, indexes=bidx, tilesize=512 ) if scale: nbands = tile.shape[0] for bdx in range(nbands): tile[bdx] = numpy.where( mask, utils.linear_rescale(tile[bdx], in_range=scale[bdx], out_range=[0, 255]), 0, ) tile = tile.astype(numpy.uint8) if colormap: colormap = utils.get_colormap(name=colormap) img = utils.array_to_img(tile, mask=mask, color_map=colormap) img_format = "JPEG2000" if ext == "jp2" else ext buffer = img_to_buffer(img, img_format) if save: with open(f"{tile_x}-{tile_y}-{tile_z}_pil.{ext}", "wb") as f: f.write(buffer)
def test_linear_rescale_valid(): """Should work as expected (read data band).""" data = np.zeros((1, 1), dtype=np.int16) + 1000 expected_value = np.zeros((1, 1), dtype=np.int16) + 25.5 assert ( utils.linear_rescale(data, in_range=(0, 10000), out_range=(0, 255)) == expected_value )
def area( scene, bbox, expression, expression_range=[-1, 1], bbox_crs="epsg:4326", out_crs="epsg:3857", ): """Area handler.""" max_img_size = 512 bands = tuple(set(re.findall(r"b(?P<bands>[0-9]{1,2})", expression))) scene_params = sentinel_parse_scene_id(scene) sentinel_address = f'{SENTINEL_BUCKET}/{scene_params["key"]}' scene_info = sentinel2_get_info(os.path.basename(SENTINEL_BUCKET), scene_params["key"], request_pays=True) addresses = [f"{sentinel_address}/B{band}.jp2" for band in bands] _worker = partial( get_area, bbox=bbox, max_img_size=max_img_size, bbox_crs=bbox_crs, out_crs=out_crs, ) with futures.ThreadPoolExecutor(max_workers=3) as executor: data = np.concatenate(list(executor.map(_worker, addresses))) if not np.any(data): raise Exception("No valid data in array") mask = np.all(data != 0, axis=0).astype(np.uint8) * 255 ctx = {} for bdx, b in enumerate(bands): ctx["b{}".format(b)] = data[bdx] ratio = np.nan_to_num(ne.evaluate(expression, local_dict=ctx)) ratio = np.where( mask, linear_rescale(ratio, in_range=expression_range, out_range=[0, 255]), 0).astype(np.uint8) img = array_to_img(ratio, mask, get_colormap(name="cfastie")) ndvi = b64_encode_img(img, "jpeg") date = (scene_params["acquisitionYear"] + "-" + scene_params["acquisitionMonth"] + "-" + scene_params["acquisitionDay"]) return { "ndvi": ndvi, "date": date, "sat": scene_info["sat"], "scene": scene, "cloud": scene_info["cloud_coverage"], }
def process(arr, skip_rescale=False, skip_alpha=False, skip_type=False): if not skip_rescale and rescale is not None: arr = linear_rescale(arr, in_range=rescale) if not skip_alpha and not with_alpha: arr[mask==0] = jpg_background if not skip_type and rgb and arr.dtype != np.uint8: arr = arr.astype(np.uint8) return arr
def area( scene, bbox, expression, expression_range=[-1, 1], bbox_crs="epsg:4326", out_crs="epsg:3857", ): """Area handler.""" max_img_size = 512 bands = tuple(set(re.findall(r"b(?P<bands>[0-9]{1,2})", expression))) scene_params = landsat_parse_scene_id(scene) meta_data = landsat_get_mtl(scene).get("L1_METADATA_FILE") landsat_address = f'{LANDSAT_BUCKET}/{scene_params["key"]}' def worker(band): """Worker.""" address = f"{landsat_address}_B{band}.TIF" sun_elev = meta_data["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"] multi_reflect = meta_data["RADIOMETRIC_RESCALING"][ f"REFLECTANCE_MULT_BAND_{band}" ] add_reflect = meta_data["RADIOMETRIC_RESCALING"][f"REFLECTANCE_ADD_BAND_{band}"] band = get_area( address, bbox, max_img_size=max_img_size, bbox_crs=bbox_crs, out_crs=out_crs ) return reflectance(band, multi_reflect, add_reflect, sun_elev, src_nodata=0) with futures.ThreadPoolExecutor(max_workers=3) as executor: data = np.concatenate(list(executor.map(worker, bands))) if not np.any(data): raise Exception("No valid data in array") mask = np.all(data != 0, axis=0).astype(np.uint8) * 255 ctx = {} for bdx, b in enumerate(bands): ctx["b{}".format(b)] = data[bdx] ratio = np.nan_to_num(ne.evaluate(expression, local_dict=ctx)) ratio = np.where( mask, linear_rescale(ratio, in_range=expression_range, out_range=[0, 255]), 0 ).astype(np.uint8) img = array_to_img(ratio, mask, get_colormap(name="cfastie")) ndvi = b64_encode_img(img, "jpeg") return { "ndvi": ndvi, "date": scene_params["date"], "scene": scene, "cloud": meta_data["IMAGE_ATTRIBUTES"]["CLOUD_COVER"], }
def tile(tile_z, tile_x, tile_y, tileformat): """Handle tile requests.""" if tileformat == 'jpg': tileformat = 'jpeg' query_args = APP.current_request.query_params query_args = query_args if isinstance(query_args, dict) else {} address = query_args['url'] ## Read Stac File with urllib.request.urlopen(address) as url: data = json.loads(url.read().decode()) asset_key = query_args.get('asset_key', 'raster') raster_address = data['assets'][asset_key]['href'] indexes = query_args.get('indexes') if indexes: indexes = tuple(int(s) for s in re.findall(r'\d+', indexes)) tilesize = query_args.get('tile', 512) tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize nodata = query_args.get('nodata') if nodata is not None: nodata = int(nodata) raster_address = data['assets'][asset_key]['href'] tile, mask = main.tile(raster_address, tile_x, tile_y, tile_z, indexes=indexes, tilesize=tilesize, nodata=nodata) linearStretch = query_args.get('linearStretch') if linearStretch is not None: if util.strtobool(linearStretch): tile = linear_rescale(tile, in_range=(np.min(tile), np.max(tile))) img = array_to_img(tile, mask=mask) str_img = b64_encode_img(img, tileformat) return ('OK', f'image/{tileformat}', str_img)
def tile( tile_z, tile_x, tile_y, tileformat, indexes=1, shadder=None, colormap=None, range=None, ): """Handle tile requests""" if tileformat == "jpg": tileformat = "jpeg" if colormap and shadder: return ("NOK", "text/plain", "Cannot pass shadder and colormap options") address = f"s3://elevation-tiles-prod/geotiff/{tile_z}/{tile_x}/{tile_y}.tif" data, mask = main.tile(address, tile_x, tile_y, tile_z, indexes=indexes, tilesize=512) if shadder: tileformat = "png" if shadder == "mapbox": tile = encoders.data_to_rgb(data[0], -10000, 1) elif shadder == "mapzen": tile = mapzen_elevation_rgb.data_to_rgb(data) else: return ("NOK", "text/plain", f"Invalid shadder mode: {shadder}") else: if not range: range = "0,10000" histoCut = list(map(int, range.split(','))) tile = numpy.where( mask, linear_rescale(data, in_range=histoCut, out_range=[0, 255]), 0) options = dict(mask=mask) if colormap: options.update(color_map=get_colormap(name=colormap)) img = array_to_img(tile, **options) return ("OK", f"image/{tileformat}", img_to_buffer(img, tileformat))
def band_worker(band, landsat_address, meta, bounds=None): """ """ address = f'{landsat_address}_B{band}.TIF' sun_elev = meta['IMAGE_ATTRIBUTES']['SUN_ELEVATION'] multi_reflect = meta['RADIOMETRIC_RESCALING'][ f'REFLECTANCE_MULT_BAND_{band}'] add_reflect = meta['RADIOMETRIC_RESCALING'][f'REFLECTANCE_ADD_BAND_{band}'] ovrSize = 1024 with rasterio.open(address) as src: if bounds: w, s, e, n = bounds with WarpedVRT(src, dst_crs='EPSG:3857', resampling=Resampling.bilinear, src_nodata=0, dst_nodata=0) as vrt: window = vrt.window(w, s, e, n, precision=21) matrix = vrt.read(window=window, boundless=True, resampling=Resampling.bilinear, out_shape=(ovrSize, ovrSize), indexes=1).astype(src.profile['dtype']) else: matrix = src.read(indexes=1, out_shape=(ovrSize, ovrSize), resampling=Resampling.bilinear).astype( src.profile['dtype']) matrix = reflectance(matrix, multi_reflect, add_reflect, sun_elev, src_nodata=0) imgRange = np.percentile(matrix[matrix > 0], (2, 98)).tolist() matrix = np.where( matrix > 0, linear_rescale(matrix, in_range=imgRange, out_range=[1, 255]), 0).astype(np.uint8) return matrix
def landsat_ratio(scene, tile_z, tile_x, tile_y, tileformat): """ Handle processing requests """ query_args = LANDSAT_APP.current_request.query_params query_args = query_args if isinstance(query_args, dict) else {} ratio_value = query_args.get('ratio', 'ndvi') if ratio_value not in RATIOS.keys(): raise LandsatTilerError('Invalid ratio: {}'.format(ratio_value)) equation = RATIOS[ratio_value]['eq'] band_names = list(set(re.findall('b[0-9]{1,2}', equation))) bands = tuple(map(lambda x: x.strip('b'), band_names)) tilesize = query_args.get('tile', 256) tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize tile = landsat8.tile(scene, tile_x, tile_y, tile_z, bands, tilesize=tilesize) for bdx, b in enumerate(band_names): globals()[b] = tile[bdx] tile = np.where( reduce(lambda x, y: x * y, [globals()[i] for i in band_names]) > 0, np.nan_to_num(ne.evaluate(equation)), -9999) range_val = equation = RATIOS[ratio_value]['rg'] rtile = np.where( tile != -9999, linear_rescale(tile, in_range=range_val, out_range=[1, 255]), 0).astype(np.uint8) tile = array_to_img(rtile, tileformat, color_map=get_colormap(name='cfastie')) if tileformat == 'jpg': tileformat = 'jpeg' return ('OK', f'image/{tileformat}', tile)
def tile(scene, tile_z, tile_x, tile_y, tileformat): """Handle tile requests.""" if tileformat == "jpg": tileformat = "jpeg" query_args = APP.current_request.query_params query_args = query_args if isinstance(query_args, dict) else {} bands = query_args.get("rgb", "04,03,02") bands = tuple(re.findall(r"[0-9A]{2}", bands)) histoCut = query_args.get("histo", "-".join(["0,16000"] * len(bands))) histoCut = re.findall(r"\d+,\d+", histoCut) histoCut = list(map(lambda x: list(map(int, x.split(","))), histoCut)) if len(bands) != len(histoCut): raise SentinelTilerError( "The number of bands doesn't match the number of histogramm values" ) tilesize = query_args.get("tile", 256) tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize tile, mask = sentinel2.tile(scene, tile_x, tile_y, tile_z, bands, tilesize=tilesize) rtile = np.zeros((len(bands), tilesize, tilesize), dtype=np.uint8) for bdx in range(len(bands)): rtile[bdx] = np.where( mask, linear_rescale(tile[bdx], in_range=histoCut[bdx], out_range=[0, 255]), 0, ) img = array_to_img(rtile, mask=mask) str_img = b64_encode_img(img, tileformat) return ("OK", f"image/{tileformat}", str_img)
def tile(scene, tile_z, tile_x, tile_y, tileformat): """Handle tile requests """ query_args = APP.current_request.query_params query_args = query_args if isinstance(query_args, dict) else {} bands = query_args.get('rgb', '04,03,02') bands = tuple(re.findall(r'[0-9A]{2}', bands)) histoCut = query_args.get('histo', '-'.join(['0,16000'] * len(bands))) histoCut = re.findall(r'\d+,\d+', histoCut) histoCut = list(map(lambda x: list(map(int, x.split(','))), histoCut)) if len(bands) != len(histoCut): raise SentinelTilerError( 'The number of bands doesn\'t match the number of histogramm values' ) tilesize = query_args.get('tile', 256) tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize tile, mask = sentinel2.tile(scene, tile_x, tile_y, tile_z, bands, tilesize=tilesize) rtile = np.zeros((len(bands), tilesize, tilesize), dtype=np.uint8) for bdx in range(len(bands)): rtile[bdx] = np.where( mask, linear_rescale(tile[bdx], in_range=histoCut[bdx], out_range=[0, 255]), 0) tile = array_to_img(rtile, tileformat, mask=mask) if tileformat == 'jpg': tileformat = 'jpeg' return ('OK', f'image/{tileformat}', tile)
def _get_tile(self, z, x, y, tileformat, color_ops=None): if tileformat == "jpg": tileformat = "jpeg" if not self.raster.tile_exists(z, x, y): raise web.HTTPError(404) data, mask = self.raster.read_tile(z, x, y) if len(data.shape) == 2: data = numpy.expand_dims(data, axis=0) if self.scale: nbands = data.shape[0] scale = self.scale if len(scale) != nbands: scale = scale * nbands for bdx in range(nbands): data[bdx] = numpy.where( mask, linear_rescale(data[bdx], in_range=scale[bdx], out_range=[0, 255]), 0, ) data = data.astype(numpy.uint8) if color_ops: data = self._apply_color_operations(data, color_ops) img = array_to_img(data, mask=mask, color_map=self.colormap) params = TileProfiles.get(tileformat) if tileformat == "jpeg": img = img.convert("RGB") sio = BytesIO() img.save(sio, tileformat.upper(), **params) sio.seek(0) return sio
def _get_tile(self, z, x, y, tileformat, color_ops=None): if tileformat == "jpg": tileformat = "jpeg" if not self.raster.tile_exists(z, x, y): raise web.HTTPError(404) data, mask = self.raster.read_tile(z, x, y) if len(data.shape) == 2: data = numpy.expand_dims(data, axis=0) if self.scale: nbands = data.shape[0] scale = self.scale if len(scale) != nbands: scale = scale * nbands for bdx in range(nbands): data[bdx] = numpy.where( mask, linear_rescale(data[bdx], in_range=scale[bdx], out_range=[0, 255]), 0, ) data = data.astype(numpy.uint8) if color_ops: data = self._apply_color_operations(data, color_ops) options = img_profiles.get(tileformat, {}) return BytesIO( array_to_image(data, mask=mask, color_map=self.colormap, img_format=tileformat, **options))
def _postprocess_tile(tile, mask, rescale=None, color_ops=None): """Tile data post-processing.""" if rescale: rescale = (tuple(map(float, rescale.split(","))), ) * tile.shape[0] for bdx in range(tile.shape[0]): tile[bdx] = numpy.where( mask, linear_rescale(tile[bdx], in_range=rescale[bdx], out_range=[0, 255]), 0, ) tile = tile.astype(numpy.uint8) if color_ops: # make sure one last time we don't have # negative value before applying color formula tile[tile < 0] = 0 for ops in parse_operations(color_ops): tile = scale_dtype(ops(to_math_type(tile)), numpy.uint8) return tile, mask
def tile(tile_z, tile_x, tile_y, tileformat): """Handle tile requests.""" if tileformat == 'jpg': tileformat = 'jpeg' query_args = APP.current_request.query_params query_args = query_args if isinstance(query_args, dict) else {} address = query_args['url'] indexes = query_args.get('indexes') if indexes: indexes = tuple(int(s) for s in re.findall(r'\d+', indexes)) tilesize = query_args.get('tile', 512) tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize nodata = query_args.get('nodata') if nodata is not None: nodata = int(nodata) tile, mask = main.tile(address, tile_x, tile_y, tile_z, indexes=indexes, tilesize=tilesize, nodata=nodata) linearStretch = query_args.get('linearStretch') if linearStretch is not None: if util.strtobool(linearStretch): tile = linear_rescale(tile, in_range=(np.min(tile), np.max(tile))) img = array_to_img(tile, mask=mask) str_img = b64_encode_img(img, tileformat) return ('OK', f'image/{tileformat}', str_img)
def _postprocess(tile, mask, tilesize, rescale=None, color_formula=None): if tile is None: # Return empty tile tile = numpy.zeros((1, tilesize, tilesize), dtype=numpy.uint8) mask = numpy.zeros((tilesize, tilesize), dtype=numpy.uint8) else: if rescale: rescale = (tuple(map(float, rescale.split(","))), ) * tile.shape[0] for bdx in range(tile.shape[0]): tile[bdx] = numpy.where( mask, linear_rescale(tile[bdx], in_range=rescale[bdx], out_range=[0, 255]), 0, ) tile = tile.astype(numpy.uint8) if color_formula: for ops in parse_operations(color_formula): tile = scale_dtype(ops(to_math_type(tile)), numpy.uint8) return tile, mask
def apply(recipes, pixels, source=None): data = pixels.data dtype_min = np.iinfo(data.dtype).min dtype_max = np.iinfo(data.dtype).max if "landsat8" in recipes: LOG.info("Applying landsat 8 recipe") out = np.ma.empty(shape=(data.shape), dtype=np.float32) for bdx, source_band in enumerate((4, 3, 2)): sun_elev = source.meta["L1_METADATA_FILE"]["IMAGE_ATTRIBUTES"][ "SUN_ELEVATION" ] multi_reflect = source.meta["L1_METADATA_FILE"][ "RADIOMETRIC_RESCALING" ].get( "REFLECTANCE_MULT_BAND_{}".format(source_band) ) add_reflect = source.meta["L1_METADATA_FILE"]["RADIOMETRIC_RESCALING"].get( "REFLECTANCE_ADD_BAND_{}".format(source_band) ) min_val = source.meta.get("values", {}).get(str(source_band), {}).get( "min", dtype_min ) max_val = source.meta.get("values", {}).get(str(source_band), {}).get( "max", dtype_max ) band_data = 10000 * reflectance.reflectance( data[bdx], multi_reflect, add_reflect, sun_elev, src_nodata=0 ) # calculate local min/max as fallbacks if ( min_val == dtype_min and max_val == dtype_max and len(data.compressed()) > 0 ): local_min, local_max = np.percentile(band_data.compressed(), (2, 98)) min_val = max(min_val, local_min) max_val = min(max_val, local_max) out[bdx] = np.ma.where( band_data > 0, utils.linear_rescale( band_data, in_range=[min_val, max_val], out_range=[0, 1] ), 0, ) data = out if "imagery" in recipes: LOG.info("Applying imagery recipe") if "rgb_bands" in recipes: data = np.ma.array([data[i - 1] for i in recipes["rgb_bands"]]) if data.shape[0] > 3: # alpha band (and beyond) present; drop it (them) # TODO use band 4 as a mask instead data = data[0:3] if "linear_stretch" in recipes: if recipes["linear_stretch"] == "global": data = utils.linear_rescale( data, in_range=(np.min(data), np.max(data)), out_range=(dtype_min, dtype_max), ) elif recipes["linear_stretch"] == "per_band": for band in xrange(0, data.shape[0]): min_val = source.meta.get("values", {}).get(band, {}).get( "min", np.min(data[band]) ) max_val = source.meta.get("values", {}).get(band, {}).get( "max", np.max(data[band]) ) data[band] = np.ma.where( data[band] > 0, utils.linear_rescale( data[band], in_range=(min_val, max_val), out_range=(dtype_min, dtype_max), ), 0, ) else: # rescale after reducing and before increasing dimensionality if data.dtype != np.uint8 and not np.issubdtype(data.dtype, float): # rescale non-8-bit sources (assuming that they're raw sensor data) for band in xrange(0, data.shape[0]): min_val = source.meta.get("values", {}).get(band, {}).get( "min", dtype_min ) max_val = source.meta.get("values", {}).get(band, {}).get( "max", dtype_max ) if ( min_val == dtype_min and max_val == dtype_max and len(data.compressed()) > 0 ): local_min, local_max = np.percentile( data[band].compressed(), (2, 98) ) min_val = max(min_val, local_min) max_val = min(max_val, local_max) data[band] = np.ma.where( data[band] > 0, utils.linear_rescale( data[band], in_range=(min_val, max_val), out_range=(dtype_min, dtype_max), ), 0, ) if data.shape[0] == 1: # likely greyscale image; use the same band on all channels data = np.ma.array([data[0], data[0], data[0]]) # normalize to 0..1 based on the range of the source type (only # for int*s) if not np.issubdtype(data.dtype, float): data = data.astype(np.float32) / np.iinfo(data.dtype).max return PixelCollection(data, pixels.bounds)
def create( scene, bands=None, expression=None, expression_range=[-1, 1], img_format="jpeg", ovrSize=512, ): """Handler.""" if img_format not in ["png", "jpeg"]: raise UserWarning(f"Invalid {img_format} extension") if not expression and not bands: raise Exception("Expression or Bands must be provided") if bands: nb_bands = len(bands) if nb_bands != 3: raise Exception("RGB combination only") if expression: bands = tuple(set(re.findall(r"b(?P<bands>[0-9]{1,2})", expression))) rgb = expression.split(",") nb_bands = len(rgb) scene_params = landsat_parse_scene_id(scene) meta_data = landsat_get_mtl(scene).get("L1_METADATA_FILE") landsat_address = f'{LANDSAT_BUCKET}/{scene_params["key"]}' _worker = partial(worker, landsat_address=landsat_address, meta=meta_data, ovr_size=ovrSize) with futures.ThreadPoolExecutor(max_workers=3) as executor: data = np.concatenate(list(executor.map(_worker, bands))) mask = np.all(data != 0, axis=0).astype(np.uint8) * 255 if expression: ctx = {} for bdx, b in enumerate(bands): ctx["b{}".format(b)] = data[bdx] data = np.array([ np.nan_to_num(ne.evaluate(bloc.strip(), local_dict=ctx)) for bloc in rgb ]) for band in range(data.shape[0]): imgRange = (expression_range if expression else np.percentile( data[band][mask > 0], (2, 98)).tolist()) data[band] = np.where( mask, linear_rescale(data[band], in_range=imgRange, out_range=[0, 255]), 0, ) data = data.squeeze() colormap = None if len(data.shape) >= 3 else get_colormap(name="cfastie") img = array_to_img(data, mask, colormap) return b64_encode_img(img, img_format)
def worker(scene, bands): """Worker.""" try: scene_params = landsat_parse_scene_id(scene) meta_data = landsat_get_mtl(scene).get("L1_METADATA_FILE") landsat_address = f'{LANDSAT_BUCKET}/{scene_params["key"]}' bqa = f"{landsat_address}_BQA.TIF" with rasterio.open(bqa) as src: ovr = src.overviews(1) ovr_width = int(src.width / ovr[0]) ovr_height = int(src.height / ovr[0]) dst_affine, width, height = calculate_default_transform( src.crs, "epsg:3857", ovr_width, ovr_height, *src.bounds) meta = { "driver": "GTiff", "count": 3, "dtype": np.uint8, "nodata": 0, "height": height, "width": width, "compress": "DEFLATE", "crs": "epsg:3857", "transform": dst_affine, } outpath = f"/tmp/{scene}.tif" with rasterio.open(outpath, "w", **meta) as dataset: sun_elev = meta_data["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"] for idx, b in enumerate(bands): with rasterio.open(f"{landsat_address}_B{b}.TIF") as src: with WarpedVRT( src, dst_crs="EPSG:3857", resampling=Resampling.bilinear, src_nodata=0, dst_nodata=0, ) as vrt: matrix = vrt.read(indexes=1, out_shape=(height, width)) multi_reflect = meta_data["RADIOMETRIC_RESCALING"][ f"REFLECTANCE_MULT_BAND_{b}"] add_reflect = meta_data["RADIOMETRIC_RESCALING"][ f"REFLECTANCE_ADD_BAND_{b}"] matrix = (reflectance( matrix, multi_reflect, add_reflect, sun_elev, src_nodata=0) * 10000) minref = (meta_data["MIN_MAX_REFLECTANCE"] [f"REFLECTANCE_MINIMUM_BAND_{b}"] * 10000) maxref = (meta_data["MIN_MAX_REFLECTANCE"] [f"REFLECTANCE_MAXIMUM_BAND_{b}"] * 10000) matrix = np.where( matrix > 0, linear_rescale(matrix, in_range=[int(minref), int(maxref)], out_range=[1, 255]), 0, ).astype(np.uint8) mask = np.ma.masked_values(matrix, 0) s = np.ma.notmasked_contiguous(mask) matrix = matrix.ravel() for sl in s: matrix[sl.start:sl.start + 5] = 0 matrix[sl.stop - 5:sl.stop] = 0 matrix = matrix.reshape((height, width)) dataset.write(matrix, indexes=idx + 1) return outpath except: return None
def tile(scene, tile_z, tile_x, tile_y, tileformat): if tileformat == 'jpg': tileformat = 'jpeg' query_args = APP.current_request.query_params query_args = query_args if isinstance(query_args, dict) else {} bands = query_args.get('rgb', '4,3,2') label_id = -1 if bands == '12,12,12': label_id = 0 bands = '4,3,2' elif bands == '13,13,13': label_id = 1 bands = '4,3,2' elif bands == '14,14,14': label_id = 2 bands = '4,3,2' elif bands == '15,15,15': label_id = 3 bands = '4,3,2' bands = tuple(re.findall(r'\d+', bands)) histoCut = query_args.get('histo', ';'.join(['0,16000'] * len(bands))) histoCut = re.findall(r'\d+,\d+', histoCut) histoCut = list(map(lambda x: list(map(int, x.split(','))), histoCut)) if len(bands) != len(histoCut): raise LandsatTilerError( 'The number of bands doesn\'t match the number of histogramm values' ) tilesize = query_args.get('tile', 256) tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize pan = True if query_args.get('pan') else False tile, mask = landsat8.tile(scene, tile_x, tile_y, tile_z, bands, pan=pan, tilesize=tilesize) rtile = np.zeros((len(bands), tilesize, tilesize), dtype=np.uint8) for bdx in range(len(bands)): rtile[bdx] = np.where( mask, linear_rescale(tile[bdx], in_range=histoCut[bdx], out_range=[0, 255]), 0) img = array_to_img(rtile, mask=mask) str_img = b64_encode_img(img, tileformat) if label_id != -1: data = { 'label_id': label_id, 'img': str_img, } r = requests.post(f"http://kursach.ngrok.io/getmask/", json=data) str_img = r.json()['mask'] img = Image.open(io.BytesIO(base64.decodebytes(eval(f"b'{str_img}'")))) img_np = np.asarray(img).T img = array_to_img(img_np, mask=mask) str_img = b64_encode_img(img, tileformat) return ('OK', f'image/{tileformat}', str_img)
if __name__ == '__main__': tile, mask = main.tile("/tmp/SRTM_cogeo.tif", 62, 44, 7, indexes=(1), tilesize=512) scale = [(0, 1000)] nbands = tile.shape[0] for bdx in range(nbands): tile[bdx] = numpy.where( mask, utils.linear_rescale(tile[bdx], in_range=scale[bdx], out_range=[0, 255]), 0, ) tile = tile.astype(numpy.uint8) pil_cmap = utils.get_colormap(name="schwarzwald") gdal_cmap = numpy.array( list(chunks(utils.get_colormap(name="schwarzwald"), 3))) def _ti(ty, img_format, number, repeat): t = timeit.Timer(f"array_to_img_{ty}('{img_format}')", setup=f"from __main__ import array_to_img_{ty}") times = t.repeat(repeat=repeat, number=number) return min(times), max(times), sum(times)
def apply(recipes, pixels, expand, source=None): data = pixels.data colormap = pixels.colormap if np.issubdtype(data.dtype, np.floating): dtype_min = np.finfo(data.dtype).min dtype_max = np.finfo(data.dtype).max else: dtype_min = np.iinfo(data.dtype).min dtype_max = np.iinfo(data.dtype).max if data.shape[0] == 1: if expand and colormap: # create a lookup table from the source's color map lut = make_colormap(colormap) # stash the mask mask = data.mask # apply the color map data = lut[data[0], :] # re-shape to match band-style data = np.ma.transpose(data, [2, 0, 1]) # re-apply the mask, merging it with pixels that were masked by the color map data.mask = data.mask | mask colormap = None if "landsat8" in recipes: LOG.info("Applying landsat 8 recipe") out = np.ma.empty(shape=(data.shape), dtype=np.float32) for bdx, source_band in enumerate((4, 3, 2)): sun_elev = source.meta["L1_METADATA_FILE"]["IMAGE_ATTRIBUTES"][ "SUN_ELEVATION" ] multi_reflect = source.meta["L1_METADATA_FILE"][ "RADIOMETRIC_RESCALING" ].get( "REFLECTANCE_MULT_BAND_{}".format(source_band) ) add_reflect = source.meta["L1_METADATA_FILE"]["RADIOMETRIC_RESCALING"].get( "REFLECTANCE_ADD_BAND_{}".format(source_band) ) min_val = source.meta.get("values", {}).get(str(source_band), {}).get( "min", dtype_min ) max_val = source.meta.get("values", {}).get(str(source_band), {}).get( "max", dtype_max ) band_data = 10000 * reflectance.reflectance( data[bdx], multi_reflect, add_reflect, sun_elev, src_nodata=0 ) # calculate local min/max as fallbacks if ( min_val == dtype_min and max_val == dtype_max and len(data.compressed()) > 0 ): local_min, local_max = np.percentile(band_data.compressed(), (2, 98)) min_val = max(min_val, local_min) max_val = min(max_val, local_max) out[bdx] = utils.linear_rescale( band_data, in_range=(min_val, max_val), out_range=(0.0, 1.0) ) data = out if "imagery" in recipes: LOG.info("Applying imagery recipe") if "rgb_bands" in recipes: data = np.ma.array( [data[i - 1] for i in recipes["rgb_bands"] if data.shape[0] >= i] ) elif "expr" in recipes: num_bands = data.shape[0] expressions = recipes["expr"].split(",") band_names = ["b" + str(i) for i in range(1, num_bands + 1)] local_dict = dict(zip(band_names, data.data)) old_mask = data.mask old_mask_shape = old_mask.shape new_mask_shape = (len(expressions), old_mask_shape[1], old_mask_shape[2]) new_mask = np.ndarray(new_mask_shape, dtype=np.bool) for (expression_index, expression) in enumerate(expressions): # identify bands used in expression band_indexes = [int(i) - 1 for i in re.findall(r"(?<=b)(\d{1,2})", expression)] old_band_masks = [old_mask[i] for i in band_indexes] new_mask[expression_index] = np.logical_and.reduce(old_band_masks) data = np.ma.array([ np.nan_to_num(ne.evaluate(expression.strip(), local_dict=local_dict)) for expression in expressions ], mask=new_mask) elif data.shape[0] > 3: # alpha(?) band (and beyond) present; drop it (them) # TODO use band 4 as an alpha channel if colorinterp == alpha instead # TODO re-order channels if BGR (whatever colorinterp says) data = data[0:3] if "linear_stretch" in recipes: # Added by Tushar Shukla # Add envi like property here to generate envi like browse tiles if recipes["linear_stretch"] == "global": data = utils.linear_rescale( data, in_range=(np.min(data), np.max(data)), out_range=(0.0, 1.0) ) elif recipes["linear_stretch"] == "per_band": out = np.ma.empty(shape=(data.shape), dtype=np.float32) for band in range(0, data.shape[0]): min_val = source.meta.get("values", {}).get(band, {}).get( "min", np.min(data[band]) ) max_val = source.meta.get("values", {}).get(band, {}).get( "max", np.max(data[band]) ) out[band] = utils.linear_rescale( data[band], in_range=(min_val, max_val), out_range=(0.0, 1.0) ) data = out else: # rescale after reducing and before increasing dimensionality if data.dtype != np.uint8: # rescale non-8-bit sources (assuming that they're raw sensor # data) and normalize to 0..1 out = np.ma.empty(shape=(data.shape), dtype=np.float32) for band in range(0, data.shape[0]): min_val = source.meta.get("values", {}).get(band, {}).get( "min", dtype_min ) max_val = source.meta.get("values", {}).get(band, {}).get( "max", dtype_max ) if ( min_val == dtype_min and max_val == dtype_max and len(data.compressed()) > 0 ): local_min, local_max = np.percentile( data[band].compressed(), (2, 98) ) min_val = max(min_val, local_min) max_val = min(max_val, local_max) out[band] = utils.linear_rescale( data[band], in_range=(min_val, max_val), out_range=(0.0, 1.0) ) data = out if not np.issubdtype(data.dtype, np.floating): # normalize to 0..1 based on the range of the source type (only # for int*s) data = data.astype(np.float32) / dtype_max if data.shape[0] == 1: # likely greyscale image; use the same band on all channels data = np.ma.array([data[0], data[0], data[0]]) return PixelCollection(data, pixels.bounds, None, colormap)
def tile(sceneid, tile_x, tile_y, tile_z, rgb=(4, 3, 2), r_bds=(0, 16000), g_bds=(0, 16000), b_bds=(0, 16000), tilesize=256, pan=False): """Create mercator tile from Landsat-8 data and encodes it in base64. Attributes ---------- sceneid : str Landsat sceneid. For scenes after May 2017, sceneid have to be LANDSAT_PRODUCT_ID. tile_x : int Mercator tile X index. tile_y : int Mercator tile Y index. tile_z : int Mercator tile ZOOM level. rgb : tuple, int, optional (default: (4, 3, 2)) Bands index for the RGB combination. r_bds : tuple, int, optional (default: (0, 16000)) First band (red) DN min and max values (DN * 10,000) used for the linear rescaling. g_bds : tuple, int, optional (default: (0, 16000)) Second band (green) DN min and max values (DN * 10,000) used for the linear rescaling. b_bds : tuple, int, optional (default: (0, 16000)) Third band (blue) DN min and max values (DN * 10,000) used for the linear rescaling. tilesize : int, optional (default: 256) Output image size. pan : boolean, optional (default: False) If True, apply pan-sharpening. Returns ------- out : numpy ndarray (type: uint8) """ scene_params = utils.landsat_parse_scene_id(sceneid) meta_data = utils.landsat_get_mtl(sceneid).get('L1_METADATA_FILE') landsat_address = '{}/{}'.format(LANDSAT_BUCKET, scene_params['key']) wgs_bounds = toa_utils._get_bounds_from_metadata( meta_data['PRODUCT_METADATA']) if not utils.tile_exists(wgs_bounds, tile_z, tile_x, tile_y): raise TileOutsideBounds('Tile {}/{}/{} is outside image bounds'.format( tile_z, tile_x, tile_y)) mercator_tile = mercantile.Tile(x=tile_x, y=tile_y, z=tile_z) tile_bounds = mercantile.xy_bounds(mercator_tile) # define a list of bands Min and Max Values (from input) histo_cuts = dict(zip(rgb, [r_bds, g_bds, b_bds])) ms_tile_size = int(tilesize / 2) if pan else tilesize addresses = ['{}_B{}.TIF'.format(landsat_address, band) for band in rgb] _tiler = partial(utils.tile_band_worker, bounds=tile_bounds, tilesize=ms_tile_size) with futures.ThreadPoolExecutor(max_workers=3) as executor: out = np.stack(list(executor.map(_tiler, addresses))) if pan: pan_address = '{}_B8.TIF'.format(landsat_address) matrix_pan = utils.tile_band_worker(pan_address, tile_bounds, tilesize) w, s, e, n = tile_bounds pan_transform = transform.from_bounds(w, s, e, n, tilesize, tilesize) vis_transform = pan_transform * Affine.scale(2.) out = pansharpen(out, vis_transform, matrix_pan, pan_transform, np.int16, 'EPSG:3857', 'EPSG:3857', 0.2, method='Brovey', src_nodata=0) sun_elev = meta_data['IMAGE_ATTRIBUTES']['SUN_ELEVATION'] for bdx, band in enumerate(rgb): multi_reflect = meta_data['RADIOMETRIC_RESCALING'].get( 'REFLECTANCE_MULT_BAND_{}'.format(band)) add_reflect = meta_data['RADIOMETRIC_RESCALING'].get( 'REFLECTANCE_ADD_BAND_{}'.format(band)) out[bdx] = 10000 * reflectance.reflectance( out[bdx], multi_reflect, add_reflect, sun_elev, src_nodata=0) out[bdx] = np.where( out[bdx] > 0, utils.linear_rescale(out[bdx], in_range=histo_cuts.get(band), out_range=[1, 255]), 0) return out.astype(np.uint8)