def test_array_to_img_colormap(): """ Should work as expected """ arr = np.random.randint(0, 255, size=(1, 512, 512), dtype=np.uint8) tileformat = 'png' utils.array_to_img(arr, tileformat, color_map=utils.get_colormap())
def test_array_to_img_bands_colormap(): """ Should raise an error with invalid format """ arr = np.random.randint(0, 255, size=(3, 512, 512), dtype=np.uint8) tileformat = 'png' with pytest.raises(InvalidFormat): utils.array_to_img(arr, tileformat, color_map=True)
def test_array_to_img_invalid_format(): """ Should raise an error with invalid format """ arr = np.random.randint(0, 255, size=(1, 512, 512), dtype=np.uint8) tileformat = 'gif' with pytest.raises(InvalidFormat): utils.array_to_img(arr, tileformat)
def test_array_to_img_cast(): """ Should work as expected """ arr = np.random.randint(0, 255, size=(1, 512, 512), dtype=np.int16) assert utils.array_to_img(arr)
def test_array_to_img_invalid_format(): """Should raise an error with invalid format """ arr = np.random.randint(0, 255, size=(1, 512, 512), dtype=np.uint8) img = utils.array_to_img(arr) with pytest.raises(InvalidFormat): utils.b64_encode_img(img, 'gif')
def pil( input, tile, ext="png", bidx=(1, 2, 3), colormap=None, scale=None, save=None, ): """Handle tile requests.""" tile_z, tile_x, tile_y = list(map(int, tile.split('-'))) tile, mask = main.tile( input, tile_x, tile_y, tile_z, indexes=bidx, tilesize=512 ) if scale: nbands = tile.shape[0] for bdx in range(nbands): tile[bdx] = numpy.where( mask, utils.linear_rescale(tile[bdx], in_range=scale[bdx], out_range=[0, 255]), 0, ) tile = tile.astype(numpy.uint8) if colormap: colormap = utils.get_colormap(name=colormap) img = utils.array_to_img(tile, mask=mask, color_map=colormap) img_format = "JPEG2000" if ext == "jp2" else ext buffer = img_to_buffer(img, img_format) if save: with open(f"{tile_x}-{tile_y}-{tile_z}_pil.{ext}", "wb") as f: f.write(buffer)
def ratio(scene, tile_z, tile_x, tile_y, tileformat): """Handle processing requests """ if tileformat == 'jpg': tileformat = 'jpeg' query_args = APP.current_request.query_params query_args = query_args if isinstance(query_args, dict) else {} ratio_value = query_args['ratio'] range_value = query_args.get('range', [-1, 1]) tilesize = query_args.get('tile', 256) tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize tile, mask = expression(scene, tile_x, tile_y, tile_z, ratio_value, tilesize=tilesize) if len(tile.shape) == 2: tile = np.expand_dims(tile, axis=0) rtile = np.where( mask, linear_rescale(tile, in_range=range_value, out_range=[0, 255]), 0).astype(np.uint8) img = array_to_img(rtile, color_map=get_colormap(name='cfastie'), mask=mask) str_img = b64_encode_img(img, tileformat) return ('OK', f'image/{tileformat}', str_img)
def test_array_to_img_valid_noband(): """ Should work as expected """ arr = np.random.randint(0, 255, size=(512, 512), dtype=np.uint8) assert utils.array_to_img(arr)
def tile(scene, tile_z, tile_x, tile_y, tileformat): """Handle tile requests """ if tileformat == 'jpg': tileformat = 'jpeg' query_args = APP.current_request.query_params query_args = query_args if isinstance(query_args, dict) else {} bands = query_args.get('rgb', '4,3,2') bands = tuple(re.findall(r'\d+', bands)) histoCut = query_args.get('histo', ';'.join(['0,16000'] * len(bands))) histoCut = re.findall(r'\d+,\d+', histoCut) histoCut = list(map(lambda x: list(map(int, x.split(','))), histoCut)) if len(bands) != len(histoCut): raise LandsatTilerError('The number of bands doesn\'t match the number of histogramm values') tilesize = query_args.get('tile', 256) tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize pan = True if query_args.get('pan') else False tile, mask = landsat8.tile(scene, tile_x, tile_y, tile_z, bands, pan=pan, tilesize=tilesize) rtile = np.zeros((len(bands), tilesize, tilesize), dtype=np.uint8) for bdx in range(len(bands)): rtile[bdx] = np.where(mask, linear_rescale(tile[bdx], in_range=histoCut[bdx], out_range=[0, 255]), 0) img = array_to_img(rtile, mask=mask) str_img = b64_encode_img(img, tileformat) return ('OK', f'image/{tileformat}', str_img)
def tile(tile_z, tile_x, tile_y, tileformat): """Handle tile requests.""" if tileformat == 'jpg': tileformat = 'jpeg' query_args = APP.current_request.query_params query_args = query_args if isinstance(query_args, dict) else {} address = query_args['url'] indexes = query_args.get('indexes') if indexes: indexes = tuple(int(s) for s in re.findall(r'\d+', indexes)) tilesize = query_args.get('tile', 512) tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize nodata = query_args.get('nodata') if nodata is not None: nodata = int(nodata) tile, mask = main.tile(address, tile_x, tile_y, tile_z, indexes=indexes, tilesize=tilesize, nodata=nodata) img = array_to_img(tile, mask=mask) str_img = b64_encode_img(img, tileformat) return ('OK', f'image/{tileformat}', str_img)
def area( scene, bbox, expression, expression_range=[-1, 1], bbox_crs="epsg:4326", out_crs="epsg:3857", ): """Area handler.""" max_img_size = 512 bands = tuple(set(re.findall(r"b(?P<bands>[0-9]{1,2})", expression))) scene_params = sentinel_parse_scene_id(scene) sentinel_address = f'{SENTINEL_BUCKET}/{scene_params["key"]}' scene_info = sentinel2_get_info(os.path.basename(SENTINEL_BUCKET), scene_params["key"], request_pays=True) addresses = [f"{sentinel_address}/B{band}.jp2" for band in bands] _worker = partial( get_area, bbox=bbox, max_img_size=max_img_size, bbox_crs=bbox_crs, out_crs=out_crs, ) with futures.ThreadPoolExecutor(max_workers=3) as executor: data = np.concatenate(list(executor.map(_worker, addresses))) if not np.any(data): raise Exception("No valid data in array") mask = np.all(data != 0, axis=0).astype(np.uint8) * 255 ctx = {} for bdx, b in enumerate(bands): ctx["b{}".format(b)] = data[bdx] ratio = np.nan_to_num(ne.evaluate(expression, local_dict=ctx)) ratio = np.where( mask, linear_rescale(ratio, in_range=expression_range, out_range=[0, 255]), 0).astype(np.uint8) img = array_to_img(ratio, mask, get_colormap(name="cfastie")) ndvi = b64_encode_img(img, "jpeg") date = (scene_params["acquisitionYear"] + "-" + scene_params["acquisitionMonth"] + "-" + scene_params["acquisitionDay"]) return { "ndvi": ndvi, "date": date, "sat": scene_info["sat"], "scene": scene, "cloud": scene_info["cloud_coverage"], }
def test_array_to_img_valid_jpg(): """ Should work as expected """ arr = np.random.randint(0, 255, size=(3, 512, 512), dtype=np.uint8) tileformat = 'jpg' assert utils.array_to_img(arr, tileformat)
def test_array_to_img_valid_mask(): """ Should work as expected """ arr = np.random.randint(0, 255, size=(3, 512, 512), dtype=np.uint8) mask = np.random.randint(0, 1, size=(512, 512), dtype=np.uint8) * 255 tileformat = 'png' assert utils.array_to_img(arr, tileformat, mask=mask)
def area( scene, bbox, expression, expression_range=[-1, 1], bbox_crs="epsg:4326", out_crs="epsg:3857", ): """Area handler.""" max_img_size = 512 bands = tuple(set(re.findall(r"b(?P<bands>[0-9]{1,2})", expression))) scene_params = landsat_parse_scene_id(scene) meta_data = landsat_get_mtl(scene).get("L1_METADATA_FILE") landsat_address = f'{LANDSAT_BUCKET}/{scene_params["key"]}' def worker(band): """Worker.""" address = f"{landsat_address}_B{band}.TIF" sun_elev = meta_data["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"] multi_reflect = meta_data["RADIOMETRIC_RESCALING"][ f"REFLECTANCE_MULT_BAND_{band}" ] add_reflect = meta_data["RADIOMETRIC_RESCALING"][f"REFLECTANCE_ADD_BAND_{band}"] band = get_area( address, bbox, max_img_size=max_img_size, bbox_crs=bbox_crs, out_crs=out_crs ) return reflectance(band, multi_reflect, add_reflect, sun_elev, src_nodata=0) with futures.ThreadPoolExecutor(max_workers=3) as executor: data = np.concatenate(list(executor.map(worker, bands))) if not np.any(data): raise Exception("No valid data in array") mask = np.all(data != 0, axis=0).astype(np.uint8) * 255 ctx = {} for bdx, b in enumerate(bands): ctx["b{}".format(b)] = data[bdx] ratio = np.nan_to_num(ne.evaluate(expression, local_dict=ctx)) ratio = np.where( mask, linear_rescale(ratio, in_range=expression_range, out_range=[0, 255]), 0 ).astype(np.uint8) img = array_to_img(ratio, mask, get_colormap(name="cfastie")) ndvi = b64_encode_img(img, "jpeg") return { "ndvi": ndvi, "date": scene_params["date"], "scene": scene, "cloud": meta_data["IMAGE_ATTRIBUTES"]["CLOUD_COVER"], }
def tile( tile_z, tile_x, tile_y, tileformat, indexes=1, shadder=None, colormap=None, range=None, ): """Handle tile requests""" if tileformat == "jpg": tileformat = "jpeg" if colormap and shadder: return ("NOK", "text/plain", "Cannot pass shadder and colormap options") address = f"s3://elevation-tiles-prod/geotiff/{tile_z}/{tile_x}/{tile_y}.tif" data, mask = main.tile(address, tile_x, tile_y, tile_z, indexes=indexes, tilesize=512) if shadder: tileformat = "png" if shadder == "mapbox": tile = encoders.data_to_rgb(data[0], -10000, 1) elif shadder == "mapzen": tile = mapzen_elevation_rgb.data_to_rgb(data) else: return ("NOK", "text/plain", f"Invalid shadder mode: {shadder}") else: if not range: range = "0,10000" histoCut = list(map(int, range.split(','))) tile = numpy.where( mask, linear_rescale(data, in_range=histoCut, out_range=[0, 255]), 0) options = dict(mask=mask) if colormap: options.update(color_map=get_colormap(name=colormap)) img = array_to_img(tile, **options) return ("OK", f"image/{tileformat}", img_to_buffer(img, tileformat))
def tile(tile_z, tile_x, tile_y, tileformat): """Handle tile requests.""" if tileformat == 'jpg': tileformat = 'jpeg' query_args = APP.current_request.query_params query_args = query_args if isinstance(query_args, dict) else {} address = query_args['url'] ## Read Stac File with urllib.request.urlopen(address) as url: data = json.loads(url.read().decode()) asset_key = query_args.get('asset_key', 'raster') raster_address = data['assets'][asset_key]['href'] indexes = query_args.get('indexes') if indexes: indexes = tuple(int(s) for s in re.findall(r'\d+', indexes)) tilesize = query_args.get('tile', 512) tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize nodata = query_args.get('nodata') if nodata is not None: nodata = int(nodata) raster_address = data['assets'][asset_key]['href'] tile, mask = main.tile(raster_address, tile_x, tile_y, tile_z, indexes=indexes, tilesize=tilesize, nodata=nodata) linearStretch = query_args.get('linearStretch') if linearStretch is not None: if util.strtobool(linearStretch): tile = linear_rescale(tile, in_range=(np.min(tile), np.max(tile))) img = array_to_img(tile, mask=mask) str_img = b64_encode_img(img, tileformat) return ('OK', f'image/{tileformat}', str_img)
def landsat_tile(scene, tile_z, tile_x, tile_y, tileformat): """ Handle tile requests """ query_args = LANDSAT_APP.current_request.query_params query_args = query_args if isinstance(query_args, dict) else {} rgb = query_args.get('rgb', '4,3,2') rgb = map(int, rgb.split(',')) if isinstance(rgb, str) else rgb rgb = tuple(rgb) r_bds = query_args.get('r_bds', '0,16000') if isinstance(r_bds, str): r_bds = map(int, r_bds.split(',')) r_bds = tuple(r_bds) g_bds = query_args.get('g_bds', '0,16000') if isinstance(g_bds, str): g_bds = map(int, g_bds.split(',')) g_bds = tuple(g_bds) b_bds = query_args.get('b_bds', '0,16000') if isinstance(b_bds, str): b_bds = map(int, b_bds.split(',')) b_bds = tuple(b_bds) tilesize = query_args.get('tile', 256) tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize pan = True if query_args.get('pan') else False tile = landsat8.tile(scene, tile_x, tile_y, tile_z, rgb, r_bds, g_bds, b_bds, pan=pan, tilesize=tilesize) tile = array_to_img(tile, tileformat) return ('OK', f'image/{tileformat}', tile)
def landsat_ratio(scene, tile_z, tile_x, tile_y, tileformat): """ Handle processing requests """ query_args = LANDSAT_APP.current_request.query_params query_args = query_args if isinstance(query_args, dict) else {} ratio_value = query_args.get('ratio', 'ndvi') if ratio_value not in RATIOS.keys(): raise LandsatTilerError('Invalid ratio: {}'.format(ratio_value)) equation = RATIOS[ratio_value]['eq'] band_names = list(set(re.findall('b[0-9]{1,2}', equation))) bands = tuple(map(lambda x: x.strip('b'), band_names)) tilesize = query_args.get('tile', 256) tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize tile = landsat8.tile(scene, tile_x, tile_y, tile_z, bands, tilesize=tilesize) for bdx, b in enumerate(band_names): globals()[b] = tile[bdx] tile = np.where( reduce(lambda x, y: x * y, [globals()[i] for i in band_names]) > 0, np.nan_to_num(ne.evaluate(equation)), -9999) range_val = equation = RATIOS[ratio_value]['rg'] rtile = np.where( tile != -9999, linear_rescale(tile, in_range=range_val, out_range=[1, 255]), 0).astype(np.uint8) tile = array_to_img(rtile, tileformat, color_map=get_colormap(name='cfastie')) if tileformat == 'jpg': tileformat = 'jpeg' return ('OK', f'image/{tileformat}', tile)
def tile(scene, tile_z, tile_x, tile_y, tileformat): """Handle tile requests.""" if tileformat == "jpg": tileformat = "jpeg" query_args = APP.current_request.query_params query_args = query_args if isinstance(query_args, dict) else {} bands = query_args.get("rgb", "04,03,02") bands = tuple(re.findall(r"[0-9A]{2}", bands)) histoCut = query_args.get("histo", "-".join(["0,16000"] * len(bands))) histoCut = re.findall(r"\d+,\d+", histoCut) histoCut = list(map(lambda x: list(map(int, x.split(","))), histoCut)) if len(bands) != len(histoCut): raise SentinelTilerError( "The number of bands doesn't match the number of histogramm values" ) tilesize = query_args.get("tile", 256) tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize tile, mask = sentinel2.tile(scene, tile_x, tile_y, tile_z, bands, tilesize=tilesize) rtile = np.zeros((len(bands), tilesize, tilesize), dtype=np.uint8) for bdx in range(len(bands)): rtile[bdx] = np.where( mask, linear_rescale(tile[bdx], in_range=histoCut[bdx], out_range=[0, 255]), 0, ) img = array_to_img(rtile, mask=mask) str_img = b64_encode_img(img, tileformat) return ("OK", f"image/{tileformat}", str_img)
def tile(scene, tile_z, tile_x, tile_y, tileformat): """Handle tile requests """ query_args = APP.current_request.query_params query_args = query_args if isinstance(query_args, dict) else {} bands = query_args.get('rgb', '04,03,02') bands = tuple(re.findall(r'[0-9A]{2}', bands)) histoCut = query_args.get('histo', '-'.join(['0,16000'] * len(bands))) histoCut = re.findall(r'\d+,\d+', histoCut) histoCut = list(map(lambda x: list(map(int, x.split(','))), histoCut)) if len(bands) != len(histoCut): raise SentinelTilerError( 'The number of bands doesn\'t match the number of histogramm values' ) tilesize = query_args.get('tile', 256) tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize tile, mask = sentinel2.tile(scene, tile_x, tile_y, tile_z, bands, tilesize=tilesize) rtile = np.zeros((len(bands), tilesize, tilesize), dtype=np.uint8) for bdx in range(len(bands)): rtile[bdx] = np.where( mask, linear_rescale(tile[bdx], in_range=histoCut[bdx], out_range=[0, 255]), 0) tile = array_to_img(rtile, tileformat, mask=mask) if tileformat == 'jpg': tileformat = 'jpeg' return ('OK', f'image/{tileformat}', tile)
def _get_tile(self, z, x, y, tileformat, color_ops=None): if tileformat == "jpg": tileformat = "jpeg" if not self.raster.tile_exists(z, x, y): raise web.HTTPError(404) data, mask = self.raster.read_tile(z, x, y) if len(data.shape) == 2: data = numpy.expand_dims(data, axis=0) if self.scale: nbands = data.shape[0] scale = self.scale if len(scale) != nbands: scale = scale * nbands for bdx in range(nbands): data[bdx] = numpy.where( mask, linear_rescale(data[bdx], in_range=scale[bdx], out_range=[0, 255]), 0, ) data = data.astype(numpy.uint8) if color_ops: data = self._apply_color_operations(data, color_ops) img = array_to_img(data, mask=mask, color_map=self.colormap) params = TileProfiles.get(tileformat) if tileformat == "jpeg": img = img.convert("RGB") sio = BytesIO() img.save(sio, tileformat.upper(), **params) sio.seek(0) return sio
def _get_tile(self, z, x, y, tileformat, color_ops=None): if tileformat == "jpg": tileformat = "jpeg" if not self.raster.tile_exists(z, x, y): raise web.HTTPError(404) data, mask = self.raster.read_tile(z, x, y) if color_ops: data = self._apply_color_operations(data, color_ops) img = array_to_img(data, mask=mask) params = TileProfiles.get(tileformat) if tileformat == "jpeg": img = img.convert("RGB") sio = BytesIO() img.save(sio, tileformat.upper(), **params) sio.seek(0) return sio
def tile(tile_z, tile_x, tile_y, tileformat): """ Handle tile requests """ query_args = APP.current_request.query_params query_args = query_args if isinstance(query_args, dict) else {} address = query_args['url'] bands = query_args.get('rgb') if bands: bands = tuple(int(s) for s in re.findall(r'\d+', bands)) tilesize = query_args.get('tile', 512) tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize nodata = query_args.get('nodata') if nodata is not None: nodata = int(nodata) alpha = query_args.get('alpha') if alpha is not None: alpha = int(alpha) tile, mask = main.tile(address, tile_x, tile_y, tile_z, bands, tilesize=tilesize, nodata=nodata, alpha=alpha) tile = array_to_img(tile, tileformat, mask=mask) if tileformat == 'jpg': tileformat = 'jpeg' return ('OK', f'image/{tileformat}', tile)
def create( scene, bands=None, expression=None, expression_range=[-1, 1], img_format="jpeg", ovrSize=512, ): """Handler.""" if img_format not in ["png", "jpeg"]: raise UserWarning(f"Invalid {img_format} extension") if not expression and not bands: raise Exception("Expression or Bands must be provided") if bands: nb_bands = len(bands) if nb_bands != 3: raise Exception("RGB combination only") if expression: bands = tuple(set(re.findall(r"b(?P<bands>[0-9]{1,2})", expression))) rgb = expression.split(",") nb_bands = len(rgb) scene_params = landsat_parse_scene_id(scene) meta_data = landsat_get_mtl(scene).get("L1_METADATA_FILE") landsat_address = f'{LANDSAT_BUCKET}/{scene_params["key"]}' _worker = partial(worker, landsat_address=landsat_address, meta=meta_data, ovr_size=ovrSize) with futures.ThreadPoolExecutor(max_workers=3) as executor: data = np.concatenate(list(executor.map(_worker, bands))) mask = np.all(data != 0, axis=0).astype(np.uint8) * 255 if expression: ctx = {} for bdx, b in enumerate(bands): ctx["b{}".format(b)] = data[bdx] data = np.array([ np.nan_to_num(ne.evaluate(bloc.strip(), local_dict=ctx)) for bloc in rgb ]) for band in range(data.shape[0]): imgRange = (expression_range if expression else np.percentile( data[band][mask > 0], (2, 98)).tolist()) data[band] = np.where( mask, linear_rescale(data[band], in_range=imgRange, out_range=[0, 255]), 0, ) data = data.squeeze() colormap = None if len(data.shape) >= 3 else get_colormap(name="cfastie") img = array_to_img(data, mask, colormap) return b64_encode_img(img, img_format)
def tile(scene, tile_z, tile_x, tile_y, tileformat): if tileformat == 'jpg': tileformat = 'jpeg' query_args = APP.current_request.query_params query_args = query_args if isinstance(query_args, dict) else {} bands = query_args.get('rgb', '4,3,2') label_id = -1 if bands == '12,12,12': label_id = 0 bands = '4,3,2' elif bands == '13,13,13': label_id = 1 bands = '4,3,2' elif bands == '14,14,14': label_id = 2 bands = '4,3,2' elif bands == '15,15,15': label_id = 3 bands = '4,3,2' bands = tuple(re.findall(r'\d+', bands)) histoCut = query_args.get('histo', ';'.join(['0,16000'] * len(bands))) histoCut = re.findall(r'\d+,\d+', histoCut) histoCut = list(map(lambda x: list(map(int, x.split(','))), histoCut)) if len(bands) != len(histoCut): raise LandsatTilerError( 'The number of bands doesn\'t match the number of histogramm values' ) tilesize = query_args.get('tile', 256) tilesize = int(tilesize) if isinstance(tilesize, str) else tilesize pan = True if query_args.get('pan') else False tile, mask = landsat8.tile(scene, tile_x, tile_y, tile_z, bands, pan=pan, tilesize=tilesize) rtile = np.zeros((len(bands), tilesize, tilesize), dtype=np.uint8) for bdx in range(len(bands)): rtile[bdx] = np.where( mask, linear_rescale(tile[bdx], in_range=histoCut[bdx], out_range=[0, 255]), 0) img = array_to_img(rtile, mask=mask) str_img = b64_encode_img(img, tileformat) if label_id != -1: data = { 'label_id': label_id, 'img': str_img, } r = requests.post(f"http://kursach.ngrok.io/getmask/", json=data) str_img = r.json()['mask'] img = Image.open(io.BytesIO(base64.decodebytes(eval(f"b'{str_img}'")))) img_np = np.asarray(img).T img = array_to_img(img_np, mask=mask) str_img = b64_encode_img(img, tileformat) return ('OK', f'image/{tileformat}', str_img)
def array_to_img_pil(ext): img = utils.array_to_img(tile, mask=mask, color_map=pil_cmap) img_format = "JPEG2000" if ext == "jp2" else ext img_to_buffer(img, img_format)
def array_to_img_pil(ext): img = utils.array_to_img(tile, mask=mask) img_format = "JPEG2000" if ext == "jp2" else ext return img_to_buffer(img, img_format)
def test_b64_encode_img_valid_webp(): """Should work as expected """ arr = np.random.randint(0, 255, size=(4, 512, 512), dtype=np.uint8) img = utils.array_to_img(arr) assert utils.b64_encode_img(img, 'webp')