def test_landsat_get_mtl_valid(urlopen): urlopen.return_value.read.return_value = LANDSAT_METADATA_RAW meta_data = utils.landsat_get_mtl(LANDSAT_SCENE_C1) assert meta_data['L1_METADATA_FILE']['METADATA_FILE_INFO'][ 'LANDSAT_SCENE_ID'] == 'LC80160372017225LGN00'
def area( scene, bbox, expression, expression_range=[-1, 1], bbox_crs="epsg:4326", out_crs="epsg:3857", ): """Area handler.""" max_img_size = 512 bands = tuple(set(re.findall(r"b(?P<bands>[0-9]{1,2})", expression))) scene_params = landsat_parse_scene_id(scene) meta_data = landsat_get_mtl(scene).get("L1_METADATA_FILE") landsat_address = f'{LANDSAT_BUCKET}/{scene_params["key"]}' def worker(band): """Worker.""" address = f"{landsat_address}_B{band}.TIF" sun_elev = meta_data["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"] multi_reflect = meta_data["RADIOMETRIC_RESCALING"][ f"REFLECTANCE_MULT_BAND_{band}" ] add_reflect = meta_data["RADIOMETRIC_RESCALING"][f"REFLECTANCE_ADD_BAND_{band}"] band = get_area( address, bbox, max_img_size=max_img_size, bbox_crs=bbox_crs, out_crs=out_crs ) return reflectance(band, multi_reflect, add_reflect, sun_elev, src_nodata=0) with futures.ThreadPoolExecutor(max_workers=3) as executor: data = np.concatenate(list(executor.map(worker, bands))) if not np.any(data): raise Exception("No valid data in array") mask = np.all(data != 0, axis=0).astype(np.uint8) * 255 ctx = {} for bdx, b in enumerate(bands): ctx["b{}".format(b)] = data[bdx] ratio = np.nan_to_num(ne.evaluate(expression, local_dict=ctx)) ratio = np.where( mask, linear_rescale(ratio, in_range=expression_range, out_range=[0, 255]), 0 ).astype(np.uint8) img = array_to_img(ratio, mask, get_colormap(name="cfastie")) ndvi = b64_encode_img(img, "jpeg") return { "ndvi": ndvi, "date": scene_params["date"], "scene": scene, "cloud": meta_data["IMAGE_ATTRIBUTES"]["CLOUD_COVER"], }
def metadata(sceneid, pmin=2, pmax=98): """ Retrieve image bounds and histogram info. Attributes ---------- sceneid : str Landsat sceneid. For scenes after May 2017, sceneid have to be LANDSAT_PRODUCT_ID. pmin : int, optional, (default: 2) Histogram minimum cut. pmax : int, optional, (default: 98) Histogram maximum cut. Returns ------- out : dict dictionary with image bounds and bands histogram cuts. """ scene_params = utils.landsat_parse_scene_id(sceneid) meta_data = utils.landsat_get_mtl(sceneid).get("L1_METADATA_FILE") landsat_address = "{}/{}".format(LANDSAT_BUCKET, scene_params["key"]) info = {"sceneid": sceneid} info["bounds"] = toa_utils._get_bounds_from_metadata( meta_data["PRODUCT_METADATA"]) bands = ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11"] _min_max_worker = partial( utils.landsat_min_max_worker, address=landsat_address, metadata=meta_data, pmin=pmin, pmax=pmax, ) with futures.ThreadPoolExecutor(max_workers=5) as executor: responses = list(executor.map(_min_max_worker, bands)) info["rgbMinMax"] = dict(zip(bands, responses)) return info
def point(scene, coordinates, expression): """Point handler.""" bands = tuple(set(re.findall(r"b(?P<bands>[0-9]{1,2})", expression))) scene_params = landsat_parse_scene_id(scene) meta_data = landsat_get_mtl(scene).get("L1_METADATA_FILE") landsat_address = f'{LANDSAT_BUCKET}/{scene_params["key"]}' def worker(band, coordinates): """Worker.""" address = f"{landsat_address}_B{band}.TIF" sun_elev = meta_data["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"] multi_reflect = meta_data["RADIOMETRIC_RESCALING"][ f"REFLECTANCE_MULT_BAND_{band}" ] add_reflect = meta_data["RADIOMETRIC_RESCALING"][f"REFLECTANCE_ADD_BAND_{band}"] with rasterio.open(address) as band: lon_srs, lat_srs = warp.transform( "EPSG:4326", band.crs, [coordinates[0]], [coordinates[1]] ) point = list(band.sample([(lon_srs[0], lat_srs[0])]))[0] return reflectance(point, multi_reflect, add_reflect, sun_elev, src_nodata=0)[0] try: _worker = partial(worker, coordinates=coordinates) with futures.ThreadPoolExecutor(max_workers=3) as executor: data = list(executor.map(_worker, bands)) ctx = {} for bdx, b in enumerate(bands): ctx["b{}".format(b)] = data[bdx] ratio = np.nan_to_num(ne.evaluate(expression, local_dict=ctx)) except IndexError: ratio = 0.0 return { "ndvi": ratio, "date": scene_params["date"], "scene": scene, "cloud": meta_data["IMAGE_ATTRIBUTES"]["CLOUD_COVER"], }
def metadata(sceneid, pmin=2, pmax=98): """Retrieve image bounds and histogram info. Attributes ---------- sceneid : str Landsat sceneid. For scenes after May 2017, sceneid have to be LANDSAT_PRODUCT_ID. pmin : int, optional, (default: 2) Histogram minimum cut. pmax : int, optional, (default: 98) Histogram maximum cut. Returns ------- out : dict dictionary with image bounds and bands histogram cuts. """ scene_params = utils.landsat_parse_scene_id(sceneid) meta_data = utils.landsat_get_mtl(sceneid).get('L1_METADATA_FILE') landsat_address = '{}/{}'.format(LANDSAT_BUCKET, scene_params['key']) info = {'sceneid': sceneid} info['bounds'] = toa_utils._get_bounds_from_metadata( meta_data['PRODUCT_METADATA']) bands = ['1', '2', '3', '4', '5', '6', '7'] _min_max_worker = partial(utils.landsat_min_max_worker, address=landsat_address, metadata=meta_data, pmin=pmin, pmax=pmax) with futures.ThreadPoolExecutor(max_workers=7) as executor: responses = list(executor.map(_min_max_worker, bands)) info['rgbMinMax'] = dict(zip(bands, responses)) return info
def bounds(sceneid): """Retrieve image bounds. Attributes ---------- sceneid : str Landsat sceneid. For scenes after May 2017, sceneid have to be LANDSAT_PRODUCT_ID. Returns ------- out : dict dictionary with image bounds. """ meta_data = utils.landsat_get_mtl(sceneid).get('L1_METADATA_FILE') info = {'sceneid': sceneid} info['bounds'] = toa_utils._get_bounds_from_metadata(meta_data['PRODUCT_METADATA']) return info
def create( scene, bands=None, expression=None, expression_range=[-1, 1], img_format="jpeg", ovrSize=512, ): """Handler.""" if img_format not in ["png", "jpeg"]: raise UserWarning(f"Invalid {img_format} extension") if not expression and not bands: raise Exception("Expression or Bands must be provided") if bands: nb_bands = len(bands) if nb_bands != 3: raise Exception("RGB combination only") if expression: bands = tuple(set(re.findall(r"b(?P<bands>[0-9]{1,2})", expression))) rgb = expression.split(",") nb_bands = len(rgb) scene_params = landsat_parse_scene_id(scene) meta_data = landsat_get_mtl(scene).get("L1_METADATA_FILE") landsat_address = f'{LANDSAT_BUCKET}/{scene_params["key"]}' _worker = partial(worker, landsat_address=landsat_address, meta=meta_data, ovr_size=ovrSize) with futures.ThreadPoolExecutor(max_workers=3) as executor: data = np.concatenate(list(executor.map(_worker, bands))) mask = np.all(data != 0, axis=0).astype(np.uint8) * 255 if expression: ctx = {} for bdx, b in enumerate(bands): ctx["b{}".format(b)] = data[bdx] data = np.array([ np.nan_to_num(ne.evaluate(bloc.strip(), local_dict=ctx)) for bloc in rgb ]) for band in range(data.shape[0]): imgRange = (expression_range if expression else np.percentile( data[band][mask > 0], (2, 98)).tolist()) data[band] = np.where( mask, linear_rescale(data[band], in_range=imgRange, out_range=[0, 255]), 0, ) data = data.squeeze() colormap = None if len(data.shape) >= 3 else get_colormap(name="cfastie") img = array_to_img(data, mask, colormap) return b64_encode_img(img, img_format)
def test_landsat_get_mtl_invalid(urlopen): urlopen.return_value.read.return_value = {} with pytest.raises(Exception): utils.landsat_get_mtl(LANDSAT_SCENE_C1)
def tile(sceneid, tile_x, tile_y, tile_z, rgb=(4, 3, 2), r_bds=(0, 16000), g_bds=(0, 16000), b_bds=(0, 16000), tilesize=256, pan=False): """Create mercator tile from Landsat-8 data and encodes it in base64. Attributes ---------- sceneid : str Landsat sceneid. For scenes after May 2017, sceneid have to be LANDSAT_PRODUCT_ID. tile_x : int Mercator tile X index. tile_y : int Mercator tile Y index. tile_z : int Mercator tile ZOOM level. rgb : tuple, int, optional (default: (4, 3, 2)) Bands index for the RGB combination. r_bds : tuple, int, optional (default: (0, 16000)) First band (red) DN min and max values (DN * 10,000) used for the linear rescaling. g_bds : tuple, int, optional (default: (0, 16000)) Second band (green) DN min and max values (DN * 10,000) used for the linear rescaling. b_bds : tuple, int, optional (default: (0, 16000)) Third band (blue) DN min and max values (DN * 10,000) used for the linear rescaling. tilesize : int, optional (default: 256) Output image size. pan : boolean, optional (default: False) If True, apply pan-sharpening. Returns ------- out : numpy ndarray (type: uint8) """ scene_params = utils.landsat_parse_scene_id(sceneid) meta_data = utils.landsat_get_mtl(sceneid).get('L1_METADATA_FILE') landsat_address = '{}/{}'.format(LANDSAT_BUCKET, scene_params['key']) wgs_bounds = toa_utils._get_bounds_from_metadata( meta_data['PRODUCT_METADATA']) if not utils.tile_exists(wgs_bounds, tile_z, tile_x, tile_y): raise TileOutsideBounds('Tile {}/{}/{} is outside image bounds'.format( tile_z, tile_x, tile_y)) mercator_tile = mercantile.Tile(x=tile_x, y=tile_y, z=tile_z) tile_bounds = mercantile.xy_bounds(mercator_tile) # define a list of bands Min and Max Values (from input) histo_cuts = dict(zip(rgb, [r_bds, g_bds, b_bds])) ms_tile_size = int(tilesize / 2) if pan else tilesize addresses = ['{}_B{}.TIF'.format(landsat_address, band) for band in rgb] _tiler = partial(utils.tile_band_worker, bounds=tile_bounds, tilesize=ms_tile_size) with futures.ThreadPoolExecutor(max_workers=3) as executor: out = np.stack(list(executor.map(_tiler, addresses))) if pan: pan_address = '{}_B8.TIF'.format(landsat_address) matrix_pan = utils.tile_band_worker(pan_address, tile_bounds, tilesize) w, s, e, n = tile_bounds pan_transform = transform.from_bounds(w, s, e, n, tilesize, tilesize) vis_transform = pan_transform * Affine.scale(2.) out = pansharpen(out, vis_transform, matrix_pan, pan_transform, np.int16, 'EPSG:3857', 'EPSG:3857', 0.2, method='Brovey', src_nodata=0) sun_elev = meta_data['IMAGE_ATTRIBUTES']['SUN_ELEVATION'] for bdx, band in enumerate(rgb): multi_reflect = meta_data['RADIOMETRIC_RESCALING'].get( 'REFLECTANCE_MULT_BAND_{}'.format(band)) add_reflect = meta_data['RADIOMETRIC_RESCALING'].get( 'REFLECTANCE_ADD_BAND_{}'.format(band)) out[bdx] = 10000 * reflectance.reflectance( out[bdx], multi_reflect, add_reflect, sun_elev, src_nodata=0) out[bdx] = np.where( out[bdx] > 0, utils.linear_rescale(out[bdx], in_range=histo_cuts.get(band), out_range=[1, 255]), 0) return out.astype(np.uint8)
def tile(sceneid, tile_x, tile_y, tile_z, rgb=(4, 3, 2), tilesize=256, pan=False): """Create mercator tile from Landsat-8 data. Attributes ---------- sceneid : str Landsat sceneid. For scenes after May 2017, sceneid have to be LANDSAT_PRODUCT_ID. tile_x : int Mercator tile X index. tile_y : int Mercator tile Y index. tile_z : int Mercator tile ZOOM level. rgb : tuple, int, optional (default: (4, 3, 2)) Bands index for the RGB combination. tilesize : int, optional (default: 256) Output image size. pan : boolean, optional (default: False) If True, apply pan-sharpening. Returns ------- data : numpy ndarray mask: numpy array """ if not isinstance(rgb, tuple): rgb = tuple((rgb, )) scene_params = utils.landsat_parse_scene_id(sceneid) meta_data = utils.landsat_get_mtl(sceneid).get('L1_METADATA_FILE') landsat_address = '{}/{}'.format(LANDSAT_BUCKET, scene_params['key']) wgs_bounds = toa_utils._get_bounds_from_metadata( meta_data['PRODUCT_METADATA']) if not utils.tile_exists(wgs_bounds, tile_z, tile_x, tile_y): raise TileOutsideBounds( 'Tile {}/{}/{} is outside image bounds'.format( tile_z, tile_x, tile_y)) mercator_tile = mercantile.Tile(x=tile_x, y=tile_y, z=tile_z) tile_bounds = mercantile.xy_bounds(mercator_tile) ms_tile_size = int(tilesize / 2) if pan else tilesize addresses = ['{}_B{}.TIF'.format(landsat_address, band) for band in rgb] _tiler = partial(utils.tile_band_worker, bounds=tile_bounds, tilesize=ms_tile_size, nodata=0) with futures.ThreadPoolExecutor(max_workers=3) as executor: data, masks = zip(*list(executor.map(_tiler, addresses))) data = np.concatenate(data) mask = np.all(masks, axis=0).astype(np.uint8) * 255 if pan: pan_address = '{}_B8.TIF'.format(landsat_address) matrix_pan, mask = utils.tile_band_worker(pan_address, tile_bounds, tilesize, nodata=0) w, s, e, n = tile_bounds pan_transform = transform.from_bounds(w, s, e, n, tilesize, tilesize) vis_transform = pan_transform * Affine.scale(2.) data = pansharpen(data, vis_transform, matrix_pan, pan_transform, np.int16, 'EPSG:3857', 'EPSG:3857', 0.2, method='Brovey', src_nodata=0) sun_elev = meta_data['IMAGE_ATTRIBUTES']['SUN_ELEVATION'] for bdx, band in enumerate(rgb): if int(band) > 9: # TIRS multi_rad = meta_data['RADIOMETRIC_RESCALING'].get( 'RADIANCE_MULT_BAND_{}'.format(band)) add_rad = meta_data['RADIOMETRIC_RESCALING'].get( 'RADIANCE_ADD_BAND_{}'.format(band)) k1 = meta_data['TIRS_THERMAL_CONSTANTS'].get( 'K1_CONSTANT_BAND_{}'.format(band)) k2 = meta_data['TIRS_THERMAL_CONSTANTS'].get( 'K2_CONSTANT_BAND_{}'.format(band)) data[bdx] = brightness_temp.brightness_temp( data[bdx], multi_rad, add_rad, k1, k2) else: multi_reflect = meta_data['RADIOMETRIC_RESCALING'].get( 'REFLECTANCE_MULT_BAND_{}'.format(band)) add_reflect = meta_data['RADIOMETRIC_RESCALING'].get( 'REFLECTANCE_ADD_BAND_{}'.format(band)) data[bdx] = 10000 * reflectance.reflectance( data[bdx], multi_reflect, add_reflect, sun_elev) return data, mask
def create_img(highres=None, bands=[5, 4, 3], min_cloud=60): """ """ max_i = 5 i = 0 while True: date_image = random_date().strftime('%Y-%m-%d') query = f'{sat_api}satellite_name=landsat-8&date={date_image}&cloud_from={min_cloud}&limit=2000' response = json.loads(urlopen(query).read()) if response.get('errorMessage'): raise Exception('What is wrong with you dude') if response.get('meta').get('found') == 0: continue scenes = response.get('results', []) index = random.randrange(0, len(scenes)) scene = scenes[index] date = time.strptime(scene['date'], '%Y-%m-%d') aws_id = re.sub(r'LGN0[0-9]', 'LGN00', scene['scene_id']) \ if date < time.strptime('2017-05-01', '%Y-%m-%d') else scene['LANDSAT_PRODUCT_ID'] lat = scene['sceneCenterLatitude'] lng = scene['sceneCenterLongitude'] try: if highres: tile = mercantile.tile(lng, lat, 9, truncate=True) bounds = mercantile.xy_bounds(tile) else: bounds = None scene_params = landsat_parse_scene_id(aws_id) meta_data = landsat_get_mtl(aws_id).get('L1_METADATA_FILE') landsat_address = f'{LANDSAT_BUCKET}/{scene_params["key"]}' _worker = partial(band_worker, landsat_address=landsat_address, meta=meta_data, bounds=bounds) with futures.ThreadPoolExecutor(max_workers=3) as executor: out = np.stack(list(executor.map(_worker, bands))) out = reshape_as_image(out) img = Image.fromarray(out, mode='RGB') im = BytesIO() img.save(im, 'jpeg', subsampling=0, quality=100) im.seek(0) except: print(f'{aws_id}: NOPE!') i += 1 if i >= max_i: raise Exception('What is wrong with you dude') continue info = {'lat': lat, 'lng': lng, 'name': get_place(lat, lng)} return aws_id, im, info
def create(scene, bands=None, expression=None): """Handler.""" scene_params = landsat_parse_scene_id(scene) meta_data = landsat_get_mtl(scene).get("L1_METADATA_FILE") landsat_address = f'{LANDSAT_BUCKET}/{scene_params["key"]}' if not expression and not bands: raise Exception("Expression or Bands must be provided") if bands: nb_bands = len(bands) data_type = np.uint16 if nb_bands != 3: raise Exception("RGB combination only") if expression: bands = list(set(re.findall(r"b(?P<bands>[0-9]{1,2})", expression))) rgb = expression.split(",") data_type = np.float32 nb_bands = len(rgb) bqa = f"{landsat_address}_BQA.TIF" with rasterio.open(bqa) as src: meta = src.meta wind = [w for ij, w in src.block_windows(1)] meta.update( nodata=0, count=nb_bands, interleave="pixel", compress="DEFLATE", photometric="MINISBLACK" if nb_bands == 1 else "RGB", dtype=data_type, ) sun_elev = meta_data["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"] memfile = MemoryFile() with memfile.open(**meta) as dataset: with contextlib.ExitStack() as stack: srcs = [ stack.enter_context( rasterio.open(f"{landsat_address}_B{band}.TIF")) for band in bands ] def get_window(idx, window): band = bands[idx] multi_reflect = meta_data["RADIOMETRIC_RESCALING"].get( f"REFLECTANCE_MULT_BAND_{band}") add_reflect = meta_data["RADIOMETRIC_RESCALING"].get( f"REFLECTANCE_ADD_BAND_{band}") data = srcs[idx].read(window=window, boundless=True, indexes=(1)) return reflectance.reflectance(data, multi_reflect, add_reflect, sun_elev) for window in wind: _worker = partial(get_window, window=window) with futures.ThreadPoolExecutor(max_workers=3) as executor: data = np.stack( list(executor.map(_worker, range(len(bands))))) if expression: ctx = {} for bdx, b in enumerate(bands): ctx["b{}".format(b)] = data[bdx] data = np.array([ np.nan_to_num( ne.evaluate(bloc.strip(), local_dict=ctx)) for bloc in rgb ]) else: data *= 10000 dataset.write(data.astype(data_type), window=window) return memfile
def worker(scene, bands): """Worker.""" try: scene_params = landsat_parse_scene_id(scene) meta_data = landsat_get_mtl(scene).get("L1_METADATA_FILE") landsat_address = f'{LANDSAT_BUCKET}/{scene_params["key"]}' bqa = f"{landsat_address}_BQA.TIF" with rasterio.open(bqa) as src: ovr = src.overviews(1) ovr_width = int(src.width / ovr[0]) ovr_height = int(src.height / ovr[0]) dst_affine, width, height = calculate_default_transform( src.crs, "epsg:3857", ovr_width, ovr_height, *src.bounds) meta = { "driver": "GTiff", "count": 3, "dtype": np.uint8, "nodata": 0, "height": height, "width": width, "compress": "DEFLATE", "crs": "epsg:3857", "transform": dst_affine, } outpath = f"/tmp/{scene}.tif" with rasterio.open(outpath, "w", **meta) as dataset: sun_elev = meta_data["IMAGE_ATTRIBUTES"]["SUN_ELEVATION"] for idx, b in enumerate(bands): with rasterio.open(f"{landsat_address}_B{b}.TIF") as src: with WarpedVRT( src, dst_crs="EPSG:3857", resampling=Resampling.bilinear, src_nodata=0, dst_nodata=0, ) as vrt: matrix = vrt.read(indexes=1, out_shape=(height, width)) multi_reflect = meta_data["RADIOMETRIC_RESCALING"][ f"REFLECTANCE_MULT_BAND_{b}"] add_reflect = meta_data["RADIOMETRIC_RESCALING"][ f"REFLECTANCE_ADD_BAND_{b}"] matrix = (reflectance( matrix, multi_reflect, add_reflect, sun_elev, src_nodata=0) * 10000) minref = (meta_data["MIN_MAX_REFLECTANCE"] [f"REFLECTANCE_MINIMUM_BAND_{b}"] * 10000) maxref = (meta_data["MIN_MAX_REFLECTANCE"] [f"REFLECTANCE_MAXIMUM_BAND_{b}"] * 10000) matrix = np.where( matrix > 0, linear_rescale(matrix, in_range=[int(minref), int(maxref)], out_range=[1, 255]), 0, ).astype(np.uint8) mask = np.ma.masked_values(matrix, 0) s = np.ma.notmasked_contiguous(mask) matrix = matrix.ravel() for sl in s: matrix[sl.start:sl.start + 5] = 0 matrix[sl.stop - 5:sl.stop] = 0 matrix = matrix.reshape((height, width)) dataset.write(matrix, indexes=idx + 1) return outpath except: return None