def point(scene, coord):
scene_params = utils.landsat_parse_scene_id(scene)
meta_data = utils.landsat_get_mtl(scene)
landsat_address = f's3://landsat-pds/{scene_params["key"]}'
band_address = f'{landsat_address}_B4.TIF'
with rio.open(band_address) as band:
lon_srs, lat_srs = warp.transform('EPSG:4326', band.crs, [coord[0]],
[coord[1]])
b4 = list(band.sample([(lon_srs[0], lat_srs[0])]))[0]
b4 = float(utils.landsat_to_toa(b4, 4, meta_data)[0])
band_address = f'{landsat_address}_B5.TIF'
with rio.open(band_address) as band:
lon_srs, lat_srs = warp.transform('EPSG:4326', band.crs, [coord[0]],
[coord[1]])
b5 = list(band.sample([(lon_srs[0], lat_srs[0])]))[0]
b5 = float(utils.landsat_to_toa(b5, 5, meta_data)[0])
if b4 * b5 > 0:
ratio = (b5 - b4) / (b5 + b4)
else:
ratio = 0
out = {
'ndvi': ratio,
'date': scene_params['date'],
'cloud': float(utils.landsat_mtl_extract(meta_data, 'CLOUD_COVER'))
}
return out
def area(scene, bbox):
max_width = 512
max_height = 512
scene_params = utils.landsat_parse_scene_id(scene)
meta_data = utils.landsat_get_mtl(scene)
landsat_address = f's3://landsat-pds/{scene_params["key"]}'
band_address = f'{landsat_address}_B4.TIF'
with rio.open(band_address) as band:
crs_bounds = warp.transform_bounds('EPSG:4326', band.crs, *bbox)
window = band.window(*crs_bounds, boundless=True)
width = window.num_cols if window.num_cols < max_width else max_width
height = window.num_rows if window.num_rows < max_width else max_width
b4 = band.read(window=window,
out_shape=(height, width),
indexes=1,
resampling=Resampling.bilinear,
boundless=True)
b4 = utils.landsat_to_toa(b4, 4, meta_data).astype(float)
band_address = f'{landsat_address}_B5.TIF'
with rio.open(band_address) as band:
crs_bounds = warp.transform_bounds('EPSG:4326', band.crs, *bbox)
window = band.window(*crs_bounds, boundless=True)
width = window.num_cols if window.num_cols < max_width else max_width
height = window.num_rows if window.num_rows < max_width else max_width
b5 = band.read(window=window,
out_shape=(height, width),
indexes=1,
resampling=Resampling.bilinear,
boundless=True)
b5 = utils.landsat_to_toa(b5, 4, meta_data).astype(float)
ratio = np.where((b5 * b4) > 0, np.nan_to_num((b5 - b4) / (b5 + b4)), -1)
ratio = np.where(
ratio > -1,
utils.linear_rescale(ratio, in_range=[-1, 1], out_range=[0, 255]), 0)
img = Image.fromarray(ratio).convert('RGB')
sio = BytesIO()
img.save(sio, 'jpeg', subsampling=0, quality=100)
sio.seek(0)
return base64.b64encode(sio.getvalue()).decode()
def create(scene, bucket, bands=[4, 3, 2]):
def worker(window, address):
with rio.open(address) as src:
return src.read(window=window, boundless=True, indexes=1)
scene_params = utils.landsat_parse_scene_id(scene)
meta_data = utils.landsat_get_mtl(scene)
landsat_address = f's3://landsat-pds/{scene_params["key"]}'
bqa = f'{landsat_address}_BQA.TIF'
with rio.open(bqa) as src:
meta = src.meta
wind = [w for ij, w in src.block_windows(1)]
meta.update(nodata=0,
count=3,
interleave='pixel',
PHOTOMETRIC='RGB',
tiled=False,
compress=None)
with MemoryFile() as memfile:
with memfile.open(**meta) as dataset:
for b in range(len(bands)):
band_address = f'{landsat_address}_B{bands[b]}.TIF'
with concurrent.futures.ThreadPoolExecutor(
max_workers=10) as executor:
future_to_window = {
executor.submit(worker, window, band_address): window
for window in wind
}
for future in concurrent.futures.as_completed(
future_to_window):
window = future_to_window[future]
result = utils.landsat_to_toa(future.result(),
bands[b], meta_data)
dataset.write(result, indexes=b + 1, window=window)
client = boto3.client('s3')
str_band = ''.join(map(str, bands))
key = f'data/landsat/{scene}_B{str_band}.tif'
response = client.put_object(ACL='public-read',
Bucket=bucket,
Key=key,
Body=memfile,
ContentType='image/tiff')
return True
def test_landsat_parse_scene_id_pre_valid():
scene = 'LC80300342017083LGN00'
expectedContent = {
'acquisitionJulianDay': '083',
'acquisitionYear': '2017',
'archiveVersion': '00',
'date': '2017-03-24',
'groundStationIdentifier': 'LGN',
'key': 'L8/030/034/LC80300342017083LGN00/LC80300342017083LGN00',
'path': '030',
'row': '034',
'satellite': '8',
'scene': 'LC80300342017083LGN00',
'sensor': 'C'
}
assert utils.landsat_parse_scene_id(scene) == expectedContent
def test_landsat_parse_scene_id_c1_valid():
scene = 'LC08_L1TP_005004_20170410_20170414_01_T1'
expectedContent = {
'acquisitionDay': '10',
'acquisitionMonth': '04',
'acquisitionYear': '2017',
'collectionCategory': 'T1',
'collectionNumber': '01',
'date': '2017-04-10',
'key':
'c1/L8/005/004/LC08_L1TP_005004_20170410_20170414_01_T1/LC08_L1TP_005004_20170410_20170414_01_T1',
'path': '005',
'processingCorrectionLevel': 'L1TP',
'processingDay': '14',
'processingMonth': '04',
'processingYear': '2017',
'row': '004',
'satellite': '08',
'scene': 'LC08_L1TP_005004_20170410_20170414_01_T1',
'sensor': 'C'
}
assert utils.landsat_parse_scene_id(scene) == expectedContent
def create(scene, bands=[4, 3, 2], img_format='jpeg', ovrSize=512):
def worker(args):
address, band, meta = args
with rio.open(address) as src:
matrix = src.read(indexes=1,
out_shape=(ovrSize, ovrSize),
resampling=Resampling.bilinear).astype(
src.profile['dtype'])
matrix = utils.landsat_to_toa(matrix, band, meta)
mask = np.ma.masked_values(matrix, 0)
s = np.ma.notmasked_contiguous(mask)
mask = None
matrix = matrix.ravel()
for sl in s:
matrix[sl.start:sl.start + 10] = 0
matrix[sl.stop - 10:sl.stop] = 0
matrix = matrix.reshape((ovrSize, ovrSize))
minRef = float(
utils.landsat_mtl_extract(
meta, f'REFLECTANCE_MINIMUM_BAND_{band}')) * 10000
maxRef = float(
utils.landsat_mtl_extract(
meta, f'REFLECTANCE_MAXIMUM_BAND_{band}')) * 10000
matrix = np.where(
matrix > 0,
utils.linear_rescale(matrix,
in_range=[int(minRef),
int(maxRef)],
out_range=[1, 255]), 0)
return matrix.astype(np.uint8)
if img_format not in ['png', 'jpeg']:
raise UserWarning(f'Invalid {img_format} extension')
scene_params = utils.landsat_parse_scene_id(scene)
meta_data = utils.landsat_get_mtl(scene)
landsat_address = f's3://landsat-pds/{scene_params["key"]}'
args = ((f'{landsat_address}_B{band}.TIF', band, meta_data)
for band in bands)
out = np.zeros((4, ovrSize, ovrSize), dtype=np.uint8)
with futures.ThreadPoolExecutor(max_workers=3) as executor:
out[0:3] = list(executor.map(worker, args))
out[-1] = np.all(np.dstack(out[:3]) != 0, axis=2).astype(np.uint8) * 255
img = Image.fromarray(np.dstack(out))
sio = BytesIO()
if img_format == 'jpeg':
img = img.convert('RGB')
img.save(sio, 'jpeg', subsampling=0, quality=100)
else:
img.save(sio, 'png', compress_level=0)
sio.seek(0)
return base64.b64encode(sio.getvalue()).decode()
def test_landsat_parse_scene_id_c1_invalid():
scene = 'LC08_005004_20170410_20170414_01_T1'
with pytest.raises(ValueError):
utils.landsat_parse_scene_id(scene)
def test_landsat_parse_scene_id_pre_invalid():
scene = 'L0300342017083LGN00'
with pytest.raises(ValueError):
utils.landsat_parse_scene_id(scene)
def worker(args):
scene, band = args
try:
scene_params = utils.landsat_parse_scene_id(scene)
meta_data = utils.landsat_get_mtl(scene)
landsat_address = f's3://landsat-pds/{scene_params["key"]}'
bqa = f'{landsat_address}_BQA.TIF'
with rio.open(bqa) as src:
ovr = src.overviews(1)
ovr_width = int(src.width / ovr[0])
ovr_height = int(src.height / ovr[0])
src_affine = transform.from_bounds(*list(src.bounds) +
[ovr_width, ovr_height])
dst_affine, width, height = calculate_default_transform(
src.crs, 'epsg:3857', ovr_width, ovr_height, *src.bounds)
outpath = f'/tmp/{scene}.tif'
with rio.open(outpath,
'w',
driver='GTiff',
count=3,
dtype=np.uint8,
nodata=0,
height=height,
width=width,
crs='epsg:3857',
transform=dst_affine) as dataset:
for b in range(len(bands)):
band_address = f'{landsat_address}_B{bands[b]}.TIF'
with rio.open(band_address) as src:
with WarpedVRT(
src,
dst_crs='EPSG:3857',
# resampling=Resampling.bilinear,
src_nodata=0,
dst_nodata=0) as vrt:
matrix = vrt.read(indexes=1,
out_shape=(height, width))
matrix = utils.landsat_to_toa(matrix, bands[b], meta_data)
minRef = float(
utils.landsat_mtl_extract(
meta_data,
f'REFLECTANCE_MINIMUM_BAND_{bands[b]}')) * 10000
maxRef = float(
utils.landsat_mtl_extract(
meta_data,
f'REFLECTANCE_MAXIMUM_BAND_{bands[b]}')) * 10000
matrix = np.where(
matrix > 0,
utils.linear_rescale(
matrix,
in_range=[int(minRef), int(maxRef)],
out_range=[1, 255]), 0)
mask = np.ma.masked_values(matrix, 0)
s = np.ma.notmasked_contiguous(mask)
mask = None
matrix = matrix.ravel()
for sl in s:
matrix[sl.start:sl.start + 7] = 0
matrix[sl.stop - 7:sl.stop] = 0
matrix = matrix.reshape((height, width))
dataset.write(matrix.astype(np.uint8), indexes=b + 1)
return outpath
except:
return None