def classify(self):
def calculate_chunks(width, height, tiles):
pixels = width * height
max_pixels = pixels / tiles
chunk_size = int(math.floor(math.sqrt(max_pixels)))
ncols = int(math.ceil(width / chunk_size))
nrows = int(math.ceil(height / chunk_size))
chunk_windows = []
for col in range(ncols):
col_offset = col * chunk_size
w = min(chunk_size, width - col_offset)
for row in range(nrows):
row_offset = row * chunk_size
h = min(chunk_size, height - row_offset)
chunk_windows.append(
((row, col), Window(col_offset, row_offset, w, h)))
return chunk_windows
with rasterio.open(self.rlayer.dataProvider().dataSourceUri()) as src:
width = src.width
height = src.height
bands = src.count
meta = src.meta
dtype = src.dtypes
self.signals.status.emit(strftime("%Y-%m-%d %H:%M:%S", gmtime()),
"Predicting image values ... ")
chunk_blocks = calculate_chunks(width, height, self.tiles)
meta.update({"count": 1, "dtype": dtype[0]})
with rasterio.open(self.outlayer, "w", **meta) as dst:
counter = 1
for idx, window in chunk_blocks:
self.signals.status.emit(
strftime("%Y-%m-%d %H:%M:%S",
gmtime()), "Processing Block: " +
str(counter) + " of " + str(len(chunk_blocks)))
img = src.read(window=window)
dtype = rasterio.dtypes.get_minimum_dtype(img)
reshaped_img = reshape_as_image(img)
rows, cols, bands_n = reshaped_img.shape
class_prediction = self.classifier.predict(
reshaped_img.reshape(-1, bands))
classification = np.zeros((rows, cols, 1)).astype(dtype)
classification[:, :, 0] = class_prediction.reshape(
reshaped_img[:, :, 1].shape).astype(dtype)
final = reshape_as_raster(classification)
dst.write(final, window=window)
counter += 1
seconds_elapsed = time() - self.starttime
self.signals.status.emit(
strftime("%Y-%m-%d %H:%M:%S", gmtime()),
"Execution completed in " +
str(np.around(seconds_elapsed, decimals=2)) + " seconds",
)
return self.outlayer
def raster_to_disk(np_array, new_rname, new_rmeta, outdir):
"""
Save a numpy array as geo-raster to disk
'p_rpath' param uses a second raster filename as prefix
*********
params:
np_array -> 3D numpy array to save as raster
new_rname -> name for the new raster
new_rmeta -> metadata for the new raster
outdir -> output directory
return:
new_rpath -> full path of the new raster
"""
assert (new_rmeta['driver'] == 'GTiff'),\
"Please use GTiff driver to write to disk. \
Passed {} instead." .format(new_rmeta['driver'])
assert (np_array.ndim == 3),\
"np_array must have ndim = 3. \
Passed np_array of dimension {} instead." .format(np_array.ndim)
name = new_rname + '.tif'
new_rpath = os.path.join(outdir, name)
with rasterio.open(new_rpath, 'w', **new_rmeta) as dst:
dst.write(reshape_as_raster(np_array))
return new_rpath
def apply_upper_thresh(t_series, hot_t_series, upper_thresh_arr,
initial_kmeans_clouds, hot_potential_clear,
hot_potential_cloudy, dn_max):
"""Applies the masking logic to refine the initial cloud
masks from k-means using the global threshold and
upper threshold computed from the time series.
Returns a time series of refined masks where 2 is cloud and 1 is clear land."""
cloud_series_mean_global = np.nanmean(hot_potential_cloudy.flatten(),
axis=0)
cloud_series_std_global = np.nanstd(hot_potential_cloudy.flatten(), axis=0)
global_cloud_thresh = cloud_series_mean_global - 1.0 * cloud_series_std_global
# 0 is where hot is below upper threshold, 1 is above
# refine initial cloud
initial_kmeans_clouds_binary = np.where(initial_kmeans_clouds > 0, 2, 1)
refined_masks = np.where(np.less(hot_potential_cloudy, upper_thresh_arr),
1, initial_kmeans_clouds_binary)
# add missed clouds
refined_masks = np.where(
np.logical_and(
np.greater(hot_potential_clear, upper_thresh_arr),
reshape_as_raster(np.greater(t_series[:, :, 3, :], dn_max * .1))),
2, refined_masks)
# global_thresh_arr = np.ones(refined_masks.shape)*global_cloud_thresh doesn't have much impact in intiial tests
refined_masks = np.where(hot_t_series > global_cloud_thresh, 2,
refined_masks)
return refined_masks
def stitch(cls, r_obj_up, r_obj_down, axis=0, write=False, outdir=None):
"""
(to be tested)
"""
height = r_obj_up.height + r_obj_down.height
width = r_obj_up.width
new_meta = copy.deepcopy(r_obj_up.meta)
new_meta.update(driver='GTiff', height=height, width=width)
arr_up = r_obj_up.load_as_arr()
arr_down = r_obj_down.load_as_arr()
arr_final = np.concatenate((arr_up, arr_down), axis=axis)
if write:
fname = os.path.basename(r_obj_up.path_to_raster).split('.')[0]+'_stitched'
if not outdir:
# set outdir as the input raster location
outdir = os.path.dirname(r_obj_up.path_to_raster)
with rasterio.open(os.path.join(outdir,fname+'.gtif'), 'w', **new_meta) as dst:
dst.write(reshape_as_raster(arr_final))
return arr_final
def proj_frames(movie, dst, prefix="proj"):
lensParameters = movie["camera_config"]["camera_type"]["lensParameters"]
lensPosition = movie["camera_config"]["lensPosition"]
src = os.path.join(movie['file']['bucket'], movie['file']['identifier'])
n = 0 # frame number
for _t, img in ORC.io.frames(src,
start_frame=0,
grayscale=True,
lens_pars=lensParameters):
# make a filename
dst_fn = os.path.join(
dst, "{:s}_{:04d}_{:06d}.tif".format(prefix, n, int(_t * 1000)))
print(f"Putting frame {n} in file {dst_fn}")
n += 1 # increase frame number by one
corr_img, transform = ORC.cv.orthorectification(
img=img,
lensPosition=lensPosition,
h_a=movie["h_a"],
bbox=movie["camera_config"]["aoi_bbox"],
resolution=movie["camera_config"]["resolution"],
**movie["camera_config"]["gcps"],
)
if len(corr_img.shape) == 3:
raster = np.int8(reshape_as_raster(corr_img))
else:
raster = np.int8(np.expand_dims(corr_img, axis=0))
# write to temporary file
ORC.io.to_geotiff(
dst_fn,
raster,
transform,
crs=f"EPSG:{movie['camera_config']['site']['crs']}",
compress="JPEG",
)
def _adjust_array(self, arr):
'''
'''
arr = arr * self.rgb_scalar
red = arr[:, :, 0] * self.red_scalar
grn = arr[:, :, 1] * self.grn_scalar
blu = arr[:, :, 2] * self.blu_scalar
arr = np.dstack((red, grn, blu))
return reshape_as_raster(arr)
def saveGeorefLabelMap(label_map, georef_filename, out_name):
# load georeference information to use
img = rio.open(georef_filename)
meta = img.meta
myLabel = reshape_as_raster(label_map)
myLabel_meta = meta.copy()
myLabel_meta.update({"dtype": rio.uint8, "count": 3, "nodata": None})
with rio.open(out_name + ".tif", "w", **myLabel_meta) as dest:
dest.write(myLabel)
def stack_t_series(paths, stackname):
""""
Stack third axis-wise. all
tifs must be same extent and in sorted order by date
"""
arrs = [ski.io.imread(path) for path in paths]
stacked = reshape_as_raster(np.dstack(arrs))
img = rio.open(paths[0])
meta = img.profile
meta.update(count=len(arrs) * arrs[0].shape[2])
with rio.open(stackname, 'w', **meta) as dst:
dst.write(stacked)
print("Saved Time Series with " + str(len(arrs)) + " images and " +
str(arrs[0].shape[2]) + " bands each")
def compute_pca_image(file, band_order, nr_components, outfile=None):
#read all bands (except B9 and B1 = 60m) and metadata
with rio.open(file) as src:
img = src.read(band_order)
profile = src.profile.copy()
arr = reshape_as_image(img).reshape(-1, len(img))
PC = PCA(n_components=nr_components, random_state=42).fit(arr)
#perform orthogonal projection/ transformation (rotation) of the array into the vector space of fitted PC
#also reduces the dimensionality (bands) into 1D space
PC_transformed = PC.transform(arr)
#reshape to 2d
PC_2d = PC_transformed.reshape(img.shape[1], img.shape[2],
nr_components)
# rescale to 0-65535 values 16bit datatype to reduce further k-means clustering time
PC_2d_Norm = np.zeros((img.shape[1], img.shape[2], len(img)))
for i in range(nr_components):
PC_2d_Norm[:, :, i] = (
(PC_2d[:, :, i] - PC_2d[:, :, i].min()) *
(1 / (PC_2d[:, :, i].max() - PC_2d[:, :, i].min()) * 65535))
#reshape to raster
pca_img = reshape_as_raster(PC_2d)[0:, :, :]
#output
profile.update({
'nodata': None,
'dtype': rio.float32,
'count': len(pca_img),
'width': img.shape[2],
'height': img.shape[1]
})
if outfile is not None:
with rio.open(outfile, 'w', **profile) as dst:
dst.write(pca_img.astype(rio.float32))
else:
return pca_img.astype(rio.float32)
def show_image(img,
transform,
polygons=[],
ax=None,
band_idx=[0],
shp_dict={
'linewidth': 1,
'facecolor': (0, 0, 0, 0),
'edgecolor': 'black'
},
cmap='Greys'):
""" """
if len(img.shape) == 3:
rioplot.show(rioplot.reshape_as_raster(img[:, :, band_idx]),
transform=transform,
ax=ax,
cmap=cmap)
elif len(img.shape) == 2:
rioplot.show(img, transform=transform, ax=ax, cmap=cmap)
else:
raise 'Wrong image dimension. Must be HxW or HxWxB'
for p in polygons:
ax.add_patch(matplotlib.patches.Polygon(p, **shp_dict))
def test_reshape_as_raster():
img_arr = np.random.randn(718, 791, 3)
rast_arr = plot.reshape_as_raster(img_arr)
assert img_arr.shape[-1] == rast_arr.shape[0]
def rescale_landsat():
'''
Runs a loop to apply offsets in header file to Collection 2 level 1 (surface refl) Landsat data. Then applies a 1-95% image stretch and resamples to uint8 for Super Resolution model. Outputs 3-band tiff of full landsat scene.
Inputs come from env variable 'three_band_full_pths_uint16' (i.e. /mnt/gcs/landsat*/*/merge-sr/*.tif)
Outputs go to a dir with the the same path as imput dir but with "-uint8" appended (i.e. /mnt/gcs/landsat[n]/[region]/merged-sr-uint8)
'''
## Loop
three_band_full_pths_uint16 = glob(
ls_dirs[0] + '/*/merge-sr/*.tif', recursive=True
) + glob(
ls_dirs[1] + '/*/merge-sr/*.tif', recursive=True
) # Search right before running loop so I don't add any new in the interim
for pth in three_band_full_pths_uint16: # e.g. /mnt/gcs/landsat5/fairbanks/merge-sr/LT05_L2SP_070014_19850831_20200918_02_T1_SR_B423.tif
## I/O
three_band_base = os.path.basename(
pth
) # basename # e.g. LT05_L2SP_070014_19850831_20200918_02_T1_SR_B423.tif
ls_id = three_band_base[:
-12] # landsat ID e.g. LT05_L2SP_070014_19850831_20200918_02_T1
## skip if commented out or not present in file_paths.yml
if ls_id not in file_inputs['bases'].keys():
print(f'Skipping ID {ls_id} bc it is not in input file yml list.')
continue
dir_out = (os.path.dirname(pth)).replace('merge-sr', 'merge-sr-uint8')
os.makedirs(
dir_out, exist_ok=True
) # make new dir e.g. /mnt/gcs/landsat5/fairbanks/merge-sr-uint8
pth_out = os.path.join(
dir_out, three_band_base
) # e.g. /mnt/gcs/landsat5/fairbanks/merge-sr-uint8/LT05_L2SP_070014_19850831_20200918_02_T1_SR_B423.tif
## Print to terminal
print(f'Reading Landsat ID: {ls_id}')
## check if output image doesn't already exist
if os.path.exists(pth_out) == True:
# print(f'Skipping output bc it exists: {pth_out}')
continue
## More pathnames (do last because glob search is slower)
tar_pth = glob(
base_dir + '/landsat*/download/' + ls_id + '.tar', recursive=True
)[0] # e.g. /mnt/gcs/landsat8/download/LC08_L2SP_070014_20210818_20210827_02_T1.tar
json_pth = ls_id + '_MTL.json' # os.path.join(tar_pth, ls_id+'_MTL.json') # relative to tar file, e.g. LC08_L2SP_070014_20210818_20210827_02_T1_MTL.json'
qa_pth = os.path.join(tar_pth, ls_id + '_QA_PIXEL.TIF')
radsat_pth = os.path.join(tar_pth, ls_id + '_QA_RADSAT.TIF')
if 'landsat5' in pth:
bands = ls_bands[0]
elif 'landsat8' in pth:
bands = ls_bands[1] # e.g. [5, 3, 4]
else:
warnings.warn('Warning (EDK): Which landsat number?')
## Parse metadata for calibration constants
with tarfile.open(tar_pth) as f:
json_file = f.extractfile(json_pth).read()
metadata = json.loads(json_file)
add = [
float(metadata['LANDSAT_METADATA_FILE']
['LEVEL1_RADIOMETRIC_RESCALING']
[f'REFLECTANCE_ADD_BAND_{bands[0]}']),
float(metadata['LANDSAT_METADATA_FILE']
['LEVEL1_RADIOMETRIC_RESCALING']
[f'REFLECTANCE_ADD_BAND_{bands[1]}']),
float(metadata['LANDSAT_METADATA_FILE']
['LEVEL1_RADIOMETRIC_RESCALING']
[f'REFLECTANCE_ADD_BAND_{bands[2]}']),
]
mult = [
float(metadata['LANDSAT_METADATA_FILE']
['LEVEL1_RADIOMETRIC_RESCALING']
[f'REFLECTANCE_MULT_BAND_{bands[0]}']),
float(metadata['LANDSAT_METADATA_FILE']
['LEVEL1_RADIOMETRIC_RESCALING']
[f'REFLECTANCE_MULT_BAND_{bands[1]}']),
float(metadata['LANDSAT_METADATA_FILE']
['LEVEL1_RADIOMETRIC_RESCALING']
[f'REFLECTANCE_MULT_BAND_{bands[2]}']),
]
elevation = float(metadata['LANDSAT_METADATA_FILE']['IMAGE_ATTRIBUTES']
['SUN_ELEVATION'])
## Load images # can easily modify to load from tarball instead
with rio.open(pth) as src:
profile = src.profile
img_u16 = src.read()
img_u16 = reshape_as_image(img_u16)
## Load mask
with rio.open('/vsitar/' + qa_pth) as src:
qa_band = src.read(1)
mask = qa_band == 1 # this is a negative mask (pixels to remove) that I will use to create nans for fill values outside of image frame
## Load RADSAT (radiometric saturation file)
with rio.open('/vsitar/' + radsat_pth) as src:
radsat_band = src.read(1)
## Apply mask
img_u16 = img_u16.astype(
'float'
) # convert to float to allow nans and for arithmetic in next section
img_u16[mask] = np.nan
## Create high saturation mask
mask_sn_cl = np.any(
dec2bin_arrayK(qa_band, qa_bit_vals), 2
) # negative mask (is 1 for all pixels that meet the cloud/snow criteria)
mask_satr = mask_sn_cl | np.any(
dec2bin_arrayK(radsat_band, radsat_bit_vals), 2
) # negative mask (is 1 for all pixels that meet the saturation criteria)
## Save mem
del (mask_sn_cl, qa_band, radsat_band)
## Apply calibration coeffs: Reshape coefficient arrays
tmp = np.zeros([1, 1, 3])
tmp[0, 0, :] = mult
mult = tmp.copy()
tmp[0, 0, :] = add
add = tmp
## Apply calibration coeffs: Matrix multiplication
img_u16 = (img_u16 * mult + add) / np.sin(elevation / 360 * 2 * np.pi)
## Find percentiles for stretch
img_u16_masked_in = img_u16[np.repeat(
(~mask & ~mask_satr)[:, :, None], 3, axis=2
)] # positive mask with no fill, clouds, snow, or saturation (flattened)
top_val = np.quantile(img_u16_masked_in, top_quantile)
btm_val = np.quantile(img_u16_masked_in, btm_quantile)
print(f'Top percentile: {top_val}\nBottom percentile: {btm_val}')
del (img_u16_masked_in)
## Apply stretch
img_u8 = rescale_reflectance_absolute(img_u16, btm_val, top_val)
img_u8[
mask] = 0 # important bc dataset doesn't originally have 0 as nodata value
## Testing: look at bin counts
# # b1 = img_u16[:,:,0].flatten()
# # counts = np.bincount(b1[~np.isnan(b1)])
# for k in np.arange(3):
# print(np.bincount(img_u8[:,:,k].flatten()))
## Write out
img_u8 = reshape_as_raster(img_u8)
profile.update(dtype='uint8', nodata=0)
with rio.open(
pth_out, 'w', **profile
) as dst: # e.g. /mnt/gcs/landsat8/fairbanks/merge-sr-uint8/LC08_L2SP_070014_20210818_20210827_02_T1_SR_B534.tif
dst.write(img_u8)
print(f'\nWrote file: {pth_out}')
def test_reshape_as_raster():
img_arr = np.random.randn(718, 791, 3)
rast_arr = plot.reshape_as_raster(img_arr)
assert img_arr.shape[-1] == rast_arr.shape[0]
def extract_project_frames(movie, prefix="proj", logger=logging):
"""
Extract frames, lens correct, greyscale correct and project to defined AOI with GCPs, water level and camera position
Results in GeoTIFF files in desired projection and resolution within bucket defined in movie
:param movie: dict, movie information
:param prefix="proj": str, prefix of file names, used in storage bucket, normally not changed by user
:param logger=logging: logger-object
:return: None
"""
# open S3 bucket
camera_config = movie["camera_config"]
s3 = utils.get_s3()
n = 0
logger.info(
f"Writing movie {movie['file']['identifier']} to {movie['file']['bucket']}"
)
# open file from bucket in memory
bucket = movie["file"]["bucket"]
fn = movie["file"]["identifier"]
# make a temporary file
s3.Bucket(bucket).download_file(fn, fn)
for _t, img in OpenRiverCam.io.frames(
fn,
grayscale=True,
lens_pars=camera_config["camera_type"]["lensParameters"]):
# filename in bucket, following template frame_{4-digit_framenumber}_{time_in_milliseconds}.jpg
dest_fn = "{:s}_{:04d}_{:06d}.tif".format(prefix, n, int(_t * 1000))
logger.debug(f"Write frame {n} in {dest_fn} to S3")
bbox = shape(
camera_config["aoi"]["bbox"]["features"][0]
["geometry"]) # extract the one and only geometry from geojson
# reproject frame with camera_config
# inputs needed
corr_img, transform = OpenRiverCam.cv.orthorectification(
img=img,
lensPosition=camera_config["lensPosition"],
h_a=movie["h_a"],
bbox=bbox,
resolution=camera_config["resolution"],
**camera_config["gcps"],
)
if len(corr_img.shape) == 3:
# RGB image
raster = np.int8(reshape_as_raster(corr_img))
else:
# b-w image (0-255) just add an axis
raster = np.int8(np.expand_dims(corr_img, axis=0))
# write to temporary file
OpenRiverCam.io.to_geotiff(
"temp.tif",
raster,
transform,
crs=camera_config["site"]["crs"],
compress="deflate",
)
# Put file in bucket
s3.Bucket(bucket).upload_file("temp.tif", dest_fn)
n += 1
# finally write last frame as .jpg for front end and write geotransform as .csv
dest_fn = "reprojection_preview.jpg"
trans_fn = "reprojection_preview.transform" # file name for geotransform
ret, im_en = cv2.imencode(".jpg", corr_img)
buf = io.BytesIO(im_en)
# Seek beginning of bytestream
buf.seek(0)
# Put file in bucket
s3.Object(bucket, dest_fn).put(Body=buf)
# write the geotransform
buf = io.BytesIO(str(transform).encode())
buf.seek(0)
s3.Object(bucket, trans_fn).put(Body=buf)
# clean up of temp file
os.remove(fn)
logger.info(f"{fn} successfully reprojected into frames in {bucket}")
from rasterio.plot import reshape_as_raster
import skimage.io as skio
import numpy as np
import rasterio as rio
path_id = "water"
def save_chips(path_id)
"""
Run after pyatsa make inputs to save each image as RGB. Make sure paths are setup before running.
"""
with rio.open('/home/rave/cloud-free-planet/cfg/buffered_stacked/'+path_id+ '_stacked.tif') as dataset:
profile = dataset.profile.copy()
meta = dataset.meta.copy()
arr = dataset.read()
arr = np.moveaxis(arr, 0, 2)
arr = np.reshape(arr, (arr.shape[0],arr.shape[1], 4, int(arr.shape[2]/4)), order='F')
for i in np.arange(arr.shape[-1]):
blue = arr[:,:,0,i]
green = arr[:,:,1,i]
red = arr[:,:,2,i]
stacked = np.stack([red, green, blue], axis = 2)
out_path = "/home/rave/cloud-free-planet/cfg/buffered_chips/scene_number_"+str(i)+"_"+path_id+"_.tif"
profile.update(count=3)
with rio.open(out_path,'w',**profile) as dst:
dst.write(reshape_as_raster(stacked))