def classify(self): def calculate_chunks(width, height, tiles): pixels = width * height max_pixels = pixels / tiles chunk_size = int(math.floor(math.sqrt(max_pixels))) ncols = int(math.ceil(width / chunk_size)) nrows = int(math.ceil(height / chunk_size)) chunk_windows = [] for col in range(ncols): col_offset = col * chunk_size w = min(chunk_size, width - col_offset) for row in range(nrows): row_offset = row * chunk_size h = min(chunk_size, height - row_offset) chunk_windows.append( ((row, col), Window(col_offset, row_offset, w, h))) return chunk_windows with rasterio.open(self.rlayer.dataProvider().dataSourceUri()) as src: width = src.width height = src.height bands = src.count meta = src.meta dtype = src.dtypes self.signals.status.emit(strftime("%Y-%m-%d %H:%M:%S", gmtime()), "Predicting image values ... ") chunk_blocks = calculate_chunks(width, height, self.tiles) meta.update({"count": 1, "dtype": dtype[0]}) with rasterio.open(self.outlayer, "w", **meta) as dst: counter = 1 for idx, window in chunk_blocks: self.signals.status.emit( strftime("%Y-%m-%d %H:%M:%S", gmtime()), "Processing Block: " + str(counter) + " of " + str(len(chunk_blocks))) img = src.read(window=window) dtype = rasterio.dtypes.get_minimum_dtype(img) reshaped_img = reshape_as_image(img) rows, cols, bands_n = reshaped_img.shape class_prediction = self.classifier.predict( reshaped_img.reshape(-1, bands)) classification = np.zeros((rows, cols, 1)).astype(dtype) classification[:, :, 0] = class_prediction.reshape( reshaped_img[:, :, 1].shape).astype(dtype) final = reshape_as_raster(classification) dst.write(final, window=window) counter += 1 seconds_elapsed = time() - self.starttime self.signals.status.emit( strftime("%Y-%m-%d %H:%M:%S", gmtime()), "Execution completed in " + str(np.around(seconds_elapsed, decimals=2)) + " seconds", ) return self.outlayer
def raster_to_disk(np_array, new_rname, new_rmeta, outdir): """ Save a numpy array as geo-raster to disk 'p_rpath' param uses a second raster filename as prefix ********* params: np_array -> 3D numpy array to save as raster new_rname -> name for the new raster new_rmeta -> metadata for the new raster outdir -> output directory return: new_rpath -> full path of the new raster """ assert (new_rmeta['driver'] == 'GTiff'),\ "Please use GTiff driver to write to disk. \ Passed {} instead." .format(new_rmeta['driver']) assert (np_array.ndim == 3),\ "np_array must have ndim = 3. \ Passed np_array of dimension {} instead." .format(np_array.ndim) name = new_rname + '.tif' new_rpath = os.path.join(outdir, name) with rasterio.open(new_rpath, 'w', **new_rmeta) as dst: dst.write(reshape_as_raster(np_array)) return new_rpath
def apply_upper_thresh(t_series, hot_t_series, upper_thresh_arr, initial_kmeans_clouds, hot_potential_clear, hot_potential_cloudy, dn_max): """Applies the masking logic to refine the initial cloud masks from k-means using the global threshold and upper threshold computed from the time series. Returns a time series of refined masks where 2 is cloud and 1 is clear land.""" cloud_series_mean_global = np.nanmean(hot_potential_cloudy.flatten(), axis=0) cloud_series_std_global = np.nanstd(hot_potential_cloudy.flatten(), axis=0) global_cloud_thresh = cloud_series_mean_global - 1.0 * cloud_series_std_global # 0 is where hot is below upper threshold, 1 is above # refine initial cloud initial_kmeans_clouds_binary = np.where(initial_kmeans_clouds > 0, 2, 1) refined_masks = np.where(np.less(hot_potential_cloudy, upper_thresh_arr), 1, initial_kmeans_clouds_binary) # add missed clouds refined_masks = np.where( np.logical_and( np.greater(hot_potential_clear, upper_thresh_arr), reshape_as_raster(np.greater(t_series[:, :, 3, :], dn_max * .1))), 2, refined_masks) # global_thresh_arr = np.ones(refined_masks.shape)*global_cloud_thresh doesn't have much impact in intiial tests refined_masks = np.where(hot_t_series > global_cloud_thresh, 2, refined_masks) return refined_masks
def stitch(cls, r_obj_up, r_obj_down, axis=0, write=False, outdir=None): """ (to be tested) """ height = r_obj_up.height + r_obj_down.height width = r_obj_up.width new_meta = copy.deepcopy(r_obj_up.meta) new_meta.update(driver='GTiff', height=height, width=width) arr_up = r_obj_up.load_as_arr() arr_down = r_obj_down.load_as_arr() arr_final = np.concatenate((arr_up, arr_down), axis=axis) if write: fname = os.path.basename(r_obj_up.path_to_raster).split('.')[0]+'_stitched' if not outdir: # set outdir as the input raster location outdir = os.path.dirname(r_obj_up.path_to_raster) with rasterio.open(os.path.join(outdir,fname+'.gtif'), 'w', **new_meta) as dst: dst.write(reshape_as_raster(arr_final)) return arr_final
def proj_frames(movie, dst, prefix="proj"): lensParameters = movie["camera_config"]["camera_type"]["lensParameters"] lensPosition = movie["camera_config"]["lensPosition"] src = os.path.join(movie['file']['bucket'], movie['file']['identifier']) n = 0 # frame number for _t, img in ORC.io.frames(src, start_frame=0, grayscale=True, lens_pars=lensParameters): # make a filename dst_fn = os.path.join( dst, "{:s}_{:04d}_{:06d}.tif".format(prefix, n, int(_t * 1000))) print(f"Putting frame {n} in file {dst_fn}") n += 1 # increase frame number by one corr_img, transform = ORC.cv.orthorectification( img=img, lensPosition=lensPosition, h_a=movie["h_a"], bbox=movie["camera_config"]["aoi_bbox"], resolution=movie["camera_config"]["resolution"], **movie["camera_config"]["gcps"], ) if len(corr_img.shape) == 3: raster = np.int8(reshape_as_raster(corr_img)) else: raster = np.int8(np.expand_dims(corr_img, axis=0)) # write to temporary file ORC.io.to_geotiff( dst_fn, raster, transform, crs=f"EPSG:{movie['camera_config']['site']['crs']}", compress="JPEG", )
def _adjust_array(self, arr): ''' ''' arr = arr * self.rgb_scalar red = arr[:, :, 0] * self.red_scalar grn = arr[:, :, 1] * self.grn_scalar blu = arr[:, :, 2] * self.blu_scalar arr = np.dstack((red, grn, blu)) return reshape_as_raster(arr)
def saveGeorefLabelMap(label_map, georef_filename, out_name): # load georeference information to use img = rio.open(georef_filename) meta = img.meta myLabel = reshape_as_raster(label_map) myLabel_meta = meta.copy() myLabel_meta.update({"dtype": rio.uint8, "count": 3, "nodata": None}) with rio.open(out_name + ".tif", "w", **myLabel_meta) as dest: dest.write(myLabel)
def stack_t_series(paths, stackname): """" Stack third axis-wise. all tifs must be same extent and in sorted order by date """ arrs = [ski.io.imread(path) for path in paths] stacked = reshape_as_raster(np.dstack(arrs)) img = rio.open(paths[0]) meta = img.profile meta.update(count=len(arrs) * arrs[0].shape[2]) with rio.open(stackname, 'w', **meta) as dst: dst.write(stacked) print("Saved Time Series with " + str(len(arrs)) + " images and " + str(arrs[0].shape[2]) + " bands each")
def compute_pca_image(file, band_order, nr_components, outfile=None): #read all bands (except B9 and B1 = 60m) and metadata with rio.open(file) as src: img = src.read(band_order) profile = src.profile.copy() arr = reshape_as_image(img).reshape(-1, len(img)) PC = PCA(n_components=nr_components, random_state=42).fit(arr) #perform orthogonal projection/ transformation (rotation) of the array into the vector space of fitted PC #also reduces the dimensionality (bands) into 1D space PC_transformed = PC.transform(arr) #reshape to 2d PC_2d = PC_transformed.reshape(img.shape[1], img.shape[2], nr_components) # rescale to 0-65535 values 16bit datatype to reduce further k-means clustering time PC_2d_Norm = np.zeros((img.shape[1], img.shape[2], len(img))) for i in range(nr_components): PC_2d_Norm[:, :, i] = ( (PC_2d[:, :, i] - PC_2d[:, :, i].min()) * (1 / (PC_2d[:, :, i].max() - PC_2d[:, :, i].min()) * 65535)) #reshape to raster pca_img = reshape_as_raster(PC_2d)[0:, :, :] #output profile.update({ 'nodata': None, 'dtype': rio.float32, 'count': len(pca_img), 'width': img.shape[2], 'height': img.shape[1] }) if outfile is not None: with rio.open(outfile, 'w', **profile) as dst: dst.write(pca_img.astype(rio.float32)) else: return pca_img.astype(rio.float32)
def show_image(img, transform, polygons=[], ax=None, band_idx=[0], shp_dict={ 'linewidth': 1, 'facecolor': (0, 0, 0, 0), 'edgecolor': 'black' }, cmap='Greys'): """ """ if len(img.shape) == 3: rioplot.show(rioplot.reshape_as_raster(img[:, :, band_idx]), transform=transform, ax=ax, cmap=cmap) elif len(img.shape) == 2: rioplot.show(img, transform=transform, ax=ax, cmap=cmap) else: raise 'Wrong image dimension. Must be HxW or HxWxB' for p in polygons: ax.add_patch(matplotlib.patches.Polygon(p, **shp_dict))
def test_reshape_as_raster(): img_arr = np.random.randn(718, 791, 3) rast_arr = plot.reshape_as_raster(img_arr) assert img_arr.shape[-1] == rast_arr.shape[0]
def rescale_landsat(): ''' Runs a loop to apply offsets in header file to Collection 2 level 1 (surface refl) Landsat data. Then applies a 1-95% image stretch and resamples to uint8 for Super Resolution model. Outputs 3-band tiff of full landsat scene. Inputs come from env variable 'three_band_full_pths_uint16' (i.e. /mnt/gcs/landsat*/*/merge-sr/*.tif) Outputs go to a dir with the the same path as imput dir but with "-uint8" appended (i.e. /mnt/gcs/landsat[n]/[region]/merged-sr-uint8) ''' ## Loop three_band_full_pths_uint16 = glob( ls_dirs[0] + '/*/merge-sr/*.tif', recursive=True ) + glob( ls_dirs[1] + '/*/merge-sr/*.tif', recursive=True ) # Search right before running loop so I don't add any new in the interim for pth in three_band_full_pths_uint16: # e.g. /mnt/gcs/landsat5/fairbanks/merge-sr/LT05_L2SP_070014_19850831_20200918_02_T1_SR_B423.tif ## I/O three_band_base = os.path.basename( pth ) # basename # e.g. LT05_L2SP_070014_19850831_20200918_02_T1_SR_B423.tif ls_id = three_band_base[: -12] # landsat ID e.g. LT05_L2SP_070014_19850831_20200918_02_T1 ## skip if commented out or not present in file_paths.yml if ls_id not in file_inputs['bases'].keys(): print(f'Skipping ID {ls_id} bc it is not in input file yml list.') continue dir_out = (os.path.dirname(pth)).replace('merge-sr', 'merge-sr-uint8') os.makedirs( dir_out, exist_ok=True ) # make new dir e.g. /mnt/gcs/landsat5/fairbanks/merge-sr-uint8 pth_out = os.path.join( dir_out, three_band_base ) # e.g. /mnt/gcs/landsat5/fairbanks/merge-sr-uint8/LT05_L2SP_070014_19850831_20200918_02_T1_SR_B423.tif ## Print to terminal print(f'Reading Landsat ID: {ls_id}') ## check if output image doesn't already exist if os.path.exists(pth_out) == True: # print(f'Skipping output bc it exists: {pth_out}') continue ## More pathnames (do last because glob search is slower) tar_pth = glob( base_dir + '/landsat*/download/' + ls_id + '.tar', recursive=True )[0] # e.g. /mnt/gcs/landsat8/download/LC08_L2SP_070014_20210818_20210827_02_T1.tar json_pth = ls_id + '_MTL.json' # os.path.join(tar_pth, ls_id+'_MTL.json') # relative to tar file, e.g. LC08_L2SP_070014_20210818_20210827_02_T1_MTL.json' qa_pth = os.path.join(tar_pth, ls_id + '_QA_PIXEL.TIF') radsat_pth = os.path.join(tar_pth, ls_id + '_QA_RADSAT.TIF') if 'landsat5' in pth: bands = ls_bands[0] elif 'landsat8' in pth: bands = ls_bands[1] # e.g. [5, 3, 4] else: warnings.warn('Warning (EDK): Which landsat number?') ## Parse metadata for calibration constants with tarfile.open(tar_pth) as f: json_file = f.extractfile(json_pth).read() metadata = json.loads(json_file) add = [ float(metadata['LANDSAT_METADATA_FILE'] ['LEVEL1_RADIOMETRIC_RESCALING'] [f'REFLECTANCE_ADD_BAND_{bands[0]}']), float(metadata['LANDSAT_METADATA_FILE'] ['LEVEL1_RADIOMETRIC_RESCALING'] [f'REFLECTANCE_ADD_BAND_{bands[1]}']), float(metadata['LANDSAT_METADATA_FILE'] ['LEVEL1_RADIOMETRIC_RESCALING'] [f'REFLECTANCE_ADD_BAND_{bands[2]}']), ] mult = [ float(metadata['LANDSAT_METADATA_FILE'] ['LEVEL1_RADIOMETRIC_RESCALING'] [f'REFLECTANCE_MULT_BAND_{bands[0]}']), float(metadata['LANDSAT_METADATA_FILE'] ['LEVEL1_RADIOMETRIC_RESCALING'] [f'REFLECTANCE_MULT_BAND_{bands[1]}']), float(metadata['LANDSAT_METADATA_FILE'] ['LEVEL1_RADIOMETRIC_RESCALING'] [f'REFLECTANCE_MULT_BAND_{bands[2]}']), ] elevation = float(metadata['LANDSAT_METADATA_FILE']['IMAGE_ATTRIBUTES'] ['SUN_ELEVATION']) ## Load images # can easily modify to load from tarball instead with rio.open(pth) as src: profile = src.profile img_u16 = src.read() img_u16 = reshape_as_image(img_u16) ## Load mask with rio.open('/vsitar/' + qa_pth) as src: qa_band = src.read(1) mask = qa_band == 1 # this is a negative mask (pixels to remove) that I will use to create nans for fill values outside of image frame ## Load RADSAT (radiometric saturation file) with rio.open('/vsitar/' + radsat_pth) as src: radsat_band = src.read(1) ## Apply mask img_u16 = img_u16.astype( 'float' ) # convert to float to allow nans and for arithmetic in next section img_u16[mask] = np.nan ## Create high saturation mask mask_sn_cl = np.any( dec2bin_arrayK(qa_band, qa_bit_vals), 2 ) # negative mask (is 1 for all pixels that meet the cloud/snow criteria) mask_satr = mask_sn_cl | np.any( dec2bin_arrayK(radsat_band, radsat_bit_vals), 2 ) # negative mask (is 1 for all pixels that meet the saturation criteria) ## Save mem del (mask_sn_cl, qa_band, radsat_band) ## Apply calibration coeffs: Reshape coefficient arrays tmp = np.zeros([1, 1, 3]) tmp[0, 0, :] = mult mult = tmp.copy() tmp[0, 0, :] = add add = tmp ## Apply calibration coeffs: Matrix multiplication img_u16 = (img_u16 * mult + add) / np.sin(elevation / 360 * 2 * np.pi) ## Find percentiles for stretch img_u16_masked_in = img_u16[np.repeat( (~mask & ~mask_satr)[:, :, None], 3, axis=2 )] # positive mask with no fill, clouds, snow, or saturation (flattened) top_val = np.quantile(img_u16_masked_in, top_quantile) btm_val = np.quantile(img_u16_masked_in, btm_quantile) print(f'Top percentile: {top_val}\nBottom percentile: {btm_val}') del (img_u16_masked_in) ## Apply stretch img_u8 = rescale_reflectance_absolute(img_u16, btm_val, top_val) img_u8[ mask] = 0 # important bc dataset doesn't originally have 0 as nodata value ## Testing: look at bin counts # # b1 = img_u16[:,:,0].flatten() # # counts = np.bincount(b1[~np.isnan(b1)]) # for k in np.arange(3): # print(np.bincount(img_u8[:,:,k].flatten())) ## Write out img_u8 = reshape_as_raster(img_u8) profile.update(dtype='uint8', nodata=0) with rio.open( pth_out, 'w', **profile ) as dst: # e.g. /mnt/gcs/landsat8/fairbanks/merge-sr-uint8/LC08_L2SP_070014_20210818_20210827_02_T1_SR_B534.tif dst.write(img_u8) print(f'\nWrote file: {pth_out}')
def test_reshape_as_raster(): img_arr = np.random.randn(718, 791, 3) rast_arr = plot.reshape_as_raster(img_arr) assert img_arr.shape[-1] == rast_arr.shape[0]
def extract_project_frames(movie, prefix="proj", logger=logging): """ Extract frames, lens correct, greyscale correct and project to defined AOI with GCPs, water level and camera position Results in GeoTIFF files in desired projection and resolution within bucket defined in movie :param movie: dict, movie information :param prefix="proj": str, prefix of file names, used in storage bucket, normally not changed by user :param logger=logging: logger-object :return: None """ # open S3 bucket camera_config = movie["camera_config"] s3 = utils.get_s3() n = 0 logger.info( f"Writing movie {movie['file']['identifier']} to {movie['file']['bucket']}" ) # open file from bucket in memory bucket = movie["file"]["bucket"] fn = movie["file"]["identifier"] # make a temporary file s3.Bucket(bucket).download_file(fn, fn) for _t, img in OpenRiverCam.io.frames( fn, grayscale=True, lens_pars=camera_config["camera_type"]["lensParameters"]): # filename in bucket, following template frame_{4-digit_framenumber}_{time_in_milliseconds}.jpg dest_fn = "{:s}_{:04d}_{:06d}.tif".format(prefix, n, int(_t * 1000)) logger.debug(f"Write frame {n} in {dest_fn} to S3") bbox = shape( camera_config["aoi"]["bbox"]["features"][0] ["geometry"]) # extract the one and only geometry from geojson # reproject frame with camera_config # inputs needed corr_img, transform = OpenRiverCam.cv.orthorectification( img=img, lensPosition=camera_config["lensPosition"], h_a=movie["h_a"], bbox=bbox, resolution=camera_config["resolution"], **camera_config["gcps"], ) if len(corr_img.shape) == 3: # RGB image raster = np.int8(reshape_as_raster(corr_img)) else: # b-w image (0-255) just add an axis raster = np.int8(np.expand_dims(corr_img, axis=0)) # write to temporary file OpenRiverCam.io.to_geotiff( "temp.tif", raster, transform, crs=camera_config["site"]["crs"], compress="deflate", ) # Put file in bucket s3.Bucket(bucket).upload_file("temp.tif", dest_fn) n += 1 # finally write last frame as .jpg for front end and write geotransform as .csv dest_fn = "reprojection_preview.jpg" trans_fn = "reprojection_preview.transform" # file name for geotransform ret, im_en = cv2.imencode(".jpg", corr_img) buf = io.BytesIO(im_en) # Seek beginning of bytestream buf.seek(0) # Put file in bucket s3.Object(bucket, dest_fn).put(Body=buf) # write the geotransform buf = io.BytesIO(str(transform).encode()) buf.seek(0) s3.Object(bucket, trans_fn).put(Body=buf) # clean up of temp file os.remove(fn) logger.info(f"{fn} successfully reprojected into frames in {bucket}")
from rasterio.plot import reshape_as_raster import skimage.io as skio import numpy as np import rasterio as rio path_id = "water" def save_chips(path_id) """ Run after pyatsa make inputs to save each image as RGB. Make sure paths are setup before running. """ with rio.open('/home/rave/cloud-free-planet/cfg/buffered_stacked/'+path_id+ '_stacked.tif') as dataset: profile = dataset.profile.copy() meta = dataset.meta.copy() arr = dataset.read() arr = np.moveaxis(arr, 0, 2) arr = np.reshape(arr, (arr.shape[0],arr.shape[1], 4, int(arr.shape[2]/4)), order='F') for i in np.arange(arr.shape[-1]): blue = arr[:,:,0,i] green = arr[:,:,1,i] red = arr[:,:,2,i] stacked = np.stack([red, green, blue], axis = 2) out_path = "/home/rave/cloud-free-planet/cfg/buffered_chips/scene_number_"+str(i)+"_"+path_id+"_.tif" profile.update(count=3) with rio.open(out_path,'w',**profile) as dst: dst.write(reshape_as_raster(stacked))