def rolling_fltr(dem, f=np.nanmedian, size=3, circular=True, origmask=False): """General rolling filter (default operator is median filter) Can input any function f Efficient for smaller arrays, correclty handles NaN, fills gaps """ print("Applying rolling filter: %s with size %s" % (f.__name__, size)) dem = malib.checkma(dem) #Convert to float32 so we can fill with nan dem = dem.astype(np.float32) newshp = (dem.size, size * size) #Force a step size of 1 t = malib.sliding_window_padded(dem.filled(np.nan), (size, size), (1, 1)) if circular: if size > 3: mask = circular_mask(size) t[:, mask] = np.nan t = t.reshape(newshp) out = f(t, axis=1).reshape(dem.shape) out = np.ma.fix_invalid(out).astype(dem.dtype) out.set_fill_value(dem.fill_value) if origmask: out = np.ma.array(out, mask=np.ma.getmaskarray(dem)) return out
def get_closest_dt_idx(dt, dt_list): """Get indices of dt_list that is closest to input dt """ from pygeotools.lib import malib dt_list = malib.checkma(dt_list, fix=False) dt_diff = np.abs(dt - dt_list) return dt_diff.argmin()
def gauss_fltr_opencv(dem, size=3, sigma=1): """OpenCV Gaussian filter Still propagates NaN values """ import cv2 dem = malib.checkma(dem) dem_cv = cv2.GaussianBlur(dem.filled(np.nan), (size, size), sigma) out = np.ma.fix_invalid(dem_cv) out.set_fill_value(dem.fill_value) return out
def gauss_fltr_pyramid(dem, size=None, full=False, origmask=False): """Pyaramidal downsampling approach for gaussian smoothing Avoids the need for large kernels, very fast Needs testing """ dem = malib.checkma(dem) levels = int(np.floor(np.log2(size))) #print levels dim = (np.floor(np.array(dem.shape) / float(2**levels) + 1) * (2**levels)).astype(int) #print dem.shape #print dim #Can do something with np.pad here #np.pad(a_fp.filled(), 1, mode='constant', constant_values=(a_fp.fill_value,)) dem2 = np.full(dim, dem.fill_value) offset = np.floor((dim - np.array(dem.shape)) / 2.0).astype(int) #print offset #dem2[0:dem.shape[0],0:dem.shape[1]] = dem.data dem2[offset[0]:dem.shape[0] + offset[0], offset[1]:dem.shape[1] + offset[1]] = dem.data dem2 = np.ma.masked_equal(dem2, dem.fill_value) #dem2 = dem for n in range(levels): print(dem2.shape) dim = (np.floor(np.array(dem2.shape) / 2.0 + 1) * 2).astype(int) #dem2 = gauss_fltr_astropy(dem2, size=5, origmask=origmask) #dem2 = gauss_fltr_astropy(dem2, size=5) dem2 = gauss_fltr_astropy(dem2, size=5) #Note: Should use zoom with same bilinear interpolation here for consistency #However, this doesn't respect nan #dem2 = zoom(dem2, 0.5, order=1, prefilter=False, cval=dem.fill_value) dem2 = dem2[::2, ::2] if full: print("Resizing to original input dimensions") from scipy.ndimage import zoom for n in range(levels): print(dem2.shape) #Note: order 1 is bilinear dem2 = zoom(dem2, 2, order=1, prefilter=False, cval=dem.fill_value) #dem2 = zoom(dem2, 2**levels, order=1, prefilter=False, cval=dem2.fill_value) print(dem2.shape) #This was for power of 2 offset #offset = (2**levels)/2 #print offset #dem2 = dem2[offset:dem.shape[0]+offset,offset:dem.shape[1]+offset] #Use original offset dem2 = dem2[offset[0]:dem.shape[0] + offset[0], offset[1]:dem.shape[1] + offset[1]] if origmask: print("Applying original mask") #Allow filling of interior holes, but use original outer edge maskfill = malib.maskfill(dem) #dem2 = np.ma.array(dem2, mask=np.ma.getmaskarray(dem)) dem2 = np.ma.array(dem2, mask=maskfill, fill_value=dem.fill_value) return dem2
def get_closest_dt_padded_idx(dt, dt_list, pad=timedelta(days=30)): """Get indices of dt_list that is closest to input dt +/- pad days """ #If pad is in decimal days if not isinstance(pad, timedelta): pad = timedelta(days=pad) from pygeotools.lib import malib dt_list = malib.checkma(dt_list, fix=False) dt_diff = np.abs(dt - dt_list) valid_idx = (dt_diff.data < pad).nonzero()[0] return valid_idx
def best_scalebar_location(a, length_pad=0.2, height_pad=0.1): """ Attempt to determine best corner for scalebar based on number of unmasked pixels """ a = malib.checkma(a) length = int(a.shape[1] * length_pad) height = int(a.shape[0] * height_pad) d = {} d['upper right'] = a[0:height, -length:].count() d['upper left'] = a[0:height, 0:length].count() d['lower right'] = a[-height:, -length:].count() d['lower left'] = a[-height:, 0:length].count() loc = min(d, key=d.get) return loc
def median_fltr_opencv(dem, size=3, iterations=1): """OpenCV median filter """ import cv2 dem = malib.checkma(dem) if size > 5: print("Need to implement iteration") n = 0 out = dem while n <= iterations: dem_cv = cv2.medianBlur(out.astype(np.float32).filled(np.nan), size) out = np.ma.fix_invalid(dem_cv) out.set_fill_value(dem.fill_value) n += 1 return out
def ndanimate(a): import matplotlib.animation as animation a = malib.checkma(a) #Compute constant scale clim = malib.calcperc(a) label = 'Elev. Diff. (m)' fig = plt.figure() ims = [] for i in a: #cmap = 'gist_rainbow_r' cmap = 'cpt_rainbow' im = plt.imshow(i, cmap=cmap, clim=clim) im.axes.patch.set_facecolor('black') #cbar = fig.colorbar(im, extend='both', shrink=0.5) #cbar.set_label(label) ims.append([im]) an = animation.ArtistAnimation(fig, ims, interval=100, blit=True) plt.show() return an
def rolling_fltr(dem, f=np.nanmedian, size=3, circular=True): """General rolling filter (default operator is median filter) Can input any function f Efficient for smaller arrays, correclty handles NaN, fills gaps """ dem = malib.checkma(dem) newshp = (dem.size, size*size) #Force a step size of 1 t = malib.sliding_window_padded(dem.filled(np.nan), (size, size), (1, 1)) if circular: mask = circular_mask(size) t[:,mask] = np.nan t = t.reshape(newshp) out = f(t, axis=1).reshape(dem.shape) out = np.ma.fix_invalid(out) out.set_fill_value(dem.fill_value) return out
def median_fltr_skimage(dem, radius=3, erode=1, origmask=False): """ Older skimage.filter.median_filter This smooths, removes noise and fills in nodata areas with median of valid pixels! Effectively an inpainting routine """ #Note, ndimage doesn't properly handle ma - convert to nan dem = malib.checkma(dem) dem = dem.astype(np.float64) #Mask islands if erode > 0: print("Eroding islands smaller than %s pixels" % (erode * 2)) dem = malib.mask_islands(dem, iterations=erode) print("Applying median filter with radius %s" % radius) #Note: this funcitonality was present in scikit-image 0.9.3 import skimage.filter dem_filt_med = skimage.filter.median_filter(dem, radius, mask=~dem.mask) #Starting in version 0.10.0, this is the new filter #This is the new filter, but only supports uint8 or unit16 #import skimage.filters #import skimage.morphology #dem_filt_med = skimage.filters.rank.median(dem, disk(radius), mask=~dem.mask) #dem_filt_med = skimage.filters.median(dem, skimage.morphology.disk(radius), mask=~dem.mask) #Now mask all nans #skimage assigns the minimum value as nodata #CHECK THIS, seems pretty hacky #Also, looks like some valid values are masked at this stage, even though they should be above min ndv = np.min(dem_filt_med) #ndv = dem_filt_med.min() + 0.001 out = np.ma.masked_less_equal(dem_filt_med, ndv) #Should probably replace the ndv with original ndv out.set_fill_value(dem.fill_value) if origmask: print("Applying original mask") #Allow filling of interior holes, but use original outer edge #maskfill = malib.maskfill(dem, iterations=radius) maskfill = malib.maskfill(dem) #dem_filt_gauss = np.ma.array(dem_filt_gauss, mask=dem.mask, fill_value=dem.fill_value) out = np.ma.array(out, mask=maskfill, fill_value=dem.fill_value) return out
def gauss_fltr_astropy(dem, size=None, sigma=None, origmask=False, fill_interior=False): """Astropy gaussian filter properly handles convolution with NaN http://stackoverflow.com/questions/23832852/by-which-measures-should-i-set-the-size-of-my-gaussian-filter-in-matlab width1 = 3; sigma1 = (width1-1) / 6; Specify width for smallest feature of interest and determine sigma appropriately sigma is width of 1 std in pixels (not multiplier) scipy and astropy both use cutoff of 4*sigma on either side of kernel - 99.994% 3*sigma on either side of kernel - 99.7% If sigma is specified, filter width will be a multiple of 8 times sigma Alternatively, specify filter size, then compute sigma: sigma = (size - 1) / 8. If size is < the required width for 6-8 sigma, need to use different mode to create kernel mode 'oversample' and 'center' are essentially identical for sigma 1, but very different for sigma 0.3 The sigma/size calculations below should work for non-integer sigma """ #import astropy.nddata import astropy.convolution dem = malib.checkma(dem) #Generate 2D gaussian kernel for input sigma and size #Default size is 8*sigma in x and y directions #kernel = astropy.nddata.make_kernel([size, size], sigma, 'gaussian') #Size must be odd if size is not None: size = int(np.floor(size/2)*2 + 1) size = max(size, 3) #Truncate the filter at this many standard deviations. Default is 4.0 truncate = 3.0 if size is not None and sigma is None: sigma = (size - 1) / (2*truncate) elif size is None and sigma is not None: #Round up to nearest odd int size = int(np.ceil((sigma * (2*truncate) + 1)/2)*2 - 1) elif size is None and sigma is None: #Use default parameters sigma = 1 size = int(np.ceil((sigma * (2*truncate) + 1)/2)*2 - 1) size = max(size, 3) kernel = astropy.convolution.Gaussian2DKernel(sigma, x_size=size, y_size=size, mode='oversample') print("Applying gaussian smoothing filter with size %i and sigma %0.3f (sum %0.3f)" % \ (size, sigma, kernel.array.sum())) #This will fill holes #np.nan is float #dem_filt_gauss = astropy.nddata.convolve(dem.astype(float).filled(np.nan), kernel, boundary='fill', fill_value=np.nan) #dem_filt_gauss = astropy.convolution.convolve(dem.astype(float).filled(np.nan), kernel, boundary='fill', fill_value=np.nan) #Added normalization to ensure filtered values are not brightened/darkened if kernelsum != 1 dem_filt_gauss = astropy.convolution.convolve(dem.astype(float).filled(np.nan), kernel, boundary='fill', fill_value=np.nan, normalize_kernel=True) #This will preserve original ndv pixels, applying original mask after filtering if origmask: print("Applying original mask") #Allow filling of interior holes, but use original outer edge if fill_interior: mask = malib.maskfill(dem) else: mask = dem.mask dem_filt_gauss = np.ma.array(dem_filt_gauss, mask=mask, fill_value=dem.fill_value) out = np.ma.fix_invalid(dem_filt_gauss, copy=False, fill_value=dem.fill_value) out.set_fill_value(dem.fill_value.astype(dem.dtype)) return out.astype(dem.dtype)
def writeGTiff(a, dst_fn, src_ds=None, bnum=1, ndv=None, gt=None, proj=None, create=False, sparse=False): """Write input array to disk as GeoTiff Parameters ---------- a : np.array or np.ma.array Input array dst_fn : str Output filename src_ds: GDAL Dataset, optional Source Dataset to use for creating copy bnum : int, optional Output band ndv : float, optional Output NoData Value gt : list, optional Output GeoTransform proj : str, optional Output Projection (OGC WKT or PROJ.4 format) create : bool, optional Create new dataset sparse : bool, optional Output should be created with sparse options """ #If input is not np.ma, this creates a new ma, which has default filL_value of 1E20 #Must manually override with ndv #Also consumes a lot of memory #Should bypass if input is bool from pygeotools.lib.malib import checkma a = checkma(a, fix=False) #Want to preserve fill_value if already specified if ndv is not None: a.set_fill_value(ndv) driver = gtif_drv #Currently only support writing singleband rasters #if a.ndim > 2: # np_nbands = a.shape[2] # if src_ds.RasterCount np_nbands: # for bnum in np_nbands: nbands = 1 np_dt = a.dtype.name if src_ds is not None: #If this is a fn, get a ds #Note: this saves a lot of unnecessary iolib.fn_getds calls if isinstance(src_ds, str): src_ds = fn_getds(src_ds) #if isinstance(src_ds, gdal.Dataset): src_dt = gdal.GetDataTypeName(src_ds.GetRasterBand(bnum).DataType) src_gt = src_ds.GetGeoTransform() #This is WKT src_proj = src_ds.GetProjection() #src_srs = osr.SpatialReference() #src_srs.ImportFromWkt(src_ds.GetProjectionRef()) #Probably a cleaner way to handle this if gt is None: gt = src_gt if proj is None: proj = src_proj #Need to create a new copy of the default options opt = list(gdal_opt) #Note: packbits is better for sparse data if sparse: opt.remove('COMPRESS=LZW') opt.append('COMPRESS=PACKBITS') #Not sure if VW can handle sparse tif #opt.append('SPARSE_OK=TRUE') #Use predictor=3 for floating point data if 'float' in np_dt.lower() and 'COMPRESS=LZW' in opt: opt.append('PREDICTOR=3') #If input ma is same as src_ds, write out array using CreateCopy from existing dataset #if not create and (src_ds is not None) and ((a.shape[0] == src_ds.RasterYSize) and (a.shape[1] == src_ds.RasterXSize) and (np_dt.lower() == src_dt.lower())): #Should compare srs.IsSame(src_srs) if not create and (src_ds is not None) and ( (a.shape[0] == src_ds.RasterYSize) and (a.shape[1] == src_ds.RasterXSize) and (np_dt.lower() == src_dt.lower())) and (src_gt == gt) and (src_proj == proj): #Note: third option is strict flag, set to false dst_ds = driver.CreateCopy(dst_fn, src_ds, 0, options=opt) #Otherwise, use Create else: a_dtype = a.dtype gdal_dtype = np2gdal_dtype(a_dtype) if a_dtype.name == 'bool': #Set ndv to 0 a.fill_value = False opt.remove('COMPRESS=LZW') opt.append('COMPRESS=DEFLATE') #opt.append('NBITS=1') #Create(fn, nx, ny, nbands, dtype, opt) dst_ds = driver.Create(dst_fn, a.shape[1], a.shape[0], nbands, gdal_dtype, options=opt) #Note: Need GeoMA here to make this work, or accept gt as argument #Could also do ds creation in calling script if gt is not None: dst_ds.SetGeoTransform(gt) if proj is not None: dst_ds.SetProjection(proj) dst_ds.GetRasterBand(bnum).WriteArray(a.filled()) dst_ds.GetRasterBand(bnum).SetNoDataValue(float(a.fill_value)) dst_ds = None
def get_closest_dt_padded_idx(dt, dt_list, pad=timedelta(days=30)): from pygeotools.lib import malib dt_list = malib.checkma(dt_list, fix=False) dt_diff = np.abs(dt - dt_list) valid_idx = (dt_diff.data < pad).nonzero()[0] return valid_idx
def extractPoint(b, x, y): b = malib.checkma(b) x = np.clip(x, 0, b.shape[1] - 1) y = np.clip(y, 0, b.shape[0] - 1) #Note: simplest way using integer indices return b[np.ma.around(y).astype(int), np.ma.around(x).astype(int)]