def rolling_fltr(dem, f=np.nanmedian, size=3, circular=True, origmask=False):
"""General rolling filter (default operator is median filter)
Can input any function f
Efficient for smaller arrays, correclty handles NaN, fills gaps
"""
print("Applying rolling filter: %s with size %s" % (f.__name__, size))
dem = malib.checkma(dem)
#Convert to float32 so we can fill with nan
dem = dem.astype(np.float32)
newshp = (dem.size, size * size)
#Force a step size of 1
t = malib.sliding_window_padded(dem.filled(np.nan), (size, size), (1, 1))
if circular:
if size > 3:
mask = circular_mask(size)
t[:, mask] = np.nan
t = t.reshape(newshp)
out = f(t, axis=1).reshape(dem.shape)
out = np.ma.fix_invalid(out).astype(dem.dtype)
out.set_fill_value(dem.fill_value)
if origmask:
out = np.ma.array(out, mask=np.ma.getmaskarray(dem))
return out
def get_closest_dt_idx(dt, dt_list):
"""Get indices of dt_list that is closest to input dt
"""
from pygeotools.lib import malib
dt_list = malib.checkma(dt_list, fix=False)
dt_diff = np.abs(dt - dt_list)
return dt_diff.argmin()
def gauss_fltr_opencv(dem, size=3, sigma=1):
"""OpenCV Gaussian filter
Still propagates NaN values
"""
import cv2
dem = malib.checkma(dem)
dem_cv = cv2.GaussianBlur(dem.filled(np.nan), (size, size), sigma)
out = np.ma.fix_invalid(dem_cv)
out.set_fill_value(dem.fill_value)
return out
def gauss_fltr_pyramid(dem, size=None, full=False, origmask=False):
"""Pyaramidal downsampling approach for gaussian smoothing
Avoids the need for large kernels, very fast
Needs testing
"""
dem = malib.checkma(dem)
levels = int(np.floor(np.log2(size)))
#print levels
dim = (np.floor(np.array(dem.shape) / float(2**levels) + 1) *
(2**levels)).astype(int)
#print dem.shape
#print dim
#Can do something with np.pad here
#np.pad(a_fp.filled(), 1, mode='constant', constant_values=(a_fp.fill_value,))
dem2 = np.full(dim, dem.fill_value)
offset = np.floor((dim - np.array(dem.shape)) / 2.0).astype(int)
#print offset
#dem2[0:dem.shape[0],0:dem.shape[1]] = dem.data
dem2[offset[0]:dem.shape[0] + offset[0],
offset[1]:dem.shape[1] + offset[1]] = dem.data
dem2 = np.ma.masked_equal(dem2, dem.fill_value)
#dem2 = dem
for n in range(levels):
print(dem2.shape)
dim = (np.floor(np.array(dem2.shape) / 2.0 + 1) * 2).astype(int)
#dem2 = gauss_fltr_astropy(dem2, size=5, origmask=origmask)
#dem2 = gauss_fltr_astropy(dem2, size=5)
dem2 = gauss_fltr_astropy(dem2, size=5)
#Note: Should use zoom with same bilinear interpolation here for consistency
#However, this doesn't respect nan
#dem2 = zoom(dem2, 0.5, order=1, prefilter=False, cval=dem.fill_value)
dem2 = dem2[::2, ::2]
if full:
print("Resizing to original input dimensions")
from scipy.ndimage import zoom
for n in range(levels):
print(dem2.shape)
#Note: order 1 is bilinear
dem2 = zoom(dem2, 2, order=1, prefilter=False, cval=dem.fill_value)
#dem2 = zoom(dem2, 2**levels, order=1, prefilter=False, cval=dem2.fill_value)
print(dem2.shape)
#This was for power of 2 offset
#offset = (2**levels)/2
#print offset
#dem2 = dem2[offset:dem.shape[0]+offset,offset:dem.shape[1]+offset]
#Use original offset
dem2 = dem2[offset[0]:dem.shape[0] + offset[0],
offset[1]:dem.shape[1] + offset[1]]
if origmask:
print("Applying original mask")
#Allow filling of interior holes, but use original outer edge
maskfill = malib.maskfill(dem)
#dem2 = np.ma.array(dem2, mask=np.ma.getmaskarray(dem))
dem2 = np.ma.array(dem2, mask=maskfill, fill_value=dem.fill_value)
return dem2
def get_closest_dt_padded_idx(dt, dt_list, pad=timedelta(days=30)):
"""Get indices of dt_list that is closest to input dt +/- pad days
"""
#If pad is in decimal days
if not isinstance(pad, timedelta):
pad = timedelta(days=pad)
from pygeotools.lib import malib
dt_list = malib.checkma(dt_list, fix=False)
dt_diff = np.abs(dt - dt_list)
valid_idx = (dt_diff.data < pad).nonzero()[0]
return valid_idx
def best_scalebar_location(a, length_pad=0.2, height_pad=0.1):
"""
Attempt to determine best corner for scalebar based on number of unmasked pixels
"""
a = malib.checkma(a)
length = int(a.shape[1] * length_pad)
height = int(a.shape[0] * height_pad)
d = {}
d['upper right'] = a[0:height, -length:].count()
d['upper left'] = a[0:height, 0:length].count()
d['lower right'] = a[-height:, -length:].count()
d['lower left'] = a[-height:, 0:length].count()
loc = min(d, key=d.get)
return loc
def median_fltr_opencv(dem, size=3, iterations=1):
"""OpenCV median filter
"""
import cv2
dem = malib.checkma(dem)
if size > 5:
print("Need to implement iteration")
n = 0
out = dem
while n <= iterations:
dem_cv = cv2.medianBlur(out.astype(np.float32).filled(np.nan), size)
out = np.ma.fix_invalid(dem_cv)
out.set_fill_value(dem.fill_value)
n += 1
return out
def ndanimate(a):
import matplotlib.animation as animation
a = malib.checkma(a)
#Compute constant scale
clim = malib.calcperc(a)
label = 'Elev. Diff. (m)'
fig = plt.figure()
ims = []
for i in a:
#cmap = 'gist_rainbow_r'
cmap = 'cpt_rainbow'
im = plt.imshow(i, cmap=cmap, clim=clim)
im.axes.patch.set_facecolor('black')
#cbar = fig.colorbar(im, extend='both', shrink=0.5)
#cbar.set_label(label)
ims.append([im])
an = animation.ArtistAnimation(fig, ims, interval=100, blit=True)
plt.show()
return an
def rolling_fltr(dem, f=np.nanmedian, size=3, circular=True):
"""General rolling filter (default operator is median filter)
Can input any function f
Efficient for smaller arrays, correclty handles NaN, fills gaps
"""
dem = malib.checkma(dem)
newshp = (dem.size, size*size)
#Force a step size of 1
t = malib.sliding_window_padded(dem.filled(np.nan), (size, size), (1, 1))
if circular:
mask = circular_mask(size)
t[:,mask] = np.nan
t = t.reshape(newshp)
out = f(t, axis=1).reshape(dem.shape)
out = np.ma.fix_invalid(out)
out.set_fill_value(dem.fill_value)
return out
def median_fltr_skimage(dem, radius=3, erode=1, origmask=False):
"""
Older skimage.filter.median_filter
This smooths, removes noise and fills in nodata areas with median of valid pixels! Effectively an inpainting routine
"""
#Note, ndimage doesn't properly handle ma - convert to nan
dem = malib.checkma(dem)
dem = dem.astype(np.float64)
#Mask islands
if erode > 0:
print("Eroding islands smaller than %s pixels" % (erode * 2))
dem = malib.mask_islands(dem, iterations=erode)
print("Applying median filter with radius %s" % radius)
#Note: this funcitonality was present in scikit-image 0.9.3
import skimage.filter
dem_filt_med = skimage.filter.median_filter(dem, radius, mask=~dem.mask)
#Starting in version 0.10.0, this is the new filter
#This is the new filter, but only supports uint8 or unit16
#import skimage.filters
#import skimage.morphology
#dem_filt_med = skimage.filters.rank.median(dem, disk(radius), mask=~dem.mask)
#dem_filt_med = skimage.filters.median(dem, skimage.morphology.disk(radius), mask=~dem.mask)
#Now mask all nans
#skimage assigns the minimum value as nodata
#CHECK THIS, seems pretty hacky
#Also, looks like some valid values are masked at this stage, even though they should be above min
ndv = np.min(dem_filt_med)
#ndv = dem_filt_med.min() + 0.001
out = np.ma.masked_less_equal(dem_filt_med, ndv)
#Should probably replace the ndv with original ndv
out.set_fill_value(dem.fill_value)
if origmask:
print("Applying original mask")
#Allow filling of interior holes, but use original outer edge
#maskfill = malib.maskfill(dem, iterations=radius)
maskfill = malib.maskfill(dem)
#dem_filt_gauss = np.ma.array(dem_filt_gauss, mask=dem.mask, fill_value=dem.fill_value)
out = np.ma.array(out, mask=maskfill, fill_value=dem.fill_value)
return out
def gauss_fltr_astropy(dem, size=None, sigma=None, origmask=False, fill_interior=False):
"""Astropy gaussian filter properly handles convolution with NaN
http://stackoverflow.com/questions/23832852/by-which-measures-should-i-set-the-size-of-my-gaussian-filter-in-matlab
width1 = 3; sigma1 = (width1-1) / 6;
Specify width for smallest feature of interest and determine sigma appropriately
sigma is width of 1 std in pixels (not multiplier)
scipy and astropy both use cutoff of 4*sigma on either side of kernel - 99.994%
3*sigma on either side of kernel - 99.7%
If sigma is specified, filter width will be a multiple of 8 times sigma
Alternatively, specify filter size, then compute sigma: sigma = (size - 1) / 8.
If size is < the required width for 6-8 sigma, need to use different mode to create kernel
mode 'oversample' and 'center' are essentially identical for sigma 1, but very different for sigma 0.3
The sigma/size calculations below should work for non-integer sigma
"""
#import astropy.nddata
import astropy.convolution
dem = malib.checkma(dem)
#Generate 2D gaussian kernel for input sigma and size
#Default size is 8*sigma in x and y directions
#kernel = astropy.nddata.make_kernel([size, size], sigma, 'gaussian')
#Size must be odd
if size is not None:
size = int(np.floor(size/2)*2 + 1)
size = max(size, 3)
#Truncate the filter at this many standard deviations. Default is 4.0
truncate = 3.0
if size is not None and sigma is None:
sigma = (size - 1) / (2*truncate)
elif size is None and sigma is not None:
#Round up to nearest odd int
size = int(np.ceil((sigma * (2*truncate) + 1)/2)*2 - 1)
elif size is None and sigma is None:
#Use default parameters
sigma = 1
size = int(np.ceil((sigma * (2*truncate) + 1)/2)*2 - 1)
size = max(size, 3)
kernel = astropy.convolution.Gaussian2DKernel(sigma, x_size=size, y_size=size, mode='oversample')
print("Applying gaussian smoothing filter with size %i and sigma %0.3f (sum %0.3f)" % \
(size, sigma, kernel.array.sum()))
#This will fill holes
#np.nan is float
#dem_filt_gauss = astropy.nddata.convolve(dem.astype(float).filled(np.nan), kernel, boundary='fill', fill_value=np.nan)
#dem_filt_gauss = astropy.convolution.convolve(dem.astype(float).filled(np.nan), kernel, boundary='fill', fill_value=np.nan)
#Added normalization to ensure filtered values are not brightened/darkened if kernelsum != 1
dem_filt_gauss = astropy.convolution.convolve(dem.astype(float).filled(np.nan), kernel, boundary='fill', fill_value=np.nan, normalize_kernel=True)
#This will preserve original ndv pixels, applying original mask after filtering
if origmask:
print("Applying original mask")
#Allow filling of interior holes, but use original outer edge
if fill_interior:
mask = malib.maskfill(dem)
else:
mask = dem.mask
dem_filt_gauss = np.ma.array(dem_filt_gauss, mask=mask, fill_value=dem.fill_value)
out = np.ma.fix_invalid(dem_filt_gauss, copy=False, fill_value=dem.fill_value)
out.set_fill_value(dem.fill_value.astype(dem.dtype))
return out.astype(dem.dtype)
def writeGTiff(a,
dst_fn,
src_ds=None,
bnum=1,
ndv=None,
gt=None,
proj=None,
create=False,
sparse=False):
"""Write input array to disk as GeoTiff
Parameters
----------
a : np.array or np.ma.array
Input array
dst_fn : str
Output filename
src_ds: GDAL Dataset, optional
Source Dataset to use for creating copy
bnum : int, optional
Output band
ndv : float, optional
Output NoData Value
gt : list, optional
Output GeoTransform
proj : str, optional
Output Projection (OGC WKT or PROJ.4 format)
create : bool, optional
Create new dataset
sparse : bool, optional
Output should be created with sparse options
"""
#If input is not np.ma, this creates a new ma, which has default filL_value of 1E20
#Must manually override with ndv
#Also consumes a lot of memory
#Should bypass if input is bool
from pygeotools.lib.malib import checkma
a = checkma(a, fix=False)
#Want to preserve fill_value if already specified
if ndv is not None:
a.set_fill_value(ndv)
driver = gtif_drv
#Currently only support writing singleband rasters
#if a.ndim > 2:
# np_nbands = a.shape[2]
# if src_ds.RasterCount np_nbands:
# for bnum in np_nbands:
nbands = 1
np_dt = a.dtype.name
if src_ds is not None:
#If this is a fn, get a ds
#Note: this saves a lot of unnecessary iolib.fn_getds calls
if isinstance(src_ds, str):
src_ds = fn_getds(src_ds)
#if isinstance(src_ds, gdal.Dataset):
src_dt = gdal.GetDataTypeName(src_ds.GetRasterBand(bnum).DataType)
src_gt = src_ds.GetGeoTransform()
#This is WKT
src_proj = src_ds.GetProjection()
#src_srs = osr.SpatialReference()
#src_srs.ImportFromWkt(src_ds.GetProjectionRef())
#Probably a cleaner way to handle this
if gt is None:
gt = src_gt
if proj is None:
proj = src_proj
#Need to create a new copy of the default options
opt = list(gdal_opt)
#Note: packbits is better for sparse data
if sparse:
opt.remove('COMPRESS=LZW')
opt.append('COMPRESS=PACKBITS')
#Not sure if VW can handle sparse tif
#opt.append('SPARSE_OK=TRUE')
#Use predictor=3 for floating point data
if 'float' in np_dt.lower() and 'COMPRESS=LZW' in opt:
opt.append('PREDICTOR=3')
#If input ma is same as src_ds, write out array using CreateCopy from existing dataset
#if not create and (src_ds is not None) and ((a.shape[0] == src_ds.RasterYSize) and (a.shape[1] == src_ds.RasterXSize) and (np_dt.lower() == src_dt.lower())):
#Should compare srs.IsSame(src_srs)
if not create and (src_ds is not None) and (
(a.shape[0] == src_ds.RasterYSize) and
(a.shape[1] == src_ds.RasterXSize) and
(np_dt.lower() == src_dt.lower())) and (src_gt == gt) and (src_proj
== proj):
#Note: third option is strict flag, set to false
dst_ds = driver.CreateCopy(dst_fn, src_ds, 0, options=opt)
#Otherwise, use Create
else:
a_dtype = a.dtype
gdal_dtype = np2gdal_dtype(a_dtype)
if a_dtype.name == 'bool':
#Set ndv to 0
a.fill_value = False
opt.remove('COMPRESS=LZW')
opt.append('COMPRESS=DEFLATE')
#opt.append('NBITS=1')
#Create(fn, nx, ny, nbands, dtype, opt)
dst_ds = driver.Create(dst_fn,
a.shape[1],
a.shape[0],
nbands,
gdal_dtype,
options=opt)
#Note: Need GeoMA here to make this work, or accept gt as argument
#Could also do ds creation in calling script
if gt is not None:
dst_ds.SetGeoTransform(gt)
if proj is not None:
dst_ds.SetProjection(proj)
dst_ds.GetRasterBand(bnum).WriteArray(a.filled())
dst_ds.GetRasterBand(bnum).SetNoDataValue(float(a.fill_value))
dst_ds = None
def get_closest_dt_padded_idx(dt, dt_list, pad=timedelta(days=30)):
from pygeotools.lib import malib
dt_list = malib.checkma(dt_list, fix=False)
dt_diff = np.abs(dt - dt_list)
valid_idx = (dt_diff.data < pad).nonzero()[0]
return valid_idx
def extractPoint(b, x, y):
b = malib.checkma(b)
x = np.clip(x, 0, b.shape[1] - 1)
y = np.clip(y, 0, b.shape[0] - 1)
#Note: simplest way using integer indices
return b[np.ma.around(y).astype(int), np.ma.around(x).astype(int)]