def median_fltr_skimage(dem, radius=3, erode=1, origmask=False):
"""
Older skimage.filter.median_filter
This smooths, removes noise and fills in nodata areas with median of valid pixels! Effectively an inpainting routine
"""
#Note, ndimage doesn't properly handle ma - convert to nan
dem = malib.checkma(dem)
dem = dem.astype(np.float64)
#Mask islands
if erode > 0:
print("Eroding islands smaller than %s pixels" % (erode * 2))
dem = malib.mask_islands(dem, iterations=erode)
print("Applying median filter with radius %s" % radius)
#Note: this funcitonality was present in scikit-image 0.9.3
import skimage.filter
dem_filt_med = skimage.filter.median_filter(dem, radius, mask=~dem.mask)
#Starting in version 0.10.0, this is the new filter
#This is the new filter, but only supports uint8 or unit16
#import skimage.filters
#import skimage.morphology
#dem_filt_med = skimage.filters.rank.median(dem, disk(radius), mask=~dem.mask)
#dem_filt_med = skimage.filters.median(dem, skimage.morphology.disk(radius), mask=~dem.mask)
#Now mask all nans
#skimage assigns the minimum value as nodata
#CHECK THIS, seems pretty hacky
#Also, looks like some valid values are masked at this stage, even though they should be above min
ndv = np.min(dem_filt_med)
#ndv = dem_filt_med.min() + 0.001
out = np.ma.masked_less_equal(dem_filt_med, ndv)
#Should probably replace the ndv with original ndv
out.set_fill_value(dem.fill_value)
if origmask:
print("Applying original mask")
#Allow filling of interior holes, but use original outer edge
#maskfill = malib.maskfill(dem, iterations=radius)
maskfill = malib.maskfill(dem)
#dem_filt_gauss = np.ma.array(dem_filt_gauss, mask=dem.mask, fill_value=dem.fill_value)
out = np.ma.array(out, mask=maskfill, fill_value=dem.fill_value)
return out
def main():
parser = getparser()
args = parser.parse_args()
dem_fn = args.dem_fn
# Write out because they will be used to mask CHM
writeall = True
# Auto compute min TOA with gaussian mixture model
auto_min_toa = args.auto_min_toa
dirname, demname = os.path.split(dem_fn)
# The subdir in which the DEM.tif sits will be the pairname
pairname = os.path.split(dirname)[1]
print("Pairname:", pairname)
if args.out_dir is not None:
# Create symlink in out_dir to: (1) original out-DEM_4m (2) *_ortho_4m.tif (3) All *.xml files
# This should look like <out_dir>/<pairname>_out-DEM_4m
dem_fn_lnk = os.path.join(args.out_dir, pairname + '_' + demname)
force_symlink(dem_fn, dem_fn_lnk)
force_symlink(os.path.join(dirname, pairname + '_ortho_4m.tif'), os.path.join(args.out_dir, pairname + '_ortho_4m.tif') )
xml_list = [f for f in os.listdir(dirname) if f.endswith('r100.xml')]
print("\nSymlinks made for:")
for x in xml_list:
print(x)
shutil.copy2(os.path.join(dirname,x), args.out_dir)
out_fn_base = os.path.splitext(dem_fn_lnk)[0]
dem_fn = dem_fn_lnk
else:
out_fn_base = os.path.splitext(dem_fn)[0]
print("\nBasename for output files:")
print(out_fn_base)
#Max Threshold value for LiDAR datset; Valid pixels under this value
lidar_fn=args.lidar_fn
max_thresh=args.max_thresh
#Need some checks on these
param = args.filt_param
if param is not None and len(param) == 1:
param = param[0]
# Get original DEM
dem = iolib.fn_getma(dem_fn)
print("\nLoading input DEM into masked array")
dem_ds = iolib.fn_getds(dem_fn)
toa_mask = None
toa_tri_mask = None # probably not used by itself; done as part of toa_mask
rough_mask = None
slope_mask = None
mask_list = [toa_tri_mask, toa_mask, rough_mask, slope_mask]
if args.filtdz:
print("\nFilter with dz from ref DEM to remove cloud returns and blunders (shadows)...")
print("Reference DEM: %s" % os.path.split(param[0])[1] )
print("Absolute dz (+/-): %s \n" % param[2] )
#May need to cast input ma as float32 so np.nan filling works
dem = dem.astype(np.float32)
#Difference filter, need to specify ref_fn and range
#Could let the user compute their own dz, then just run a standard range or absrange filter
ref_fn = param[0]
ref_ds = warplib.memwarp_multi_fn([ref_fn,], res=dem_ds, extent=dem_ds, t_srs=dem_ds)[0]
ref = iolib.ds_getma(ref_ds)
param = map(float, param[1:])
# A dem that has been masked based on the dz filter
dem = filtlib.dz_fltr_ma(dem, ref, rangelim=param)
if writeall:
#out_fn = os.path.splitext(dem_fn)[0]+'_dzfilt.tif'
out_fn = os.path.join(out_fn_base +'_dzfilt.tif')
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(dem, out_fn, src_ds=dem_ds, ndv=args.ndv)
#Initialize a control mask that we'll update
#True (1) represents "valid" unmasked pixel, False (0) represents "invalid" pixel to be masked
controlmask = ~(np.ma.getmaskarray(dem))
# DEM masking: Each block returns a masked output (not a mask)
# TOA: mask dark and/or smooth areas (shadows and/or water)
# Roughness
# Slope
if args.toamask or args.toatrimask:
#try:
print("\nCompute TOA from ortho...\n")
toa_fn = get_toa_fn(out_fn_base + '.tif') ##--->dem_fn
print(toa_fn)
print("\nWarp TOA to DEM...\n")
toa_ds = warplib.memwarp_multi_fn([toa_fn,], res=dem_ds, extent=dem_ds, t_srs=dem_ds)[0]
if args.toamask:
if auto_min_toa:
# Compute a good min TOA value
m,s = get_min_gaus(toa_fn, 50, 4)
min_toa = m + s
min_toa = m
else:
min_toa = args.min_toa
with open(os.path.join(os.path.split(toa_fn)[0], "min_toa_" + pairname + ".txt"), "w") as text_file:
text_file.write(os.path.basename(__file__))
text_file.write("\nMinimum TOA used for mask:\n{0}".format(min_toa))
# Should mask dark areas and dilate
toa_mask = get_toa_mask(toa_ds, min_toa)
#Dilate the mask
if args.dilate_toa is not None:
niter = args.dilate_toa
print("Dilating TOA mask with %i iterations" % niter)
from scipy import ndimage
toa_mask = ~(ndimage.morphology.binary_dilation(~toa_mask, iterations=niter))
controlmask = np.logical_and(toa_mask, controlmask)
# Mask islands here
controlmask = malib.mask_islands(controlmask, 5)
if writeall:
#out_fn = out_fn_base+'_toamask.tif'
out_fn = os.path.join(out_fn_base +'_toamask.tif')
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(toa_mask, out_fn, src_ds=dem_ds)
if args.toatrimask:
# Should mask smooth areas (measures local variance)
toa_tri_mask = get_tri_mask(toa_ds, args.min_toatri)
controlmask = np.logical_and(toa_tri_mask, controlmask)
if writeall:
#out_fn = out_fn_base+'_toatrimask.tif'
out_fn = os.path.join(out_fn_base +'_toatrimask.tif')
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(toa_tri_mask, out_fn, src_ds=dem_ds)
#except Exception, e:
#print "\tFailed to apply TOA masking.\n"
if args.slopemask:
slope_mask = get_slope_mask(dem_ds, args.max_slope)
controlmask = np.logical_and(slope_mask, controlmask)
#if args.slopemaskcoarse:
#dem_fn2 = args.dem_coarscomp_fn
#print("\nLoading input coarse DEM into masked array")
#dem2_ds = iolib.fn_getds(dem_fn2)
#slope_mask = get_slope_mask(dem2_ds, args.max_slope)
#controlmask = np.logical_and(slope_mask, controlmask)
if writeall:
#out_fn = out_fn_base+'_slopemask.tif'
out_fn = os.path.join(out_fn_base +'_slopemask.tif')
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(slope_mask, out_fn, src_ds=dem_ds)
if args.lidar_fn:
try:
print("Masking DEM file based on Lidar Dataset\n")
print("\nWarp Lidar Raster to DEM...\n")
lidar_ds=warplib.memwarp_multi_fn([lidar_fn,],r='near', res=dem_ds, extent=dem_ds, t_srs=dem_ds)[0]
lidarmask = get_lidar_mask(dem_ds, lidar_ds, max_thresh)
controlmask = np.logical_and(lidarmask, controlmask)
if writeall:
out_fn=out_fn_base+'_lidarmask.tif'
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(lidarmask, out_fn, src_ds=dem_ds)
except Exception as e:
print("\tFailed to Apply Lidar Mask")
# CHM mask will be a subset of the Control mask; slope_mask, toa_mask, toa_tri_mask
chmmask = controlmask
print("Generating final CHM mask to apply later")
#out_fn = out_fn_base+'_chmmask.tif'
out_fn = os.path.join(out_fn_base +'_chmmask.tif')
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(chmmask, out_fn, src_ds=dem_ds)
if args.roughmask:
rough_mask = get_rough_mask(dem_ds, args.max_rough)
controlmask = np.logical_and(rough_mask, controlmask)
if writeall:
out_fn = os.path.join(out_fn_base +'_roughmask.tif')
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(rough_mask, out_fn, src_ds=dem_ds)
print("Generating final mask to use for reference surfaces, and applying to input DEM")
#Now invert to use to create final masked array
# This steps results in the areas to be removed being set to a valid value
controlmask = ~controlmask
#Dilate the mask
if args.dilate_con is not None:
niter = args.dilate_con
print("Dilating control mask with %i iterations" % niter)
from scipy import ndimage
#
# So, this should work too.... controlmask = ndimage.morphology.binary_dilation(controlmask, iterations=niter))
controlmask = ~(ndimage.morphology.binary_dilation(~controlmask, iterations=niter)) # This steps results in the areas to be removed being set to a valid value, again
print("\nApply mask to original DEM - use these control surfaces for co-registration...")
newdem = np.ma.array(dem, mask=controlmask) # This sets the valid values of the controlmask to the 'mask' of the DEM, which turns them into NaN values
if True:
print("\nStats of valid DEM with masks applied:")
valid_stats = malib.print_stats(newdem)
valid_stats_med = valid_stats[5]
print("\nWriting DEM control surfaces:")
#if args.out_dir is not None:
# dst_fn = os.path.join(args.out_dir, os.path.split(dirname)[1] + os.path.splitext(demname)[0]+'_control.tif')
#else:
# dst_fn = os.path.splitext(dem_fn)[0]+'_control.tif'
dst_fn = os.path.join(out_fn_base +'_control.tif')
print(dst_fn)
iolib.writeGTiff(newdem, dst_fn, dem_ds)
return dst_fn
def main():
parser = getparser()
args = parser.parse_args()
dem_fn = args.dem_fn
# Write out because they will be used to mask CHM
writeall = True
# Auto compute min TOA with gaussian mixture model
compute_min_toa = True
#Basename for output files
out_fn_base = os.path.splitext(dem_fn)[0]
#Need some checks on these
param = args.filt_param
if param is not None and len(param) == 1:
param = param[0]
# Get original DEM
dem = iolib.fn_getma(dem_fn)
print("\nLoading input DEM into masked array")
dem_ds = iolib.fn_getds(dem_fn)
toa_mask = None
toa_tri_mask = None # probably not used by itself; done as part of toa_mask
rough_mask = None
slope_mask = None
mask_list = [toa_tri_mask, toa_mask, rough_mask, slope_mask]
if args.filtdz:
print(
"\nFilter with dz from ref DEM to remove cloud returns and blunders (shadows)..."
)
print("Reference DEM: %s" % os.path.split(param[0])[1])
print("Absolute dz (+/-): %s \n" % param[2])
#May need to cast input ma as float32 so np.nan filling works
dem = dem.astype(np.float32)
#Difference filter, need to specify ref_fn and range
#Could let the user compute their own dz, then just run a standard range or absrange filter
ref_fn = param[0]
ref_ds = warplib.memwarp_multi_fn([
ref_fn,
],
res=dem_ds,
extent=dem_ds,
t_srs=dem_ds)[0]
ref = iolib.ds_getma(ref_ds)
param = map(float, param[1:])
# A dem that has been masked based on the dz filter
dem = filtlib.dz_fltr_ma(dem, ref, rangelim=param)
if writeall:
out_fn = os.path.splitext(dem_fn)[0] + '_dzfilt.tif'
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(dem, out_fn, src_ds=dem_ds, ndv=args.ndv)
#Initialize a control mask that we'll update
#True (1) represents "valid" unmasked pixel, False (0) represents "invalid" pixel to be masked
controlmask = ~(np.ma.getmaskarray(dem))
# DEM masking: Each block returns a masked output (not a mask)
# TOA: mask dark and/or smooth areas (shadows and/or water)
# Roughness
# Slope
if args.toamask or args.toatrimask:
print("\nCompute TOA from ortho...\n")
toa_fn = get_toa_fn(dem_fn)
print("\nWarp TOA to DEM...\n")
toa_ds = warplib.memwarp_multi_fn([
toa_fn,
],
res=dem_ds,
extent=dem_ds,
t_srs=dem_ds)[0]
if args.toamask:
if compute_min_toa:
# Compute a good min TOA value
m, s = get_min_gaus(toa_fn, 50, 4)
min_toa = m + s
min_toa = m
else:
min_toa = args.min_toa
with open(os.path.join(os.path.split(toa_fn)[0], "min_toa.txt"),
"w") as text_file:
text_file.write(os.path.basename(__file__))
text_file.write(
"\nMinimum TOA used for mask:\n{0}".format(min_toa))
# Should mask dark areas and dilate
toa_mask = get_toa_mask(toa_ds, min_toa)
#Dilate the mask
if args.dilate_toa is not None:
niter = args.dilate_toa
print("Dilating TOA mask with %i iterations" % niter)
from scipy import ndimage
toa_mask = ~(ndimage.morphology.binary_dilation(
~toa_mask, iterations=niter))
controlmask = np.logical_and(toa_mask, controlmask)
# Mask islands here
controlmask = malib.mask_islands(controlmask, 5)
if writeall:
out_fn = out_fn_base + '_toamask.tif'
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(toa_mask, out_fn, src_ds=dem_ds)
if args.toatrimask:
# Should mask smooth areas (measures local variance)
toa_tri_mask = get_tri_mask(toa_ds, args.min_toatri)
controlmask = np.logical_and(toa_tri_mask, controlmask)
if writeall:
out_fn = out_fn_base + '_toatrimask.tif'
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(toa_tri_mask, out_fn, src_ds=dem_ds)
if args.slopemask:
slope_mask = get_slope_mask(dem_ds, args.max_slope)
controlmask = np.logical_and(slope_mask, controlmask)
if writeall:
out_fn = out_fn_base + '_slopemask.tif'
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(slope_mask, out_fn, src_ds=dem_ds)
# CHM mask will be a subset of the Control mask; slope_mask, toa_mask, toa_tri_mask
chmmask = controlmask
print("Generating final CHM mask to apply later")
out_fn = out_fn_base + '_chmmask.tif'
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(chmmask, out_fn, src_ds=dem_ds)
if args.roughmask:
rough_mask = get_rough_mask(dem_ds, args.max_rough)
controlmask = np.logical_and(rough_mask, controlmask)
if writeall:
out_fn = out_fn_base + '_roughmask.tif'
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(rough_mask, out_fn, src_ds=dem_ds)
print(
"Generating final mask to use for reference surfaces, and applying to input DEM"
)
#Now invert to use to create final masked array
controlmask = ~controlmask
#Dilate the mask
if args.dilate_con is not None:
niter = args.dilate_con
print("Dilating control mask with %i iterations" % niter)
from scipy import ndimage
controlmask = ~(ndimage.morphology.binary_dilation(~controlmask,
iterations=niter))
#Apply mask to original DEM - use these surfaces for co-registration
newdem = np.ma.array(dem, mask=controlmask)
if True:
print("\nStats of valid DEM with maskes applied:")
valid_stats = malib.print_stats(newdem)
valid_stats_med = valid_stats[5]
print("\nWriting DEM control surfaces:")
dst_fn = os.path.splitext(dem_fn)[0] + '_control.tif'
print(dst_fn)
iolib.writeGTiff(newdem, dst_fn, dem_ds)
return dst_fn