def main():
parser = getparser()
args = parser.parse_args()
dem_fn = args.dem_fn
# Write out because they will be used to mask CHM
writeall = True
# Auto compute min TOA with gaussian mixture model
auto_min_toa = args.auto_min_toa
dirname, demname = os.path.split(dem_fn)
# The subdir in which the DEM.tif sits will be the pairname
pairname = os.path.split(dirname)[1]
print("Pairname:", pairname)
if args.out_dir is not None:
# Create symlink in out_dir to: (1) original out-DEM_4m (2) *_ortho_4m.tif (3) All *.xml files
# This should look like <out_dir>/<pairname>_out-DEM_4m
dem_fn_lnk = os.path.join(args.out_dir, pairname + '_' + demname)
force_symlink(dem_fn, dem_fn_lnk)
force_symlink(os.path.join(dirname, pairname + '_ortho_4m.tif'), os.path.join(args.out_dir, pairname + '_ortho_4m.tif') )
xml_list = [f for f in os.listdir(dirname) if f.endswith('r100.xml')]
print("\nSymlinks made for:")
for x in xml_list:
print(x)
shutil.copy2(os.path.join(dirname,x), args.out_dir)
out_fn_base = os.path.splitext(dem_fn_lnk)[0]
dem_fn = dem_fn_lnk
else:
out_fn_base = os.path.splitext(dem_fn)[0]
print("\nBasename for output files:")
print(out_fn_base)
#Max Threshold value for LiDAR datset; Valid pixels under this value
lidar_fn=args.lidar_fn
max_thresh=args.max_thresh
#Need some checks on these
param = args.filt_param
if param is not None and len(param) == 1:
param = param[0]
# Get original DEM
dem = iolib.fn_getma(dem_fn)
print("\nLoading input DEM into masked array")
dem_ds = iolib.fn_getds(dem_fn)
toa_mask = None
toa_tri_mask = None # probably not used by itself; done as part of toa_mask
rough_mask = None
slope_mask = None
mask_list = [toa_tri_mask, toa_mask, rough_mask, slope_mask]
if args.filtdz:
print("\nFilter with dz from ref DEM to remove cloud returns and blunders (shadows)...")
print("Reference DEM: %s" % os.path.split(param[0])[1] )
print("Absolute dz (+/-): %s \n" % param[2] )
#May need to cast input ma as float32 so np.nan filling works
dem = dem.astype(np.float32)
#Difference filter, need to specify ref_fn and range
#Could let the user compute their own dz, then just run a standard range or absrange filter
ref_fn = param[0]
ref_ds = warplib.memwarp_multi_fn([ref_fn,], res=dem_ds, extent=dem_ds, t_srs=dem_ds)[0]
ref = iolib.ds_getma(ref_ds)
param = map(float, param[1:])
# A dem that has been masked based on the dz filter
dem = filtlib.dz_fltr_ma(dem, ref, rangelim=param)
if writeall:
#out_fn = os.path.splitext(dem_fn)[0]+'_dzfilt.tif'
out_fn = os.path.join(out_fn_base +'_dzfilt.tif')
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(dem, out_fn, src_ds=dem_ds, ndv=args.ndv)
#Initialize a control mask that we'll update
#True (1) represents "valid" unmasked pixel, False (0) represents "invalid" pixel to be masked
controlmask = ~(np.ma.getmaskarray(dem))
# DEM masking: Each block returns a masked output (not a mask)
# TOA: mask dark and/or smooth areas (shadows and/or water)
# Roughness
# Slope
if args.toamask or args.toatrimask:
#try:
print("\nCompute TOA from ortho...\n")
toa_fn = get_toa_fn(out_fn_base + '.tif') ##--->dem_fn
print(toa_fn)
print("\nWarp TOA to DEM...\n")
toa_ds = warplib.memwarp_multi_fn([toa_fn,], res=dem_ds, extent=dem_ds, t_srs=dem_ds)[0]
if args.toamask:
if auto_min_toa:
# Compute a good min TOA value
m,s = get_min_gaus(toa_fn, 50, 4)
min_toa = m + s
min_toa = m
else:
min_toa = args.min_toa
with open(os.path.join(os.path.split(toa_fn)[0], "min_toa_" + pairname + ".txt"), "w") as text_file:
text_file.write(os.path.basename(__file__))
text_file.write("\nMinimum TOA used for mask:\n{0}".format(min_toa))
# Should mask dark areas and dilate
toa_mask = get_toa_mask(toa_ds, min_toa)
#Dilate the mask
if args.dilate_toa is not None:
niter = args.dilate_toa
print("Dilating TOA mask with %i iterations" % niter)
from scipy import ndimage
toa_mask = ~(ndimage.morphology.binary_dilation(~toa_mask, iterations=niter))
controlmask = np.logical_and(toa_mask, controlmask)
# Mask islands here
controlmask = malib.mask_islands(controlmask, 5)
if writeall:
#out_fn = out_fn_base+'_toamask.tif'
out_fn = os.path.join(out_fn_base +'_toamask.tif')
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(toa_mask, out_fn, src_ds=dem_ds)
if args.toatrimask:
# Should mask smooth areas (measures local variance)
toa_tri_mask = get_tri_mask(toa_ds, args.min_toatri)
controlmask = np.logical_and(toa_tri_mask, controlmask)
if writeall:
#out_fn = out_fn_base+'_toatrimask.tif'
out_fn = os.path.join(out_fn_base +'_toatrimask.tif')
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(toa_tri_mask, out_fn, src_ds=dem_ds)
#except Exception, e:
#print "\tFailed to apply TOA masking.\n"
if args.slopemask:
slope_mask = get_slope_mask(dem_ds, args.max_slope)
controlmask = np.logical_and(slope_mask, controlmask)
#if args.slopemaskcoarse:
#dem_fn2 = args.dem_coarscomp_fn
#print("\nLoading input coarse DEM into masked array")
#dem2_ds = iolib.fn_getds(dem_fn2)
#slope_mask = get_slope_mask(dem2_ds, args.max_slope)
#controlmask = np.logical_and(slope_mask, controlmask)
if writeall:
#out_fn = out_fn_base+'_slopemask.tif'
out_fn = os.path.join(out_fn_base +'_slopemask.tif')
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(slope_mask, out_fn, src_ds=dem_ds)
if args.lidar_fn:
try:
print("Masking DEM file based on Lidar Dataset\n")
print("\nWarp Lidar Raster to DEM...\n")
lidar_ds=warplib.memwarp_multi_fn([lidar_fn,],r='near', res=dem_ds, extent=dem_ds, t_srs=dem_ds)[0]
lidarmask = get_lidar_mask(dem_ds, lidar_ds, max_thresh)
controlmask = np.logical_and(lidarmask, controlmask)
if writeall:
out_fn=out_fn_base+'_lidarmask.tif'
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(lidarmask, out_fn, src_ds=dem_ds)
except Exception as e:
print("\tFailed to Apply Lidar Mask")
# CHM mask will be a subset of the Control mask; slope_mask, toa_mask, toa_tri_mask
chmmask = controlmask
print("Generating final CHM mask to apply later")
#out_fn = out_fn_base+'_chmmask.tif'
out_fn = os.path.join(out_fn_base +'_chmmask.tif')
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(chmmask, out_fn, src_ds=dem_ds)
if args.roughmask:
rough_mask = get_rough_mask(dem_ds, args.max_rough)
controlmask = np.logical_and(rough_mask, controlmask)
if writeall:
out_fn = os.path.join(out_fn_base +'_roughmask.tif')
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(rough_mask, out_fn, src_ds=dem_ds)
print("Generating final mask to use for reference surfaces, and applying to input DEM")
#Now invert to use to create final masked array
# This steps results in the areas to be removed being set to a valid value
controlmask = ~controlmask
#Dilate the mask
if args.dilate_con is not None:
niter = args.dilate_con
print("Dilating control mask with %i iterations" % niter)
from scipy import ndimage
#
# So, this should work too.... controlmask = ndimage.morphology.binary_dilation(controlmask, iterations=niter))
controlmask = ~(ndimage.morphology.binary_dilation(~controlmask, iterations=niter)) # This steps results in the areas to be removed being set to a valid value, again
print("\nApply mask to original DEM - use these control surfaces for co-registration...")
newdem = np.ma.array(dem, mask=controlmask) # This sets the valid values of the controlmask to the 'mask' of the DEM, which turns them into NaN values
if True:
print("\nStats of valid DEM with masks applied:")
valid_stats = malib.print_stats(newdem)
valid_stats_med = valid_stats[5]
print("\nWriting DEM control surfaces:")
#if args.out_dir is not None:
# dst_fn = os.path.join(args.out_dir, os.path.split(dirname)[1] + os.path.splitext(demname)[0]+'_control.tif')
#else:
# dst_fn = os.path.splitext(dem_fn)[0]+'_control.tif'
dst_fn = os.path.join(out_fn_base +'_control.tif')
print(dst_fn)
iolib.writeGTiff(newdem, dst_fn, dem_ds)
return dst_fn
def main():
parser = getparser()
args = parser.parse_args()
fn = args.fn
if not iolib.fn_check(fn):
sys.exit("Unable to locate input file: %s" % fn)
#Need some checks on these
param = args.param
print("Loading input raster into masked array")
ds = iolib.fn_getds(fn)
#Currently supports only single band operations
r = iolib.ds_getma(ds, 1)
#May need to cast input ma as float32 so np.nan filling works
#r = r.astype(np.float32)
#Want function that checks and returns float32 if necessary
#Should filter, then return original dtype
r_fltr = r
#Loop through all specified input filters
#for filt in args.filt:
filt = args.filt[0]
if len(param) == 1:
param = param[0]
param_str = ''
if filt == 'range':
#Range filter
param = [float(i) for i in param[1:]]
r_fltr = filtlib.range_fltr(r_fltr, param)
param_str = '_{0:0.2f}-{1:0.2f}'.format(*param)
elif filt == 'absrange':
#Range filter of absolute values
param = [float(i) for i in param[1:]]
r_fltr = filtlib.absrange_fltr(r_fltr, param)
param_str = '_{0:0.2f}-{1:0.2f}'.format(*param)
elif filt == 'perc':
#Percentile filter
param = [float(i) for i in param[1:]]
r_fltr = filtlib.perc_fltr(r, perc=param)
param_str = '_{0:0.2f}-{1:0.2f}'.format(*param)
elif filt == 'med':
#Median filter
param = int(param)
r_fltr = filtlib.rolling_fltr(r_fltr, f=np.nanmedian, size=param)
#r_fltr = filtlib.median_fltr(r_fltr, fsize=param, origmask=True)
#r_fltr = filtlib.median_fltr_skimage(r_fltr, radius=4, origmask=True)
param_str = '_%ipx' % param
elif filt == 'gauss':
#Gaussian filter (default)
param = int(param)
r_fltr = filtlib.gauss_fltr_astropy(r_fltr,
size=param,
origmask=False,
fill_interior=False)
param_str = '_%ipx' % param
elif filt == 'highpass':
#High pass filter
param = int(param)
r_fltr = filtlib.highpass(r_fltr, size=param)
param_str = '_%ipx' % param
elif filt == 'sigma':
#n*sigma filter, remove outliers
param = int(param)
r_fltr = filtlib.sigma_fltr(r_fltr, n=param)
param_str = '_n%i' % param
elif filt == 'mad':
#n*mad filter, remove outliers
#Maybe better to use a percentile filter
param = int(param)
r_fltr = filtlib.mad_fltr(r_fltr, n=param)
param_str = '_n%i' % param
elif filt == 'dz':
#Difference filter, need to specify ref_fn and range
#Could let the user compute their own dz, then just run a standard range or absrange filter
ref_fn = param[0]
ref_ds = warplib.memwarp_multi_fn([
ref_fn,
],
res=ds,
extent=ds,
t_srs=ds)[0]
ref = iolib.ds_getma(ref_ds)
param = [float(i) for i in param[1:]]
r_fltr = filtlib.dz_fltr_ma(r, ref, rangelim=param)
#param_str = '_{0:0.2f}-{1:0.2f}'.format(*param)
param_str = '_{0:0.0f}_{1:0.0f}'.format(*param)
else:
sys.exit("No filter type specified")
#Compute and print stats before/after
if args.stats:
print("Input stats:")
malib.print_stats(r)
print("Filtered stats:")
malib.print_stats(r_fltr)
#Write out
dst_fn = os.path.splitext(fn)[0] + '_%sfilt%s.tif' % (filt, param_str)
if args.outdir is not None:
outdir = args.outdir
if not os.path.exists(outdir):
os.makedirs(outdir)
dst_fn = os.path.join(outdir, os.path.split(dst_fn)[-1])
print("Writing out filtered raster: %s" % dst_fn)
iolib.writeGTiff(r_fltr, dst_fn, ds)
def main():
parser = getparser()
args = parser.parse_args()
dem_fn = args.dem_fn
# Write out because they will be used to mask CHM
writeall = True
# Auto compute min TOA with gaussian mixture model
compute_min_toa = True
#Basename for output files
out_fn_base = os.path.splitext(dem_fn)[0]
#Need some checks on these
param = args.filt_param
if param is not None and len(param) == 1:
param = param[0]
# Get original DEM
dem = iolib.fn_getma(dem_fn)
print("\nLoading input DEM into masked array")
dem_ds = iolib.fn_getds(dem_fn)
toa_mask = None
toa_tri_mask = None # probably not used by itself; done as part of toa_mask
rough_mask = None
slope_mask = None
mask_list = [toa_tri_mask, toa_mask, rough_mask, slope_mask]
if args.filtdz:
print(
"\nFilter with dz from ref DEM to remove cloud returns and blunders (shadows)..."
)
print("Reference DEM: %s" % os.path.split(param[0])[1])
print("Absolute dz (+/-): %s \n" % param[2])
#May need to cast input ma as float32 so np.nan filling works
dem = dem.astype(np.float32)
#Difference filter, need to specify ref_fn and range
#Could let the user compute their own dz, then just run a standard range or absrange filter
ref_fn = param[0]
ref_ds = warplib.memwarp_multi_fn([
ref_fn,
],
res=dem_ds,
extent=dem_ds,
t_srs=dem_ds)[0]
ref = iolib.ds_getma(ref_ds)
param = map(float, param[1:])
# A dem that has been masked based on the dz filter
dem = filtlib.dz_fltr_ma(dem, ref, rangelim=param)
if writeall:
out_fn = os.path.splitext(dem_fn)[0] + '_dzfilt.tif'
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(dem, out_fn, src_ds=dem_ds, ndv=args.ndv)
#Initialize a control mask that we'll update
#True (1) represents "valid" unmasked pixel, False (0) represents "invalid" pixel to be masked
controlmask = ~(np.ma.getmaskarray(dem))
# DEM masking: Each block returns a masked output (not a mask)
# TOA: mask dark and/or smooth areas (shadows and/or water)
# Roughness
# Slope
if args.toamask or args.toatrimask:
print("\nCompute TOA from ortho...\n")
toa_fn = get_toa_fn(dem_fn)
print("\nWarp TOA to DEM...\n")
toa_ds = warplib.memwarp_multi_fn([
toa_fn,
],
res=dem_ds,
extent=dem_ds,
t_srs=dem_ds)[0]
if args.toamask:
if compute_min_toa:
# Compute a good min TOA value
m, s = get_min_gaus(toa_fn, 50, 4)
min_toa = m + s
min_toa = m
else:
min_toa = args.min_toa
with open(os.path.join(os.path.split(toa_fn)[0], "min_toa.txt"),
"w") as text_file:
text_file.write(os.path.basename(__file__))
text_file.write(
"\nMinimum TOA used for mask:\n{0}".format(min_toa))
# Should mask dark areas and dilate
toa_mask = get_toa_mask(toa_ds, min_toa)
#Dilate the mask
if args.dilate_toa is not None:
niter = args.dilate_toa
print("Dilating TOA mask with %i iterations" % niter)
from scipy import ndimage
toa_mask = ~(ndimage.morphology.binary_dilation(
~toa_mask, iterations=niter))
controlmask = np.logical_and(toa_mask, controlmask)
# Mask islands here
controlmask = malib.mask_islands(controlmask, 5)
if writeall:
out_fn = out_fn_base + '_toamask.tif'
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(toa_mask, out_fn, src_ds=dem_ds)
if args.toatrimask:
# Should mask smooth areas (measures local variance)
toa_tri_mask = get_tri_mask(toa_ds, args.min_toatri)
controlmask = np.logical_and(toa_tri_mask, controlmask)
if writeall:
out_fn = out_fn_base + '_toatrimask.tif'
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(toa_tri_mask, out_fn, src_ds=dem_ds)
if args.slopemask:
slope_mask = get_slope_mask(dem_ds, args.max_slope)
controlmask = np.logical_and(slope_mask, controlmask)
if writeall:
out_fn = out_fn_base + '_slopemask.tif'
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(slope_mask, out_fn, src_ds=dem_ds)
# CHM mask will be a subset of the Control mask; slope_mask, toa_mask, toa_tri_mask
chmmask = controlmask
print("Generating final CHM mask to apply later")
out_fn = out_fn_base + '_chmmask.tif'
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(chmmask, out_fn, src_ds=dem_ds)
if args.roughmask:
rough_mask = get_rough_mask(dem_ds, args.max_rough)
controlmask = np.logical_and(rough_mask, controlmask)
if writeall:
out_fn = out_fn_base + '_roughmask.tif'
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(rough_mask, out_fn, src_ds=dem_ds)
print(
"Generating final mask to use for reference surfaces, and applying to input DEM"
)
#Now invert to use to create final masked array
controlmask = ~controlmask
#Dilate the mask
if args.dilate_con is not None:
niter = args.dilate_con
print("Dilating control mask with %i iterations" % niter)
from scipy import ndimage
controlmask = ~(ndimage.morphology.binary_dilation(~controlmask,
iterations=niter))
#Apply mask to original DEM - use these surfaces for co-registration
newdem = np.ma.array(dem, mask=controlmask)
if True:
print("\nStats of valid DEM with maskes applied:")
valid_stats = malib.print_stats(newdem)
valid_stats_med = valid_stats[5]
print("\nWriting DEM control surfaces:")
dst_fn = os.path.splitext(dem_fn)[0] + '_control.tif'
print(dst_fn)
iolib.writeGTiff(newdem, dst_fn, dem_ds)
return dst_fn