def median_fltr_skimage(dem, radius=3, erode=1, origmask=False): """ Older skimage.filter.median_filter This smooths, removes noise and fills in nodata areas with median of valid pixels! Effectively an inpainting routine """ #Note, ndimage doesn't properly handle ma - convert to nan dem = malib.checkma(dem) dem = dem.astype(np.float64) #Mask islands if erode > 0: print("Eroding islands smaller than %s pixels" % (erode * 2)) dem = malib.mask_islands(dem, iterations=erode) print("Applying median filter with radius %s" % radius) #Note: this funcitonality was present in scikit-image 0.9.3 import skimage.filter dem_filt_med = skimage.filter.median_filter(dem, radius, mask=~dem.mask) #Starting in version 0.10.0, this is the new filter #This is the new filter, but only supports uint8 or unit16 #import skimage.filters #import skimage.morphology #dem_filt_med = skimage.filters.rank.median(dem, disk(radius), mask=~dem.mask) #dem_filt_med = skimage.filters.median(dem, skimage.morphology.disk(radius), mask=~dem.mask) #Now mask all nans #skimage assigns the minimum value as nodata #CHECK THIS, seems pretty hacky #Also, looks like some valid values are masked at this stage, even though they should be above min ndv = np.min(dem_filt_med) #ndv = dem_filt_med.min() + 0.001 out = np.ma.masked_less_equal(dem_filt_med, ndv) #Should probably replace the ndv with original ndv out.set_fill_value(dem.fill_value) if origmask: print("Applying original mask") #Allow filling of interior holes, but use original outer edge #maskfill = malib.maskfill(dem, iterations=radius) maskfill = malib.maskfill(dem) #dem_filt_gauss = np.ma.array(dem_filt_gauss, mask=dem.mask, fill_value=dem.fill_value) out = np.ma.array(out, mask=maskfill, fill_value=dem.fill_value) return out
def main(): parser = getparser() args = parser.parse_args() dem_fn = args.dem_fn # Write out because they will be used to mask CHM writeall = True # Auto compute min TOA with gaussian mixture model auto_min_toa = args.auto_min_toa dirname, demname = os.path.split(dem_fn) # The subdir in which the DEM.tif sits will be the pairname pairname = os.path.split(dirname)[1] print("Pairname:", pairname) if args.out_dir is not None: # Create symlink in out_dir to: (1) original out-DEM_4m (2) *_ortho_4m.tif (3) All *.xml files # This should look like <out_dir>/<pairname>_out-DEM_4m dem_fn_lnk = os.path.join(args.out_dir, pairname + '_' + demname) force_symlink(dem_fn, dem_fn_lnk) force_symlink(os.path.join(dirname, pairname + '_ortho_4m.tif'), os.path.join(args.out_dir, pairname + '_ortho_4m.tif') ) xml_list = [f for f in os.listdir(dirname) if f.endswith('r100.xml')] print("\nSymlinks made for:") for x in xml_list: print(x) shutil.copy2(os.path.join(dirname,x), args.out_dir) out_fn_base = os.path.splitext(dem_fn_lnk)[0] dem_fn = dem_fn_lnk else: out_fn_base = os.path.splitext(dem_fn)[0] print("\nBasename for output files:") print(out_fn_base) #Max Threshold value for LiDAR datset; Valid pixels under this value lidar_fn=args.lidar_fn max_thresh=args.max_thresh #Need some checks on these param = args.filt_param if param is not None and len(param) == 1: param = param[0] # Get original DEM dem = iolib.fn_getma(dem_fn) print("\nLoading input DEM into masked array") dem_ds = iolib.fn_getds(dem_fn) toa_mask = None toa_tri_mask = None # probably not used by itself; done as part of toa_mask rough_mask = None slope_mask = None mask_list = [toa_tri_mask, toa_mask, rough_mask, slope_mask] if args.filtdz: print("\nFilter with dz from ref DEM to remove cloud returns and blunders (shadows)...") print("Reference DEM: %s" % os.path.split(param[0])[1] ) print("Absolute dz (+/-): %s \n" % param[2] ) #May need to cast input ma as float32 so np.nan filling works dem = dem.astype(np.float32) #Difference filter, need to specify ref_fn and range #Could let the user compute their own dz, then just run a standard range or absrange filter ref_fn = param[0] ref_ds = warplib.memwarp_multi_fn([ref_fn,], res=dem_ds, extent=dem_ds, t_srs=dem_ds)[0] ref = iolib.ds_getma(ref_ds) param = map(float, param[1:]) # A dem that has been masked based on the dz filter dem = filtlib.dz_fltr_ma(dem, ref, rangelim=param) if writeall: #out_fn = os.path.splitext(dem_fn)[0]+'_dzfilt.tif' out_fn = os.path.join(out_fn_base +'_dzfilt.tif') print("Writing out %s\n" % out_fn) iolib.writeGTiff(dem, out_fn, src_ds=dem_ds, ndv=args.ndv) #Initialize a control mask that we'll update #True (1) represents "valid" unmasked pixel, False (0) represents "invalid" pixel to be masked controlmask = ~(np.ma.getmaskarray(dem)) # DEM masking: Each block returns a masked output (not a mask) # TOA: mask dark and/or smooth areas (shadows and/or water) # Roughness # Slope if args.toamask or args.toatrimask: #try: print("\nCompute TOA from ortho...\n") toa_fn = get_toa_fn(out_fn_base + '.tif') ##--->dem_fn print(toa_fn) print("\nWarp TOA to DEM...\n") toa_ds = warplib.memwarp_multi_fn([toa_fn,], res=dem_ds, extent=dem_ds, t_srs=dem_ds)[0] if args.toamask: if auto_min_toa: # Compute a good min TOA value m,s = get_min_gaus(toa_fn, 50, 4) min_toa = m + s min_toa = m else: min_toa = args.min_toa with open(os.path.join(os.path.split(toa_fn)[0], "min_toa_" + pairname + ".txt"), "w") as text_file: text_file.write(os.path.basename(__file__)) text_file.write("\nMinimum TOA used for mask:\n{0}".format(min_toa)) # Should mask dark areas and dilate toa_mask = get_toa_mask(toa_ds, min_toa) #Dilate the mask if args.dilate_toa is not None: niter = args.dilate_toa print("Dilating TOA mask with %i iterations" % niter) from scipy import ndimage toa_mask = ~(ndimage.morphology.binary_dilation(~toa_mask, iterations=niter)) controlmask = np.logical_and(toa_mask, controlmask) # Mask islands here controlmask = malib.mask_islands(controlmask, 5) if writeall: #out_fn = out_fn_base+'_toamask.tif' out_fn = os.path.join(out_fn_base +'_toamask.tif') print("Writing out %s\n" % out_fn) iolib.writeGTiff(toa_mask, out_fn, src_ds=dem_ds) if args.toatrimask: # Should mask smooth areas (measures local variance) toa_tri_mask = get_tri_mask(toa_ds, args.min_toatri) controlmask = np.logical_and(toa_tri_mask, controlmask) if writeall: #out_fn = out_fn_base+'_toatrimask.tif' out_fn = os.path.join(out_fn_base +'_toatrimask.tif') print("Writing out %s\n" % out_fn) iolib.writeGTiff(toa_tri_mask, out_fn, src_ds=dem_ds) #except Exception, e: #print "\tFailed to apply TOA masking.\n" if args.slopemask: slope_mask = get_slope_mask(dem_ds, args.max_slope) controlmask = np.logical_and(slope_mask, controlmask) #if args.slopemaskcoarse: #dem_fn2 = args.dem_coarscomp_fn #print("\nLoading input coarse DEM into masked array") #dem2_ds = iolib.fn_getds(dem_fn2) #slope_mask = get_slope_mask(dem2_ds, args.max_slope) #controlmask = np.logical_and(slope_mask, controlmask) if writeall: #out_fn = out_fn_base+'_slopemask.tif' out_fn = os.path.join(out_fn_base +'_slopemask.tif') print("Writing out %s\n" % out_fn) iolib.writeGTiff(slope_mask, out_fn, src_ds=dem_ds) if args.lidar_fn: try: print("Masking DEM file based on Lidar Dataset\n") print("\nWarp Lidar Raster to DEM...\n") lidar_ds=warplib.memwarp_multi_fn([lidar_fn,],r='near', res=dem_ds, extent=dem_ds, t_srs=dem_ds)[0] lidarmask = get_lidar_mask(dem_ds, lidar_ds, max_thresh) controlmask = np.logical_and(lidarmask, controlmask) if writeall: out_fn=out_fn_base+'_lidarmask.tif' print("Writing out %s\n" % out_fn) iolib.writeGTiff(lidarmask, out_fn, src_ds=dem_ds) except Exception as e: print("\tFailed to Apply Lidar Mask") # CHM mask will be a subset of the Control mask; slope_mask, toa_mask, toa_tri_mask chmmask = controlmask print("Generating final CHM mask to apply later") #out_fn = out_fn_base+'_chmmask.tif' out_fn = os.path.join(out_fn_base +'_chmmask.tif') print("Writing out %s\n" % out_fn) iolib.writeGTiff(chmmask, out_fn, src_ds=dem_ds) if args.roughmask: rough_mask = get_rough_mask(dem_ds, args.max_rough) controlmask = np.logical_and(rough_mask, controlmask) if writeall: out_fn = os.path.join(out_fn_base +'_roughmask.tif') print("Writing out %s\n" % out_fn) iolib.writeGTiff(rough_mask, out_fn, src_ds=dem_ds) print("Generating final mask to use for reference surfaces, and applying to input DEM") #Now invert to use to create final masked array # This steps results in the areas to be removed being set to a valid value controlmask = ~controlmask #Dilate the mask if args.dilate_con is not None: niter = args.dilate_con print("Dilating control mask with %i iterations" % niter) from scipy import ndimage # # So, this should work too.... controlmask = ndimage.morphology.binary_dilation(controlmask, iterations=niter)) controlmask = ~(ndimage.morphology.binary_dilation(~controlmask, iterations=niter)) # This steps results in the areas to be removed being set to a valid value, again print("\nApply mask to original DEM - use these control surfaces for co-registration...") newdem = np.ma.array(dem, mask=controlmask) # This sets the valid values of the controlmask to the 'mask' of the DEM, which turns them into NaN values if True: print("\nStats of valid DEM with masks applied:") valid_stats = malib.print_stats(newdem) valid_stats_med = valid_stats[5] print("\nWriting DEM control surfaces:") #if args.out_dir is not None: # dst_fn = os.path.join(args.out_dir, os.path.split(dirname)[1] + os.path.splitext(demname)[0]+'_control.tif') #else: # dst_fn = os.path.splitext(dem_fn)[0]+'_control.tif' dst_fn = os.path.join(out_fn_base +'_control.tif') print(dst_fn) iolib.writeGTiff(newdem, dst_fn, dem_ds) return dst_fn
def main(): parser = getparser() args = parser.parse_args() dem_fn = args.dem_fn # Write out because they will be used to mask CHM writeall = True # Auto compute min TOA with gaussian mixture model compute_min_toa = True #Basename for output files out_fn_base = os.path.splitext(dem_fn)[0] #Need some checks on these param = args.filt_param if param is not None and len(param) == 1: param = param[0] # Get original DEM dem = iolib.fn_getma(dem_fn) print("\nLoading input DEM into masked array") dem_ds = iolib.fn_getds(dem_fn) toa_mask = None toa_tri_mask = None # probably not used by itself; done as part of toa_mask rough_mask = None slope_mask = None mask_list = [toa_tri_mask, toa_mask, rough_mask, slope_mask] if args.filtdz: print( "\nFilter with dz from ref DEM to remove cloud returns and blunders (shadows)..." ) print("Reference DEM: %s" % os.path.split(param[0])[1]) print("Absolute dz (+/-): %s \n" % param[2]) #May need to cast input ma as float32 so np.nan filling works dem = dem.astype(np.float32) #Difference filter, need to specify ref_fn and range #Could let the user compute their own dz, then just run a standard range or absrange filter ref_fn = param[0] ref_ds = warplib.memwarp_multi_fn([ ref_fn, ], res=dem_ds, extent=dem_ds, t_srs=dem_ds)[0] ref = iolib.ds_getma(ref_ds) param = map(float, param[1:]) # A dem that has been masked based on the dz filter dem = filtlib.dz_fltr_ma(dem, ref, rangelim=param) if writeall: out_fn = os.path.splitext(dem_fn)[0] + '_dzfilt.tif' print("Writing out %s\n" % out_fn) iolib.writeGTiff(dem, out_fn, src_ds=dem_ds, ndv=args.ndv) #Initialize a control mask that we'll update #True (1) represents "valid" unmasked pixel, False (0) represents "invalid" pixel to be masked controlmask = ~(np.ma.getmaskarray(dem)) # DEM masking: Each block returns a masked output (not a mask) # TOA: mask dark and/or smooth areas (shadows and/or water) # Roughness # Slope if args.toamask or args.toatrimask: print("\nCompute TOA from ortho...\n") toa_fn = get_toa_fn(dem_fn) print("\nWarp TOA to DEM...\n") toa_ds = warplib.memwarp_multi_fn([ toa_fn, ], res=dem_ds, extent=dem_ds, t_srs=dem_ds)[0] if args.toamask: if compute_min_toa: # Compute a good min TOA value m, s = get_min_gaus(toa_fn, 50, 4) min_toa = m + s min_toa = m else: min_toa = args.min_toa with open(os.path.join(os.path.split(toa_fn)[0], "min_toa.txt"), "w") as text_file: text_file.write(os.path.basename(__file__)) text_file.write( "\nMinimum TOA used for mask:\n{0}".format(min_toa)) # Should mask dark areas and dilate toa_mask = get_toa_mask(toa_ds, min_toa) #Dilate the mask if args.dilate_toa is not None: niter = args.dilate_toa print("Dilating TOA mask with %i iterations" % niter) from scipy import ndimage toa_mask = ~(ndimage.morphology.binary_dilation( ~toa_mask, iterations=niter)) controlmask = np.logical_and(toa_mask, controlmask) # Mask islands here controlmask = malib.mask_islands(controlmask, 5) if writeall: out_fn = out_fn_base + '_toamask.tif' print("Writing out %s\n" % out_fn) iolib.writeGTiff(toa_mask, out_fn, src_ds=dem_ds) if args.toatrimask: # Should mask smooth areas (measures local variance) toa_tri_mask = get_tri_mask(toa_ds, args.min_toatri) controlmask = np.logical_and(toa_tri_mask, controlmask) if writeall: out_fn = out_fn_base + '_toatrimask.tif' print("Writing out %s\n" % out_fn) iolib.writeGTiff(toa_tri_mask, out_fn, src_ds=dem_ds) if args.slopemask: slope_mask = get_slope_mask(dem_ds, args.max_slope) controlmask = np.logical_and(slope_mask, controlmask) if writeall: out_fn = out_fn_base + '_slopemask.tif' print("Writing out %s\n" % out_fn) iolib.writeGTiff(slope_mask, out_fn, src_ds=dem_ds) # CHM mask will be a subset of the Control mask; slope_mask, toa_mask, toa_tri_mask chmmask = controlmask print("Generating final CHM mask to apply later") out_fn = out_fn_base + '_chmmask.tif' print("Writing out %s\n" % out_fn) iolib.writeGTiff(chmmask, out_fn, src_ds=dem_ds) if args.roughmask: rough_mask = get_rough_mask(dem_ds, args.max_rough) controlmask = np.logical_and(rough_mask, controlmask) if writeall: out_fn = out_fn_base + '_roughmask.tif' print("Writing out %s\n" % out_fn) iolib.writeGTiff(rough_mask, out_fn, src_ds=dem_ds) print( "Generating final mask to use for reference surfaces, and applying to input DEM" ) #Now invert to use to create final masked array controlmask = ~controlmask #Dilate the mask if args.dilate_con is not None: niter = args.dilate_con print("Dilating control mask with %i iterations" % niter) from scipy import ndimage controlmask = ~(ndimage.morphology.binary_dilation(~controlmask, iterations=niter)) #Apply mask to original DEM - use these surfaces for co-registration newdem = np.ma.array(dem, mask=controlmask) if True: print("\nStats of valid DEM with maskes applied:") valid_stats = malib.print_stats(newdem) valid_stats_med = valid_stats[5] print("\nWriting DEM control surfaces:") dst_fn = os.path.splitext(dem_fn)[0] + '_control.tif' print(dst_fn) iolib.writeGTiff(newdem, dst_fn, dem_ds) return dst_fn