def main():
parser = getparser()
args = parser.parse_args()
ras_fn = args.ras_fn
min = args.min
max = args.max
print("Loading dz raster into masked array")
ras_ds = iolib.fn_getds(ras_fn)
ras = iolib.ds_getma(ras_ds, 1)
#Cast input ma as float32 so np.nan filling works
ras = ras.astype(np.float32)
ras_fltr = ras
#Absolute range filter
ras_fltr = filtlib.range_fltr(ras_fltr, (min, max))
if args.stats:
print("Input dz raster stats:")
malib.print_stats(ras)
print("Filtered dz raster stats:")
malib.print_stats(ras_fltr)
#Output filename will have 'filt' appended
dst_fn = os.path.splitext(ras_fn)[0] + '_filt.tif'
print("Writing out filtered dz raster: %s" % dst_fn)
#Note: writeGTiff writes ras_fltr.filled()
iolib.writeGTiff(ras_fltr, dst_fn, ras_ds)
def gen_ts_fn(fn, dt_ref=None, ma=False):
from osgeo import gdal
from pygeotools.lib import iolib
print("Generating timestamp for: %s" % fn)
fn_ts = os.path.splitext(fn)[0] + '_ts.tif'
if not os.path.exists(fn_ts) or dt_ref is not None:
ds = gdal.Open(fn)
#Should be ok with float ordinals here
a = iolib.ds_getma(ds)
ts = fn_getdatetime(fn)
#Want to check that dt_ref is valid datetime object
if dt_ref is not None:
t = ts - dt_ref
t = t.total_seconds() / 86400.
fn_ts = os.path.splitext(fn)[0] + '_ts_rel.tif'
else:
t = dt2o(ts)
a[~np.ma.getmaskarray(a)] = t
#Probably want to be careful about ndv here - could be 0 for rel
#ndv = 1E20
ndv = -9999.0
a.set_fill_value(ndv)
iolib.writeGTiff(a, fn_ts, ds)
if ma:
return a
else:
return fn_ts
def get_nlcd_mask(nlcd_ds, filter='not_forest', out_fn=None):
"""Generate raster mask for specified NLCD LULC filter
"""
print("Loading NLCD LULC")
b = nlcd_ds.GetRasterBand(1)
l = b.ReadAsArray()
print("Filtering NLCD LULC with: %s" % filter)
#Original nlcd products have nan as ndv
#12 - ice
#31 - rock
#11 - open water, includes rivers
#52 - shrub, <5 m tall, >20%
#42 - evergreeen forest
#Should use data dictionary here for general masking
#Using 'rock+ice+water' preserves the most pixels, although could be problematic over areas with lakes
if filter == 'rock':
mask = (l==31)
elif filter == 'rock+ice':
mask = np.logical_or((l==31),(l==12))
elif filter == 'rock+ice+water':
mask = np.logical_or(np.logical_or((l==31),(l==12)),(l==11))
elif filter == 'not_forest':
mask = ~(np.logical_or(np.logical_or((l==41),(l==42)),(l==43)))
elif filter == 'not_forest+not_water':
mask = ~(np.logical_or(np.logical_or(np.logical_or((l==41),(l==42)),(l==43)),(l==11)))
else:
print("Invalid mask type")
mask = None
#Write out original data
if out_fn is not None:
print("Writing out %s" % out_fn)
iolib.writeGTiff(l, out_fn, nlcd_ds)
l = None
return mask
def main():
parser = argparse.ArgumentParser(description="Clip input raster by input shp polygons")
#Should add support for similar arguments as in warplib - arbitrary extent, res, etc
parser.add_argument('-extent', type=str, default='raster', choices=['raster','shp'], \
help='Desired output extent')
parser.add_argument('r_fn', type=str, help='Input raster filename')
parser.add_argument('shp_fn', type=str, help='Input shp filename')
args = parser.parse_args()
r_fn = args.r_fn
if not os.path.exists(args.r_fn):
sys.exit("Unable to find r_fn: %s" % r_fn)
shp_fn = args.shp_fn
if shp_fn == 'RGI' or shp_fn == 'rgi':
from demcoreg.dem_mask import get_glacier_poly
rgi_fn = get_glacier_poly()
shp_fn = rgi_fn
if not os.path.exists(shp_fn):
sys.exit("Unable to find shp_fn: %s" % shp_fn)
extent=args.extent
#Do the clipping
r = raster_shpclip(r_fn, shp_fn, extent)
#Write out
out_fn = os.path.splitext(r_fn)[0]+'_shpclip.tif'
#Note: passing r_fn here as the src_ds
iolib.writeGTiff(r, out_fn, r_fn)
def main():
parser = getparser()
args = parser.parse_args()
r_fn = args.r_fn
if not os.path.exists(r_fn):
sys.exit("Unable to find r_fn: %s" % r_fn)
shp_fn = args.shp_fn
#Convenience shortcut to clip to glacier polygons (global shp)
#Requires demcoreg package: https://github.com/dshean/demcoreg
if shp_fn == 'RGI' or shp_fn == 'rgi':
from demcoreg.dem_mask import get_glacier_poly
rgi_fn = get_glacier_poly()
shp_fn = rgi_fn
if not os.path.exists(shp_fn):
sys.exit("Unable to find shp_fn: %s" % shp_fn)
extent = args.extent
#Do the clipping
r = geolib.raster_shpclip(r_fn, shp_fn, extent)
#Write out
out_fn = os.path.splitext(r_fn)[0] + '_shpclip.tif'
#Note: passing r_fn here as the src_ds
iolib.writeGTiff(r, out_fn, r_fn)
def main():
parser = getparser()
args = parser.parse_args()
src_fn = args.src_fn
new_ndv = args.new_ndv
#Input argument is a string, which is not recognized by set_fill_value
#Must use np.nan object
if new_ndv == 'nan' or new_ndv == 'np.nan':
new_ndv = np.nan
else:
new_ndv = float(new_ndv)
#Output filename will have ndv appended
if args.overwrite:
out_fn = src_fn
else:
out_fn = os.path.splitext(src_fn)[0] + '_ndv.tif'
ds = gdal.Open(src_fn)
b = ds.GetRasterBand(1)
#Extract old ndv
old_ndv = iolib.get_ndv_b(b)
print(src_fn)
print("Replacing old ndv %s with new ndv %s" % (old_ndv, new_ndv))
#Load masked array
bma = iolib.ds_getma(ds)
#Set new fill value
bma.set_fill_value(new_ndv)
#Fill ma with new value and write out
iolib.writeGTiff(bma.filled(), out_fn, ds, ndv=new_ndv)
def main():
parser = getparser()
args = parser.parse_args()
r_fn = args.r_fn
if not os.path.exists(r_fn):
sys.exit("Unable to find r_fn: %s" % r_fn)
# r_fn to r_ds
r_ds = iolib.fn_getds(r_fn)
# ...to array
r_arr = r_ds.GetRasterBand(1).ReadAsArray()
r_arr = r_arr * args.scale_factor
arr_out = r_arr.astype(getattr(
np, args.out_type)) # this works like r_arr.astype(np.unit16)
print "\tOutput array type: %s" % (arr_out.dtype)
print "\tOutput nodata value: %s" % (args.nodata_val)
# Set No Data Value: Deal with negatives and nan
arr_out = np.where(arr_out < 0, args.nodata_val, arr_out)
arr_out = np.where(np.isnan(arr_out), args.nodata_val, arr_out)
#Write out
out_fn = os.path.splitext(r_fn)[0] + '_' + args.out_type + '.tif'
print "\tOutput file: %s" % (out_fn)
#Note: passing r_fn here as the src_ds
iolib.writeGTiff(arr_out, out_fn, r_fn)
def main():
parser = getparser()
args = parser.parse_args()
mask_list = []
if args.toa: mask_list.append('toa')
if args.snodas: mask_list.append('snodas')
if args.modscag: mask_list.append('modscag')
if args.bareground: mask_list.append('bareground')
if args.glaciers: mask_list.append('glaciers')
if args.nlcd: mask_list.append('nlcd')
if not mask_list:
parser.print_help()
sys.exit("Must specify at least one mask type")
#This directory should or will contain the relevant data products
#if args.datadir is None:
# datadir = iolib.get_datadir()
dem_fn = args.dem_fn
dem_ds = gdal.Open(dem_fn)
print(dem_fn)
#Get DEM masked array
dem = iolib.ds_getma(dem_ds)
print("%i valid pixels in original input tif" % dem.count())
#Set up cascading mask preparation
#True (1) represents "valid" unmasked pixel, False (0) represents "invalid" pixel to be masked
#Initialize the mask
#newmask = ~(np.ma.getmaskarray(dem))
#Basename for output files
if args.outdir is not None:
if not os.path.exists(args.outdir):
os.makedirs(args.outdir)
else:
args.outdir = os.path.split(dem_fn)[0]
newmask = get_mask(dem_ds, mask_list, dem_fn=dem_fn, writeout=args.writeout, outdir=args.outdir, args=args)
#Apply mask to original DEM - use these surfaces for co-registration
newdem = np.ma.array(dem, mask=newmask)
#Check that we have enough pixels, good distribution
min_validpx_count = 100
min_validpx_std = 10
validpx_count = newdem.count()
validpx_std = newdem.std()
print("%i valid pixels in masked output tif to be used as ref" % validpx_count)
print("%0.2f std in masked output tif to be used as ref" % validpx_std)
#if (validpx_count > min_validpx_count) and (validpx_std > min_validpx_std):
if (validpx_count > min_validpx_count):
out_fn = os.path.join(args.outdir, os.path.splitext(dem_fn)[0]+'_ref.tif')
print("Writing out %s" % out_fn)
iolib.writeGTiff(newdem, out_fn, src_ds=dem_ds)
else:
print("Not enough valid pixels!")
def main():
parser = getparser()
args = parser.parse_args()
dem_fn = args.dem_fn
ndv = args.ndv
max_slope = args.max_slope
reduce_pct = args.reduce_pct
#slope_res = args.slope_res
slopelim = (0.1, max_slope)
out_base = os.path.splitext(dem_fn)[0]
print "\n\tGetting a slopemasked dem...\n"
if reduce_pct != 100:
print "\tCoarsening by %s.." % (reduce_pct)
out_fn = out_base + '_pct' + str(reduce_pct) + '.tif'
if not os.path.exists(out_fn):
red_pct_str = str(reduce_pct) + "% " + str(reduce_pct) + "%"
cmdStr = " ".join([
"gdal_translate", "-r", "cubic", "-outsize", red_pct_str,
dem_fn, out_fn
])
print cmdStr
#cmdStr = "gdal_translate -r cubic -outsize " + str(reduce_pct) + "% " + str(reduce_pct) + "%"
#cmdStr += dem_fn + out_fn
cmd = subprocess.Popen(cmdStr, stdout=subprocess.PIPE, shell=True)
stdOut, err = cmd.communicate()
dem_fn = out_fn
# Get slope as a masked array
dem_slope_ma = geolib.gdaldem_wrapper(dem_fn,
product='slope',
returnma=True)
# Get reduced dem ds
dem_ds = iolib.fn_getds(dem_fn)
dem_ma = iolib.ds_getma(dem_ds)
# Get a new slope ma using max slope
slopemask = (dem_slope_ma > max_slope)
# Get whichever dem ma was there in the first place
demmask = (dem_ma == ndv)
newmask = np.logical_or(demmask, slopemask)
# Apply mask from slope to slope
newdem = np.ma.array(dem_ma, mask=newmask)
# Save the new masked DEM
dst_fn = out_base + '_slopemasked.tif'
iolib.writeGTiff(newdem, dst_fn, dem_ds, ndv=ndv)
return dst_fn
def mask_nlcd(ds,
valid='rock+ice+water',
datadir=None,
mask_glaciers=True,
out_fn=None):
"""Generate raster mask for exposed rock in NLCD data
"""
print("Loading NLCD LULC")
b = ds.GetRasterBand(1)
l = b.ReadAsArray()
print("Filtering NLCD LULC")
#Original nlcd products have nan as ndv
#12 - ice
#31 - rock
#11 - open water, includes rivers
#52 - shrub, <5 m tall, >20%
#42 - evergreeen forest
#Should use data dictionary here for general masking
#Using 'rock+ice+water' preserves the most pixels, although could be problematic over areas with lakes
if valid == 'rock':
mask = (l == 31)
elif valid == 'rock+ice':
mask = np.logical_or((l == 31), (l == 12))
elif valid == 'rock+ice+water':
mask = np.logical_or(np.logical_or((l == 31), (l == 12)), (l == 11))
elif valid == 'not_forest':
mask = ~(np.logical_or(np.logical_or((l == 41), (l == 42)), (l == 43)))
elif valid == 'not_forest+not_water':
mask = ~(np.logical_or(
np.logical_or(np.logical_or((l == 41), (l == 42)), (l == 43)),
(l == 11)))
else:
print("Invalid mask type")
mask = None
#Write out original data
if out_fn is not None:
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(l, out_fn, ds)
l = None
if mask_glaciers:
if datadir is None:
datadir = iolib.get_datadir()
#Note: RGI6.0 includes Fountain's 24K polygons, no longer need to maintain two separate shp
#Use updated 24k glacier outlines
#glac_shp_fn = os.path.join(datadir, 'conus_glacierpoly_24k/conus_glacierpoly_24k_32610.shp')
#if not os.path.exists(glac_shp_fn):
# glac_shp_fn = None
glac_shp_fn = None
icemask = get_icemask(ds, datadir=datadir, glac_shp_fn=glac_shp_fn)
if icemask is not None:
mask *= icemask
return mask
def proc_modscag(fn_list, extent=None, t_srs=None):
"""Process the MODSCAG products for full date range, create composites and reproject
"""
#Use cubic spline here for improve upsampling
ds_list = warplib.memwarp_multi_fn(fn_list, res='min', extent=extent, t_srs=t_srs, r='cubicspline')
stack_fn = os.path.splitext(fn_list[0])[0] + '_' + os.path.splitext(os.path.split(fn_list[-1])[1])[0] + '_stack_%i' % len(fn_list)
#Create stack here - no need for most of mastack machinery, just make 3D array
#Mask values greater than 100% (clouds, bad pixels, etc)
ma_stack = np.ma.array([np.ma.masked_greater(iolib.ds_getma(ds), 100) for ds in np.array(ds_list)], dtype=np.uint8)
stack_count = np.ma.masked_equal(ma_stack.count(axis=0), 0).astype(np.uint8)
stack_count.set_fill_value(0)
stack_min = ma_stack.min(axis=0).astype(np.uint8)
stack_min.set_fill_value(0)
stack_max = ma_stack.max(axis=0).astype(np.uint8)
stack_max.set_fill_value(0)
stack_med = np.ma.median(ma_stack, axis=0).astype(np.uint8)
stack_med.set_fill_value(0)
out_fn = stack_fn + '_count.tif'
iolib.writeGTiff(stack_count, out_fn, ds_list[0])
out_fn = stack_fn + '_max.tif'
iolib.writeGTiff(stack_max, out_fn, ds_list[0])
out_fn = stack_fn + '_min.tif'
iolib.writeGTiff(stack_min, out_fn, ds_list[0])
out_fn = stack_fn + '_med.tif'
iolib.writeGTiff(stack_med, out_fn, ds_list[0])
ds = gdal.Open(out_fn)
return ds
def domask(tile_fn):
prefix = '-'.join(tile_fn.split('-')[:-1])
print("\nLoading: %s" % tile_fn)
#dem_fn = prefix+'-median.tif'
dem_fn = tile_fn
dem_ds = iolib.fn_getds(dem_fn)
dem = iolib.ds_getma(dem_ds)
#Get original mask, True where masked
mask = np.ma.getmaskarray(dem)
valid_px_count = (~mask).sum()
if valid_px_count < 1:
print("No valid pixels remain")
else:
print("Valid pixel count: %i" % valid_px_count)
min_count = 2
count_fn = prefix + '-count.tif'
print("Loading: %s" % count_fn)
count = iolib.fn_getma(count_fn)
print("min: %i max: %i" % (count.min(), count.max()))
print("Masking: (count < %i)" % min_count)
mask = np.logical_or(mask, (count < min_count))
valid_px_count = (~mask).sum()
if valid_px_count < 1:
print("No valid pixels remain")
else:
print("Valid pixel count: %i" % valid_px_count)
max_std = 3.0
#std_fn = prefix+'-std.tif'
std_fn = prefix + '-nmad.tif'
print("Loading: %s" % std_fn)
std = iolib.fn_getma(std_fn)
print("min: %i max: %i" % (std.min(), std.max()))
print("Masking: (std/nmad >= %i)" % max_std)
mask = np.logical_or(mask, (std >= max_std))
valid_px_count = (~mask).sum()
if valid_px_count < 1:
print("No valid pixels remain")
else:
print("Valid pixel count: %i" % valid_px_count)
#Modified so we always write out, even if empty tif
#Easier for tracking progress
print("Applying mask")
dem_masked = np.ma.array(dem, mask=mask)
out_fn = os.path.splitext(dem_fn)[0] + '_masked.tif'
print("Writing: %s" % out_fn)
iolib.writeGTiff(dem_masked, out_fn, dem_ds)
def get_bareground_mask(bareground_ds, bareground_thresh=60, out_fn=None):
"""Generate raster mask for exposed bare ground from global bareground data
"""
print("Loading bareground")
b = bareground_ds.GetRasterBand(1)
l = b.ReadAsArray()
print("Masking pixels with <%0.1f%% bare ground" % bareground_thresh)
if bareground_thresh < 0.0 or bareground_thresh > 100.0:
sys.exit("Invalid bare ground percentage")
mask = (l > bareground_thresh)
#Write out original data
if out_fn is not None:
print("Writing out %s" % out_fn)
iolib.writeGTiff(l, out_fn, bareground_ds)
l = None
return mask
def main():
parser = getparser()
args = parser.parse_args()
dem_fn = args.dem_fn
ndv = args.ndv
max_slope = args.max_slope
reduce_pct = args.reduce_pct
#slope_res = args.slope_res
slopelim = (0.1, max_slope)
print "\n\tGetting masked slope of input dem..."
#Get a coarsened version of DEM on which to calc slope
dem_fn_reduced = os.path.splitext(dem_fn)[0] + '_' + str(
reduce_pct) + 'pct.vrt'
#dem_fn_reduced = os.path.splitext(dem_fn)[0]+'_'+ str(slope_res) +'m.vrt'
#print "\tReducing percent by %s..." %(str(reduce_pct))
run_os("gdal_translate -of VRT -r cubic -outsize " + str(reduce_pct) +
"% " + str(reduce_pct) + "% " + dem_fn + " " + dem_fn_reduced)
#run_os("gdal_translate -of VRT -r cubic -tr " + str(slope_res) + "% " + str(slope_res) + "% " + dem_fn + " " + dem_fn_reduced)
# Run slope
dem_slope_fn = geolib.gdaldem_wrapper(dem_fn_reduced,
product='slope',
returnma=False)
# Get original ma and ds
dem_slope = iolib.fn_getma(dem_slope_fn)
dem_slope_ds = iolib.fn_getds(dem_slope_fn)
# Apply mask from slope to slope
dem_slope = np.ma.array(dem_slope,
mask=np.ma.masked_outside(dem_slope,
*slopelim).mask,
keep_mask=True,
fill_value=dem_slope.fill_value)
# Save the filtered slope dataset
dst_fn = os.path.splitext(dem_slope_fn)[0] + '_mask.tif'
iolib.writeGTiff(dem_slope, dst_fn, dem_slope_ds, ndv=ndv)
run_os("rm -fv " + dem_slope_fn)
return dst_fn
def mask_slope(ds, maxslope=10, out_fn=None):
"""Generate raster mask for slope from input slope
"""
print("Loading slope")
b = ds.GetRasterBand(1)
l = b.ReadAsArray()
print("Masking pixels with >%0.1f%% slopes\n" % maxslope)
if maxslope < 0.0 or maxslope > 90.0:
sys.exit("Invalid slope in degrees")
mask = (l < maxslope)
#Write out original data
if out_fn is not None:
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(l, out_fn, ds)
l = None
return mask
def main():
parser = getparser()
args = parser.parse_args()
dem_fn = args.dem_fn
weight = args.weight
poly_order = args.order
# Get original DEM
array_dem = iolib.fn_getma(dem_fn)
dem_ds = iolib.fn_getds(dem_fn)
# Create empty meshgrid
x = range (0, dem_ds.RasterXSize, 1)
y = range (0, dem_ds.RasterYSize, 1)
X, Y = np.meshgrid(x, y)
print "\nGet the gradient of the dem..."
dzdy,dzdx = np.gradient(array_dem)
print "\nCreate copies for modification, and set the masked areas to 0 in the gradients..."
dzdx_mod = dzdx.copy()
dzdy_mod = dzdy.copy()
print "\nOsmanoglu Algorithm..."
#routine to get the scale corrected surface back from gradients.
#Note that this is with the original slopes, I want the result to be close to the original DEM as possible
fco_dem = frankotchellappaosmanoglu(dzdx, dzdy) ; #FrankotChellappa removes long wavelength trends
print "\nSubtract the recovered DEM from the original and estimate a surface..."
planefit, fitfunc = fitSurface(X.ravel(), Y.ravel(), (array_dem - fco_dem).ravel())
print "\nAdd the surface to the masked gradient derived DEM..."
dem_interp = frankotchellappaosmanoglu(dzdx_mod, dzdy_mod) + fitfunc(planefit, X, Y, weight, poly_order)
print("\nWriting DEM with interpolated surfaces:")
dst_fn = os.path.splitext(dem_fn)[0]+'_interp.tif'
print(dst_fn)
iolib.writeGTiff(dem_interp, dst_fn, dem_ds)
return dst_fn
# Return a numpy masked array
return dem_interp
def main():
parser = getparser()
args = parser.parse_args()
src_fn = args.src_fn
if not iolib.fn_check(src_fn):
sys.exit("Unable to find src_fn: %s" % src_fn)
#This is a wrapper around gdal.Open()
src_ds = iolib.fn_getds(src_fn)
src_gt = src_ds.GetGeoTransform()
print("Loading input raster into masked array")
bma = iolib.ds_getma(src_ds)
print("Computing min/max indices for mask")
edge_env = malib.edgefind2(bma, intround=True)
print("Updating output geotransform")
out_gt = list(src_gt)
#This should be OK, as edge_env values are integer multiples, and the initial gt values are upper left pixel corner
#Update UL_X
out_gt[0] = src_gt[0] + src_gt[1] * edge_env[2]
#Update UL_Y, note src_gt[5] is negative
out_gt[3] = src_gt[3] + src_gt[5] * edge_env[0]
out_gt = tuple(out_gt)
#debug
#print([0, bma.shape[0], 0, bma.shape[1]])
#print(edge_env)
#print(src_gt)
#print(out_gt)
out_fn = os.path.splitext(src_fn)[0] + '_trim.tif'
print("Writing out: %s" % out_fn)
#Extract valid subsection from input array
#indices+1 are necessary to include valid row/col on right and bottom edges
iolib.writeGTiff(bma[edge_env[0]:edge_env[1] + 1,
edge_env[2]:edge_env[3] + 1],
out_fn,
src_ds,
gt=out_gt)
bma = None
def main():
parser = getparser()
args = parser.parse_args()
chm_fn = args.chm_fn
outdir = args.outdir
ndv = args.ndv
min_height = args.min_height
max_height = args.max_height
if not iolib.fn_check(chm_fn):
sys.exit("Unable to find chm_fn: %s" % chm_fn)
out_base = os.path.splitext(chm_fn)[0]
if outdir is not None:
inputdir, chmname = os.path.split(chm_fn)
out_base = os.path.join(outdir, os.path.splitext(chmname)[0])
# Get chm ds and ma
chm_ds = iolib.fn_getds(chm_fn)
chm_ma = iolib.ds_getma(chm_ds)
# Get a new chm ma using max height
heightmasklo = (chm_ma < min_height)
heightmaskhi = (chm_ma > max_height)
# Get whichever dem ma was there in the first place
chmmask = (chm_ma == ndv)
# https://stackoverflow.com/questions/20528328/numpy-logical-or-for-more-than-two-arguments
newmask = np.logical_or.reduce(
(chmmask, heightmasklo,
heightmaskhi)) #np.logical_or(heightmasklo, heightmaskhi)
# Apply mask
newchm = np.ma.array(chm_ma, mask=newmask)
# Save the new masked CHM
dst_fn = out_base + '_htmasked.tif'
iolib.writeGTiff(newchm, dst_fn, chm_ds, ndv=ndv)
print(dst_fn)
return dst_fn
def make_dem_mosaic_index_ts(index_tif_fn):
index_txt_fn = index_tif_fn + '-index-map.txt'
out_fn = os.path.splitext(index_tif_fn)[0] + '_ts.tif'
if not os.path.exists(index_tif_fn):
print("Unable to find input file: %s" % index_tif_fn)
return False
if not os.path.exists(index_txt_fn):
print("Unable to find input file: %s" % index_txt_fn)
return False
if os.path.exists(out_fn):
print("Existing ts output found: %s" % out_fn)
return True
#Load dem_mosaic index tif
index_tif_ds = iolib.fn_getds(index_tif_fn)
index_tif = iolib.ds_getma(index_tif_ds)
#Create output array
#index_ts_tif = np.zeros(index_tif.shape, dtype=np.float32)
#index_ts_tif = np.ma.masked_all_like(index_tif)
index_ts_tif = np.ma.masked_all(index_tif.shape, dtype=np.float32)
#Read in dem_mosaic index txt file - should be "fn, index" for each record
index_txt = np.atleast_2d(np.genfromtxt(index_txt_fn, dtype=str))
#Extract Python datetime object from filename (should pull out YYYYMMDD_HHMM)
index_ts = [timelib.fn_getdatetime(fn) for fn in index_txt[:, 0]]
#Convert to desired output timestamp format
#Python ordinal
#index_ts = timelib.dt2o(index_ts)
#YYYYMMDD integer
#index_ts = [ts.strftime('%Y%m%d') for ts in index_ts]
#Decimal year
index_ts = [timelib.dt2decyear(ts) for ts in index_ts]
for n, dt in enumerate(index_ts):
index_ts_tif[index_tif == n] = dt
iolib.writeGTiff(index_ts_tif, out_fn, index_tif_ds, ndv=0)
return True
def mask_bareground(ds, minperc=60, mask_glaciers=True, out_fn=None):
"""Generate raster mask for exposed bare ground from global bareground data
"""
print("Loading bareground")
b = ds.GetRasterBand(1)
l = b.ReadAsArray()
print("Masking pixels with <%0.1f%% bare ground\n" % minperc)
if minperc < 0.0 or minperc > 100.0:
sys.exit("Invalid bare ground percentage")
mask = (l > minperc)
#Write out original data
if out_fn is not None:
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(l, out_fn, ds)
l = None
if mask_glaciers:
icemask = get_icemask(ds)
if icemask is not None:
mask *= icemask
return mask
def dz_fltr_dir(dem_fn_list, refdem_fn, abs_dz_lim, out_dir):
#names = dem_fn_list[0]
for names in dem_fn_list:
#print("Loading ouput DEM into masked array")
dem_ds = iolib.fn_getds(names)
dem_fltr = iolib.ds_getma(dem_ds, 1)
#Difference filter, need to specify refdem_fn
dem_fltr = filtlib.dz_fltr(names, refdem_fn, abs_dz_lim)
# create output directory and file name
parts = names.split('/')
file_name = parts[len(parts) - 1]
dst_fn = out_dir + file_name.split(
'.')[0] + '_filt%ipx.tif' % abs_dz_lim[1]
print("Writing out filtered DEM: %s" % dst_fn)
#Note: writeGTiff writes dem_fltr.filled()
iolib.writeGTiff(dem_fltr, dst_fn, dem_ds)
def main():
parser = getparser()
args = parser.parse_args()
r_fn = args.r_fn
if not os.path.exists(r_fn):
sys.exit("Unable to find r_fn: %s" % r_fn)
windowsize=args.windowsize
# r_fn to r_ds
r_ds = iolib.fn_getds(r_fn)
r_arr = r_ds.GetRasterBand(1).ReadAsArray()
# Creating data range
#r_arr = np.ma.masked_outside(r_arr,0,100) # mask all values outside this interval
#r_arr = np.ma.masked_invalid(r_arr) # mask all nan and inf values
# Forget about masked arrays...just use np.where to put 0 for all invalid vals
r_arr = np.where((r_arr > args.min) & (r_arr <= args.max), r_arr, 0)
#myType=float
#r_arr = r_arr.astype(myType)
print "\tDoing moving window on array..."
arr_out = moving_window(r_arr.astype(np.uint16), func=np.sum, window_size=windowsize)
#r = filtlib.rolling_fltr(r_arr, f=np.sum, size=windowsize, circular=True)
#r = gauss_fltr(r_arr, sigma=1)
print "\tCheck array type OUTPUT from filter: %s" %(arr_out.dtype)
# Deal with negatives and nan
arr_out = np.where(arr_out < 0, -99, arr_out)
arr_out = np.where(np.isnan(arr_out),-99, arr_out)
#Write out
win_str = "%02d" % (int(windowsize))
out_fn = os.path.splitext(r_fn)[0]+'_win'+win_str+'sum.tif'
#Note: passing r_fn here as the src_ds
iolib.writeGTiff(arr_out, out_fn, r_fn)
def main():
start_time = timer()
parser = getparser()
args = parser.parse_args()
r_fn = args.r_fn
metric_list = args.metric_list.split(" ")
win_size_list = args.win_size_list.split(" ")
distance_list = args.distance_list.split(" ")
print "\tGLCM metrics to be processed: %s" % metric_list
if not os.path.exists(r_fn):
sys.exit("Unable to find r_fn: %s" % r_fn)
# r_fn to r_ds
r_ds = iolib.fn_getds(r_fn)
# ...to array
r_arr = r_ds.GetRasterBand(1).ReadAsArray()
# Forget about masked arrays...just use np.where to put 0 for all invalid vals
r_arr = np.where((r_arr > 0.0) & (r_arr <= 1.0), r_arr, np.nan)
# A way of checking the histograms of the orig image, and the scaled image
# scale and set to to byte
##fit_GMM(r_arr, os.path.split(r_fn)[0], os.path.split(r_fn)[1], 5, 'float')
##r_arr = img_as_ubyte(r_arr)
##r_arr = img_as_uint(r_arr)
##fit_GMM(r_arr, os.path.split(r_fn)[0], os.path.split(r_fn)[1], 5, 'byte')
end_readdata = timer()
print "\tData: %s" % (os.path.basename(r_fn))
print "\n\tTime to read in data: {} minutes\n".format(
round(find_elapsed_time(start_time, end_readdata), 3))
print "\tWindow sizes to be processed: %s" % win_size_list
print "\tDistances to be processed: %s" % distance_list
for win_size in win_size_list:
print "\n\tWindow size: %s" % win_size
win_size = int(win_size)
for distance in distance_list:
print "\tDistance: %s" % distance
start_win_dist = timer()
distance = int(distance)
# Set up arrays to hold GLCM output
# these need to be float
con = np.copy(r_arr).astype(np.float32)
con[:] = 0
dis = np.copy(r_arr).astype(np.float32)
dis[:] = 0
cor = np.copy(r_arr).astype(np.float32)
cor[:] = 0
asm = np.copy(r_arr).astype(np.float32)
asm[:] = 0
start_glcm = timer()
print "\tCalculating GLCM and its properties..."
# Loop over pixel windows (row and col, by win_size
for i in range(con.shape[0]):
print i,
for j in range(con.shape[1]):
#windows needs to fit completely in image
if i < ((win_size - 1) / 2) or j < ((win_size - 1) / 2):
continue
if i > (con.shape[0] -
((win_size + 1) / 2)) or j > (con.shape[0] - (
(win_size + 1) / 2)):
continue
#Calculate GLCM on a window
#
# converting to byte array right before GLCM processing
# output arrays still with input precision
in_arr = img_as_ubyte(r_arr)
in_glcm_window_arr = in_arr[(i - ((win_size - 1) / 2)):(
i + ((win_size + 1) / 2)), (j - ((win_size - 1) / 2)):(
j + ((win_size + 1) / 2))]
del in_arr
out_glcm_window_arr = greycomatrix(in_glcm_window_arr, \
distances=[distance],\
#angles=[0, np.pi/4, np.pi/2, 3*np.pi/4],\
angles=[0],\
levels=256, symmetric=True, normed=True )
con[i, j], dis[i, j], cor[i, j], asm[i, j] = [
greycoprops(out_glcm_window_arr, metric)
for metric in metric_list
]
del out_glcm_window_arr
end_glcm = timer()
print "\n\tTime to compute this GLCM and its properties: {} minutes\n".format(
round(find_elapsed_time(start_glcm, end_glcm), 3))
out_glcm_list = [con, dis, cor, asm]
for num, metric in enumerate(metric_list):
out_fn = os.path.splitext(r_fn)[0] + '_TEXTij_win' + str(
win_size) + '_' + 'dist' + str(
distance) + '_' + metric + '.tif'
print '\tWriting: %s' % (out_fn)
iolib.writeGTiff(out_glcm_list[num], out_fn, r_fn)
end_win_dist = timer()
print "\tTotal compute time for GLCM on this window & distance: {} minutes\n".format(
round(find_elapsed_time(start_win_dist, end_win_dist), 3))
def main(argv=None):
parser = getparser()
args = parser.parse_args()
#Should check that files exist
ref_dem_fn = args.ref_fn
src_dem_fn = args.src_fn
mode = args.mode
mask_list = args.mask_list
max_offset = args.max_offset
max_dz = args.max_dz
slope_lim = tuple(args.slope_lim)
tiltcorr = args.tiltcorr
polyorder = args.polyorder
res = args.res
#Maximum number of iterations
max_iter = args.max_iter
#These are tolerances (in meters) to stop iteration
tol = args.tol
min_dx = tol
min_dy = tol
min_dz = tol
outdir = args.outdir
if outdir is None:
outdir = os.path.splitext(src_dem_fn)[0] + '_dem_align'
if tiltcorr:
outdir += '_tiltcorr'
tiltcorr_done = False
#Relax tolerance for initial round of co-registration
#tiltcorr_tol = 0.1
#if tol < tiltcorr_tol:
# tol = tiltcorr_tol
if not os.path.exists(outdir):
os.makedirs(outdir)
outprefix = '%s_%s' % (os.path.splitext(os.path.split(src_dem_fn)[-1])[0], \
os.path.splitext(os.path.split(ref_dem_fn)[-1])[0])
outprefix = os.path.join(outdir, outprefix)
print("\nReference: %s" % ref_dem_fn)
print("Source: %s" % src_dem_fn)
print("Mode: %s" % mode)
print("Output: %s\n" % outprefix)
src_dem_ds = gdal.Open(src_dem_fn)
ref_dem_ds = gdal.Open(ref_dem_fn)
#Get local cartesian coordinate system
#local_srs = geolib.localtmerc_ds(src_dem_ds)
#Use original source dataset coordinate system
#Potentially issues with distortion and xyz/tiltcorr offsets for DEM with large extent
local_srs = geolib.get_ds_srs(src_dem_ds)
#local_srs = geolib.get_ds_srs(ref_dem_ds)
#Resample to common grid
ref_dem_res = float(geolib.get_res(ref_dem_ds, t_srs=local_srs, square=True)[0])
#Create a copy to be updated in place
src_dem_ds_align = iolib.mem_drv.CreateCopy('', src_dem_ds, 0)
src_dem_res = float(geolib.get_res(src_dem_ds, t_srs=local_srs, square=True)[0])
src_dem_ds = None
#Resample to user-specified resolution
ref_dem_ds, src_dem_ds_align = warplib.memwarp_multi([ref_dem_ds, src_dem_ds_align], \
extent='intersection', res=args.res, t_srs=local_srs, r='cubic')
res = float(geolib.get_res(src_dem_ds_align, square=True)[0])
print("\nReference DEM res: %0.2f" % ref_dem_res)
print("Source DEM res: %0.2f" % src_dem_res)
print("Resolution for coreg: %s (%0.2f m)\n" % (args.res, res))
#Iteration number
n = 1
#Cumulative offsets
dx_total = 0
dy_total = 0
dz_total = 0
#Now iteratively update geotransform and vertical shift
while True:
print("*** Iteration %i ***" % n)
dx, dy, dz, static_mask, fig = compute_offset(ref_dem_ds, src_dem_ds_align, src_dem_fn, mode, max_offset, \
mask_list=mask_list, max_dz=max_dz, slope_lim=slope_lim, plot=True)
xyz_shift_str_iter = "dx=%+0.2fm, dy=%+0.2fm, dz=%+0.2fm" % (dx, dy, dz)
print("Incremental offset: %s" % xyz_shift_str_iter)
dx_total += dx
dy_total += dy
dz_total += dz
xyz_shift_str_cum = "dx=%+0.2fm, dy=%+0.2fm, dz=%+0.2fm" % (dx_total, dy_total, dz_total)
print("Cumulative offset: %s" % xyz_shift_str_cum)
#String to append to output filenames
xyz_shift_str_cum_fn = '_%s_x%+0.2f_y%+0.2f_z%+0.2f' % (mode, dx_total, dy_total, dz_total)
#Should make an animation of this converging
if n == 1:
#static_mask_orig = static_mask
if fig is not None:
dst_fn = outprefix + '_%s_iter%02i_plot.png' % (mode, n)
print("Writing offset plot: %s" % dst_fn)
fig.gca().set_title("Incremental: %s\nCumulative: %s" % (xyz_shift_str_iter, xyz_shift_str_cum))
fig.savefig(dst_fn, dpi=300)
#Apply the horizontal shift to the original dataset
src_dem_ds_align = coreglib.apply_xy_shift(src_dem_ds_align, dx, dy, createcopy=False)
#Should
src_dem_ds_align = coreglib.apply_z_shift(src_dem_ds_align, dz, createcopy=False)
n += 1
print("\n")
#If magnitude of shift in all directions is less than tol
#if n > max_iter or (abs(dx) <= min_dx and abs(dy) <= min_dy and abs(dz) <= min_dz):
#If magnitude of shift is less than tol
dm = np.sqrt(dx**2 + dy**2 + dz**2)
dm_total = np.sqrt(dx_total**2 + dy_total**2 + dz_total**2)
if dm_total > max_offset:
sys.exit("Total offset exceeded specified max_offset (%0.2f m). Consider increasing -max_offset argument" % max_offset)
#Stop iteration
if n > max_iter or dm < tol:
if fig is not None:
dst_fn = outprefix + '_%s_iter%02i_plot.png' % (mode, n)
print("Writing offset plot: %s" % dst_fn)
fig.gca().set_title("Incremental:%s\nCumulative:%s" % (xyz_shift_str_iter, xyz_shift_str_cum))
fig.savefig(dst_fn, dpi=300)
#Compute final elevation difference
if True:
ref_dem_clip_ds_align, src_dem_clip_ds_align = warplib.memwarp_multi([ref_dem_ds, src_dem_ds_align], \
res=res, extent='intersection', t_srs=local_srs, r='cubic')
ref_dem_align = iolib.ds_getma(ref_dem_clip_ds_align, 1)
src_dem_align = iolib.ds_getma(src_dem_clip_ds_align, 1)
ref_dem_clip_ds_align = None
diff_align = src_dem_align - ref_dem_align
src_dem_align = None
ref_dem_align = None
#Get updated, final mask
static_mask_final = get_mask(src_dem_clip_ds_align, mask_list, src_dem_fn)
static_mask_final = np.logical_or(np.ma.getmaskarray(diff_align), static_mask_final)
#Final stats, before outlier removal
diff_align_compressed = diff_align[~static_mask_final]
diff_align_stats = malib.get_stats_dict(diff_align_compressed, full=True)
#Prepare filtered version for tiltcorr fit
diff_align_filt = np.ma.array(diff_align, mask=static_mask_final)
diff_align_filt = outlier_filter(diff_align_filt, f=3, max_dz=max_dz)
#diff_align_filt = outlier_filter(diff_align_filt, perc=(12.5, 87.5), max_dz=max_dz)
slope = get_filtered_slope(src_dem_clip_ds_align)
diff_align_filt = np.ma.array(diff_align_filt, mask=np.ma.getmaskarray(slope))
diff_align_filt_stats = malib.get_stats_dict(diff_align_filt, full=True)
#Fit 2D polynomial to residuals and remove
#To do: add support for along-track and cross-track artifacts
if tiltcorr and not tiltcorr_done:
print("\n************")
print("Calculating 'tiltcorr' 2D polynomial fit to residuals with order %i" % polyorder)
print("************\n")
gt = src_dem_clip_ds_align.GetGeoTransform()
#Need to apply the mask here, so we're only fitting over static surfaces
#Note that the origmask=False will compute vals for all x and y indices, which is what we want
vals, resid, coeff = geolib.ma_fitpoly(diff_align_filt, order=polyorder, gt=gt, perc=(0,100), origmask=False)
#vals, resid, coeff = geolib.ma_fitplane(diff_align_filt, gt, perc=(12.5, 87.5), origmask=False)
#Should write out coeff or grid with correction
vals_stats = malib.get_stats_dict(vals)
#Want to have max_tilt check here
#max_tilt = 4.0 #m
#Should do percentage
#vals.ptp() > max_tilt
#Note: dimensions of ds and vals will be different as vals are computed for clipped intersection
#Need to recompute planar offset for full src_dem_ds_align extent and apply
xgrid, ygrid = geolib.get_xy_grids(src_dem_ds_align)
valgrid = geolib.polyval2d(xgrid, ygrid, coeff)
#For results of ma_fitplane
#valgrid = coeff[0]*xgrid + coeff[1]*ygrid + coeff[2]
src_dem_ds_align = coreglib.apply_z_shift(src_dem_ds_align, -valgrid, createcopy=False)
if True:
print("Creating plot of polynomial fit to residuals")
fig, axa = plt.subplots(1,2, figsize=(8, 4))
dz_clim = malib.calcperc_sym(vals, (2, 98))
ax = pltlib.iv(diff_align_filt, ax=axa[0], cmap='RdBu', clim=dz_clim, \
label='Residual dz (m)', scalebar=False)
ax = pltlib.iv(valgrid, ax=axa[1], cmap='RdBu', clim=dz_clim, \
label='Polyfit dz (m)', ds=src_dem_ds_align)
#if tiltcorr:
#xyz_shift_str_cum_fn += "_tiltcorr"
tiltcorr_fig_fn = outprefix + '%s_polyfit.png' % xyz_shift_str_cum_fn
print("Writing out figure: %s\n" % tiltcorr_fig_fn)
fig.savefig(tiltcorr_fig_fn, dpi=300)
print("Applying tilt correction to difference map")
diff_align -= vals
#Should iterate until tilts are below some threshold
#For now, only do one tiltcorr
tiltcorr_done=True
#Now use original tolerance, and number of iterations
tol = args.tol
max_iter = n + args.max_iter
else:
break
if True:
#Write out aligned difference map for clipped extent with vertial offset removed
align_diff_fn = outprefix + '%s_align_diff.tif' % xyz_shift_str_cum_fn
print("Writing out aligned difference map with median vertical offset removed")
iolib.writeGTiff(diff_align, align_diff_fn, src_dem_clip_ds_align)
if True:
#Write out fitered aligned difference map
align_diff_filt_fn = outprefix + '%s_align_diff_filt.tif' % xyz_shift_str_cum_fn
print("Writing out filtered aligned difference map with median vertical offset removed")
iolib.writeGTiff(diff_align_filt, align_diff_filt_fn, src_dem_clip_ds_align)
#Extract final center coordinates for intersection
center_coord_ll = geolib.get_center(src_dem_clip_ds_align, t_srs=geolib.wgs_srs)
center_coord_xy = geolib.get_center(src_dem_clip_ds_align)
src_dem_clip_ds_align = None
#Write out final aligned src_dem
align_fn = outprefix + '%s_align.tif' % xyz_shift_str_cum_fn
print("Writing out shifted src_dem with median vertical offset removed: %s" % align_fn)
#Open original uncorrected dataset at native resolution
src_dem_ds = gdal.Open(src_dem_fn)
src_dem_ds_align = iolib.mem_drv.CreateCopy('', src_dem_ds, 0)
#Apply final horizontal and vertial shift to the original dataset
#Note: potentially issues if we used a different projection during coregistration!
src_dem_ds_align = coreglib.apply_xy_shift(src_dem_ds_align, dx_total, dy_total, createcopy=False)
src_dem_ds_align = coreglib.apply_z_shift(src_dem_ds_align, dz_total, createcopy=False)
if tiltcorr:
xgrid, ygrid = geolib.get_xy_grids(src_dem_ds_align)
valgrid = geolib.polyval2d(xgrid, ygrid, coeff)
#For results of ma_fitplane
#valgrid = coeff[0]*xgrid + coeff[1]*ygrid + coeff[2]
src_dem_ds_align = coreglib.apply_z_shift(src_dem_ds_align, -valgrid, createcopy=False)
#Might be cleaner way to write out MEM ds directly to disk
src_dem_full_align = iolib.ds_getma(src_dem_ds_align)
iolib.writeGTiff(src_dem_full_align, align_fn, src_dem_ds_align)
if True:
#Output final aligned src_dem, masked so only best pixels are preserved
#Useful if creating a new reference product
#Can also use apply_mask.py
print("Applying filter to shiftec src_dem")
align_diff_filt_full_ds = warplib.memwarp_multi_fn([align_diff_filt_fn,], res=src_dem_ds_align, extent=src_dem_ds_align, \
t_srs=src_dem_ds_align)[0]
align_diff_filt_full = iolib.ds_getma(align_diff_filt_full_ds)
align_diff_filt_full_ds = None
align_fn_masked = outprefix + '%s_align_filt.tif' % xyz_shift_str_cum_fn
iolib.writeGTiff(np.ma.array(src_dem_full_align, mask=np.ma.getmaskarray(align_diff_filt_full)), \
align_fn_masked, src_dem_ds_align)
src_dem_full_align = None
src_dem_ds_align = None
#Compute original elevation difference
if True:
ref_dem_clip_ds, src_dem_clip_ds = warplib.memwarp_multi([ref_dem_ds, src_dem_ds], \
res=res, extent='intersection', t_srs=local_srs, r='cubic')
src_dem_ds = None
ref_dem_ds = None
ref_dem_orig = iolib.ds_getma(ref_dem_clip_ds)
src_dem_orig = iolib.ds_getma(src_dem_clip_ds)
#Needed for plotting
ref_dem_hs = geolib.gdaldem_mem_ds(ref_dem_clip_ds, processing='hillshade', returnma=True, computeEdges=True)
src_dem_hs = geolib.gdaldem_mem_ds(src_dem_clip_ds, processing='hillshade', returnma=True, computeEdges=True)
diff_orig = src_dem_orig - ref_dem_orig
#Only compute stats over valid surfaces
static_mask_orig = get_mask(src_dem_clip_ds, mask_list, src_dem_fn)
#Note: this doesn't include outlier removal or slope mask!
static_mask_orig = np.logical_or(np.ma.getmaskarray(diff_orig), static_mask_orig)
#For some reason, ASTER DEM diff have a spike near the 0 bin, could be an issue with masking?
diff_orig_compressed = diff_orig[~static_mask_orig]
diff_orig_stats = malib.get_stats_dict(diff_orig_compressed, full=True)
#Prepare filtered version for comparison
diff_orig_filt = np.ma.array(diff_orig, mask=static_mask_orig)
diff_orig_filt = outlier_filter(diff_orig_filt, f=3, max_dz=max_dz)
#diff_orig_filt = outlier_filter(diff_orig_filt, perc=(12.5, 87.5), max_dz=max_dz)
slope = get_filtered_slope(src_dem_clip_ds)
diff_orig_filt = np.ma.array(diff_orig_filt, mask=np.ma.getmaskarray(slope))
diff_orig_filt_stats = malib.get_stats_dict(diff_orig_filt, full=True)
#Write out original difference map
print("Writing out original difference map for common intersection before alignment")
orig_diff_fn = outprefix + '_orig_diff.tif'
iolib.writeGTiff(diff_orig, orig_diff_fn, ref_dem_clip_ds)
src_dem_clip_ds = None
ref_dem_clip_ds = None
if True:
align_stats_fn = outprefix + '%s_align_stats.json' % xyz_shift_str_cum_fn
align_stats = {}
align_stats['src_fn'] = src_dem_fn
align_stats['ref_fn'] = ref_dem_fn
align_stats['align_fn'] = align_fn
align_stats['res'] = {}
align_stats['res']['src'] = src_dem_res
align_stats['res']['ref'] = ref_dem_res
align_stats['res']['coreg'] = res
align_stats['center_coord'] = {'lon':center_coord_ll[0], 'lat':center_coord_ll[1], \
'x':center_coord_xy[0], 'y':center_coord_xy[1]}
align_stats['shift'] = {'dx':dx_total, 'dy':dy_total, 'dz':dz_total, 'dm':dm_total}
#This tiltcorr flag gets set to false, need better flag
if tiltcorr:
align_stats['tiltcorr'] = {}
align_stats['tiltcorr']['coeff'] = coeff.tolist()
align_stats['tiltcorr']['val_stats'] = vals_stats
align_stats['before'] = diff_orig_stats
align_stats['before_filt'] = diff_orig_filt_stats
align_stats['after'] = diff_align_stats
align_stats['after_filt'] = diff_align_filt_stats
import json
with open(align_stats_fn, 'w') as f:
json.dump(align_stats, f)
#Create output plot
if True:
print("Creating final plot")
kwargs = {'interpolation':'none'}
#f, axa = plt.subplots(2, 4, figsize=(11, 8.5))
f, axa = plt.subplots(2, 4, figsize=(16, 8))
for ax in axa.ravel()[:-1]:
ax.set_facecolor('k')
pltlib.hide_ticks(ax)
dem_clim = malib.calcperc(ref_dem_orig, (2,98))
axa[0,0].imshow(ref_dem_hs, cmap='gray', **kwargs)
im = axa[0,0].imshow(ref_dem_orig, cmap='cpt_rainbow', clim=dem_clim, alpha=0.6, **kwargs)
pltlib.add_cbar(axa[0,0], im, arr=ref_dem_orig, clim=dem_clim, label=None)
pltlib.add_scalebar(axa[0,0], res=res)
axa[0,0].set_title('Reference DEM')
axa[0,1].imshow(src_dem_hs, cmap='gray', **kwargs)
im = axa[0,1].imshow(src_dem_orig, cmap='cpt_rainbow', clim=dem_clim, alpha=0.6, **kwargs)
pltlib.add_cbar(axa[0,1], im, arr=src_dem_orig, clim=dem_clim, label=None)
axa[0,1].set_title('Source DEM')
#axa[0,2].imshow(~static_mask_orig, clim=(0,1), cmap='gray')
axa[0,2].imshow(~static_mask, clim=(0,1), cmap='gray', **kwargs)
axa[0,2].set_title('Surfaces for co-registration')
dz_clim = malib.calcperc_sym(diff_orig_compressed, (5, 95))
im = axa[1,0].imshow(diff_orig, cmap='RdBu', clim=dz_clim)
pltlib.add_cbar(axa[1,0], im, arr=diff_orig, clim=dz_clim, label=None)
axa[1,0].set_title('Elev. Diff. Before (m)')
im = axa[1,1].imshow(diff_align, cmap='RdBu', clim=dz_clim)
pltlib.add_cbar(axa[1,1], im, arr=diff_align, clim=dz_clim, label=None)
axa[1,1].set_title('Elev. Diff. After (m)')
#tight_dz_clim = (-1.0, 1.0)
tight_dz_clim = (-2.0, 2.0)
#tight_dz_clim = (-10.0, 10.0)
#tight_dz_clim = malib.calcperc_sym(diff_align_filt, (5, 95))
im = axa[1,2].imshow(diff_align_filt, cmap='RdBu', clim=tight_dz_clim)
pltlib.add_cbar(axa[1,2], im, arr=diff_align_filt, clim=tight_dz_clim, label=None)
axa[1,2].set_title('Elev. Diff. After (m)')
#Tried to insert Nuth fig here
#ax_nuth.change_geometry(1,2,1)
#f.axes.append(ax_nuth)
bins = np.linspace(dz_clim[0], dz_clim[1], 128)
axa[1,3].hist(diff_orig_compressed, bins, color='g', label='Before', alpha=0.5)
axa[1,3].hist(diff_align_compressed, bins, color='b', label='After', alpha=0.5)
axa[1,3].set_xlim(*dz_clim)
axa[1,3].axvline(0, color='k', linewidth=0.5, linestyle=':')
axa[1,3].set_xlabel('Elev. Diff. (m)')
axa[1,3].set_ylabel('Count (px)')
axa[1,3].set_title("Source - Reference")
before_str = 'Before\nmed: %0.2f\nnmad: %0.2f' % (diff_orig_stats['med'], diff_orig_stats['nmad'])
axa[1,3].text(0.05, 0.95, before_str, va='top', color='g', transform=axa[1,3].transAxes, fontsize=8)
after_str = 'After\nmed: %0.2f\nnmad: %0.2f' % (diff_align_stats['med'], diff_align_stats['nmad'])
axa[1,3].text(0.65, 0.95, after_str, va='top', color='b', transform=axa[1,3].transAxes, fontsize=8)
#This is empty
axa[0,3].axis('off')
suptitle = '%s\nx: %+0.2fm, y: %+0.2fm, z: %+0.2fm' % (os.path.split(outprefix)[-1], dx_total, dy_total, dz_total)
f.suptitle(suptitle)
f.tight_layout()
plt.subplots_adjust(top=0.90)
fig_fn = outprefix + '%s_align.png' % xyz_shift_str_cum_fn
print("Writing out figure: %s" % fig_fn)
f.savefig(fig_fn, dpi=300)
def main():
parser = getparser()
args = parser.parse_args()
refdem = args.refdem
srcdem = args.srcdem
outfolder = '{}__{}_comparison_stats'.format(
os.path.splitext(os.path.basename(refdem))[0],
os.path.splitext(os.path.basename(srcdem))[0])
header_str = '{}__{}'.format(
os.path.splitext(os.path.basename(refdem))[0],
os.path.splitext(os.path.basename(srcdem))[0])
if not os.path.exists(outfolder):
os.makedirs(outfolder)
if args.local_ortho == 1:
temp_ds = warplib.memwarp_multi_fn([refdem, srcdem])[0]
bbox = geolib.ds_extent(temp_ds)
geo_crs = temp_ds.GetProjection()
print('Bounding box lon_lat is{}'.format(bbox))
bound_poly = Polygon([[bbox[0], bbox[3]], [bbox[2], bbox[3]],
[bbox[2], bbox[1]], [bbox[0], bbox[1]]])
bound_shp = gpd.GeoDataFrame(index=[0],
geometry=[bound_poly],
crs=geo_crs)
bound_centroid = bound_shp.centroid
cx = bound_centroid.x.values[0]
cy = bound_centroid.y.values[0]
pad = np.ptp([bbox[3], bbox[1]]) / 6.0
lat_1 = bbox[1] + pad
lat_2 = bbox[3] - pad
local_ortho = "+proj=ortho +lat_1={} +lat_2={} +lat_0={} +lon_0={} +x_0=0 +y_0=0 +ellps=WGS84 +datum=WGS84 +units=m +no_defs".format(
lat_1, lat_2, cy, cx)
logging.info('Local Ortho projection is {}'.format(local_ortho))
t_srs = local_ortho
else:
t_srs = 'first'
# this step performs the desired warping operation
ds_list = warplib.memwarp_multi_fn([refdem, srcdem],
res=args.comparison_res,
t_srs=t_srs)
refma = iolib.ds_getma(ds_list[0])
srcma = iolib.ds_getma(ds_list[1])
init_diff = refma - srcma
init_stats = malib.get_stats_dict(init_diff)
print("Original descriptive statistics {}".format(init_stats))
init_diff_json_fn = os.path.join(
outfolder, '{}_precoreg_descriptive_stats.json'.format(header_str))
init_diff_json = json.dumps(init_stats)
with open(init_diff_json_fn, 'w') as f:
f.write(init_diff_json)
logging.info("Saved initial stats at {}".format(init_diff_json))
refslope = gdaldem(ds_list[0])
# stats for elevation difference vs reference DEM elevation
elev_bin, diff_mean, diff_median, diff_std, diff_perc = cummulative_profile(
refma, init_diff, args.elev_bin_width)
# stats for elevation difference vs reference DEM slope
slope_bin, diff_mean_s, diff_median_s, diff_std_s, diff_perc_s = cummulative_profile(
refslope, init_diff, args.slope_bin_width)
f, ax = plt.subplots(1, 2, figsize=(10, 4))
im = ax[0].scatter(elev_bin, diff_mean, c=diff_perc, cmap='inferno')
ax[0].set_xlabel('Elevation (m)')
divider = make_axes_locatable(ax[0])
cax = divider.append_axes('right', size='2.5%', pad=0.05)
plt.colorbar(im,
cax=cax,
orientation='vertical',
label='pixel count percentage')
im2 = ax[1].scatter(slope_bin, diff_mean_s, c=diff_perc_s, cmap='inferno')
ax[1].set_xlabel('Slope (degrees)')
divider = make_axes_locatable(ax[1])
cax = divider.append_axes('right', size='2.5%', pad=0.05)
plt.colorbar(im2,
cax=cax,
orientation='vertical',
label='pixel count percentage')
for axa in ax.ravel():
axa.axhline(y=0, c='k')
axa.set_ylabel('Elevation Difference (m)')
plt.tight_layout()
precoreg_plot = os.path.join(outfolder,
header_str + '_precoreg_binned_plot.png')
f.savefig(precoreg_plot, dpi=300, bbox_inches='tight', pad_inches=0.1)
logging.info("Saved binned plot at {}".format(precoreg_plot))
if args.coreg == 1:
logging.info("will attempt coregisteration")
if args.local_ortho == 1:
ref_local_ortho = os.path.splitext(refdem)[0] + '_local_ortho.tif'
src_local_ortho = os.path.splitext(srcdem)[0] + '_local_ortho.tif'
# coregisteration works best at mean resolution
# we will rewarp if the initial args.res was not mean
if args.comparison_res != 'mean':
ds_list = warplib.memwarp_multi_fn([refdem, srcdem],
res='mean',
t_srs=t_srs)
refma = iolib.ds_getma(ds_list[0])
srcma = iolib.ds_getma(ds_list[1])
iolib.writeGTiff(refma, ref_local_ortho, ds_list[0])
iolib.writeGTiff(srcma, src_local_ortho, ds_list[1])
coreg_ref = ref_local_ortho
src_ref = src_local_ortho
else:
coreg_ref = refdem
src_ref = srcdem
demcoreg_dir = os.path.join(outfolder, 'coreg_results')
align_opts = [
'-mode', 'nuth', '-max_iter', '12', '-max_offset', '400',
'-outdir', demcoreg_dir
]
align_args = [coreg_ref, src_ref]
align_cmd = ['dem_align.py'] + align_opts + align_args
subprocess.call(align_cmd)
#ah final round of warping and stats calculation
try:
srcdem_align = glob.glob(os.path.join(demcoreg_dir,
'*align.tif'))[0]
logging.info(
"Attempting stats calculation for aligned DEM {}".format(
srcdem_align))
ds_list = warplib.memwarp_multi_fn([args.refdem, srcdem_align],
res=args.comparison_res,
t_srs=t_srs)
refma = iolib.ds_getma(ds_list[0])
srcma = iolib.ds_getma(ds_list[1])
# this is creepy, but I am recycling variable names to save on memory
init_diff = refma - srcma
init_stats = malib.get_stats_dict(init_diff)
print("Final descriptive statistics {}".format(init_stats))
init_diff_json_fn = os.path.join(
outfolder,
'{}_postcoreg_descriptive_stats.json'.format(header_str))
init_diff_json = json.dumps(init_stats)
with open(init_diff_json_fn, 'w') as f:
f.write(init_diff_json)
logging.info("Saved final stats at {}".format(init_diff_json))
refslope = gdaldem(ds_list[0])
# stats for elevation difference vs reference DEM elevation
elev_bin, diff_mean, diff_median, diff_std, diff_perc = cummulative_profile(
refma, init_diff, args.elev_bin_width)
# stats for elevation difference vs reference DEM slope
slope_bin, diff_mean_s, diff_median_s, diff_std_s, diff_perc_s = cummulative_profile(
refslope, init_diff, args.slope_bin_width)
f, ax = plt.subplots(1, 2, figsize=(10, 4))
im = ax[0].scatter(elev_bin,
diff_mean,
c=diff_perc,
cmap='inferno')
ax[0].set_xlabel('Elevation (m)')
divider = make_axes_locatable(ax[0])
cax = divider.append_axes('right', size='2.5%', pad=0.05)
plt.colorbar(im,
cax=cax,
orientation='vertical',
label='pixel count percentage')
im2 = ax[1].scatter(slope_bin,
diff_mean_s,
c=diff_perc_s,
cmap='inferno')
ax[1].set_xlabel('Slope (degrees)')
divider = make_axes_locatable(ax[1])
cax = divider.append_axes('right', size='2.5%', pad=0.05)
plt.colorbar(im2,
cax=cax,
orientation='vertical',
label='pixel count percentage')
for axa in ax.ravel():
axa.axhline(y=0, c='k')
axa.set_ylabel('Elevation Difference (m)')
plt.tight_layout()
precoreg_plot = os.path.join(
outfolder, header_str + '_postcoreg_binned_plot.png')
f.savefig(precoreg_plot,
dpi=300,
bbox_inches='tight',
pad_inches=0.1)
except:
logging.info(
"Failed to compute post coreg stats, see corresponding job log"
)
logging.info("Script is complete !")
def main():
parser = getparser()
args = parser.parse_args()
src_fn = args.src_fn
if not iolib.fn_check(src_fn):
sys.exit("Unable to find src_fn: %s" % src_fn)
mask_fn = args.mask_fn
if not iolib.fn_check(mask_fn):
sys.exit("Unable to find mask_fn: %s" % mask_fn)
#Determine output extent, default is input raster extent
extent = args.extent
if extent == 'raster':
extent = src_fn
elif extent == 'mask':
extent = mask_fn
else:
#This is a hack for intersection computation
src_ds_list = [
gdal.Open(fn, gdal.GA_ReadOnly) for fn in [src_fn, mask_fn]
]
#t_srs = geolib.get_ds_srs(src_ds_list[0])
extent = warplib.parse_extent(extent, src_ds_list, src_fn)
print("Warping mask_fn")
mask_ds = warplib.memwarp_multi_fn([
mask_fn,
],
res=src_fn,
extent=extent,
t_srs=src_fn)[0]
print("Loading mask array")
mask_ma_full = iolib.ds_getma(mask_ds)
mask_ds = None
print("Extracting mask")
mask = np.ma.getmaskarray(mask_ma_full)
#Free up memory
mask_ma_full = None
#Add dilation step for buffer
#newmask = np.logical_or(np.ma.getmaskarray(src_ma_full), mask)
if args.invert:
print("Inverting mask")
mask = ~(mask)
print("Loading src array and applying updated mask")
if extent == src_fn:
src_ds = gdal.Open(src_fn)
else:
src_ds = warplib.memwarp_multi_fn([
src_fn,
],
res=src_fn,
extent=extent,
t_srs=src_fn)[0]
#Now load source array with new mask
src_ma_full = np.ma.array(iolib.ds_getma(src_ds), mask=mask)
mask = None
if args.out_fn is not None:
src_fn_masked = args.out_fn
else:
src_fn_masked = os.path.splitext(src_fn)[0] + '_masked.tif'
print("Writing out masked version of input raster: %s" % src_fn_masked)
iolib.writeGTiff(src_ma_full, src_fn_masked, src_ds, create=True)
print('Mean mb: %0.2f mwe/yr' % mb_mean)
print('Sum/Area mb: %0.2f mwe/yr' % (mb_sum / glac_area))
print('Mean mb * Area: %0.2f mwe/yr' % dmbdt_total_myr)
print('Sum mb: %0.2f mwe/yr' % mb_sum)
print('-------------------------------')
#Write to master list
out.append(outlist)
#Write to temporary file
writer.writerow(outlist)
f.flush()
min_glac_area_writeout = 2.0
if writeout and (glac_area / 1E6 > min_glac_area_writeout):
out_dz_fn = os.path.join(outdir, feat_fn + '_dz.tif')
iolib.writeGTiff(dz, out_dz_fn, ds_list[0])
out_z1_fn = os.path.join(outdir, feat_fn + '_z1.tif')
iolib.writeGTiff(z1, out_z1_fn, ds_list[0])
out_z2_fn = os.path.join(outdir, feat_fn + '_z2.tif')
iolib.writeGTiff(z2, out_z2_fn, ds_list[0])
if site == 'conus':
out_z1_date_fn = os.path.join(outdir, feat_fn + '_ned_date.tif')
iolib.writeGTiff(z1_date, out_z1_date_fn, ds_list[0])
out_prism_ppt_annual_fn = os.path.join(
outdir, feat_fn + '_precip_annual.tif')
iolib.writeGTiff(prism_ppt_annual, out_prism_ppt_annual_fn,
ds_list[0])
#dem1_tide_fn = dem1_fn_base+'_tidecorr.tif'
#dem2_tide_fn = dem2_fn_base+'_tidecorr.tif'
dem1_tide_fn = findfile(dem1_fn_base, 'tidecorr.tif')
if not dem1_tide_fn:
dem1_tide_fn = findfile(dem1_fn_base, 'tidemodel_smooth_full_clip.tif')
dem2_tide_fn = findfile(dem2_fn_base, 'tidecorr.tif')
if not dem2_tide_fn:
dem2_tide_fn = findfile(dem2_fn_base, 'tidemodel_smooth_full_clip.tif')
if dem1_tide_fn and dem2_tide_fn:
dem1_tide_ds, dem2_tide_ds = warplib.memwarp_multi_fn(
[dem1_tide_fn, dem2_tide_fn], extent=dem1_ds, res=dem1_ds)
dem1_tide = iolib.ds_getma(dem1_tide_ds)
dem2_tide = iolib.ds_getma(dem2_tide_ds)
tide_diff = dem2_tide - dem1_tide
dst_fn = os.path.join(outdir, outprefix + '_tide_diff.tif')
iolib.writeGTiff(tide_diff, dst_fn, dem1_ds, ndv=diffndv)
#These values are tide prediction, to remove, want to subtract from observed elevation
#Need to fill with 0 to prevent clipping to floating ice
dem1 -= dem1_tide.filled(0)
dem2 -= dem2_tide.filled(0)
firnair = True
#This is constant value for PIG
dem1_firnair = 15.0
dem2_firnair = dem1_firnair
firnair_diff = dem2_firnair - dem1_firnair
if firnair:
#dem1_firnair_fn = dem1_fn_base+'_racmo_FirnAir.tif'
#dem2_firnair_fn = dem2_fn_base+'_racmo_FirnAir.tif'
dem1_firnair_fn = findfile(dem1_fn_base, 'racmo_FirnAir.tif')
dem2_firnair_fn = findfile(dem2_fn_base, 'racmo_FirnAir.tif')
def get_mask(dem_ds,
mask_list,
dem_fn=None,
writeout=False,
outdir=None,
args=None):
mask_list = check_mask_list(mask_list)
if not mask_list or 'none' in mask_list:
newmask = False
else:
#Basename for output files
if outdir is not None:
if not os.path.exists(outdir):
os.makedirs(outdir)
else:
outdir = os.path.split(os.path.realpath(dem_fn))[0]
if dem_fn is not None:
#Extract DEM timestamp
dem_dt = timelib.fn_getdatetime(dem_fn)
out_fn_base = os.path.join(
outdir,
os.path.splitext(os.path.split(dem_fn)[-1])[0])
if args is None:
#Get default values
parser = getparser()
args = parser.parse_args([
'',
])
newmask = True
if 'glaciers' in mask_list:
icemask = get_icemask(dem_ds)
if writeout:
out_fn = out_fn_base + '_ice_mask.tif'
print("Writing out %s" % out_fn)
iolib.writeGTiff(icemask, out_fn, src_ds=dem_ds)
newmask = np.logical_and(icemask, newmask)
#Need to process NLCD separately, with nearest neighbor inteprolatin
if 'nlcd' in mask_list and args.nlcd_filter != 'none':
rs = 'near'
nlcd_ds = gdal.Open(get_nlcd_fn())
nlcd_ds_warp = warplib.memwarp_multi([
nlcd_ds,
],
res=dem_ds,
extent=dem_ds,
t_srs=dem_ds,
r=rs)[0]
out_fn = None
if writeout:
out_fn = out_fn_base + '_nlcd.tif'
nlcdmask = get_nlcd_mask(nlcd_ds_warp,
filter=args.nlcd_filter,
out_fn=out_fn)
if writeout:
out_fn = os.path.splitext(out_fn)[0] + '_mask.tif'
print("Writing out %s" % out_fn)
iolib.writeGTiff(nlcdmask, out_fn, src_ds=dem_ds)
newmask = np.logical_and(nlcdmask, newmask)
if 'bareground' in mask_list and args.bareground_thresh > 0:
bareground_ds = gdal.Open(get_bareground_fn())
bareground_ds_warp = warplib.memwarp_multi([
bareground_ds,
],
res=dem_ds,
extent=dem_ds,
t_srs=dem_ds,
r='cubicspline')[0]
out_fn = None
if writeout:
out_fn = out_fn_base + '_bareground.tif'
baregroundmask = get_bareground_mask(
bareground_ds_warp,
bareground_thresh=args.bareground_thresh,
out_fn=out_fn)
if writeout:
out_fn = os.path.splitext(out_fn)[0] + '_mask.tif'
print("Writing out %s" % out_fn)
iolib.writeGTiff(baregroundmask, out_fn, src_ds=dem_ds)
newmask = np.logical_and(baregroundmask, newmask)
if 'snodas' in mask_list and args.snodas_thresh > 0:
#Get SNODAS snow depth products for DEM timestamp
snodas_min_dt = datetime(2003, 9, 30)
if dem_dt >= snodas_min_dt:
snodas_ds = get_snodas_ds(dem_dt)
if snodas_ds is not None:
snodas_ds_warp = warplib.memwarp_multi([
snodas_ds,
],
res=dem_ds,
extent=dem_ds,
t_srs=dem_ds,
r='cubicspline')[0]
#snow depth values are mm, convert to meters
snodas_depth = iolib.ds_getma(snodas_ds_warp) / 1000.
if snodas_depth.count() > 0:
print(
"Applying SNODAS snow depth filter (masking values >= %0.2f m)"
% args.snodas_thresh)
out_fn = None
if writeout:
out_fn = out_fn_base + '_snodas_depth.tif'
print("Writing out %s" % out_fn)
iolib.writeGTiff(snodas_depth,
out_fn,
src_ds=dem_ds)
snodas_mask = np.ma.masked_greater(
snodas_depth, args.snodas_thresh)
snodas_mask = ~(np.ma.getmaskarray(snodas_mask))
if writeout:
out_fn = os.path.splitext(out_fn)[0] + '_mask.tif'
print("Writing out %s" % out_fn)
iolib.writeGTiff(snodas_mask,
out_fn,
src_ds=dem_ds)
newmask = np.logical_and(snodas_mask, newmask)
else:
print(
"SNODAS grid for input location and timestamp is empty"
)
#These tiles cover CONUS
#tile_list=('h08v04', 'h09v04', 'h10v04', 'h08v05', 'h09v05')
if 'modscag' in mask_list and args.modscag_thresh > 0:
modscag_min_dt = datetime(2000, 2, 24)
if dem_dt < modscag_min_dt:
print("Warning: DEM timestamp (%s) is before earliest MODSCAG timestamp (%s)" \
% (dem_dt, modscag_min_dt))
else:
tile_list = get_modis_tile_list(dem_ds)
print(tile_list)
pad_days = 7
modscag_fn_list = get_modscag_fn_list(dem_dt,
tile_list=tile_list,
pad_days=pad_days)
if modscag_fn_list:
modscag_ds = proc_modscag(modscag_fn_list,
extent=dem_ds,
t_srs=dem_ds)
modscag_ds_warp = warplib.memwarp_multi([
modscag_ds,
],
res=dem_ds,
extent=dem_ds,
t_srs=dem_ds,
r='cubicspline')[0]
print(
"Applying MODSCAG fractional snow cover percent filter (masking values >= %0.1f%%)"
% args.modscag_thresh)
modscag_fsca = iolib.ds_getma(modscag_ds_warp)
out_fn = None
if writeout:
out_fn = out_fn_base + '_modscag_fsca.tif'
print("Writing out %s" % out_fn)
iolib.writeGTiff(modscag_fsca, out_fn, src_ds=dem_ds)
modscag_mask = (modscag_fsca.filled(0) >=
args.modscag_thresh)
modscag_mask = ~(modscag_mask)
if writeout:
out_fn = os.path.splitext(out_fn)[0] + '_mask.tif'
print("Writing out %s" % out_fn)
iolib.writeGTiff(modscag_mask, out_fn, src_ds=dem_ds)
newmask = np.logical_and(modscag_mask, newmask)
#Use reflectance values to estimate snowcover
if 'toa' in mask_list:
#Use top of atmosphere scaled reflectance values (0-1)
toa_ds = gdal.Open(get_toa_fn(dem_fn))
toa_ds_warp = warplib.memwarp_multi([
toa_ds,
],
res=dem_ds,
extent=dem_ds,
t_srs=dem_ds)[0]
toa_mask = get_toa_mask(toa_ds_warp, args.toa_thresh)
if writeout:
out_fn = out_fn_base + '_toa_mask.tif'
print("Writing out %s" % out_fn)
iolib.writeGTiff(toa_mask, out_fn, src_ds=dem_ds)
newmask = np.logical_and(toa_mask, newmask)
if False:
#Filter based on expected snowline
#Simplest approach uses altitude cutoff
max_elev = 1500
newdem = np.ma.masked_greater(dem, max_elev)
newmask = np.ma.getmaskarray(newdem)
print(
"Generating final mask to use for reference surfaces, and applying to input DEM"
)
#Now invert to use to create final masked array
#True (1) represents "invalid" pixel to match numpy ma convetion
newmask = ~newmask
#Dilate the mask
if args.dilate is not None:
niter = args.dilate
print("Dilating mask with %i iterations" % niter)
from scipy import ndimage
newmask = ~(ndimage.morphology.binary_dilation(~newmask,
iterations=niter))
return newmask
def compute_offset(ref_dem_ds, src_dem_ds, src_dem_fn, mode='nuth', remove_outliers=True, max_offset=100, \
max_dz=100, slope_lim=(0.1, 40), mask_list=['glaciers',], plot=True):
#Make sure the input datasets have the same resolution/extent
#Use projection of source DEM
ref_dem_clip_ds, src_dem_clip_ds = warplib.memwarp_multi([ref_dem_ds, src_dem_ds], \
res='max', extent='intersection', t_srs=src_dem_ds, r='cubic')
#Compute size of NCC and SAD search window in pixels
res = float(geolib.get_res(ref_dem_clip_ds, square=True)[0])
max_offset_px = (max_offset/res) + 1
#print(max_offset_px)
pad = (int(max_offset_px), int(max_offset_px))
#This will be updated geotransform for src_dem
src_dem_gt = np.array(src_dem_clip_ds.GetGeoTransform())
#Load the arrays
ref_dem = iolib.ds_getma(ref_dem_clip_ds, 1)
src_dem = iolib.ds_getma(src_dem_clip_ds, 1)
print("Elevation difference stats for uncorrected input DEMs (src - ref)")
diff = src_dem - ref_dem
static_mask = get_mask(src_dem_clip_ds, mask_list, src_dem_fn)
diff = np.ma.array(diff, mask=static_mask)
if diff.count() == 0:
sys.exit("No overlapping, unmasked pixels shared between input DEMs")
if remove_outliers:
diff = outlier_filter(diff, f=3, max_dz=max_dz)
#Want to use higher quality DEM, should determine automatically from original res/count
#slope = get_filtered_slope(ref_dem_clip_ds, slope_lim=slope_lim)
slope = get_filtered_slope(src_dem_clip_ds, slope_lim=slope_lim)
print("Computing aspect")
#aspect = geolib.gdaldem_mem_ds(ref_dem_clip_ds, processing='aspect', returnma=True, computeEdges=False)
aspect = geolib.gdaldem_mem_ds(src_dem_clip_ds, processing='aspect', returnma=True, computeEdges=False)
ref_dem_clip_ds = None
src_dem_clip_ds = None
#Apply slope filter to diff
#Note that we combine masks from diff and slope in coreglib
diff = np.ma.array(diff, mask=np.ma.getmaskarray(slope))
#Get final mask after filtering
static_mask = np.ma.getmaskarray(diff)
#Compute stats for new masked difference map
print("Filtered difference map")
diff_stats = malib.print_stats(diff)
dz = diff_stats[5]
print("Computing sub-pixel offset between DEMs using mode: %s" % mode)
#By default, don't create output figure
fig = None
#Default horizntal shift is (0,0)
dx = 0
dy = 0
#Sum of absolute differences
if mode == "sad":
ref_dem = np.ma.array(ref_dem, mask=static_mask)
src_dem = np.ma.array(src_dem, mask=static_mask)
m, int_offset, sp_offset = coreglib.compute_offset_sad(ref_dem, src_dem, pad=pad)
#Geotransform has negative y resolution, so don't need negative sign
#np array is positive down
#GDAL coordinates are positive up
dx = sp_offset[1]*src_dem_gt[1]
dy = sp_offset[0]*src_dem_gt[5]
#Normalized cross-correlation of clipped, overlapping areas
elif mode == "ncc":
ref_dem = np.ma.array(ref_dem, mask=static_mask)
src_dem = np.ma.array(src_dem, mask=static_mask)
m, int_offset, sp_offset, fig = coreglib.compute_offset_ncc(ref_dem, src_dem, \
pad=pad, prefilter=False, plot=plot)
dx = sp_offset[1]*src_dem_gt[1]
dy = sp_offset[0]*src_dem_gt[5]
#Nuth and Kaab (2011)
elif mode == "nuth":
#Compute relationship between elevation difference, slope and aspect
fit_param, fig = coreglib.compute_offset_nuth(diff, slope, aspect, plot=plot)
if fit_param is None:
print("Failed to calculate horizontal shift")
else:
#fit_param[0] is magnitude of shift vector
#fit_param[1] is direction of shift vector
#fit_param[2] is mean bias divided by tangent of mean slope
#print(fit_param)
dx = fit_param[0]*np.sin(np.deg2rad(fit_param[1]))
dy = fit_param[0]*np.cos(np.deg2rad(fit_param[1]))
med_slope = malib.fast_median(slope)
nuth_dz = fit_param[2]*np.tan(np.deg2rad(med_slope))
print('Median dz: %0.2f\nNuth dz: %0.2f' % (dz, nuth_dz))
#dz = nuth_dz
elif mode == "all":
print("Not yet implemented")
#Want to compare all methods, average offsets
#m, int_offset, sp_offset = coreglib.compute_offset_sad(ref_dem, src_dem)
#m, int_offset, sp_offset = coreglib.compute_offset_ncc(ref_dem, src_dem)
elif mode == "none":
print("Skipping alignment, writing out DEM with median bias over static surfaces removed")
dst_fn = outprefix+'_med%0.1f.tif' % dz
iolib.writeGTiff(src_dem_orig + dz, dst_fn, src_dem_ds)
sys.exit()
#Note: minus signs here since we are computing dz=(src-ref), but adjusting src
return -dx, -dy, -dz, static_mask, fig
