def proc_modscag(fn_list, extent=None, t_srs=None):
"""Process the MODSCAG products for full date range, create composites and reproject
"""
#Use cubic spline here for improve upsampling
ds_list = warplib.memwarp_multi_fn(fn_list, res='min', extent=extent, t_srs=t_srs, r='cubicspline')
stack_fn = os.path.splitext(fn_list[0])[0] + '_' + os.path.splitext(os.path.split(fn_list[-1])[1])[0] + '_stack_%i' % len(fn_list)
#Create stack here - no need for most of mastack machinery, just make 3D array
#Mask values greater than 100% (clouds, bad pixels, etc)
ma_stack = np.ma.array([np.ma.masked_greater(iolib.ds_getma(ds), 100) for ds in np.array(ds_list)], dtype=np.uint8)
stack_count = np.ma.masked_equal(ma_stack.count(axis=0), 0).astype(np.uint8)
stack_count.set_fill_value(0)
stack_min = ma_stack.min(axis=0).astype(np.uint8)
stack_min.set_fill_value(0)
stack_max = ma_stack.max(axis=0).astype(np.uint8)
stack_max.set_fill_value(0)
stack_med = np.ma.median(ma_stack, axis=0).astype(np.uint8)
stack_med.set_fill_value(0)
out_fn = stack_fn + '_count.tif'
iolib.writeGTiff(stack_count, out_fn, ds_list[0])
out_fn = stack_fn + '_max.tif'
iolib.writeGTiff(stack_max, out_fn, ds_list[0])
out_fn = stack_fn + '_min.tif'
iolib.writeGTiff(stack_min, out_fn, ds_list[0])
out_fn = stack_fn + '_med.tif'
iolib.writeGTiff(stack_med, out_fn, ds_list[0])
ds = gdal.Open(out_fn)
return ds
def dz_fltr(dem_fn, refdem_fn, perc=None, abs_dz_lim=(0, 30), smooth=True):
"""Absolute elevation difference range filter using values from a source raster file and a reference raster file
"""
try:
open(refdem_fn)
except IOError:
sys.exit('Unable to open reference DEM: %s' % refdem_fn)
dem_ds, refdem_ds = warplib.memwarp_multi_fn([dem_fn, refdem_fn], res='first', extent='first', t_srs='first')
dem = iolib.ds_getma(dem_ds)
refdem = iolib.ds_getma(refdem_ds)
out = dz_fltr_ma(dem, refdem, perc, abs_dz_lim, smooth)
return out
def plot_composite_fig(ortho, dem, count, nmad, outfn, product='triplet'):
"""
Plot the gallery figure for final DEM products
Parameters
------------
ortho: str
path to orthoimage
dem: str
path to dem
count: str
path to count map
nmad: str
path to NMAD
outfn: str
path to save output figure
ortho: str
product to plot (triplet/video)
"""
if product == 'triplet':
figsize = (10, 8)
else:
figsize = (10, 3)
f, ax = plt.subplots(1, 4, figsize=figsize)
ds_list = warplib.memwarp_multi_fn([ortho, dem, count, nmad], res='max')
ortho, dem, count, nmad = [iolib.ds_getma(x) for x in ds_list]
pltlib.iv(ortho,
ax=ax[0],
cmap='gray',
scalebar=True,
cbar=False,
ds=ds_list[0],
skinny=False)
pltlib.iv(dem,
ax=ax[1],
hillshade=True,
scalebar=False,
ds=ds_list[1],
label='Elevation (m WGS84)',
skinny=False)
pltlib.iv(count, ax=ax[2], cmap='YlOrRd', label='DEM count', skinny=False)
pltlib.iv(nmad,
ax=ax[3],
cmap='inferno',
clim=(0, 10),
label='Elevation NMAD (m)',
skinny=False)
plt.tight_layout()
f.savefig(outfn, dpi=300, bbox_inches='tight', pad_inches=0.1)
def abs_range_fltr_lowresDEM(dem_fn, refdem_fn, pad=30):
try:
open(refdem_fn)
except IOError:
sys.exit('Unable to open reference DEM: %s' % refdem_fn)
dem_ds, refdem_ds = warplib.memwarp_multi_fn([dem_fn, refdem_fn], res='first', extent='first', t_srs='first')
dem = iolib.ds_getma(dem_ds)
refdem = iolib.ds_getma(refdem_ds)
rangelim = (refdem.min(), refdem.max())
rangelim = (rangelim[0] - pad, rangelim[1] + pad)
print('Excluding values outside of padded ({0:0.1f} m) lowres DEM range: {1:0.1f} to {2:0.1f} m'.format(pad, *rangelim))
out = range_fltr(dem, rangelim)
return out
def get_t_factor_fn(fn1, fn2, ds=None):
t_factor = None
#Extract timestamps from input filenames
t1 = fn_getdatetime(fn1)
t2 = fn_getdatetime(fn2)
t_factor = get_t_factor(t1,t2)
#Attempt to load timestamp arrays (for mosaics with variable timestamps)
t1_fn = os.path.splitext(fn1)[0]+'_ts.tif'
t2_fn = os.path.splitext(fn2)[0]+'_ts.tif'
if os.path.exists(t1_fn) and os.path.exists(t2_fn) and ds is not None:
print("Preparing timestamp arrays")
from pygeotools.lib import warplib
t1_ds, t2_ds = warplib.memwarp_multi_fn([t1_fn, t2_fn], extent=ds, res=ds)
print("Loading timestamps into masked arrays")
from pygeotools.lib import iolib
t1 = iolib.ds_getma(t1_ds)
t2 = iolib.ds_getma(t2_ds)
#This is a new masked array
t_factor = (t2 - t1) / 365.25
return t_factor
def get_mask(ds, dem_fn):
#Mask glaciers, vegetated slopes
static_mask = dem_mask.get_lulc_mask(ds,
mask_glaciers=True,
filter='not_forest',
bareground_thresh=60)
#Mask glaciers only
#static_mask = dem_mask.get_icemask(ds)
#Top-of-atmosphere reflectance threshold (requires orthoimage and output from toa.sh)
toa_fn = dem_mask.get_toa_fn(dem_fn)
if toa_fn is not None:
toa_ds = warplib.memwarp_multi_fn([
toa_fn,
],
res=ds,
extent=ds,
t_srs=ds,
r='cubicspline')[0]
toa_mask = dem_mask.get_toa_mask(toa_ds)
static_mask = np.logical_and(static_mask, toa_mask)
#Return final mask, ready to be applied
return ~(static_mask)
else:
feat_fn = str(glacnum)
print("\n%i of %i: %s\n" % (n + 1, feat_count, feat_fn))
glac_geom = feat.GetGeometryRef()
glac_geom.AssignSpatialReference(glac_shp_srs)
glac_geom_extent = geolib.geom_extent(glac_geom)
glac_area = glac_geom.GetArea()
if glac_area / 1E6 < min_glac_area:
print("Glacier area below %0.1f km2 threshold" % min_glac_area)
continue
cx, cy = glac_geom.Centroid().GetPoint_2D()
#Warp everything to common res/extent/proj
ds_list = warplib.memwarp_multi_fn([z1_fn, z2_fn], res='min', \
extent=glac_geom_extent, t_srs=aea_srs, verbose=False)
if site == 'conus':
#Add prism datasets
prism_fn_list = [prism_ppt_annual_fn, prism_tmean_annual_fn]
prism_fn_list.extend([
prism_ppt_summer_fn, prism_ppt_winter_fn, prism_tmean_summer_fn,
prism_tmean_winter_fn
])
ds_list.extend(
warplib.memwarp_multi_fn(prism_fn_list,
res=ds_list[0],
extent=glac_geom_extent,
t_srs=aea_srs,
verbose=False))
def main(argv=None):
parser = getparser()
args = parser.parse_args()
#Should check that files exist
ref_dem_fn = args.ref_fn
src_dem_fn = args.src_fn
mode = args.mode
mask_list = args.mask_list
max_offset = args.max_offset
max_dz = args.max_dz
slope_lim = tuple(args.slope_lim)
tiltcorr = args.tiltcorr
polyorder = args.polyorder
res = args.res
#Maximum number of iterations
max_iter = args.max_iter
#These are tolerances (in meters) to stop iteration
tol = args.tol
min_dx = tol
min_dy = tol
min_dz = tol
outdir = args.outdir
if outdir is None:
outdir = os.path.splitext(src_dem_fn)[0] + '_dem_align'
if tiltcorr:
outdir += '_tiltcorr'
tiltcorr_done = False
#Relax tolerance for initial round of co-registration
#tiltcorr_tol = 0.1
#if tol < tiltcorr_tol:
# tol = tiltcorr_tol
if not os.path.exists(outdir):
os.makedirs(outdir)
outprefix = '%s_%s' % (os.path.splitext(os.path.split(src_dem_fn)[-1])[0], \
os.path.splitext(os.path.split(ref_dem_fn)[-1])[0])
outprefix = os.path.join(outdir, outprefix)
print("\nReference: %s" % ref_dem_fn)
print("Source: %s" % src_dem_fn)
print("Mode: %s" % mode)
print("Output: %s\n" % outprefix)
src_dem_ds = gdal.Open(src_dem_fn)
ref_dem_ds = gdal.Open(ref_dem_fn)
#Get local cartesian coordinate system
#local_srs = geolib.localtmerc_ds(src_dem_ds)
#Use original source dataset coordinate system
#Potentially issues with distortion and xyz/tiltcorr offsets for DEM with large extent
local_srs = geolib.get_ds_srs(src_dem_ds)
#local_srs = geolib.get_ds_srs(ref_dem_ds)
#Resample to common grid
ref_dem_res = float(geolib.get_res(ref_dem_ds, t_srs=local_srs, square=True)[0])
#Create a copy to be updated in place
src_dem_ds_align = iolib.mem_drv.CreateCopy('', src_dem_ds, 0)
src_dem_res = float(geolib.get_res(src_dem_ds, t_srs=local_srs, square=True)[0])
src_dem_ds = None
#Resample to user-specified resolution
ref_dem_ds, src_dem_ds_align = warplib.memwarp_multi([ref_dem_ds, src_dem_ds_align], \
extent='intersection', res=args.res, t_srs=local_srs, r='cubic')
res = float(geolib.get_res(src_dem_ds_align, square=True)[0])
print("\nReference DEM res: %0.2f" % ref_dem_res)
print("Source DEM res: %0.2f" % src_dem_res)
print("Resolution for coreg: %s (%0.2f m)\n" % (args.res, res))
#Iteration number
n = 1
#Cumulative offsets
dx_total = 0
dy_total = 0
dz_total = 0
#Now iteratively update geotransform and vertical shift
while True:
print("*** Iteration %i ***" % n)
dx, dy, dz, static_mask, fig = compute_offset(ref_dem_ds, src_dem_ds_align, src_dem_fn, mode, max_offset, \
mask_list=mask_list, max_dz=max_dz, slope_lim=slope_lim, plot=True)
xyz_shift_str_iter = "dx=%+0.2fm, dy=%+0.2fm, dz=%+0.2fm" % (dx, dy, dz)
print("Incremental offset: %s" % xyz_shift_str_iter)
dx_total += dx
dy_total += dy
dz_total += dz
xyz_shift_str_cum = "dx=%+0.2fm, dy=%+0.2fm, dz=%+0.2fm" % (dx_total, dy_total, dz_total)
print("Cumulative offset: %s" % xyz_shift_str_cum)
#String to append to output filenames
xyz_shift_str_cum_fn = '_%s_x%+0.2f_y%+0.2f_z%+0.2f' % (mode, dx_total, dy_total, dz_total)
#Should make an animation of this converging
if n == 1:
#static_mask_orig = static_mask
if fig is not None:
dst_fn = outprefix + '_%s_iter%02i_plot.png' % (mode, n)
print("Writing offset plot: %s" % dst_fn)
fig.gca().set_title("Incremental: %s\nCumulative: %s" % (xyz_shift_str_iter, xyz_shift_str_cum))
fig.savefig(dst_fn, dpi=300)
#Apply the horizontal shift to the original dataset
src_dem_ds_align = coreglib.apply_xy_shift(src_dem_ds_align, dx, dy, createcopy=False)
#Should
src_dem_ds_align = coreglib.apply_z_shift(src_dem_ds_align, dz, createcopy=False)
n += 1
print("\n")
#If magnitude of shift in all directions is less than tol
#if n > max_iter or (abs(dx) <= min_dx and abs(dy) <= min_dy and abs(dz) <= min_dz):
#If magnitude of shift is less than tol
dm = np.sqrt(dx**2 + dy**2 + dz**2)
dm_total = np.sqrt(dx_total**2 + dy_total**2 + dz_total**2)
if dm_total > max_offset:
sys.exit("Total offset exceeded specified max_offset (%0.2f m). Consider increasing -max_offset argument" % max_offset)
#Stop iteration
if n > max_iter or dm < tol:
if fig is not None:
dst_fn = outprefix + '_%s_iter%02i_plot.png' % (mode, n)
print("Writing offset plot: %s" % dst_fn)
fig.gca().set_title("Incremental:%s\nCumulative:%s" % (xyz_shift_str_iter, xyz_shift_str_cum))
fig.savefig(dst_fn, dpi=300)
#Compute final elevation difference
if True:
ref_dem_clip_ds_align, src_dem_clip_ds_align = warplib.memwarp_multi([ref_dem_ds, src_dem_ds_align], \
res=res, extent='intersection', t_srs=local_srs, r='cubic')
ref_dem_align = iolib.ds_getma(ref_dem_clip_ds_align, 1)
src_dem_align = iolib.ds_getma(src_dem_clip_ds_align, 1)
ref_dem_clip_ds_align = None
diff_align = src_dem_align - ref_dem_align
src_dem_align = None
ref_dem_align = None
#Get updated, final mask
static_mask_final = get_mask(src_dem_clip_ds_align, mask_list, src_dem_fn)
static_mask_final = np.logical_or(np.ma.getmaskarray(diff_align), static_mask_final)
#Final stats, before outlier removal
diff_align_compressed = diff_align[~static_mask_final]
diff_align_stats = malib.get_stats_dict(diff_align_compressed, full=True)
#Prepare filtered version for tiltcorr fit
diff_align_filt = np.ma.array(diff_align, mask=static_mask_final)
diff_align_filt = outlier_filter(diff_align_filt, f=3, max_dz=max_dz)
#diff_align_filt = outlier_filter(diff_align_filt, perc=(12.5, 87.5), max_dz=max_dz)
slope = get_filtered_slope(src_dem_clip_ds_align)
diff_align_filt = np.ma.array(diff_align_filt, mask=np.ma.getmaskarray(slope))
diff_align_filt_stats = malib.get_stats_dict(diff_align_filt, full=True)
#Fit 2D polynomial to residuals and remove
#To do: add support for along-track and cross-track artifacts
if tiltcorr and not tiltcorr_done:
print("\n************")
print("Calculating 'tiltcorr' 2D polynomial fit to residuals with order %i" % polyorder)
print("************\n")
gt = src_dem_clip_ds_align.GetGeoTransform()
#Need to apply the mask here, so we're only fitting over static surfaces
#Note that the origmask=False will compute vals for all x and y indices, which is what we want
vals, resid, coeff = geolib.ma_fitpoly(diff_align_filt, order=polyorder, gt=gt, perc=(0,100), origmask=False)
#vals, resid, coeff = geolib.ma_fitplane(diff_align_filt, gt, perc=(12.5, 87.5), origmask=False)
#Should write out coeff or grid with correction
vals_stats = malib.get_stats_dict(vals)
#Want to have max_tilt check here
#max_tilt = 4.0 #m
#Should do percentage
#vals.ptp() > max_tilt
#Note: dimensions of ds and vals will be different as vals are computed for clipped intersection
#Need to recompute planar offset for full src_dem_ds_align extent and apply
xgrid, ygrid = geolib.get_xy_grids(src_dem_ds_align)
valgrid = geolib.polyval2d(xgrid, ygrid, coeff)
#For results of ma_fitplane
#valgrid = coeff[0]*xgrid + coeff[1]*ygrid + coeff[2]
src_dem_ds_align = coreglib.apply_z_shift(src_dem_ds_align, -valgrid, createcopy=False)
if True:
print("Creating plot of polynomial fit to residuals")
fig, axa = plt.subplots(1,2, figsize=(8, 4))
dz_clim = malib.calcperc_sym(vals, (2, 98))
ax = pltlib.iv(diff_align_filt, ax=axa[0], cmap='RdBu', clim=dz_clim, \
label='Residual dz (m)', scalebar=False)
ax = pltlib.iv(valgrid, ax=axa[1], cmap='RdBu', clim=dz_clim, \
label='Polyfit dz (m)', ds=src_dem_ds_align)
#if tiltcorr:
#xyz_shift_str_cum_fn += "_tiltcorr"
tiltcorr_fig_fn = outprefix + '%s_polyfit.png' % xyz_shift_str_cum_fn
print("Writing out figure: %s\n" % tiltcorr_fig_fn)
fig.savefig(tiltcorr_fig_fn, dpi=300)
print("Applying tilt correction to difference map")
diff_align -= vals
#Should iterate until tilts are below some threshold
#For now, only do one tiltcorr
tiltcorr_done=True
#Now use original tolerance, and number of iterations
tol = args.tol
max_iter = n + args.max_iter
else:
break
if True:
#Write out aligned difference map for clipped extent with vertial offset removed
align_diff_fn = outprefix + '%s_align_diff.tif' % xyz_shift_str_cum_fn
print("Writing out aligned difference map with median vertical offset removed")
iolib.writeGTiff(diff_align, align_diff_fn, src_dem_clip_ds_align)
if True:
#Write out fitered aligned difference map
align_diff_filt_fn = outprefix + '%s_align_diff_filt.tif' % xyz_shift_str_cum_fn
print("Writing out filtered aligned difference map with median vertical offset removed")
iolib.writeGTiff(diff_align_filt, align_diff_filt_fn, src_dem_clip_ds_align)
#Extract final center coordinates for intersection
center_coord_ll = geolib.get_center(src_dem_clip_ds_align, t_srs=geolib.wgs_srs)
center_coord_xy = geolib.get_center(src_dem_clip_ds_align)
src_dem_clip_ds_align = None
#Write out final aligned src_dem
align_fn = outprefix + '%s_align.tif' % xyz_shift_str_cum_fn
print("Writing out shifted src_dem with median vertical offset removed: %s" % align_fn)
#Open original uncorrected dataset at native resolution
src_dem_ds = gdal.Open(src_dem_fn)
src_dem_ds_align = iolib.mem_drv.CreateCopy('', src_dem_ds, 0)
#Apply final horizontal and vertial shift to the original dataset
#Note: potentially issues if we used a different projection during coregistration!
src_dem_ds_align = coreglib.apply_xy_shift(src_dem_ds_align, dx_total, dy_total, createcopy=False)
src_dem_ds_align = coreglib.apply_z_shift(src_dem_ds_align, dz_total, createcopy=False)
if tiltcorr:
xgrid, ygrid = geolib.get_xy_grids(src_dem_ds_align)
valgrid = geolib.polyval2d(xgrid, ygrid, coeff)
#For results of ma_fitplane
#valgrid = coeff[0]*xgrid + coeff[1]*ygrid + coeff[2]
src_dem_ds_align = coreglib.apply_z_shift(src_dem_ds_align, -valgrid, createcopy=False)
#Might be cleaner way to write out MEM ds directly to disk
src_dem_full_align = iolib.ds_getma(src_dem_ds_align)
iolib.writeGTiff(src_dem_full_align, align_fn, src_dem_ds_align)
if True:
#Output final aligned src_dem, masked so only best pixels are preserved
#Useful if creating a new reference product
#Can also use apply_mask.py
print("Applying filter to shiftec src_dem")
align_diff_filt_full_ds = warplib.memwarp_multi_fn([align_diff_filt_fn,], res=src_dem_ds_align, extent=src_dem_ds_align, \
t_srs=src_dem_ds_align)[0]
align_diff_filt_full = iolib.ds_getma(align_diff_filt_full_ds)
align_diff_filt_full_ds = None
align_fn_masked = outprefix + '%s_align_filt.tif' % xyz_shift_str_cum_fn
iolib.writeGTiff(np.ma.array(src_dem_full_align, mask=np.ma.getmaskarray(align_diff_filt_full)), \
align_fn_masked, src_dem_ds_align)
src_dem_full_align = None
src_dem_ds_align = None
#Compute original elevation difference
if True:
ref_dem_clip_ds, src_dem_clip_ds = warplib.memwarp_multi([ref_dem_ds, src_dem_ds], \
res=res, extent='intersection', t_srs=local_srs, r='cubic')
src_dem_ds = None
ref_dem_ds = None
ref_dem_orig = iolib.ds_getma(ref_dem_clip_ds)
src_dem_orig = iolib.ds_getma(src_dem_clip_ds)
#Needed for plotting
ref_dem_hs = geolib.gdaldem_mem_ds(ref_dem_clip_ds, processing='hillshade', returnma=True, computeEdges=True)
src_dem_hs = geolib.gdaldem_mem_ds(src_dem_clip_ds, processing='hillshade', returnma=True, computeEdges=True)
diff_orig = src_dem_orig - ref_dem_orig
#Only compute stats over valid surfaces
static_mask_orig = get_mask(src_dem_clip_ds, mask_list, src_dem_fn)
#Note: this doesn't include outlier removal or slope mask!
static_mask_orig = np.logical_or(np.ma.getmaskarray(diff_orig), static_mask_orig)
#For some reason, ASTER DEM diff have a spike near the 0 bin, could be an issue with masking?
diff_orig_compressed = diff_orig[~static_mask_orig]
diff_orig_stats = malib.get_stats_dict(diff_orig_compressed, full=True)
#Prepare filtered version for comparison
diff_orig_filt = np.ma.array(diff_orig, mask=static_mask_orig)
diff_orig_filt = outlier_filter(diff_orig_filt, f=3, max_dz=max_dz)
#diff_orig_filt = outlier_filter(diff_orig_filt, perc=(12.5, 87.5), max_dz=max_dz)
slope = get_filtered_slope(src_dem_clip_ds)
diff_orig_filt = np.ma.array(diff_orig_filt, mask=np.ma.getmaskarray(slope))
diff_orig_filt_stats = malib.get_stats_dict(diff_orig_filt, full=True)
#Write out original difference map
print("Writing out original difference map for common intersection before alignment")
orig_diff_fn = outprefix + '_orig_diff.tif'
iolib.writeGTiff(diff_orig, orig_diff_fn, ref_dem_clip_ds)
src_dem_clip_ds = None
ref_dem_clip_ds = None
if True:
align_stats_fn = outprefix + '%s_align_stats.json' % xyz_shift_str_cum_fn
align_stats = {}
align_stats['src_fn'] = src_dem_fn
align_stats['ref_fn'] = ref_dem_fn
align_stats['align_fn'] = align_fn
align_stats['res'] = {}
align_stats['res']['src'] = src_dem_res
align_stats['res']['ref'] = ref_dem_res
align_stats['res']['coreg'] = res
align_stats['center_coord'] = {'lon':center_coord_ll[0], 'lat':center_coord_ll[1], \
'x':center_coord_xy[0], 'y':center_coord_xy[1]}
align_stats['shift'] = {'dx':dx_total, 'dy':dy_total, 'dz':dz_total, 'dm':dm_total}
#This tiltcorr flag gets set to false, need better flag
if tiltcorr:
align_stats['tiltcorr'] = {}
align_stats['tiltcorr']['coeff'] = coeff.tolist()
align_stats['tiltcorr']['val_stats'] = vals_stats
align_stats['before'] = diff_orig_stats
align_stats['before_filt'] = diff_orig_filt_stats
align_stats['after'] = diff_align_stats
align_stats['after_filt'] = diff_align_filt_stats
import json
with open(align_stats_fn, 'w') as f:
json.dump(align_stats, f)
#Create output plot
if True:
print("Creating final plot")
kwargs = {'interpolation':'none'}
#f, axa = plt.subplots(2, 4, figsize=(11, 8.5))
f, axa = plt.subplots(2, 4, figsize=(16, 8))
for ax in axa.ravel()[:-1]:
ax.set_facecolor('k')
pltlib.hide_ticks(ax)
dem_clim = malib.calcperc(ref_dem_orig, (2,98))
axa[0,0].imshow(ref_dem_hs, cmap='gray', **kwargs)
im = axa[0,0].imshow(ref_dem_orig, cmap='cpt_rainbow', clim=dem_clim, alpha=0.6, **kwargs)
pltlib.add_cbar(axa[0,0], im, arr=ref_dem_orig, clim=dem_clim, label=None)
pltlib.add_scalebar(axa[0,0], res=res)
axa[0,0].set_title('Reference DEM')
axa[0,1].imshow(src_dem_hs, cmap='gray', **kwargs)
im = axa[0,1].imshow(src_dem_orig, cmap='cpt_rainbow', clim=dem_clim, alpha=0.6, **kwargs)
pltlib.add_cbar(axa[0,1], im, arr=src_dem_orig, clim=dem_clim, label=None)
axa[0,1].set_title('Source DEM')
#axa[0,2].imshow(~static_mask_orig, clim=(0,1), cmap='gray')
axa[0,2].imshow(~static_mask, clim=(0,1), cmap='gray', **kwargs)
axa[0,2].set_title('Surfaces for co-registration')
dz_clim = malib.calcperc_sym(diff_orig_compressed, (5, 95))
im = axa[1,0].imshow(diff_orig, cmap='RdBu', clim=dz_clim)
pltlib.add_cbar(axa[1,0], im, arr=diff_orig, clim=dz_clim, label=None)
axa[1,0].set_title('Elev. Diff. Before (m)')
im = axa[1,1].imshow(diff_align, cmap='RdBu', clim=dz_clim)
pltlib.add_cbar(axa[1,1], im, arr=diff_align, clim=dz_clim, label=None)
axa[1,1].set_title('Elev. Diff. After (m)')
#tight_dz_clim = (-1.0, 1.0)
tight_dz_clim = (-2.0, 2.0)
#tight_dz_clim = (-10.0, 10.0)
#tight_dz_clim = malib.calcperc_sym(diff_align_filt, (5, 95))
im = axa[1,2].imshow(diff_align_filt, cmap='RdBu', clim=tight_dz_clim)
pltlib.add_cbar(axa[1,2], im, arr=diff_align_filt, clim=tight_dz_clim, label=None)
axa[1,2].set_title('Elev. Diff. After (m)')
#Tried to insert Nuth fig here
#ax_nuth.change_geometry(1,2,1)
#f.axes.append(ax_nuth)
bins = np.linspace(dz_clim[0], dz_clim[1], 128)
axa[1,3].hist(diff_orig_compressed, bins, color='g', label='Before', alpha=0.5)
axa[1,3].hist(diff_align_compressed, bins, color='b', label='After', alpha=0.5)
axa[1,3].set_xlim(*dz_clim)
axa[1,3].axvline(0, color='k', linewidth=0.5, linestyle=':')
axa[1,3].set_xlabel('Elev. Diff. (m)')
axa[1,3].set_ylabel('Count (px)')
axa[1,3].set_title("Source - Reference")
before_str = 'Before\nmed: %0.2f\nnmad: %0.2f' % (diff_orig_stats['med'], diff_orig_stats['nmad'])
axa[1,3].text(0.05, 0.95, before_str, va='top', color='g', transform=axa[1,3].transAxes, fontsize=8)
after_str = 'After\nmed: %0.2f\nnmad: %0.2f' % (diff_align_stats['med'], diff_align_stats['nmad'])
axa[1,3].text(0.65, 0.95, after_str, va='top', color='b', transform=axa[1,3].transAxes, fontsize=8)
#This is empty
axa[0,3].axis('off')
suptitle = '%s\nx: %+0.2fm, y: %+0.2fm, z: %+0.2fm' % (os.path.split(outprefix)[-1], dx_total, dy_total, dz_total)
f.suptitle(suptitle)
f.tight_layout()
plt.subplots_adjust(top=0.90)
fig_fn = outprefix + '%s_align.png' % xyz_shift_str_cum_fn
print("Writing out figure: %s" % fig_fn)
f.savefig(fig_fn, dpi=300)
#Note: velocity is expected to be a 2-band file
#for i in *mos_vx.tif ; do
#base=$(echo $i | awk -F'_v' '{print $1}')
#gdal_merge.py -separate -a_nodata -2000000000 -o ${base}_vxy.tif ${base}_vx.tif ${base}_vy.tif
#done
print "Warping DEMs to same res/extent/proj"
#Might want to limit intersection to dem1 and dem2, as vmap could be significantly smaller
disp_ds = None
if len(sys.argv) == 4:
disp_fn = sys.argv[3]
if 'track' in disp_fn or 'tsx' in disp_fn:
vel_input = True
fn_list.append(disp_fn)
dem1_ds, dem2_ds, disp_ds = warplib.memwarp_multi_fn(fn_list,
extent='intersection',
res='max')
else:
dem1_ds, dem2_ds = warplib.memwarp_multi_fn(fn_list,
extent='intersection',
res='max')
outdir = os.path.split(dem1_fn)[0]
outprefix = os.path.splitext(
os.path.split(dem1_fn)[1])[0] + '_' + os.path.splitext(
os.path.split(dem2_fn)[1])[0]
#Load input DEMs into masked arrays
print "Loading input DEMs into masked arrays"
dem1 = iolib.ds_getma(dem1_ds, 1)
dem2 = iolib.ds_getma(dem2_ds, 1)
def main():
parser = getparser()
args = parser.parse_args()
fn = args.fn
if not iolib.fn_check(fn):
sys.exit("Unable to locate input file: %s" % fn)
#Need some checks on these
param = args.param
print("Loading input raster into masked array")
ds = iolib.fn_getds(fn)
#Currently supports only single band operations
r = iolib.ds_getma(ds, 1)
#May need to cast input ma as float32 so np.nan filling works
#r = r.astype(np.float32)
#Want function that checks and returns float32 if necessary
#Should filter, then return original dtype
r_fltr = r
#Loop through all specified input filters
#for filt in args.filt:
filt = args.filt[0]
if len(param) == 1:
param = param[0]
param_str = ''
if filt == 'range':
#Range filter
param = [float(i) for i in param[1:]]
r_fltr = filtlib.range_fltr(r_fltr, param)
param_str = '_{0:0.2f}-{1:0.2f}'.format(*param)
elif filt == 'absrange':
#Range filter of absolute values
param = [float(i) for i in param[1:]]
r_fltr = filtlib.absrange_fltr(r_fltr, param)
param_str = '_{0:0.2f}-{1:0.2f}'.format(*param)
elif filt == 'perc':
#Percentile filter
param = [float(i) for i in param[1:]]
r_fltr = filtlib.perc_fltr(r, perc=param)
param_str = '_{0:0.2f}-{1:0.2f}'.format(*param)
elif filt == 'med':
#Median filter
param = int(param)
r_fltr = filtlib.rolling_fltr(r_fltr, f=np.nanmedian, size=param)
#r_fltr = filtlib.median_fltr(r_fltr, fsize=param, origmask=True)
#r_fltr = filtlib.median_fltr_skimage(r_fltr, radius=4, origmask=True)
param_str = '_%ipx' % param
elif filt == 'gauss':
#Gaussian filter (default)
param = int(param)
r_fltr = filtlib.gauss_fltr_astropy(r_fltr,
size=param,
origmask=False,
fill_interior=False)
param_str = '_%ipx' % param
elif filt == 'highpass':
#High pass filter
param = int(param)
r_fltr = filtlib.highpass(r_fltr, size=param)
param_str = '_%ipx' % param
elif filt == 'sigma':
#n*sigma filter, remove outliers
param = int(param)
r_fltr = filtlib.sigma_fltr(r_fltr, n=param)
param_str = '_n%i' % param
elif filt == 'mad':
#n*mad filter, remove outliers
#Maybe better to use a percentile filter
param = int(param)
r_fltr = filtlib.mad_fltr(r_fltr, n=param)
param_str = '_n%i' % param
elif filt == 'dz':
#Difference filter, need to specify ref_fn and range
#Could let the user compute their own dz, then just run a standard range or absrange filter
ref_fn = param[0]
ref_ds = warplib.memwarp_multi_fn([
ref_fn,
],
res=ds,
extent=ds,
t_srs=ds)[0]
ref = iolib.ds_getma(ref_ds)
param = [float(i) for i in param[1:]]
r_fltr = filtlib.dz_fltr_ma(r, ref, rangelim=param)
#param_str = '_{0:0.2f}-{1:0.2f}'.format(*param)
param_str = '_{0:0.0f}_{1:0.0f}'.format(*param)
else:
sys.exit("No filter type specified")
#Compute and print stats before/after
if args.stats:
print("Input stats:")
malib.print_stats(r)
print("Filtered stats:")
malib.print_stats(r_fltr)
#Write out
dst_fn = os.path.splitext(fn)[0] + '_%sfilt%s.tif' % (filt, param_str)
if args.outdir is not None:
outdir = args.outdir
if not os.path.exists(outdir):
os.makedirs(outdir)
dst_fn = os.path.join(outdir, os.path.split(dst_fn)[-1])
print("Writing out filtered raster: %s" % dst_fn)
iolib.writeGTiff(r_fltr, dst_fn, ds)
def warpMultipleFiles(filenames,srs_template): from pygeotools.lib import warplib return(warplib.memwarp_multi_fn(filenames, extent='intersection', res='min', t_srs=srs_template))
def main():
parser = getparser()
args = parser.parse_args()
dem_fn = args.dem_fn
# Write out because they will be used to mask CHM
writeall = True
# Auto compute min TOA with gaussian mixture model
auto_min_toa = args.auto_min_toa
dirname, demname = os.path.split(dem_fn)
# The subdir in which the DEM.tif sits will be the pairname
pairname = os.path.split(dirname)[1]
print("Pairname:", pairname)
if args.out_dir is not None:
# Create symlink in out_dir to: (1) original out-DEM_4m (2) *_ortho_4m.tif (3) All *.xml files
# This should look like <out_dir>/<pairname>_out-DEM_4m
dem_fn_lnk = os.path.join(args.out_dir, pairname + '_' + demname)
force_symlink(dem_fn, dem_fn_lnk)
force_symlink(os.path.join(dirname, pairname + '_ortho_4m.tif'), os.path.join(args.out_dir, pairname + '_ortho_4m.tif') )
xml_list = [f for f in os.listdir(dirname) if f.endswith('r100.xml')]
print("\nSymlinks made for:")
for x in xml_list:
print(x)
shutil.copy2(os.path.join(dirname,x), args.out_dir)
out_fn_base = os.path.splitext(dem_fn_lnk)[0]
dem_fn = dem_fn_lnk
else:
out_fn_base = os.path.splitext(dem_fn)[0]
print("\nBasename for output files:")
print(out_fn_base)
#Max Threshold value for LiDAR datset; Valid pixels under this value
lidar_fn=args.lidar_fn
max_thresh=args.max_thresh
#Need some checks on these
param = args.filt_param
if param is not None and len(param) == 1:
param = param[0]
# Get original DEM
dem = iolib.fn_getma(dem_fn)
print("\nLoading input DEM into masked array")
dem_ds = iolib.fn_getds(dem_fn)
toa_mask = None
toa_tri_mask = None # probably not used by itself; done as part of toa_mask
rough_mask = None
slope_mask = None
mask_list = [toa_tri_mask, toa_mask, rough_mask, slope_mask]
if args.filtdz:
print("\nFilter with dz from ref DEM to remove cloud returns and blunders (shadows)...")
print("Reference DEM: %s" % os.path.split(param[0])[1] )
print("Absolute dz (+/-): %s \n" % param[2] )
#May need to cast input ma as float32 so np.nan filling works
dem = dem.astype(np.float32)
#Difference filter, need to specify ref_fn and range
#Could let the user compute their own dz, then just run a standard range or absrange filter
ref_fn = param[0]
ref_ds = warplib.memwarp_multi_fn([ref_fn,], res=dem_ds, extent=dem_ds, t_srs=dem_ds)[0]
ref = iolib.ds_getma(ref_ds)
param = map(float, param[1:])
# A dem that has been masked based on the dz filter
dem = filtlib.dz_fltr_ma(dem, ref, rangelim=param)
if writeall:
#out_fn = os.path.splitext(dem_fn)[0]+'_dzfilt.tif'
out_fn = os.path.join(out_fn_base +'_dzfilt.tif')
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(dem, out_fn, src_ds=dem_ds, ndv=args.ndv)
#Initialize a control mask that we'll update
#True (1) represents "valid" unmasked pixel, False (0) represents "invalid" pixel to be masked
controlmask = ~(np.ma.getmaskarray(dem))
# DEM masking: Each block returns a masked output (not a mask)
# TOA: mask dark and/or smooth areas (shadows and/or water)
# Roughness
# Slope
if args.toamask or args.toatrimask:
#try:
print("\nCompute TOA from ortho...\n")
toa_fn = get_toa_fn(out_fn_base + '.tif') ##--->dem_fn
print(toa_fn)
print("\nWarp TOA to DEM...\n")
toa_ds = warplib.memwarp_multi_fn([toa_fn,], res=dem_ds, extent=dem_ds, t_srs=dem_ds)[0]
if args.toamask:
if auto_min_toa:
# Compute a good min TOA value
m,s = get_min_gaus(toa_fn, 50, 4)
min_toa = m + s
min_toa = m
else:
min_toa = args.min_toa
with open(os.path.join(os.path.split(toa_fn)[0], "min_toa_" + pairname + ".txt"), "w") as text_file:
text_file.write(os.path.basename(__file__))
text_file.write("\nMinimum TOA used for mask:\n{0}".format(min_toa))
# Should mask dark areas and dilate
toa_mask = get_toa_mask(toa_ds, min_toa)
#Dilate the mask
if args.dilate_toa is not None:
niter = args.dilate_toa
print("Dilating TOA mask with %i iterations" % niter)
from scipy import ndimage
toa_mask = ~(ndimage.morphology.binary_dilation(~toa_mask, iterations=niter))
controlmask = np.logical_and(toa_mask, controlmask)
# Mask islands here
controlmask = malib.mask_islands(controlmask, 5)
if writeall:
#out_fn = out_fn_base+'_toamask.tif'
out_fn = os.path.join(out_fn_base +'_toamask.tif')
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(toa_mask, out_fn, src_ds=dem_ds)
if args.toatrimask:
# Should mask smooth areas (measures local variance)
toa_tri_mask = get_tri_mask(toa_ds, args.min_toatri)
controlmask = np.logical_and(toa_tri_mask, controlmask)
if writeall:
#out_fn = out_fn_base+'_toatrimask.tif'
out_fn = os.path.join(out_fn_base +'_toatrimask.tif')
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(toa_tri_mask, out_fn, src_ds=dem_ds)
#except Exception, e:
#print "\tFailed to apply TOA masking.\n"
if args.slopemask:
slope_mask = get_slope_mask(dem_ds, args.max_slope)
controlmask = np.logical_and(slope_mask, controlmask)
#if args.slopemaskcoarse:
#dem_fn2 = args.dem_coarscomp_fn
#print("\nLoading input coarse DEM into masked array")
#dem2_ds = iolib.fn_getds(dem_fn2)
#slope_mask = get_slope_mask(dem2_ds, args.max_slope)
#controlmask = np.logical_and(slope_mask, controlmask)
if writeall:
#out_fn = out_fn_base+'_slopemask.tif'
out_fn = os.path.join(out_fn_base +'_slopemask.tif')
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(slope_mask, out_fn, src_ds=dem_ds)
if args.lidar_fn:
try:
print("Masking DEM file based on Lidar Dataset\n")
print("\nWarp Lidar Raster to DEM...\n")
lidar_ds=warplib.memwarp_multi_fn([lidar_fn,],r='near', res=dem_ds, extent=dem_ds, t_srs=dem_ds)[0]
lidarmask = get_lidar_mask(dem_ds, lidar_ds, max_thresh)
controlmask = np.logical_and(lidarmask, controlmask)
if writeall:
out_fn=out_fn_base+'_lidarmask.tif'
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(lidarmask, out_fn, src_ds=dem_ds)
except Exception as e:
print("\tFailed to Apply Lidar Mask")
# CHM mask will be a subset of the Control mask; slope_mask, toa_mask, toa_tri_mask
chmmask = controlmask
print("Generating final CHM mask to apply later")
#out_fn = out_fn_base+'_chmmask.tif'
out_fn = os.path.join(out_fn_base +'_chmmask.tif')
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(chmmask, out_fn, src_ds=dem_ds)
if args.roughmask:
rough_mask = get_rough_mask(dem_ds, args.max_rough)
controlmask = np.logical_and(rough_mask, controlmask)
if writeall:
out_fn = os.path.join(out_fn_base +'_roughmask.tif')
print("Writing out %s\n" % out_fn)
iolib.writeGTiff(rough_mask, out_fn, src_ds=dem_ds)
print("Generating final mask to use for reference surfaces, and applying to input DEM")
#Now invert to use to create final masked array
# This steps results in the areas to be removed being set to a valid value
controlmask = ~controlmask
#Dilate the mask
if args.dilate_con is not None:
niter = args.dilate_con
print("Dilating control mask with %i iterations" % niter)
from scipy import ndimage
#
# So, this should work too.... controlmask = ndimage.morphology.binary_dilation(controlmask, iterations=niter))
controlmask = ~(ndimage.morphology.binary_dilation(~controlmask, iterations=niter)) # This steps results in the areas to be removed being set to a valid value, again
print("\nApply mask to original DEM - use these control surfaces for co-registration...")
newdem = np.ma.array(dem, mask=controlmask) # This sets the valid values of the controlmask to the 'mask' of the DEM, which turns them into NaN values
if True:
print("\nStats of valid DEM with masks applied:")
valid_stats = malib.print_stats(newdem)
valid_stats_med = valid_stats[5]
print("\nWriting DEM control surfaces:")
#if args.out_dir is not None:
# dst_fn = os.path.join(args.out_dir, os.path.split(dirname)[1] + os.path.splitext(demname)[0]+'_control.tif')
#else:
# dst_fn = os.path.splitext(dem_fn)[0]+'_control.tif'
dst_fn = os.path.join(out_fn_base +'_control.tif')
print(dst_fn)
iolib.writeGTiff(newdem, dst_fn, dem_ds)
return dst_fn
def main():
parser = getparser()
#Create dictionary of arguments
args = vars(parser.parse_args())
#Want to enable -full when -of is specified, probably a fancy way to do this with argparse
if args['of']:
args['full'] = True
args['imshow_kwargs'] = pltlib.imshow_kwargs
#Need to implement better extent handling for link and overlay
#Can use warplib extent parsing
extent = 'first'
#extent = 'union'
#Should accept 'ts' or 'fn' or string here, default is 'ts'
#Can also accept list for subplots
title = args['title']
if args['link']:
fig = plt.figure(0)
n_ax = len(args['filelist'])
src_ds_list = [gdal.Open(fn) for fn in args['filelist']]
t_srs = geolib.get_ds_srs(src_ds_list[0])
res_stats = geolib.get_res_stats(src_ds_list, t_srs=t_srs)
#Use min res
res = res_stats[0]
extent = 'intersection'
extent = geolib.ds_geom_union_extent(src_ds_list, t_srs=t_srs)
#extent = geolib.ds_geom_intersection_extent(src_ds_list, t_srs=t_srs)
#print(res, extent)
for n, fn in enumerate(args['filelist']):
if not iolib.fn_check(fn):
print('Unable to open input file: %s' % fn)
continue
if title == 'ts':
ts = timelib.fn_getdatetime_list(fn)
if ts:
print("Timestamp list: ", ts)
if len(ts) == 1:
args['title'] = ts[0].date()
elif len(ts) > 1:
args['title'] = "%s to %s" % (ts[0].date(), ts[1].date())
else:
print("Unable to extract timestamp")
args['title'] = None
elif title == 'fn':
args['title'] = fn
#if title is not None:
# plt.title(title, fontdict={'fontsize':12})
#Note: this won't work if img1 has 1 band and img2 has 3 bands
#Hack for now
if not args['link']:
fig = plt.figure(n)
n_ax = 1
#fig.set_facecolor('black')
fig.set_facecolor('white')
fig.canvas.set_window_title(os.path.split(fn)[1])
#fig.suptitle(os.path.split(fn)[1], fontsize=10)
if args['overlay']:
#Should automatically search for shaded relief with same base fn
#bg_fn = os.path.splitext(fn)[0]+'_hs_az315.tif'
#Clip/warp background dataset to match overlay dataset
src_ds, bg_ds = warplib.memwarp_multi_fn([fn, args['overlay']],
extent=extent,
res='max')
#Want to load up the unique bg array for each input
args['bg'] = get_bma(bg_ds, 1, args['full'])
else:
src_ds = gdal.Open(fn, gdal.GA_ReadOnly)
if args['link']:
src_ds = warplib.memwarp_multi([
src_ds,
],
res=res,
extent=extent,
t_srs=t_srs)[0]
args['cbar_kwargs'] = pltlib.cbar_kwargs
if args['no_cbar']:
args['cbar_kwargs'] = None
nbands = src_ds.RasterCount
b = src_ds.GetRasterBand(1)
dt = gdal.GetDataTypeName(b.DataType)
#Eventually, check dt of each band
print("%s (%i bands)" % (fn, nbands))
#Singleband raster
if (nbands == 1):
if args['cmap'] is None:
#Special case to handle ASP float32 grayscale data
if '-L_sub' in fn or '-R_sub' in fn:
args['cmap'] = 'gray'
else:
if (dt == 'Float64') or (dt == 'Float32') or (dt
== 'Int32'):
args['cmap'] = 'cpt_rainbow'
#This is for WV images
elif (dt == 'UInt16'):
args['cmap'] = 'gray'
elif (dt == 'Byte'):
args['cmap'] = 'gray'
else:
args['cmap'] = 'cpt_rainbow'
"""
if 'count' in fn:
args['clim_perc'] = (0,100)
cbar_kwargs['extend'] = 'neither'
args['cmap'] = 'cpt_rainbow'
if 'mask' in fn:
args['clim'] = (0, 1)
#Could be (0, 255)
#args['clim_perc'] = (0,100)
#Want absolute clim of 0, then perc of 100
cbar_kwargs['extend'] = 'neither'
args['cmap'] = 'gray'
"""
bma = get_bma(src_ds, 1, args['full'])
if args['invert']:
bma *= -1
#Note n+1 here ensures we're assigning subplot correctly here (n is 0-relative, subplot is 1)
bma_fig(fig, bma, n_subplt=n_ax, subplt=n + 1, ds=src_ds, **args)
#3-band raster, likely disparity map
#This doesn't work when alpha band is present
elif (nbands == 3) and (dt == 'Byte'):
#For some reason, tifs are vertically flipped
if (os.path.splitext(fn)[1] == '.tif'):
args['imshow_kwargs']['origin'] = 'lower'
#Use gdal dataset here instead of imread(fn)?
imgplot = plt.imshow(plt.imread(fn), **args['imshow_kwargs'])
pltlib.hide_ticks(imgplot.axes)
#Handle the 3-band disparity map case here
#elif ((dt == 'Float32') or (dt == 'Int32')):
else:
if args['cmap'] is None:
args['cmap'] = 'cpt_rainbow'
bn = 1
while bn <= nbands:
bma = get_bma(src_ds, bn, args['full'])
bma_fig(fig,
bma,
n_subplt=nbands,
subplt=bn,
ds=src_ds,
**args)
bn += 1
#Want to be better about this else case - lazy for now
#else:
# bma = get_bma(src_ds, 1, args['full'])
# bma_fig(fig, bma, **args)
plt.tight_layout()
#Write out the file
#Note: make sure display is local for savefig
if args['of']:
outf = str(os.path.splitext(fn)[0]) + '_fig.' + args['of']
#outf = str(os.path.splitext(fn)[0])+'_'+str(os.path.splitext(args['overlay'])[0])+'_fig.'+args['of']
#Note: need to account for colorbar (12%) and title - some percentage of axes beyond bma dimensions
#Should specify minimum text size for output
max_size = np.array((10.0, 10.0))
max_dpi = 300.0
#If both outsize and dpi are specified, don't try to change, just make the figure
if (args['outsize'] is None) and (args['dpi'] is None):
args['dpi'] = 150.0
#Unspecified out figure size for a given dpi
if (args['outsize'] is None) and (args['dpi'] is not None):
args['outsize'] = np.array(bma.shape[::-1]) / args['dpi']
if np.any(np.array(args['outsize']) > max_size):
args['outsize'] = max_size
#Specified output figure size, no specified dpi
elif (args['outsize'] is not None) and (args['dpi'] is None):
args['dpi'] = np.min([
np.max(
np.array(bma.shape[::-1]) / np.array(args['outsize'])),
max_dpi
])
print()
print("Saving output figure:")
print("Filename: ", outf)
print("Size (in): ", args['outsize'])
print("DPI (px/in): ", args['dpi'])
print("Input dimensions (px): ", bma.shape[::-1])
print("Output dimensions (px): ",
tuple(np.array(args['outsize']) * args['dpi']))
print()
fig.set_size_inches(args['outsize'])
#fig.set_size_inches(54.427, 71.87)
#fig.set_size_inches(40, 87)
fig.savefig(outf,
dpi=args['dpi'],
bbox_inches='tight',
pad_inches=0,
facecolor=fig.get_facecolor(),
edgecolor='none')
#fig.savefig(outf, dpi=args['dpi'], facecolor=fig.get_facecolor(), edgecolor='none')
#Show the plot - want to show all at once
if not args['of']:
plt.show()
def main():
parser = getparser()
#Create dictionary of arguments
args = vars(parser.parse_args())
#Want to enable -full when -of is specified, probably a fancy way to do this with argparse
if args['of']:
args['full'] = True
#Note, imshow has many interpolation types:
#'none', 'nearest', 'bilinear', 'bicubic', 'spline16', 'spline36', 'hanning', 'hamming',
#'hermite', 'kaiser', 'quadric', 'catrom', 'gaussian', 'bessel', 'mitchell', 'sinc', 'lanczos'
#{'interpolation':'bicubic', 'aspect':'auto'}
#args['imshow_kwargs']={'interpolation':'bicubic'}
args['imshow_kwargs']={'interpolation':'none'}
if args['clipped'] and args['overlay'] is None:
sys.exit("Must specify an overlay filename with option 'clipped'")
#Set this as the background numpy array
args['bg'] = None
if args['shp'] is not None:
print args['shp']
if args['link']:
fig = plt.figure(0)
n_ax = len(args['filelist'])
src_ds_list = [gdal.Open(fn) for fn in args['filelist']]
t_srs = geolib.get_ds_srs(src_ds_list[0])
res_stats = geolib.get_res_stats(src_ds_list, t_srs=t_srs)
#Use min res
res = res_stats[0]
extent = geolib.ds_geom_union_extent(src_ds_list, t_srs=t_srs)
#print res, extent
for n,fn in enumerate(args['filelist']):
if not iolib.fn_check(fn):
print 'Unable to open input file: %s' % fn
continue
#Note: this won't work if img1 has 1 band and img2 has 3 bands
#Hack for now
if not args['link']:
fig = plt.figure(n)
n_ax = 1
#fig.set_facecolor('black')
fig.set_facecolor('white')
fig.canvas.set_window_title(os.path.split(fn)[1])
#fig.suptitle(os.path.split(fn)[1], fontsize=10)
#Note: warplib SHOULD internally check to see if extent/resolution/projection are identical
#This eliminates the need for a clipped flag
#If user has already warped the background and source data
if args['overlay']:
if args['clipped']:
src_ds = gdal.Open(fn, gdal.GA_ReadOnly)
#Only load up the bg array once
if args['bg'] is None:
#Need to check that background fn exists
print "%s background" % args['overlay']
bg_ds = gdal.Open(args['overlay'], gdal.GA_ReadOnly)
#Check image dimensions
args['bg'] = get_bma(bg_ds, 1, args['full'])
else:
#Clip/warp background dataset to match overlay dataset
#src_ds, bg_ds = warplib.memwarp_multi_fn([fn, args['overlay']], extent='union')
src_ds, bg_ds = warplib.memwarp_multi_fn([fn, args['overlay']], extent='first')
#src_ds, bg_ds = warplib.memwarp_multi_fn([fn, args['overlay']], res='min', extent='first')
#Want to load up the unique bg array for each input
args['bg'] = get_bma(bg_ds, 1, args['full'])
else:
src_ds = gdal.Open(fn, gdal.GA_ReadOnly)
if args['link']:
#Not sure why, but this still warps all linked ds, even when identical res/extent/srs
#src_ds = warplib.warp(src_ds, res=res, extent=extent, t_srs=t_srs)
src_ds = warplib.memwarp_multi([src_ds,], res=res, extent=extent, t_srs=t_srs)[0]
cbar_kwargs={'extend':'both', 'orientation':'vertical', 'shrink':0.7, 'fraction':0.12, 'pad':0.02}
nbands = src_ds.RasterCount
b = src_ds.GetRasterBand(1)
dt = gdal.GetDataTypeName(b.DataType)
#Eventually, check dt of each band
print
print "%s (%i bands)" % (fn, nbands)
#Singleband raster
if (nbands == 1):
if args['cmap'] is None:
#Special case to handle ASP float32 grayscale data
if '-L_sub' in fn or '-R_sub' in fn:
args['cmap'] = 'gray'
else:
if (dt == 'Float64') or (dt == 'Float32') or (dt == 'Int32'):
args['cmap'] = 'cpt_rainbow'
#This is for WV images
elif (dt == 'UInt16'):
args['cmap'] = 'gray'
elif (dt == 'Byte'):
args['cmap'] = 'gray'
else:
args['cmap'] = 'cpt_rainbow'
"""
if 'count' in fn:
args['clim_perc'] = (0,100)
cbar_kwargs['extend'] = 'neither'
args['cmap'] = 'cpt_rainbow'
if 'mask' in fn:
args['clim'] = (0, 1)
#Could be (0, 255)
#args['clim_perc'] = (0,100)
#Want absolute clim of 0, then perc of 100
cbar_kwargs['extend'] = 'neither'
args['cmap'] = 'gray'
"""
args['cbar_kwargs'] = cbar_kwargs
bma = get_bma(src_ds, 1, args['full'])
#Note n+1 here ensures we're assigning subplot correctly here (n is 0-relative, subplot is 1)
bma_fig(fig, bma, n_subplt=n_ax, subplt=n+1, ds=src_ds, **args)
#3-band raster, likely disparity map
#This doesn't work when alpha band is present
elif (nbands == 3) and (dt == 'Byte'):
#For some reason, tifs are vertically flipped
if (os.path.splitext(fn)[1] == '.tif'):
args['imshow_kwargs']['origin'] = 'lower'
#Use gdal dataset here instead of imread(fn)?
imgplot = plt.imshow(plt.imread(fn), **args['imshow_kwargs'])
pltlib.hide_ticks(imgplot.axes)
#Handle the 3-band disparity map case here
#elif ((dt == 'Float32') or (dt == 'Int32')):
else:
if args['cmap'] is None:
args['cmap'] = 'cpt_rainbow'
bn = 1
while bn <= nbands:
bma = get_bma(src_ds, bn, args['full'])
bma_fig(fig, bma, n_subplt=nbands, subplt=bn, ds=src_ds, **args)
bn += 1
#Want to be better about this else case - lazy for now
#else:
# bma = get_bma(src_ds, 1, args['full'])
# bma_fig(fig, bma, **args)
ts = timelib.fn_getdatetime_list(fn)
if ts:
print "Timestamp list: ", ts
"""
if len(ts) == 1:
plt.title(ts[0].date())
elif len(ts) == 2:
plt.title("%s to %s" % (ts[0].date(), ts[1].date()))
"""
plt.tight_layout()
#Write out the file
#Note: make sure display is local for savefig
if args['of']:
outf = str(os.path.splitext(fn)[0])+'_fig.'+args['of']
#outf = str(os.path.splitext(fn)[0])+'_'+str(os.path.splitext(args['overlay'])[0])+'_fig.'+args['of']
#Note: need to account for colorbar (12%) and title - some percentage of axes beyond bma dimensions
#Should specify minimum text size for output
max_size = np.array((10.0,10.0))
max_dpi = 300.0
#If both outsize and dpi are specified, don't try to change, just make the figure
if (args['outsize'] is None) and (args['dpi'] is None):
args['dpi'] = 150.0
#Unspecified out figure size for a given dpi
if (args['outsize'] is None) and (args['dpi'] is not None):
args['outsize'] = np.array(bma.shape[::-1])/args['dpi']
if np.any(np.array(args['outsize']) > max_size):
args['outsize'] = max_size
#Specified output figure size, no specified dpi
elif (args['outsize'] is not None) and (args['dpi'] is None):
args['dpi'] = np.min([np.max(np.array(bma.shape[::-1])/np.array(args['outsize'])), max_dpi])
print
print "Saving output figure:"
print "Filename: ", outf
print "Size (in): ", args['outsize']
print "DPI (px/in): ", args['dpi']
print "Input dimensions (px): ", bma.shape[::-1]
print "Output dimensions (px): ", tuple(np.array(args['outsize'])*args['dpi'])
print
fig.set_size_inches(args['outsize'])
#fig.set_size_inches(54.427, 71.87)
#fig.set_size_inches(40, 87)
fig.savefig(outf, dpi=args['dpi'], bbox_inches='tight', pad_inches=0, facecolor=fig.get_facecolor(), edgecolor='none')
#Show the plot - want to show all at once
if not args['of']:
plt.show()
def main():
parser = getparser()
args = parser.parse_args()
refdem = args.refdem
srcdem = args.srcdem
outfolder = '{}__{}_comparison_stats'.format(
os.path.splitext(os.path.basename(refdem))[0],
os.path.splitext(os.path.basename(srcdem))[0])
header_str = '{}__{}'.format(
os.path.splitext(os.path.basename(refdem))[0],
os.path.splitext(os.path.basename(srcdem))[0])
if not os.path.exists(outfolder):
os.makedirs(outfolder)
if args.local_ortho == 1:
temp_ds = warplib.memwarp_multi_fn([refdem, srcdem])[0]
bbox = geolib.ds_extent(temp_ds)
geo_crs = temp_ds.GetProjection()
print('Bounding box lon_lat is{}'.format(bbox))
bound_poly = Polygon([[bbox[0], bbox[3]], [bbox[2], bbox[3]],
[bbox[2], bbox[1]], [bbox[0], bbox[1]]])
bound_shp = gpd.GeoDataFrame(index=[0],
geometry=[bound_poly],
crs=geo_crs)
bound_centroid = bound_shp.centroid
cx = bound_centroid.x.values[0]
cy = bound_centroid.y.values[0]
pad = np.ptp([bbox[3], bbox[1]]) / 6.0
lat_1 = bbox[1] + pad
lat_2 = bbox[3] - pad
local_ortho = "+proj=ortho +lat_1={} +lat_2={} +lat_0={} +lon_0={} +x_0=0 +y_0=0 +ellps=WGS84 +datum=WGS84 +units=m +no_defs".format(
lat_1, lat_2, cy, cx)
logging.info('Local Ortho projection is {}'.format(local_ortho))
t_srs = local_ortho
else:
t_srs = 'first'
# this step performs the desired warping operation
ds_list = warplib.memwarp_multi_fn([refdem, srcdem],
res=args.comparison_res,
t_srs=t_srs)
refma = iolib.ds_getma(ds_list[0])
srcma = iolib.ds_getma(ds_list[1])
init_diff = refma - srcma
init_stats = malib.get_stats_dict(init_diff)
print("Original descriptive statistics {}".format(init_stats))
init_diff_json_fn = os.path.join(
outfolder, '{}_precoreg_descriptive_stats.json'.format(header_str))
init_diff_json = json.dumps(init_stats)
with open(init_diff_json_fn, 'w') as f:
f.write(init_diff_json)
logging.info("Saved initial stats at {}".format(init_diff_json))
refslope = gdaldem(ds_list[0])
# stats for elevation difference vs reference DEM elevation
elev_bin, diff_mean, diff_median, diff_std, diff_perc = cummulative_profile(
refma, init_diff, args.elev_bin_width)
# stats for elevation difference vs reference DEM slope
slope_bin, diff_mean_s, diff_median_s, diff_std_s, diff_perc_s = cummulative_profile(
refslope, init_diff, args.slope_bin_width)
f, ax = plt.subplots(1, 2, figsize=(10, 4))
im = ax[0].scatter(elev_bin, diff_mean, c=diff_perc, cmap='inferno')
ax[0].set_xlabel('Elevation (m)')
divider = make_axes_locatable(ax[0])
cax = divider.append_axes('right', size='2.5%', pad=0.05)
plt.colorbar(im,
cax=cax,
orientation='vertical',
label='pixel count percentage')
im2 = ax[1].scatter(slope_bin, diff_mean_s, c=diff_perc_s, cmap='inferno')
ax[1].set_xlabel('Slope (degrees)')
divider = make_axes_locatable(ax[1])
cax = divider.append_axes('right', size='2.5%', pad=0.05)
plt.colorbar(im2,
cax=cax,
orientation='vertical',
label='pixel count percentage')
for axa in ax.ravel():
axa.axhline(y=0, c='k')
axa.set_ylabel('Elevation Difference (m)')
plt.tight_layout()
precoreg_plot = os.path.join(outfolder,
header_str + '_precoreg_binned_plot.png')
f.savefig(precoreg_plot, dpi=300, bbox_inches='tight', pad_inches=0.1)
logging.info("Saved binned plot at {}".format(precoreg_plot))
if args.coreg == 1:
logging.info("will attempt coregisteration")
if args.local_ortho == 1:
ref_local_ortho = os.path.splitext(refdem)[0] + '_local_ortho.tif'
src_local_ortho = os.path.splitext(srcdem)[0] + '_local_ortho.tif'
# coregisteration works best at mean resolution
# we will rewarp if the initial args.res was not mean
if args.comparison_res != 'mean':
ds_list = warplib.memwarp_multi_fn([refdem, srcdem],
res='mean',
t_srs=t_srs)
refma = iolib.ds_getma(ds_list[0])
srcma = iolib.ds_getma(ds_list[1])
iolib.writeGTiff(refma, ref_local_ortho, ds_list[0])
iolib.writeGTiff(srcma, src_local_ortho, ds_list[1])
coreg_ref = ref_local_ortho
src_ref = src_local_ortho
else:
coreg_ref = refdem
src_ref = srcdem
demcoreg_dir = os.path.join(outfolder, 'coreg_results')
align_opts = [
'-mode', 'nuth', '-max_iter', '12', '-max_offset', '400',
'-outdir', demcoreg_dir
]
align_args = [coreg_ref, src_ref]
align_cmd = ['dem_align.py'] + align_opts + align_args
subprocess.call(align_cmd)
#ah final round of warping and stats calculation
try:
srcdem_align = glob.glob(os.path.join(demcoreg_dir,
'*align.tif'))[0]
logging.info(
"Attempting stats calculation for aligned DEM {}".format(
srcdem_align))
ds_list = warplib.memwarp_multi_fn([args.refdem, srcdem_align],
res=args.comparison_res,
t_srs=t_srs)
refma = iolib.ds_getma(ds_list[0])
srcma = iolib.ds_getma(ds_list[1])
# this is creepy, but I am recycling variable names to save on memory
init_diff = refma - srcma
init_stats = malib.get_stats_dict(init_diff)
print("Final descriptive statistics {}".format(init_stats))
init_diff_json_fn = os.path.join(
outfolder,
'{}_postcoreg_descriptive_stats.json'.format(header_str))
init_diff_json = json.dumps(init_stats)
with open(init_diff_json_fn, 'w') as f:
f.write(init_diff_json)
logging.info("Saved final stats at {}".format(init_diff_json))
refslope = gdaldem(ds_list[0])
# stats for elevation difference vs reference DEM elevation
elev_bin, diff_mean, diff_median, diff_std, diff_perc = cummulative_profile(
refma, init_diff, args.elev_bin_width)
# stats for elevation difference vs reference DEM slope
slope_bin, diff_mean_s, diff_median_s, diff_std_s, diff_perc_s = cummulative_profile(
refslope, init_diff, args.slope_bin_width)
f, ax = plt.subplots(1, 2, figsize=(10, 4))
im = ax[0].scatter(elev_bin,
diff_mean,
c=diff_perc,
cmap='inferno')
ax[0].set_xlabel('Elevation (m)')
divider = make_axes_locatable(ax[0])
cax = divider.append_axes('right', size='2.5%', pad=0.05)
plt.colorbar(im,
cax=cax,
orientation='vertical',
label='pixel count percentage')
im2 = ax[1].scatter(slope_bin,
diff_mean_s,
c=diff_perc_s,
cmap='inferno')
ax[1].set_xlabel('Slope (degrees)')
divider = make_axes_locatable(ax[1])
cax = divider.append_axes('right', size='2.5%', pad=0.05)
plt.colorbar(im2,
cax=cax,
orientation='vertical',
label='pixel count percentage')
for axa in ax.ravel():
axa.axhline(y=0, c='k')
axa.set_ylabel('Elevation Difference (m)')
plt.tight_layout()
precoreg_plot = os.path.join(
outfolder, header_str + '_postcoreg_binned_plot.png')
f.savefig(precoreg_plot,
dpi=300,
bbox_inches='tight',
pad_inches=0.1)
except:
logging.info(
"Failed to compute post coreg stats, see corresponding job log"
)
logging.info("Script is complete !")
def main():
parser = getparser()
args = parser.parse_args()
if args.seedmode == 'existing_velocity':
if args.vx_fn is None or args.vy_fn is None:
parser.error('"-seedmode existing_velocity" requires "-vx_fn" and "-vy_fn"')
print('\n%s' % datetime.now())
print('%s UTC\n' % datetime.utcnow())
align = args.align
seedmode = args.seedmode
spr = args.refinement
erode = args.erode
#Correlator tile timeout
#With proper seeding, correlation should be very fast
#timeout = 360
timeout = 1200
threads = args.threads
kernel = (args.kernel, args.kernel)
#SGM correlator
if spr > 3:
#kernel = (7,7)
kernel = (11,11)
erode = 0
#Smooth the output F.tif
smoothF = args.filter
res = args.tr
#Resample input to something easier to work with
#res = 4.0
#Open input files
fn1 = args.fn1
fn2 = args.fn2
if not iolib.fn_check(fn1) or not iolib.fn_check(fn2):
sys.exit("Unable to locate input files")
if args.outdir is not None:
outdir = args.outdir
else:
outdir = '%s__%s_vmap_%sm_%ipx_spm%i' % (os.path.splitext(os.path.split(fn1)[1])[0], \
os.path.splitext(os.path.split(fn2)[1])[0], res, kernel[0], spr)
#Note, can encounter filename length issues in boost, just use vmap prefix
outprefix = '%s/vmap' % (outdir)
if not os.path.exists(outdir):
os.makedirs(outdir)
#Check to see if inputs have geolocation and projection information
ds1 = iolib.fn_getds(fn1)
ds2 = iolib.fn_getds(fn2)
if geolib.srs_check(ds1) and geolib.srs_check(ds2):
ds1_clip_fn = os.path.join(outdir, os.path.splitext(os.path.basename(fn1))[0]+'_warp.tif')
ds2_clip_fn = os.path.join(outdir, os.path.splitext(os.path.basename(fn2))[0]+'_warp.tif')
if not os.path.exists(ds1_clip_fn) or not os.path.exists(ds2_clip_fn):
#This should write out files to new subdir
ds1_clip, ds2_clip = warplib.diskwarp_multi_fn([fn1, fn2], extent='intersection', res=res, r='average', outdir=outdir)
ds1_clip = None
ds2_clip = None
#However, if inputs have identical extent/res/proj, then link to original files
if not os.path.exists(ds1_clip_fn):
os.symlink(os.path.abspath(fn1), ds1_clip_fn)
if not os.path.exists(ds2_clip_fn):
os.symlink(os.path.abspath(fn2), ds2_clip_fn)
align = 'None'
#Mask support - limit correlation only to rock/ice surfaces, no water/veg
#This masks input images - guarantee we won't waste time correlating over vegetation
#TODO: Add support to load arbitrary raster or shp mask
if args.mask_input:
ds1_masked_fn = os.path.splitext(ds1_clip_fn)[0]+'_masked.tif'
ds2_masked_fn = os.path.splitext(ds2_clip_fn)[0]+'_masked.tif'
if not os.path.exists(ds1_masked_fn) or not os.path.exists(ds2_masked_fn):
#Load NLCD or bareground mask
from demcoreg.dem_mask import get_lulc_mask
ds1_clip = iolib.fn_getds(ds1_clip_fn)
lulc_mask_fn = os.path.join(outdir, 'lulc_mask.tif')
#if not os.path.exists(nlcd_mask_fn):
lulc_mask = get_lulc_mask(ds1_clip, mask_glaciers=False, filter='not_forest')
iolib.writeGTiff(lulc_mask, lulc_mask_fn, ds1_clip)
ds1_clip = None
#Now apply to original images
#This could be problematic for huge inputs, see apply_mask.py
#lulc_mask = lulc_mask.astype(int)
for fn in (ds1_clip_fn, ds2_clip_fn):
ds = iolib.fn_getds(fn)
a = iolib.ds_getma(ds)
a = np.ma.array(a, mask=~(lulc_mask))
if a.count() > 0:
out_fn = os.path.splitext(fn)[0]+'_masked.tif'
iolib.writeGTiff(a,out_fn,ds)
a = None
else:
sys.exit("No unmasked pixels over bare earth")
ds1_clip_fn = ds1_masked_fn
ds2_clip_fn = ds2_masked_fn
else:
ds1_clip_fn = fn1
ds2_clip_fn = fn2
#Now let user specify alignment methods as option - don't hardcode
#align = 'Homography'
#align = 'AffineEpipolar'
ds1 = None
ds2 = None
#Should have extra kwargs option here
stereo_opt = get_stereo_opt(threads=threads, kernel=kernel, timeout=timeout, \
erode=erode, spr=spr, align=align)
#Stereo arguments
#Latest version of ASP should accept tif without camera models
#stereo_args = [ds1_clip_fn, ds2_clip_fn, outprefix]
#Nope - still need to provide dummy camera models, and they must be unique files
#Use the dummy.tsai file bundled in the vmap repo
dummy_tsai = os.path.join(os.path.split(os.path.realpath(__file__))[0], 'dummy.tsai')
dummy_tsai2 = os.path.splitext(dummy_tsai)[0]+'2.tsai'
if not os.path.exists(dummy_tsai2):
dummy_tsai2 = os.symlink(dummy_tsai, os.path.splitext(dummy_tsai)[0]+'2.tsai')
stereo_args = [ds1_clip_fn, ds2_clip_fn, dummy_tsai, dummy_tsai2, outprefix]
#Run stereo_pprc
if not os.path.exists(outprefix+'-R_sub.tif'):
run_cmd('stereo_pprc', stereo_opt+stereo_args, msg='0: Preprocessing')
#Copy proj info to outputs, this should happen automatically now?
for ext in ('L', 'R', 'L_sub', 'R_sub', 'lMask', 'rMask', 'lMask_sub', 'rMask_sub'):
geolib.copyproj(ds1_clip_fn, '%s-%s.tif' % (outprefix,ext))
#Prepare seeding for stereo_corr
#TODO: these are untested after refactoring
if not os.path.exists(outprefix+'_D_sub.tif'):
#Don't need to do anything for default seed-mode 1
if seedmode == 'sparse_disp':
#Sparse correlation of full-res images
stereo_opt.extend(['--corr-seed-mode', '3'])
sparse_disp_opt = []
sparse_disp_opt.extend(['--Debug', '--coarse', '512', '--fine', '256', '--no_epipolar_fltr'])
sparse_disp_opt.extend(['-P', str(threads)])
sparse_disp_args = [outprefix+'-L.tif', outprefix+'-R.tif', outprefix]
run_cmd('sparse_disp', sparse_disp_opt+sparse_disp_args, msg='0.5: D_sub generation')
elif seedmode == 'existing_velocity':
#User-input low-res velocity maps for seeding
#TODO: Add functions that fetch best available velocities for Ant/GrIS or user-defined low-res velocities
#Automatically query GoLive velocities here
vx_fn = args.vx_fn
vy_fn = args.vy_fn
#Check for existence
#HMA seeding
vdir = '/nobackup/deshean/rpcdem/hma/velocity_jpl_amaury_2013-2015'
vx_fn = os.path.join(vdir, 'PKH_WRS2_B8_2013_2015_snr5_n1_r170_res12.x_vel.TIF')
vy_fn = os.path.join(vdir, 'PKH_WRS2_B8_2013_2015_snr5_n1_r170_res12.y_vel.TIF')
if os.path.exists(vx_fn) and os.path.exists(vy_fn):
ds1_clip = iolib.fn_getds(ds1_clip_fn)
ds1_res = geolib.get_res(ds1_clip, square=True)[0]
#Compute L_sub res - use this for output dimensions
L_sub_fn = outprefix+'-L_sub.tif'
L_sub_ds = gdal.Open(L_sub_fn)
L_sub_x_scale = float(ds1_clip.RasterXSize) / L_sub_ds.RasterXSize
L_sub_y_scale = float(ds1_clip.RasterYSize) / L_sub_ds.RasterYSize
L_sub_scale = np.max([L_sub_x_scale, L_sub_y_scale])
L_sub_res = ds1_res * L_sub_scale
#Since we are likely upsampling here, use cubicspline
vx_ds_clip, vy_ds_clip = warplib.memwarp_multi_fn([vx_fn, vy_fn], extent=ds1_clip, \
t_srs=ds1_clip, res=L_sub_res, r='cubicspline')
ds1_clip = None
#Get vx and vy arrays
vx = iolib.ds_getma(vx_ds_clip)
vy = iolib.ds_getma(vy_ds_clip)
#Determine time interval between inputs
#Use to scaling of known low-res velocities
t_factor = get_t_factor_fn(ds1_clip_fn, ds2_clip_fn, ds=vx_ds_clip)
if t_factor is not None:
#Compute expected offset in scaled pixels
dx = (vx*t_factor)/L_sub_res
dy = (vy*t_factor)/L_sub_res
#Note: Joughin and Rignot's values are positive y up!
#ASP is positive y down, so need to multiply these values by -1
#dy = -(vy*t_factor)/L_sub_res
#Should smooth/fill dx and dy
#If absolute search window is only 30x30
#Don't seed, just use fixed search window
#search_window_area_thresh = 900
search_window_area_thresh = 0
search_window = np.array([dx.min(), dy.min(), dx.max(), dy.max()])
dx_p = calcperc(dx, perc=(0.5, 99.5))
dy_p = calcperc(dy, perc=(0.5, 99.5))
search_window = np.array([dx_p[0], dy_p[0], dx_p[1], dy_p[1]])
search_window_area = (search_window[2]-search_window[0]) * (search_window[3]-search_window[1])
if search_window_area < search_window_area_thresh:
stereo_opt.extend(['--corr-seed-mode', '0'])
stereo_opt.append('--corr-search')
stereo_opt.extend([str(x) for x in search_window])
#pad_perc=0.1
#stereo_opt.extend(['--corr-sub-seed-percent', str(pad_perc)]
#Otherwise, generate a D_sub map from low-res velocity
else:
stereo_opt.extend(['--corr-seed-mode', '3'])
#This is relative to the D_sub scaled disparities
d_sub_fn = L_sub_fn.split('-L_sub')[0]+'-D_sub.tif'
gen_d_sub(d_sub_fn, dx, dy)
#If the above didn't generate a D_sub.tif for seeding, run stereo_corr to generate Low-res D_sub.tif
if not os.path.exists(outprefix+'-D_sub.tif'):
newopt = ['--compute-low-res-disparity-only',]
run_cmd('stereo_corr', newopt+stereo_opt+stereo_args, msg='1.1: Low-res Correlation')
#Copy projection info to D_sub
geolib.copyproj(outprefix+'-L_sub.tif', outprefix+'-D_sub.tif')
#Mask D_sub to limit correlation over bare earth surfaces
#This _should_ be a better approach than masking input images, but stereo_corr doesn't honor D_sub
#Still need to mask input images before stereo_pprc
#Left this in here for reference, or if this changes in ASP
if False:
D_sub_ds = gdal.Open(outprefix+'-D_sub.tif', gdal.GA_Update)
#Mask support - limit correlation only to rock/ice surfaces, no water/veg
from demcoreg.dem_mask import get_nlcd, mask_nlcd
nlcd_fn = get_nlcd()
nlcd_ds = warplib.diskwarp_multi_fn([nlcd_fn,], extent=D_sub_ds, res=D_sub_ds, t_srs=D_sub_ds, r='near', outdir=outdir)[0]
#validmask = mask_nlcd(nlcd_ds, valid='rock+ice')
validmask = mask_nlcd(nlcd_ds, valid='not_forest', mask_glaciers=False)
nlcd_mask_fn = os.path.join(outdir, 'nlcd_validmask.tif')
iolib.writeGTiff(validmask, nlcd_mask_fn, nlcd_ds)
#Now apply to D_sub (band 3 is valid mask)
#validmask = validmask.astype(int)
for b in (1,2,3):
dsub = iolib.ds_getma(D_sub_ds, b)
dsub = np.ma.array(dsub, mask=~(validmask))
D_sub_ds.GetRasterBand(b).WriteArray(dsub.filled())
D_sub_ds = None
#OK, finally run stereo_corr full-res integer correlation with appropriate seeding
if not os.path.exists(outprefix+'-D.tif'):
run_cmd('stereo_corr', stereo_opt+stereo_args, msg='1: Correlation')
geolib.copyproj(ds1_clip_fn, outprefix+'-D.tif')
#Run stereo_rfne
if spr > 0:
if not os.path.exists(outprefix+'-RD.tif'):
run_cmd('stereo_rfne', stereo_opt+stereo_args, msg='2: Refinement')
geolib.copyproj(ds1_clip_fn, outprefix+'-RD.tif')
d_fn = make_ln(outdir, outprefix, '-RD.tif')
else:
ln_fn = outprefix+'-RD.tif'
if os.path.lexists(ln_fn):
os.remove(ln_fn)
os.symlink(os.path.split(outprefix)[1]+'-D.tif', ln_fn)
#Run stereo_fltr
if not os.path.exists(outprefix+'-F.tif'):
run_cmd('stereo_fltr', stereo_opt+stereo_args, msg='3: Filtering')
geolib.copyproj(ds1_clip_fn, outprefix+'-F.tif')
d_fn = make_ln(outdir, outprefix, '-F.tif')
if smoothF and not os.path.exists(outprefix+'-F_smooth.tif'):
print('Smoothing F.tif')
from pygeotools.lib import filtlib
#Fill holes and smooth F
F_fill_fn = outprefix+'-F_smooth.tif'
F_ds = gdal.Open(outprefix+'-F.tif', gdal.GA_ReadOnly)
#import dem_downsample_fill
#F_fill_ds = dem_downsample_fill.gdalfill_ds(F_fill_ds)
print('Creating F_smooth.tif')
F_fill_ds = iolib.gtif_drv.CreateCopy(F_fill_fn, F_ds, 0, options=iolib.gdal_opt)
F_ds = None
for n in (1, 2):
print('Smoothing band %i' % n)
b = F_fill_ds.GetRasterBand(n)
b_fill_bma = iolib.b_getma(b)
#b_fill_bma = iolib.b_getma(dem_downsample_fill.gdalfill(b))
#Filter extreme values (careful, could lose areas of valid data with fastest v)
#b_fill_bma = filtlib.perc_fltr(b_fill_bma, perc=(0.01, 99.99))
#These filters remove extreme values and fill data gaps
#b_fill_bma = filtlib.median_fltr_skimage(b_fill_bma, radius=7, erode=0)
#b_fill_bma = filtlib.median_fltr(b_fill_bma, fsize=7, origmask=True)
#Gaussian filter
b_fill_bma = filtlib.gauss_fltr_astropy(b_fill_bma, size=9)
b.WriteArray(b_fill_bma)
F_fill_ds = None
d_fn = make_ln(outdir, outprefix, '-F_smooth.tif')
print('\n%s' % datetime.now())
print('%s UTC\n' % datetime.utcnow())
#If time interval is specified, convert pixel displacements to rates
if args.dt != 'none':
#Check if vm.tif already exists
#Should probably just overwrite by default
#if os.path.exists(os.path.splitext(d_fn)[0]+'_vm.tif'):
# print("\nFound existing velocity magnitude map!\n"
#else:
#Generate output velocity products and figure
#Requires that vmap repo is in PATH
cmd = ['disp2v.py', d_fn]
#Note: this will attempt to automatically determine control surfaces
#disp2v.py will accept arbitrary mask, could pass through here
if args.remove_offsets:
cmd.append('-remove_offsets')
cmd.extend(['-dt', args.dt])
print("Converting disparities to velocities")
print(cmd)
subprocess.call(cmd)
def main():
parser = getparser()
args = parser.parse_args()
src_fn = args.src_fn
if not iolib.fn_check(src_fn):
sys.exit("Unable to find src_fn: %s" % src_fn)
mask_fn = args.mask_fn
if not iolib.fn_check(mask_fn):
sys.exit("Unable to find mask_fn: %s" % mask_fn)
#Determine output extent, default is input raster extent
extent = args.extent
if extent == 'raster':
extent = src_fn
elif extent == 'mask':
extent = mask_fn
else:
#This is a hack for intersection computation
src_ds_list = [
gdal.Open(fn, gdal.GA_ReadOnly) for fn in [src_fn, mask_fn]
]
#t_srs = geolib.get_ds_srs(src_ds_list[0])
extent = warplib.parse_extent(extent, src_ds_list, src_fn)
print("Warping mask_fn")
mask_ds = warplib.memwarp_multi_fn([
mask_fn,
],
res=src_fn,
extent=extent,
t_srs=src_fn)[0]
print("Loading mask array")
mask_ma_full = iolib.ds_getma(mask_ds)
mask_ds = None
print("Extracting mask")
mask = np.ma.getmaskarray(mask_ma_full)
#Free up memory
mask_ma_full = None
#Add dilation step for buffer
#newmask = np.logical_or(np.ma.getmaskarray(src_ma_full), mask)
if args.invert:
print("Inverting mask")
mask = ~(mask)
print("Loading src array and applying updated mask")
if extent == src_fn:
src_ds = gdal.Open(src_fn)
else:
src_ds = warplib.memwarp_multi_fn([
src_fn,
],
res=src_fn,
extent=extent,
t_srs=src_fn)[0]
#Now load source array with new mask
src_ma_full = np.ma.array(iolib.ds_getma(src_ds), mask=mask)
mask = None
if args.out_fn is not None:
src_fn_masked = args.out_fn
else:
src_fn_masked = os.path.splitext(src_fn)[0] + '_masked.tif'
print("Writing out masked version of input raster: %s" % src_fn_masked)
iolib.writeGTiff(src_ma_full, src_fn_masked, src_ds, create=True)
def mb_calc(gf, z1_date=z1_date, z2_date=z2_date, verbose=verbose):
#print("\n%i of %i: %s\n" % (n+1, len(glacfeat_list), gf.feat_fn))
print(gf.feat_fn)
#This should already be handled by earlier attribute filter, but RGI area could be wrong
#24k shp has area in m^2, RGI in km^2
#if gf.glac_area/1E6 < min_glac_area:
if gf.glac_area < min_glac_area:
if verbose:
print("Glacier area below %0.1f km2 threshold" % min_glac_area)
return None
#Warp everything to common res/extent/proj
ds_list = warplib.memwarp_multi_fn([z1_fn, z2_fn], res='min', \
extent=gf.glac_geom_extent, t_srs=aea_srs, verbose=verbose)
if site == 'conus':
#Add prism datasets
prism_fn_list = [prism_ppt_annual_fn, prism_tmean_annual_fn]
prism_fn_list.extend([
prism_ppt_summer_fn, prism_ppt_winter_fn, prism_tmean_summer_fn,
prism_tmean_winter_fn
])
ds_list.extend(warplib.memwarp_multi_fn(prism_fn_list, res=ds_list[0], \
extent=gf.glac_geom_extent, t_srs=aea_srs, verbose=verbose))
if site == 'hma':
#Add debris cover datasets
#Should tar this up, and extract only necessary file
#Downloaded from: http://mountainhydrology.org/data-nature-2017/
kra_nature_dir = '/nobackup/deshean/data/Kraaijenbrink_hma/regions/out'
#This assumes that numbers are identical between RGI50 and RGI60
debris_class_fn = os.path.join(
kra_nature_dir, 'RGI50-%s/classification.tif' % gf.glacnum)
debris_thick_fn = os.path.join(
kra_nature_dir, 'RGI50-%s/debris-thickness-50cm.tif' % gf.glacnum)
ice_thick_fn = os.path.join(kra_nature_dir,
'RGI50-%s/ice-thickness.tif' % gf.glacnum)
hma_fn_list = []
if os.path.exists(debris_class_fn):
hma_fn_list.append(debris_class_fn)
if os.path.exists(debris_thick_fn):
hma_fn_list.append(debris_thick_fn)
if os.path.exists(ice_thick_fn):
hma_fn_list.append(ice_thick_fn)
if len(hma_fn_list) > 0:
#Add velocity
hma_fn_list.extend([vx_fn, vy_fn])
ds_list.extend(warplib.memwarp_multi_fn(hma_fn_list, res=ds_list[0], \
extent=gf.glac_geom_extent, t_srs=aea_srs, verbose=verbose))
#Check to see if z2 is empty, as z1 should be continuous
gf.z2 = iolib.ds_getma(ds_list[1])
if gf.z2.count() == 0:
if verbose:
print("No z2 pixels")
return None
glac_geom_mask = geolib.geom2mask(gf.glac_geom, ds_list[0])
gf.z1 = np.ma.array(iolib.ds_getma(ds_list[0]), mask=glac_geom_mask)
#Apply SRTM penetration correction
if z1_srtm_penetration_corr:
gf.z1 = srtm_corr(gf.z1)
if z2_srtm_penetration_corr:
gf.z2 = srtm_corr(gf.z2)
gf.z2 = np.ma.array(gf.z2, mask=glac_geom_mask)
gf.dz = gf.z2 - gf.z1
if gf.dz.count() == 0:
if verbose:
print("No valid dz pixels")
return None
#Should add better filtering here
#Elevation dependent abs. threshold filter?
filter_outliers = True
#Remove clearly bogus pixels
if filter_outliers:
bad_perc = (0.1, 99.9)
#bad_perc = (1, 99)
rangelim = malib.calcperc(gf.dz, bad_perc)
gf.dz = np.ma.masked_outside(gf.dz, *rangelim)
gf.res = geolib.get_res(ds_list[0])
valid_area = gf.dz.count() * gf.res[0] * gf.res[1]
valid_area_perc = valid_area / gf.glac_area
if valid_area_perc < min_valid_area_perc:
if verbose:
print(
"Not enough valid pixels. %0.1f%% percent of glacier polygon area"
% (100 * valid_area_perc))
return None
#Filter dz - throw out abs differences >150 m
#Compute dz, volume change, mass balance and stats
gf.z1_stats = malib.get_stats(gf.z1)
gf.z2_stats = malib.get_stats(gf.z2)
z2_elev_med = gf.z2_stats[5]
z2_elev_p16 = gf.z2_stats[11]
z2_elev_p84 = gf.z2_stats[12]
#Caluclate stats for aspect and slope using z2
#Requires GDAL 2.1+
gf.z2_aspect = np.ma.array(geolib.gdaldem_mem_ds(ds_list[1],
processing='aspect',
returnma=True),
mask=glac_geom_mask)
gf.z2_aspect_stats = malib.get_stats(gf.z2_aspect)
z2_aspect_med = gf.z2_aspect_stats[5]
gf.z2_slope = np.ma.array(geolib.gdaldem_mem_ds(ds_list[1],
processing='slope',
returnma=True),
mask=glac_geom_mask)
gf.z2_slope_stats = malib.get_stats(gf.z2_slope)
z2_slope_med = gf.z2_slope_stats[5]
#Rasterize source dates
if z1_date is None:
z1_date = get_date_a(ds_list[0], z1_date_shp_lyr, glac_geom_mask,
z1_datefield)
gf.t1 = z1_date.mean()
else:
gf.t1 = z1_date
if z2_date is None:
z2_date = get_date_a(ds_list[0], z2_date_shp_lyr, glac_geom_mask,
z2_datefield)
#Attempt to use YYYYMMDD string
#z2_dta = np.datetime64(z2_date.astype("S8").tolist())
gf.t2 = z2_date.mean()
else:
gf.t2 = z2_date
if isinstance(gf.t1, datetime):
gf.t1 = timelib.dt2decyear(gf.t1)
if isinstance(gf.t2, datetime):
gf.t2 = timelib.dt2decyear(gf.t2)
gf.t1 = float(gf.t1)
gf.t2 = float(gf.t2)
#Calculate dt grids
#gf.dt = z2_date - z1_date
#gf.dt = gf.dt.mean()
#This should be decimal years
gf.dt = gf.t2 - gf.t1
#if isinstance(gf.dt, timedelta):
# gf.dt = gf.dt.total_seconds()/timelib.spy
#Calculate dh/dt, in m/yr
gf.dhdt = gf.dz / gf.dt
gf.dhdt_stats = malib.get_stats(gf.dhdt)
dhdt_mean = gf.dhdt_stats[3]
dhdt_med = gf.dhdt_stats[5]
rho_i = 0.91
rho_s = 0.50
rho_f = 0.60
#This is recommendation by Huss et al (2013)
rho_is = 0.85
rho_sigma = 0.06
#Can estimate ELA values computed from hypsometry and typical AAR
#For now, assume ELA is mean
gf.z1_ela = None
gf.z1_ela = gf.z1_stats[3]
gf.z2_ela = gf.z2_stats[3]
#Note: in theory, the ELA should get higher with mass loss
#In practice, using mean and same polygon, ELA gets lower as glacier surface thins
if verbose:
print("ELA(t1): %0.1f" % gf.z1_ela)
print("ELA(t2): %0.1f" % gf.z2_ela)
if gf.z1_ela > gf.z2_ela:
min_ela = gf.z2_ela
max_ela = gf.z1_ela
else:
min_ela = gf.z1_ela
max_ela = gf.z2_ela
#Calculate mass balance map from dhdt
gf.mb = gf.dhdt * rho_is
"""
# This attempted to assign different densities above and below ELA
if gf.z1_ela is None:
gf.mb = gf.dhdt * rho_is
else:
#Initiate with average density
gf.mb = gf.dhdt*(rho_is + rho_f)/2.
#Everything that is above ELA at t2 is elevation change over firn, use firn density
accum_mask = (gf.z2 > gf.z2_ela).filled(0).astype(bool)
gf.mb[accum_mask] = (gf.dhdt*rho_f)[accum_mask]
#Everything that is below ELA at t1 is elevation change over ice, use ice density
abl_mask = (gf.z1 <= gf.z1_ela).filled(0).astype(bool)
gf.mb[abl_mask] = (gf.dhdt*rho_is)[abl_mask]
#Everything in between, use average of ice and firn density
#mb[(z1 > z1_ela) || (z2 <= z2_ela)] = dhdt*(rhois + rho_f)/2.
#Linear ramp
#rho_f + z2*((rho_is - rho_f)/(z2_ela - z1_ela))
#mb = np.where(dhdt < ela, dhdt*rho_i, dhdt*rho_s)
"""
#Use this for winter balance
#mb = dhdt * rho_s
gf.mb_stats = malib.get_stats(gf.mb)
gf.mb_mean = gf.mb_stats[3]
#Calculate uncertainty of total elevation change
#TODO: Better spatial distribution characterization
#Add slope-dependent component here
dz_sigma = np.sqrt(z1_sigma**2 + z2_sigma**2)
#Uncrtainty of dh/dt
dhdt_sigma = dz_sigma / gf.dt
#This is mb uncertainty map
gf.mb_sigma = np.ma.abs(gf.mb) * np.sqrt((rho_sigma / rho_is)**2 +
(dhdt_sigma / gf.dhdt)**2)
gf.mb_sigma_stats = malib.get_stats(gf.mb_sigma)
#This is average mb uncertainty
gf.mb_mean_sigma = gf.mb_sigma_stats[3]
#Now calculate mb for entire polygon
area_sigma_perc = 0.09
gf.mb_mean_totalarea = gf.mb_mean * gf.glac_area
#Already have area uncertainty as percentage, just use directly
gf.mb_mean_totalarea_sigma = np.ma.abs(gf.mb_mean_totalarea) * np.sqrt(
(gf.mb_mean_sigma / gf.mb_mean)**2 + area_sigma_perc**2)
mb_sum = np.sum(gf.mb) * gf.res[0] * gf.res[1]
outlist = [gf.glacnum, gf.cx, gf.cy, z2_elev_med, z2_elev_p16, z2_elev_p84, z2_slope_med, z2_aspect_med, \
gf.mb_mean, gf.mb_mean_sigma, gf.glac_area, gf.mb_mean_totalarea, gf.mb_mean_totalarea_sigma, \
gf.t1, gf.t2, gf.dt]
if site == 'conus':
prism_ppt_annual = np.ma.array(iolib.ds_getma(ds_list[2]),
mask=glac_geom_mask) / 1000.
prism_ppt_annual_stats = malib.get_stats(prism_ppt_annual)
prism_ppt_annual_mean = prism_ppt_annual_stats[3]
prism_tmean_annual = np.ma.array(iolib.ds_getma(ds_list[3]),
mask=glac_geom_mask)
prism_tmean_annual_stats = malib.get_stats(prism_tmean_annual)
prism_tmean_annual_mean = prism_tmean_annual_stats[3]
outlist.extend([prism_ppt_annual_mean, prism_tmean_annual_mean])
#This is mean monthly summer precip, need to multiply by nmonths to get cumulative
n_summer = 4
prism_ppt_summer = n_summer * np.ma.array(iolib.ds_getma(ds_list[4]),
mask=glac_geom_mask) / 1000.
prism_ppt_summer_stats = malib.get_stats(prism_ppt_summer)
prism_ppt_summer_mean = prism_ppt_summer_stats[3]
n_winter = 8
prism_ppt_winter = n_winter * np.ma.array(iolib.ds_getma(ds_list[5]),
mask=glac_geom_mask) / 1000.
prism_ppt_winter_stats = malib.get_stats(prism_ppt_winter)
prism_ppt_winter_mean = prism_ppt_winter_stats[3]
prism_tmean_summer = np.ma.array(iolib.ds_getma(ds_list[6]),
mask=glac_geom_mask)
prism_tmean_summer_stats = malib.get_stats(prism_tmean_summer)
prism_tmean_summer_mean = prism_tmean_summer_stats[3]
prism_tmean_winter = np.ma.array(iolib.ds_getma(ds_list[7]),
mask=glac_geom_mask)
prism_tmean_winter_stats = malib.get_stats(prism_tmean_winter)
prism_tmean_winter_mean = prism_tmean_winter_stats[3]
outlist.extend([
prism_ppt_summer_mean, prism_ppt_winter_mean,
prism_tmean_summer_mean, prism_tmean_winter_mean
])
if site == 'hma':
#Classes are: 1 = clean ice, 2 = debris, 3 = pond
#Load up debris cover maps, ice thickness
if len(ds_list) > 2:
gf.debris_class = np.ma.array(iolib.ds_getma(ds_list[2]),
mask=glac_geom_mask)
gf.debris_thick = np.ma.array(iolib.ds_getma(ds_list[3]),
mask=glac_geom_mask)
#Load ice thickness from glabtop2
gf.H = np.ma.array(iolib.ds_getma(ds_list[4]), mask=glac_geom_mask)
#Load surface velocity maps from Dehecq
gf.vx = np.ma.array(iolib.ds_getma(ds_list[5]),
mask=glac_geom_mask)
gf.vy = np.ma.array(iolib.ds_getma(ds_list[6]),
mask=glac_geom_mask)
gf.vm = np.ma.sqrt(gf.vx**2 + gf.vy**2)
v_col_factor = 0.8
#Should smooth, better handling of data gaps
gf.divU = np.gradient(v_col_factor * gf.vx)[1] + np.gradient(
v_col_factor * gf.vy)[0]
gf.divQ = gf.H * gf.divU
#Compute debris/pond/clean percentages for entire polygon
if gf.debris_class.count() > 0:
gf.perc_clean = 100. * (gf.debris_class
== 1).sum() / gf.debris_class.count()
gf.perc_debris = 100. * (gf.debris_class
== 2).sum() / gf.debris_class.count()
gf.perc_pond = 100. * (gf.debris_class
== 3).sum() / gf.debris_class.count()
outlist.extend([
gf.H.mean(),
gf.debris_thick.mean(), gf.perc_debris, gf.perc_pond,
gf.perc_clean
])
if verbose:
print('Mean mb: %0.2f +/- %0.2f mwe/yr' %
(gf.mb_mean, gf.mb_mean_sigma))
print('Sum/Area mb: %0.2f mwe/yr' % (mb_sum / gf.glac_area))
print('Mean mb * Area: %0.2f +/- %0.2f mwe/yr' %
(gf.mb_mean_totalarea, gf.mb_mean_totalarea_sigma))
print('Sum mb: %0.2f mwe/yr' % mb_sum)
#print('-------------------------------')
#Write to master list
#out.append(outlist)
#Write to temporary file
#writer.writerow(outlist)
#f.flush()
if writeout and (gf.glac_area / 1E6 > min_glac_area_writeout):
out_dz_fn = os.path.join(outdir, gf.feat_fn + '_dz.tif')
iolib.writeGTiff(gf.dz, out_dz_fn, ds_list[0])
out_z1_fn = os.path.join(outdir, gf.feat_fn + '_z1.tif')
iolib.writeGTiff(gf.z1, out_z1_fn, ds_list[0])
out_z2_fn = os.path.join(outdir, gf.feat_fn + '_z2.tif')
iolib.writeGTiff(gf.z2, out_z2_fn, ds_list[0])
temp_fn = os.path.join(outdir, gf.feat_fn + '_z2_aspect.tif')
iolib.writeGTiff(gf.z2_aspect, temp_fn, ds_list[0])
temp_fn = os.path.join(outdir, gf.feat_fn + '_z2_slope.tif')
iolib.writeGTiff(gf.z2_slope, temp_fn, ds_list[0])
#Need to fix this - write out constant date arrays regardless of source
#out_z1_date_fn = os.path.join(outdir, gf.feat_fn+'_ned_date.tif')
#iolib.writeGTiff(z1_date, out_z1_date_fn, ds_list[0])
if site == 'conus':
out_prism_ppt_annual_fn = os.path.join(
outdir, gf.feat_fn + '_precip_annual.tif')
iolib.writeGTiff(prism_ppt_annual, out_prism_ppt_annual_fn,
ds_list[0])
out_prism_tmean_annual_fn = os.path.join(
outdir, gf.feat_fn + '_tmean_annual.tif')
iolib.writeGTiff(prism_tmean_annual, out_prism_tmean_annual_fn,
ds_list[0])
out_prism_ppt_summer_fn = os.path.join(
outdir, gf.feat_fn + '_precip_summer.tif')
iolib.writeGTiff(prism_ppt_summer, out_prism_ppt_summer_fn,
ds_list[0])
out_prism_ppt_winter_fn = os.path.join(
outdir, gf.feat_fn + '_precip_winter.tif')
iolib.writeGTiff(prism_ppt_winter, out_prism_ppt_winter_fn,
ds_list[0])
out_prism_tmean_summer_fn = os.path.join(
outdir, gf.feat_fn + '_tmean_summer.tif')
iolib.writeGTiff(prism_tmean_summer, out_prism_tmean_summer_fn,
ds_list[0])
out_prism_tmean_winter_fn = os.path.join(
outdir, gf.feat_fn + '_tmean_winter.tif')
iolib.writeGTiff(prism_tmean_winter, out_prism_tmean_winter_fn,
ds_list[0])
if site == 'hma':
if gf.H is not None:
temp_fn = os.path.join(outdir, gf.feat_fn + '_H.tif')
iolib.writeGTiff(gf.H, temp_fn, ds_list[0])
if gf.debris_thick is not None:
temp_fn = os.path.join(outdir,
gf.feat_fn + '_debris_thick.tif')
iolib.writeGTiff(gf.debris_thick, temp_fn, ds_list[0])
if gf.debris_class is not None:
temp_fn = os.path.join(outdir,
gf.feat_fn + '_debris_class.tif')
iolib.writeGTiff(gf.debris_class, temp_fn, ds_list[0])
if gf.vm is not None:
temp_fn = os.path.join(outdir, gf.feat_fn + '_vm.tif')
iolib.writeGTiff(gf.vm, temp_fn, ds_list[0])
if gf.divQ is not None:
temp_fn = os.path.join(outdir, gf.feat_fn + '_divQ.tif')
iolib.writeGTiff(gf.divQ, temp_fn, ds_list[0])
#Do AED for all
#Compute mb using scaled AED vs. polygon
#Check for valid pixel count vs. feature area, fill if appropriate
if mb_plot and (gf.glac_area / 1E6 > min_glac_area_writeout):
z_bin_edges = hist_plot(gf, outdir)
gf.z1_hs = geolib.gdaldem_mem_ds(ds_list[0],
processing='hillshade',
returnma=True)
gf.z2_hs = geolib.gdaldem_mem_ds(ds_list[1],
processing='hillshade',
returnma=True)
map_plot(gf, z_bin_edges, outdir)
return outlist, gf
def main():
parser = getparser()
args = parser.parse_args()
t_unit = args.dt
plot = args.plot
remove_offsets = args.remove_offsets
mask_fn = args.mask_fn
if mask_fn is not None:
remove_offsets = True
#Input is 3-band disparity map, extract bands directly
src_fn = args.disp_fn
if not iolib.fn_check(src_fn):
sys.exit("Unable to locate input file: %s" % src_fn)
src_ds = iolib.fn_getds(src_fn)
if src_ds.RasterCount != 3:
sys.exit("Input file must be ASP disparity map (3 bands: x, y, mask)")
#Extract pixel resolution
h_res, v_res = geolib.get_res(src_ds)
#Horizontal scale factor
#If running on disparity_view output (gdal_translate -outsize 5% 5% F.tif F_5.tif)
#h_res /= 20
#v_res /= 20
#Load horizontal and vertical disparities
h = iolib.ds_getma(src_ds, bnum=1)
v = iolib.ds_getma(src_ds, bnum=2)
#ASP output has northward motion as negative values in band 2
v *= -1
t1, t2 = timelib.fn_getdatetime_list(src_fn)
dt = t2 - t1
#Default t_factor is in 1/years
t_factor = timelib.get_t_factor(t1, t2)
#Input timestamp arrays if inputs are mosaics
if False:
t1_fn = ''
t2_fn = ''
if os.path.exists(t1_fn) and os.path.exists(t2_fn):
t_factor = timelib.get_t_factor_fn(t1_fn, t2_fn)
if t_factor is None:
sys.exit("Unable to determine input timestamps")
if t_unit == 'day':
t_factor *= 365.25
print("Input dates:")
print(t1)
print(t2)
print(dt)
print(t_factor, t_unit)
#Scale values for polar stereographic distortion
srs = geolib.get_ds_srs(src_ds)
proj_scale_factor = 1.0
#Want to scale to get correct distances for polar sterographic
if srs.IsSame(geolib.nps_srs) or srs.IsSame(geolib.sps_srs):
proj_scale_factor = geolib.scale_ps_ds(src_ds)
#Convert disparity values in pixels to m/t_unit
h_myr = h * h_res * proj_scale_factor / t_factor
h = None
v_myr = v * v_res * proj_scale_factor / t_factor
v = None
#Velocity Magnitude
m = np.ma.sqrt(h_myr**2 + v_myr**2)
print("Velocity Magnitude stats")
malib.print_stats(m)
#Remove x and y offsets over control surfaces
offset_str = ''
if remove_offsets:
if mask_fn is None:
from demcoreg.dem_mask import get_mask
print(
"\nUsing demcoreg to prepare mask of stable control surfaces\n"
)
#TODO: Accept mask_list as in demcoreg
#mask_list = args.mask_list
# for now keep it simple, limit to non-glacier surfaces
mask_list = [
'glaciers',
]
mask = get_mask(src_ds, mask_list=mask_list, dem_fn=src_fn)
else:
print("\nWarping input raster mask")
#This can be from previous dem_mask.py run (e.g. *rockmask.tif)
mask_ds = warplib.memwarp_multi_fn([
mask_fn,
],
res=src_ds,
extent=src_ds,
t_srs=src_ds)[0]
mask = iolib.ds_getma(mask_ds)
#The default from ds_getma is a masked array, so need to isolate boolean mask
#Assume input is 0 for masked, 1 for unmasked (valid control surface)
mask = mask.filled().astype('bool')
#This should work, as the *rockmask.py is 1 for unmasked, 0 for masked, with ndv=0
#mask = np.ma.getmaskarray(mask)
#Vector mask - untested
if os.path.splitext(mask_fn)[1] == 'shp':
mask = geolib.shp2array(mask_fn, src_ds)
print("\nRemoving median x and y offset over static control surfaces")
h_myr_count = h_myr.count()
h_myr_static_count = np.ma.array(h_myr, mask=mask).count()
h_myr_mad, h_myr_med = malib.mad(np.ma.array(h_myr, mask=mask),
return_med=True)
v_myr_mad, v_myr_med = malib.mad(np.ma.array(v_myr, mask=mask),
return_med=True)
print("Static pixel count: %i (%0.1f%%)" %
(h_myr_static_count,
100 * float(h_myr_static_count) / h_myr_count))
print("median (+/-NMAD)")
print("x velocity offset: %0.2f (+/-%0.2f) m/%s" %
(h_myr_med, h_myr_mad, t_unit))
print("y velocity offset: %0.2f (+/-%0.2f) m/%s" %
(v_myr_med, v_myr_mad, t_unit))
h_myr -= h_myr_med
v_myr -= v_myr_med
offset_str = '_offsetcorr_h%0.2f_v%0.2f' % (h_myr_med, v_myr_med)
#Velocity Magnitude
m = np.ma.sqrt(h_myr**2 + v_myr**2)
print("Velocity Magnitude stats after correction")
malib.print_stats(m)
if plot:
fig_fn = os.path.splitext(src_fn)[0] + '.png'
label = 'Velocity (m/%s)' % t_unit
f, ax = make_plot(m, fig_fn, label)
plotvec(h_myr, v_myr)
plt.tight_layout()
plt.savefig(fig_fn,
dpi=300,
bbox_inches='tight',
pad_inches=0,
edgecolor='none')
print("Writing out files")
gt = src_ds.GetGeoTransform()
proj = src_ds.GetProjection()
dst_fn = os.path.splitext(src_fn)[0] + '_vm%s.tif' % offset_str
iolib.writeGTiff(m, dst_fn, create=True, gt=gt, proj=proj)
dst_fn = os.path.splitext(src_fn)[0] + '_vx%s.tif' % offset_str
iolib.writeGTiff(h_myr, dst_fn, create=True, gt=gt, proj=proj)
dst_fn = os.path.splitext(src_fn)[0] + '_vy%s.tif' % offset_str
iolib.writeGTiff(v_myr, dst_fn, create=True, gt=gt, proj=proj)
src_ds = None
main_glac_thickness['RGIId'] = main_glac_rgi.RGIId.values
main_glac_width['RGIId'] = main_glac_rgi.RGIId.values
main_glac_length['RGIId'] = main_glac_rgi.RGIId.values
main_glac_slope['RGIId'] = main_glac_rgi.RGIId.values
# ===== PROCESS EACH GLACIER ======
for nglac, glacno in enumerate(glacno_list):
# print(nglac, glacno)
thickness_fn = thickness_fp + 'RGI60-' + glacno + '_thickness.tif'
dem_farinotti_fn = dem_farinotti_fp + 'surface_DEM_RGI60-' + glacno + '.tif'
# Reproject, resample, warp rasters to common extent, grid size, etc.
# note: use thickness for the reference to avoid unrealistic extrapolations, e.g., negative thicknesses
# also using equal area increases areas significantly compared to RGI
raster_fn_list = [dem_ref_fn, dem_farinotti_fn, thickness_fn]
ds_list = warplib.memwarp_multi_fn(raster_fn_list, extent='intersection', res='min', t_srs=thickness_fn)
# print('\n\nSWITCH BACK TO THICKNESS_FN AFTER OTHERS CORRECTED!\n\n')
# ds_list = warplib.memwarp_multi_fn(raster_fn_list, extent='intersection', res='min', t_srs=dem_ref_fn)
# masked arrays using ice thickness estimates
dem_ref_raw, dem_far_raw, thickness = [iolib.ds_getma(i) for i in ds_list]
dem_ref = dem_ref_raw.copy()
dem_ref.mask = thickness.mask
dem_far = dem_far_raw.copy()
dem_far.mask = thickness.mask
# DEM selection for binning computations
# if exceeds threshold, then use the reference
if (abs(main_glac_rgi.loc[nglac,'Zmin'] - dem_far.min()) > dem_poorquality_threshold or
abs(main_glac_rgi.loc[nglac,'Zmax'] - dem_far.max()) > dem_poorquality_threshold):
print(' Check Glacier ' + glacno + ': use Christian DEM instead of Farinotti')
def main():
parser = getparser()
args = parser.parse_args()
src_fn = args.src_fn
if not iolib.fn_check(src_fn):
sys.exit("Unable to find src_fn: %s" % src_fn)
mask_fn = args.mask_fn
if not iolib.fn_check(mask_fn):
sys.exit("Unable to find mask_fn: %s" % mask_fn)
#Determine output extent, default is input raster extent
extent = args.extent
if extent == 'raster':
extent = src_fn
elif extent == 'mask':
extent = mask_fn
else:
#This is a hack for intersection computation
src_ds_list = [
gdal.Open(fn, gdal.GA_ReadOnly) for fn in [src_fn, mask_fn]
]
#t_srs = geolib.get_ds_srs(src_ds_list[0])
extent = warplib.parse_extent(extent, src_ds_list, src_fn)
#Set resampling algorithm appropriately
r = 'cubic'
mask_val = args.mask_val
if mask_val is not None:
r = 'near'
print("Warping mask_fn")
mask_ds = warplib.memwarp_multi_fn([
mask_fn,
],
res=src_fn,
extent=extent,
t_srs=src_fn,
r=r)[0]
print("Loading mask array")
mask_ma_full = iolib.ds_getma(mask_ds)
mask_ds = None
print("Extracting mask")
if mask_val is not None:
#Use specified value
mask = ~((mask_ma_full == mask_val).data)
elif mask_ma_full.std() != 0:
#Input mask filename is a raster, or other masked array
#Just need to extract mask
mask = np.ma.getmaskarray(mask_ma_full)
else:
#Input mask filename is a mask, use directly
#If input mask values are zero, valid values are nonzero
#Bool True == 1, so need to invert
if mask_ma_full.fill_value == 0:
mask = ~((mask_ma_full.data).astype(bool))
else:
mask = (mask_ma_full.data).astype(bool)
#Free up memory
mask_ma_full = None
#Add dilation step for buffer
#newmask = np.logical_or(np.ma.getmaskarray(src_ma_full), mask)
if args.invert:
print("Inverting mask")
mask = ~(mask)
print("Loading src array and applying updated mask")
if extent == src_fn:
src_ds = gdal.Open(src_fn)
else:
src_ds = warplib.memwarp_multi_fn([
src_fn,
],
res=src_fn,
extent=extent,
t_srs=src_fn)[0]
#Now load source array with new mask
src_ma_full = np.ma.array(iolib.ds_getma(src_ds), mask=mask)
mask = None
if args.out_fn is not None:
src_fn_masked = args.out_fn
else:
src_fn_masked = os.path.splitext(src_fn)[0] + '_masked.tif'
print("Writing out masked version of input raster: %s" % src_fn_masked)
iolib.writeGTiff(src_ma_full, src_fn_masked, src_ds, create=True)
def crop_sim_res_extent(img_list, outfol, vrt=False, rpc=False):
"""
Warp images to common 'finest' resolution and intersecting extent
This is useful for stereo processing with mapprojected imagery with the skysat pairs
Parameters
----------
img_list: list
list containing two images
outfol: str
path to folder where warped images will be saved
vrt: bool
Produce warped VRT instead of geotiffs if True
rpc: bool
Copy RPC information to warped images if True
Returns
----------
out: list
list containing the two warped images, first entry (left image) is the image which was of finer resolution (more nadir) initially
If the images do not intersect, two None objects are returned in the list
"""
resample_alg = 'lanczos'
img1 = img_list[0]
img2 = img_list[1]
img1_ds = iolib.fn_getds(img1)
img2_ds = iolib.fn_getds(img2)
res1 = geolib.get_res(img1_ds, square=True)[0]
res2 = geolib.get_res(img2_ds, square=True)[0]
# set left image as higher resolution, this is repeated for video, but
# good for triplet with no gsd information
if res1 < res2:
l_img = img1
r_img = img2
res = res1
else:
l_img = img2
r_img = img1
res = res2
# ASP stereo command expects the input to be .tif/.tiff, complains for .vrt
# Try to save with vrt driver but a tif extension ?
l_img_warp = os.path.join(
outfol,
os.path.splitext(os.path.basename(l_img))[0] + '_warp.tif')
r_img_warp = os.path.join(
outfol,
os.path.splitext(os.path.basename(r_img))[0] + '_warp.tif')
if not (os.path.exists(l_img_warp)):
# can turn on verbose during qa/qc
# Better to turn off during large runs, writing takes time
verbose = False
if not os.path.exists(outfol):
os.makedirs(outfol)
try:
#this will simply break and continue if the images do not intersect
ds_list = warplib.memwarp_multi_fn([l_img, r_img],
r=resample_alg,
verbose=verbose,
res='min',
extent='intersection')
if vrt:
extent = geolib.ds_extent(ds_list[0])
res = geolib.get_res(ds_list[0], square=True)
vrt_options = gdal.BuildVRTOptions(resampleAlg='average',
resolution='user',
xRes=res[0],
yRes=res[1],
outputBounds=tuple(extent))
l_vrt = gdal.BuildVRT(l_img_warp, [
l_img,
],
options=vrt_options)
r_vrt = gdal.BuildVRT(r_img_warp, [
r_img,
],
options=vrt_options)
# close vrt to save to disk
l_vrt = None
r_vrt = None
out = [l_img_warp, r_img_warp]
else:
# I am opting out of writing out vrt, to prevent correlation
# artifacts. GeoTiffs will be written out in the meantime
l_img_ma = iolib.ds_getma(ds_list[0])
r_img_ma = iolib.ds_getma(ds_list[1])
iolib.writeGTiff(l_img_ma, l_img_warp, ds_list[0])
iolib.writeGTiff(r_img_ma, r_img_warp, ds_list[1])
out = [l_img_warp, r_img_warp]
del (ds_list)
if rpc:
copy_rpc(l_img, l_img_warp)
copy_rpc(r_img, r_img_warp)
except BaseException:
out = None
else:
out = [l_img_warp, r_img_warp]
return out
#def main():
parser = getparser()
args = parser.parse_args()
dem1_fn = args.dem1_fn
dem1_ts = timelib.fn_getdatetime(dem1_fn)
res = 'min'
save = True
#For testing
#res = 64
if args.dem2_fn is not None:
dem2_fn = args.dem2_fn
print("Warping DEMs to same res/extent/proj")
#This will check input param for validity, could do beforehand
dem1_ds, dem2_ds = warplib.memwarp_multi_fn([dem1_fn, dem2_fn], extent='intersection', res=res, t_srs='first')
print("Loading input DEMs into masked arrays")
dem1 = iolib.ds_getma(dem1_ds, 1)
dem2 = iolib.ds_getma(dem2_ds, 1)
dem2_ts = timelib.fn_getdatetime(dem2_fn)
dz = dem2 - dem1
outprefix = os.path.splitext(os.path.split(dem1_fn)[1])[0]+'_'+os.path.splitext(os.path.split(dem2_fn)[1])[0]
elif args.dz_fn is not None:
dz_fn = args.dz_fn
dem1_ds, dz_ds = warplib.memwarp_multi_fn([dem1_fn, dz_fn], extent='intersection', res=res, t_srs='first')
print("Loading input DEM and Snow depth into masked arrays")
dem1 = iolib.ds_getma(dem1_ds, 1)
dz = iolib.ds_getma(dz_ds, 1)
#Try to pull out second timestamp from dz_fn
dem2_ts = timelib.fn_getdatetime_list(dz_fn)[-1]
outprefix = os.path.splitext(os.path.split(dz_fn)[1])[0]
xlim = (100000, 300000)
b[:, 1][(b[:, 1] > xlim[1]) | (b[:, 1] < xlim[0])] = np.ma.masked
ylim = (4100000, 4500000)
b[:, 2][(b[:, 2] > ylim[1]) | (b[:, 2] < ylim[0])] = np.ma.masked
#Only pass pits with valid x and y coord
b = b[b[:, 1:3].count(axis=1) == 2]
#Stereo2SWE preliminary products
dem_fn = '/Users/dshean/Documents/UW/SnowEx/preliminary_mos_20170504/gm_8m-tile-0.tif'
hs_fn = '/Users/dshean/Documents/UW/SnowEx/preliminary_mos_20170504/gm_8m-tile-0_hs_az315.tif'
#snowdepth_fn = '/Users/dshean/Documents/UW/SnowEx/preliminary_snowdepth_20170606/snowdepth_20170201-20170317_mos-tile-0.tif'
snowdepth_fn = '/Users/dshean/Documents/UW/SnowEx/preliminary_snowdepth_20170606/snowdepth_tif/snowdepth_20170201-20170317_mos-tile-0_filt5px.tif'
#Load and clip to common extent
dem_ds, hs_ds, snowdepth_ds = warplib.memwarp_multi_fn(
[dem_fn, hs_fn, snowdepth_fn], extent='union')
dem = iolib.ds_getma(dem_ds)
hs = iolib.ds_getma(hs_ds)
snowdepth = iolib.ds_getma(snowdepth_ds)
#Pixel coordinates of sample sites
x, y = geolib.mapToPixel(b[:, 1], b[:, 2], dem_ds.GetGeoTransform())
depth = b[:, 3]
rho = b[:, 4]
#Sample DEM snow depth
samp = geolib.sample(snowdepth_ds, b[:, 1], b[:, 2], pad=5)
#Filter to throw out samples with significant roughness over sampled area
samp_perc_thresh = 0.3
samp_mask = (samp[:, 1] / samp[:, 0]) > samp_perc_thresh
def main():
"""
# chm_refine.py
# Main goals:
# (1)fix non-forest "heights"
# (2)fix dense interior forest height estimates
# (3)remove water
# Basic logic
#Divide HRSI CHM into forest and non-forest;
#to estimate max canopy height, within the forest mask run a 'max'filter (filtlib)
#to remove spurious 'heights' in the non-forest using a 'min' filter (filtlib)
#
#(1) invert roughmask to get 'forest'
# -run a 'max' filter,
#(2) use roughmask (get_lo_rough_mask) to get 'non-forest' pixels
# -run a 'min' filter,
# -maybe a small (3 pix) window filter
#(3) then mask the result with the toamask (this removes water and other dark (shadow) areas
# -remove dark and smooth (water)
# -smooth is non-veg land
# -dark and rough is woody veg land
#
#(4) for later: then mask with the slopemask, toatrimask
# step 1
# get_dark_mask from ortho_toa --> remove the areas that are TOA dark (water,shadow) from the chm
# step 2
# get_hi_slope_mask from DEM --> remove areas of high slopes from chm
# step 3
# get_lo_rough_mask from DEM --> remove areas that are NOT rough (aka remove non-forest)
# run a max filter on remaining pixels
* mask outputs should all consistently show the 'masked' area as valid
"""
parser = getparser()
args = parser.parse_args()
outdir = args.outdir
pairname = args.pairname
if not os.path.exists(outdir):
os.mkdir(outdir)
outfolder = os.path.join(outdir, pairname)
if not os.path.exists(outfolder):
os.mkdir(outfolder)
auto_min_toa = args.auto_min_toa
#Symlink files to working directory.
#symlinks=['out-DEM_1m.tif','{}_ortho.tif'.format(pairname)]
print("\nSymlinking Files to Working Directory\n")
cmd = "ln -sf /att/pubrepo/DEM/hrsi_dsm/v2/{}/*ortho*tif {}".format(
pairname, outfolder)
subprocess.call(cmd, shell=True)
cmd = "ln -sf /att/pubrepo/DEM/hrsi_dsm/v2/{}/out-DEM*m.tif {}".format(
pairname, outfolder)
subprocess.call(cmd, shell=True)
cmd = "xml_fn_list=$(ls /att/pubrepo/DEM/hrsi_dsm/v2/{}/*.xml);ln -sf $xml_fn_list {}".format(
pairname, outfolder)
subprocess.call(cmd, shell=True)
#dsm_maindir='/att/pubrepo/DEM/hrsi_dsm/v2/'
#dsm_dir=os.path.join(dsm_maindir,pairname)
chm_dir = '/att/gpfsfs/briskfs01/ppl/pmontesa/chm_work/hrsi_chm_sgm_filt/chm'
chm_name = '{}_sr05_4m-sr05-min_1m-sr05-max_dz_eul.tif'.format(pairname)
chm_fn = os.path.join(chm_dir, chm_name)
print("[1]\nLoading Input CHM into masked array\n")
chm_ds = iolib.fn_getds(chm_fn)
print("[2]\nGetting Dark Mask from Ortho TOA\n")
#May need to include get_toa_fn from DEM_control.py
print("\n\t-Compute TOA from Ortho\n")
dem_fn = os.path.join(outfolder, 'out-DEM_1m.tif')
toa_fn = get_toa_fn(dem_fn)
print("\nt-Warp TOA to CHM...\n")
toa_ds = warplib.memwarp_multi_fn([
toa_fn,
],
res=chm_ds,
extent=chm_ds,
t_srs=chm_ds)[0]
#Determine from inputs or calculate lowest acceptable TOA valuesfor masking
if auto_min_toa:
# Compute a good min TOA value
m, s = get_min_gaus(toa_fn, 50, 4)
min_toa = m + s
min_toa = m
else:
min_toa = args.min_toa
#Write TOA Mins for reference
with open(
os.path.join(
os.path.split(toa_fn)[0], "min_toa_" + pairname + ".txt"),
"w") as text_file:
text_file.write(os.path.basename(__file__))
text_file.write("\nMinimum TOA used for mask:\n{0}".format(min_toa))
# Should mask dark areas and dilate
dark_mask = get_dark_mask(toa_ds, min_toa)
print("\n\t-Completed Calculating Dark Mask\n")
print("[3]\nGetting High Slope Mask from DEM\n")
max_slope = args.max_slope
dem_ds = iolib.fn_getds(dem_fn)
slope_mask = get_hi_slope_mask(dem_ds, max_slope)
print("\n\t-Completed Sploe Masking\n")
print("[4]\nGetting Roughness for Forest/Non-Forest Classification\n")
#NOTE: Not sure which mask we want to use. Will write up both
min_rough = args.min_rough
lo_rough_mask = get_lo_rough_mask(dem_ds, min_rough)
#Areas less than min_rough is masked#
min_tri = args.min_tri
lo_tri_mask = get_lo_tri_mask(dem_ds, min_tri)
#Valid areas are forest
forest_mask = np.logical_or(lo_rough_mask, log_tri_mask)
ground_mask = ~forest_mask
def main2(args):
#Should check that files exist
dem1_fn = args.ref_fn
dem2_fn = args.src_fn
mode = args.mode
apply_mask = not args.nomask
max_offset_m = args.max_offset
tiltcorr = args.tiltcorr
#These are tolerances (in meters) to stop iteration
tol = args.tol
min_dx = tol
min_dy = tol
min_dz = tol
#Maximum number of iterations
max_n = 10
outdir = args.outdir
if outdir is None:
outdir = os.path.splitext(dem2_fn)[0] + '_dem_align'
if not os.path.exists(outdir):
os.makedirs(outdir)
outprefix = '%s_%s' % (os.path.splitext(os.path.split(dem2_fn)[-1])[0], \
os.path.splitext(os.path.split(dem1_fn)[-1])[0])
outprefix = os.path.join(outdir, outprefix)
print("\nReference: %s" % dem1_fn)
print("Source: %s" % dem2_fn)
print("Mode: %s" % mode)
print("Output: %s\n" % outprefix)
dem2_ds = gdal.Open(dem2_fn, gdal.GA_ReadOnly)
#Often the "ref" DEM is high-res lidar or similar
#This is a shortcut to resample to match "source" DEM
dem1_ds = warplib.memwarp_multi_fn([
dem1_fn,
],
res=dem2_ds,
extent=dem2_ds,
t_srs=dem2_ds)[0]
#dem1_ds = gdal.Open(dem1_fn, gdal.GA_ReadOnly)
#Create a copy to be updated in place
dem2_ds_align = iolib.mem_drv.CreateCopy('', dem2_ds, 0)
#dem2_ds_align = dem2_ds
#Iteration number
n = 1
#Cumulative offsets
dx_total = 0
dy_total = 0
dz_total = 0
#Now iteratively update geotransform and vertical shift
while True:
print("*** Iteration %i ***" % n)
dx, dy, dz, static_mask, fig = compute_offset(dem1_ds,
dem2_ds_align,
dem2_fn,
mode,
max_offset_m,
apply_mask=apply_mask)
if n == 1:
static_mask_orig = static_mask
xyz_shift_str_iter = "dx=%+0.2fm, dy=%+0.2fm, dz=%+0.2fm" % (dx, dy,
dz)
print("Incremental offset: %s" % xyz_shift_str_iter)
#Should make an animation of this converging
if fig is not None:
dst_fn = outprefix + '_%s_iter%i_plot.png' % (mode, n)
print("Writing offset plot: %s" % dst_fn)
fig.gca().set_title(xyz_shift_str_iter)
fig.savefig(dst_fn, dpi=300, bbox_inches='tight', pad_inches=0.1)
#Apply the horizontal shift to the original dataset
dem2_ds_align = coreglib.apply_xy_shift(dem2_ds_align,
dx,
dy,
createcopy=False)
dem2_ds_align = coreglib.apply_z_shift(dem2_ds_align,
dz,
createcopy=False)
dx_total += dx
dy_total += dy
dz_total += dz
print("Cumulative offset: dx=%+0.2fm, dy=%+0.2fm, dz=%+0.2fm" %
(dx_total, dy_total, dz_total))
#Fit plane to residuals and remove
#Might be better to do this after converging
"""
if tiltcorr:
print("Applying planar tilt correction")
gt = dem2_ds_align.GetGeoTransform()
#Need to compute diff_euler here
#Copy portions of compute_offset, create new function
vals, resid, coeff = geolib.ma_fitplane(diff_euler_align, gt, perc=(4, 96))
dem2_ds_align = coreglib.apply_z_shift(dem2_ds_align, -vals, createcopy=False)
"""
n += 1
print("\n")
#If magnitude of shift in all directions is less than tol
#if n > max_n or (abs(dx) <= min_dx and abs(dy) <= min_dy and abs(dz) <= min_dz):
#If magnitude of shift is less than tol
dm = np.sqrt(dx**2 + dy**2 + dz**2)
if n > max_n or dm < tol:
break
#String to append to output filenames
xyz_shift_str_cum = '_%s_x%+0.2f_y%+0.2f_z%+0.2f' % (mode, dx_total,
dy_total, dz_total)
if tiltcorr:
xyz_shift_str_cum += "_tiltcorr"
#Compute original elevation difference
if True:
dem1_clip_ds, dem2_clip_ds = warplib.memwarp_multi([dem1_ds, dem2_ds], \
res='max', extent='intersection', t_srs=dem2_ds)
dem1_orig = iolib.ds_getma(dem1_clip_ds, 1)
dem2_orig = iolib.ds_getma(dem2_clip_ds, 1)
diff_euler_orig = dem2_orig - dem1_orig
if not apply_mask:
static_mask_orig = np.ma.getmaskarray(diff_euler_orig)
diff_euler_orig_compressed = diff_euler_orig[~static_mask_orig]
diff_euler_orig_stats = np.array(
malib.print_stats(diff_euler_orig_compressed))
#Write out original eulerian difference map
print(
"Writing out original euler difference map for common intersection before alignment"
)
dst_fn = outprefix + '_orig_dz_eul.tif'
iolib.writeGTiff(diff_euler_orig, dst_fn, dem1_clip_ds)
#Compute final elevation difference
if True:
dem1_clip_ds_align, dem2_clip_ds_align = warplib.memwarp_multi([dem1_ds, dem2_ds_align], \
res='max', extent='intersection', t_srs=dem2_ds_align)
dem1_align = iolib.ds_getma(dem1_clip_ds_align, 1)
dem2_align = iolib.ds_getma(dem2_clip_ds_align, 1)
diff_euler_align = dem2_align - dem1_align
if not apply_mask:
static_mask = np.ma.getmaskarray(diff_euler_align)
diff_euler_align_compressed = diff_euler_align[~static_mask]
diff_euler_align_stats = np.array(
malib.print_stats(diff_euler_align_compressed))
#Fit plane to residuals and remove
if tiltcorr:
print("Applying planar tilt correction")
gt = dem1_clip_ds_align.GetGeoTransform()
#Need to apply the mask here, so we're only fitting over static surfaces
#Note that the origmask=False will compute vals for all x and y indices, which is what we want
vals, resid, coeff = geolib.ma_fitplane(np.ma.array(diff_euler_align, mask=static_mask), \
gt, perc=(4, 96), origmask=False)
#Remove planar offset from difference map
diff_euler_align -= vals
#Remove planar offset from aligned dem2
#Note: dimensions of ds and vals will be different as vals are computed for clipped intersection
#Recompute planar offset for dem2_ds_align extent
xgrid, ygrid = geolib.get_xy_grids(dem2_ds_align)
vals = coeff[0] * xgrid + coeff[1] * ygrid + coeff[2]
dem2_ds_align = coreglib.apply_z_shift(dem2_ds_align,
-vals,
createcopy=False)
if not apply_mask:
static_mask = np.ma.getmaskarray(diff_euler_align)
diff_euler_align_compressed = diff_euler_align[~static_mask]
diff_euler_align_stats = np.array(
malib.print_stats(diff_euler_align_compressed))
print("Creating fitplane plot")
fig, ax = plt.subplots(figsize=(6, 6))
fitplane_clim = malib.calcperc(vals, (2, 98))
im = ax.imshow(vals, cmap='cpt_rainbow', clim=fitplane_clim)
res = float(geolib.get_res(dem2_clip_ds, square=True)[0])
pltlib.add_scalebar(ax, res=res)
pltlib.hide_ticks(ax)
pltlib.add_cbar(ax, im, label='Fit plane residuals (m)')
fig.tight_layout()
dst_fn1 = outprefix + '%s_align_dz_eul_fitplane.png' % xyz_shift_str_cum
print("Writing out figure: %s" % dst_fn1)
fig.savefig(dst_fn1, dpi=300, bbox_inches='tight', pad_inches=0.1)
#Compute higher-order fits?
#Could also attempt to model along-track and cross-track artifacts
#Write out aligned eulerian difference map for clipped extent with vertial offset removed
dst_fn = outprefix + '%s_align_dz_eul.tif' % xyz_shift_str_cum
print(
"Writing out aligned difference map with median vertical offset removed"
)
iolib.writeGTiff(diff_euler_align, dst_fn, dem1_clip_ds)
#Write out aligned dem_2 with vertial offset removed
if True:
dst_fn2 = outprefix + '%s_align.tif' % xyz_shift_str_cum
print(
"Writing out shifted dem2 with median vertical offset removed: %s"
% dst_fn2)
#Might be cleaner way to write out MEM ds directly to disk
dem2_align = iolib.ds_getma(dem2_ds_align)
iolib.writeGTiff(dem2_align, dst_fn2, dem2_ds_align)
dem2_ds_align = None
#Create output plot
if True:
print("Creating final plot")
dem1_hs = geolib.gdaldem_mem_ma(dem1_orig, dem1_clip_ds, returnma=True)
dem2_hs = geolib.gdaldem_mem_ma(dem2_orig, dem2_clip_ds, returnma=True)
f, axa = plt.subplots(2, 3, figsize=(11, 8.5))
for ax in axa.ravel()[:-1]:
ax.set_facecolor('k')
pltlib.hide_ticks(ax)
dem_clim = malib.calcperc(dem1_orig, (2, 98))
axa[0, 0].imshow(dem1_hs, cmap='gray')
axa[0, 0].imshow(dem1_orig,
cmap='cpt_rainbow',
clim=dem_clim,
alpha=0.6)
res = float(geolib.get_res(dem1_clip_ds, square=True)[0])
pltlib.add_scalebar(axa[0, 0], res=res)
axa[0, 0].set_title('Reference DEM')
axa[0, 1].imshow(dem2_hs, cmap='gray')
axa[0, 1].imshow(dem2_orig,
cmap='cpt_rainbow',
clim=dem_clim,
alpha=0.6)
axa[0, 1].set_title('Source DEM')
axa[0, 2].imshow(~static_mask_orig, clim=(0, 1), cmap='gray')
axa[0, 2].set_title('Surfaces for co-registration')
dz_clim = malib.calcperc_sym(diff_euler_orig_compressed, (5, 95))
im = axa[1, 0].imshow(diff_euler_orig, cmap='RdBu', clim=dz_clim)
pltlib.add_cbar(axa[1, 0], im, label=None)
axa[1, 0].set_title('Elev. Diff. Before (m)')
im = axa[1, 1].imshow(diff_euler_align, cmap='RdBu', clim=dz_clim)
pltlib.add_cbar(axa[1, 1], im, label=None)
axa[1, 1].set_title('Elev. Diff. After (m)')
#Tried to insert Nuth fig here
#ax_nuth.change_geometry(1,2,1)
#f.axes.append(ax_nuth)
bins = np.linspace(dz_clim[0], dz_clim[1], 128)
axa[1, 2].hist(diff_euler_orig_compressed,
bins,
color='g',
label='Before',
alpha=0.5)
axa[1, 2].hist(diff_euler_align_compressed,
bins,
color='b',
label='After',
alpha=0.5)
axa[1, 2].axvline(0, color='k', linewidth=0.5, linestyle=':')
axa[1, 2].set_xlabel('Elev. Diff. (m)')
axa[1, 2].set_ylabel('Count (px)')
axa[1, 2].set_title("Source - Reference")
#axa[1,2].legend(loc='upper right')
#before_str = 'Before\nmean: %0.2f\nstd: %0.2f\nmed: %0.2f\nnmad: %0.2f' % tuple(diff_euler_orig_stats[np.array((3,4,5,6))])
#after_str = 'After\nmean: %0.2f\nstd: %0.2f\nmed: %0.2f\nnmad: %0.2f' % tuple(diff_euler_align_stats[np.array((3,4,5,6))])
before_str = 'Before\nmed: %0.2f\nnmad: %0.2f' % tuple(
diff_euler_orig_stats[np.array((5, 6))])
axa[1, 2].text(0.05,
0.95,
before_str,
va='top',
color='g',
transform=axa[1, 2].transAxes)
after_str = 'After\nmed: %0.2f\nnmad: %0.2f' % tuple(
diff_euler_align_stats[np.array((5, 6))])
axa[1, 2].text(0.65,
0.95,
after_str,
va='top',
color='b',
transform=axa[1, 2].transAxes)
suptitle = '%s\nx: %+0.2fm, y: %+0.2fm, z: %+0.2fm' % (
os.path.split(outprefix)[-1], dx_total, dy_total, dz_total)
f.suptitle(suptitle)
f.tight_layout()
plt.subplots_adjust(top=0.90)
dst_fn = outprefix + '%s_align.png' % xyz_shift_str_cum
print("Writing out figure: %s" % dst_fn)
f.savefig(dst_fn, dpi=300, bbox_inches='tight', pad_inches=0.1)
#Removing residual planar tilt can introduce additional slope/aspect dependent offset
#Want to run another round of main dem_align after removing planar tilt
if tiltcorr:
print("\n Rerunning after applying tilt correction \n")
#Create copy of original arguments
import copy
args2 = copy.copy(args)
#Use aligned, tilt-corrected DEM as input src_fn for second round
args2.src_fn = dst_fn2
#Assume we've already corrected most of the tilt during first round (also prevents endless loop)
args2.tiltcorr = False
main2(args2)
fig.tight_layout()
fig.colorbar(im_list[0], ax=axa.ravel().tolist(), label=label, extend='both', shrink=0.5)
if fn is not None:
fig.savefig(fn, bbox_inches='tight', pad_inches=0, dpi=150)
! ./Users/elischwat/Documents/UW/pygeotools/pygeotools/ogr_merge.sh
#Input DEM filenames
dem2007_fn = 'rainier_dem_differencing/07_joined.tif'
dem2009_fn = 'rainier_dem_differencing/09_joined.tif'
dem_fn_list = [dem2007_fn,dem2009_fn]
#This will return warped, in-memory GDAL dataset objects
#Can also resample all inputs to a lower resolution (res=256)
ds_list = warplib.memwarp_multi_fn(dem_fn_list, extent='intersection', res='min')
#Load datasets to NumPy arrays
dem_2007, dem_2009 = [iolib.ds_getma(i) for i in ds_list]
dem_list = [dem_2007, dem_2009]
#dem_list = [iolib.ds_getma(i) for i in ds_list]
import matplotlib
matplotlib.pyplot.imshow(dem_2007)
matplotlib.pyplot.imshow(dem_2009)
titles = ['2007', '2009']
clim = malib.calcperc(dem_list[0], (2,98))
plot_panels(2, dem_list, clim, titles, 'inferno', 'Elevation (m WGS84)', fn='dem.png')
def main():
parser = getparser()
args = parser.parse_args()
#This is output ndv, avoid using 0 for differences
diffndv = -9999
dem1_fn = args.fn1
dem2_fn = args.fn2
if dem1_fn == dem2_fn:
sys.exit('Input filenames are identical')
fn_list = [dem1_fn, dem2_fn]
print("Warping DEMs to same res/extent/proj")
#This will check input param for validity, could do beforehand
dem1_ds, dem2_ds = warplib.memwarp_multi_fn(fn_list, extent=args.te, res=args.tr, t_srs=args.t_srs)
outdir = args.outdir
if outdir is None:
outdir = os.path.dirname(os.path.abspath(dem1_fn))
if not os.path.exists(outdir):
os.makedirs(outdir)
outprefix = os.path.splitext(os.path.split(dem1_fn)[1])[0]+'_'+os.path.splitext(os.path.split(dem2_fn)[1])[0]
print("Loading input DEMs into masked arrays")
dem1 = iolib.ds_getma(dem1_ds, 1)
dem2 = iolib.ds_getma(dem2_ds, 1)
#Compute dz/dt rates if possible, in m/yr
rates = True
if rates:
#Extract basename
#This was a hack to work with timestamp array filenames that have geoid offset applied
adj = ''
if '-adj' in dem1_fn:
adj = '-adj'
dem1_fn_base = re.sub(adj, '', os.path.splitext(dem1_fn)[0])
dem2_fn_base = re.sub(adj, '', os.path.splitext(dem2_fn)[0])
#Attempt to load timestamp arrays (for mosaics) if present
t1_fn = dem1_fn_base+'_ts.tif'
t2_fn = dem2_fn_base+'_ts.tif'
if os.path.exists(t1_fn) and os.path.exists(t2_fn):
print("Preparing timestamp arrays")
t1_ds, t2_ds = warplib.memwarp_multi_fn([t1_fn, t2_fn], extent=dem1_ds, res=dem1_ds)
print("Loading timestamps into masked arrays")
t1 = iolib.ds_getma(t1_ds)
t2 = iolib.ds_getma(t2_ds)
#Compute dt in days
t_factor = t2 - t1
t_factor /= 365.25
else:
#Attempt to extract timestamps from input filenames
t1 = timelib.fn_getdatetime(dem1_fn)
t2 = timelib.fn_getdatetime(dem2_fn)
if t1 is not None and t2 is not None and t1 != t2:
dt = t2 - t1
year = timedelta(days=365.25)
t_factor = abs(dt.total_seconds()/year.total_seconds())
print("Time differences is %s, dh/%0.3f" % (dt, t_factor))
else:
print("Unable to extract timestamps for input images")
rates = False
#Compute relative elevation difference with Eulerian approach
print("Computing eulerian elevation difference")
diff_euler = dem2 - dem1
#Check to make sure inputs actually intersect
#if not np.any(~dem1.mask*~dem2.mask):
if diff_euler.count() == 0:
sys.exit("No valid overlap between input DEMs")
if True:
print("Eulerian elevation difference stats:")
diff_euler_stats = malib.print_stats(diff_euler)
diff_euler_med = diff_euler_stats[5]
if True:
print("Writing Eulerian elevation difference map")
dst_fn = os.path.join(outdir, outprefix+'_dz_eul.tif')
print(dst_fn)
iolib.writeGTiff(diff_euler, dst_fn, dem1_ds, ndv=diffndv)
if rates:
print("Writing Eulerian rate map")
dst_fn = os.path.join(outdir, outprefix+'_dz_eul_rate.tif')
print(dst_fn)
iolib.writeGTiff(diff_euler/t_factor, dst_fn, dem1_ds, ndv=diffndv)
if False:
print("Writing Eulerian relative elevation difference map")
diff_euler_rel = diff_euler - diff_euler_med
dst_fn = os.path.join(outdir, outprefix+'_dz_eul_rel.tif')
print(dst_fn)
iolib.writeGTiff(diff_euler_rel, dst_fn, dem1_ds, ndv=diffndv)
if False:
print("Writing out DEM2 with median elevation difference removed")
dst_fn = os.path.splitext(dem2_fn)[0]+'_med'+diff_euler_med+'.tif'
print(dst_fn)
iolib.writeGTiff(dem2 - diff_euler_med, dst_fn, dem1_ds, ndv=diffndv)
if False:
print("Writing Eulerian elevation difference percentage map")
diff_euler_perc = 100.0*diff_euler/dem1
dst_fn = os.path.join(outdir, outprefix+'_dz_eul_perc.tif')
print(dst_fn)
iolib.writeGTiff(diff_euler_perc, dst_fn, dem1_ds, ndv=diffndv)
def ols(x, **kwargs):
fit = sm.OLS(
y,
x,
missing='drop',
).fit()
return fit.intercept
if __name__ == '__main__':
files = RasterSource(
'/Volumes/warehouse/projects/UofU/geohackweek/galcier_data/khumbu_DEM_32m/'
).file_list()
raster_list = warplib.memwarp_multi_fn(files,
extent='union',
res='min',
t_srs=files[0])
band_data = [iolib.ds_getma(i).filled(np.NaN) for i in raster_list]
x, y = geolib.get_xy_grids(raster_list[0])
time_list = np.array([timelib.fn_getdatetime(fn) for fn in files])
x_band_data = xr.DataArray(np.stack(band_data),
coords={
'lat': y[::, 0],
'lon': x[0, ::],
'time': time_list
},
dims=('time', 'lat', 'lon'),
name='elevation')
from pygeotools.lib import iolib, warplib, malib
fn1 = 'D:\\shapedata\\result\\2016-spectral16d366ae8.tif'
fn2 = 'D:\\shapedata\\result\\2016-spectral16d366ae8.tif'
ds_list = warplib.memwarp_multi_fn([fn1, fn2],
res='max',
extent='intersection',
t_srs='first',
r='cubic')
r1 = iolib.ds_getma(ds_list[0])
r2 = iolib.ds_getma(ds_list[1])
rdiff = r1 - r2
malib.print_stats(rdiff)
out_fn = 'D:\\shapedata\\result\\2016-3456564344343434.tif'
iolib.writeGTiff(rdiff, out_fn, ds_list[0])
