def raster_shpclip(r_fn, shp_fn, extent='raster'):
r_ds = iolib.fn_getds(r_fn)
r_srs = geolib.get_ds_srs(r_ds)
r_extent = geolib.ds_extent(r_ds)
shp_ds = ogr.Open(shp_fn)
lyr = shp_ds.GetLayer()
shp_srs = lyr.GetSpatialRef()
shp_extent = lyr.GetExtent()
#Define the output - can set to either raster or shp
#Accept as cl arg
out_srs = r_srs
if extent == 'raster':
out_extent = r_extent
elif extent == 'shp':
out_extent = shp_extent
#r = iolib.ds_getma(r_ds)
r_ds = warplib.memwarp(r_ds, extent=out_extent, t_srs=out_srs, r='cubic')
r = iolib.ds_getma(r_ds)
mask = geolib.shp2array(shp_fn, r_ds)
r = np.ma.array(r, mask=mask)
return r
def site_filter_extent_ds(ds, pad=None):
"""
Filter available sites for a given dataset
"""
snotel_srs = geolib.wgs_srs
ds_srs = geolib.get_ds_srs(ds)
extent = geolib.ds_extent(ds)
#extent = geolib.ds_extent(ds, snotel_srs)
#geom = geolib.get_outline(ds)
return site_filter_extent(extent, ds_srs, pad)
def get_cam2rpc_opts(t='pinhole', dem=None, gsd=None, num_samples=50):
"""
generates cmd for ASP cam2rpc
This generates rpc camera models from the optimized frame camera models
See documentation here: https://stereopipeline.readthedocs.io/en/latest/tools/cam2rpc.html
Parameters
----------
t: str
session, or for here, type of input camera, default: pinhole
dem: str
path to DEM which will be used for calculating RPC polynomials
gsd: float
Expected ground-samplind distance
num_samples: int
Sampling for RPC approximation calculation (default=50)
Returns
----------
cam2rpc_opts: list
A list of arguments for cam2rpc call.
"""
cam2rpc_opts = []
cam2rpc_opts.extend(['--dem-file', dem])
dem_ds = iolib.fn_getds(dem)
dem_proj = dem_ds.GetProjection()
dem = iolib.ds_getma(dem_ds)
min_height, max_height = np.percentile(dem.compressed(), (0.01, 0.99))
tsrs = epsg2geolib(4326)
xmin, ymin, xmax, ymax = geolib.ds_extent(ds, tsrs)
cam2rpc_opts.extend(['--height-range', str(min_height), str(max_height)])
cam2rpc_opts.extend(
['--lon-lat-range',
str(xmin),
str(ymin),
str(xmax),
str(ymax)])
if gsd:
cam2rpc_opts.extend(['--gsd', str(gsd)])
cam2rpc_opts.extend(['--session', t])
cam2rpc_opts.extend(['--num-samples', str(num_samples)])
return cam2rpc_opts
def main():
parser = getparser()
args = parser.parse_args()
refdem = args.refdem
srcdem = args.srcdem
outfolder = '{}__{}_comparison_stats'.format(
os.path.splitext(os.path.basename(refdem))[0],
os.path.splitext(os.path.basename(srcdem))[0])
header_str = '{}__{}'.format(
os.path.splitext(os.path.basename(refdem))[0],
os.path.splitext(os.path.basename(srcdem))[0])
if not os.path.exists(outfolder):
os.makedirs(outfolder)
if args.local_ortho == 1:
temp_ds = warplib.memwarp_multi_fn([refdem, srcdem])[0]
bbox = geolib.ds_extent(temp_ds)
geo_crs = temp_ds.GetProjection()
print('Bounding box lon_lat is{}'.format(bbox))
bound_poly = Polygon([[bbox[0], bbox[3]], [bbox[2], bbox[3]],
[bbox[2], bbox[1]], [bbox[0], bbox[1]]])
bound_shp = gpd.GeoDataFrame(index=[0],
geometry=[bound_poly],
crs=geo_crs)
bound_centroid = bound_shp.centroid
cx = bound_centroid.x.values[0]
cy = bound_centroid.y.values[0]
pad = np.ptp([bbox[3], bbox[1]]) / 6.0
lat_1 = bbox[1] + pad
lat_2 = bbox[3] - pad
local_ortho = "+proj=ortho +lat_1={} +lat_2={} +lat_0={} +lon_0={} +x_0=0 +y_0=0 +ellps=WGS84 +datum=WGS84 +units=m +no_defs".format(
lat_1, lat_2, cy, cx)
logging.info('Local Ortho projection is {}'.format(local_ortho))
t_srs = local_ortho
else:
t_srs = 'first'
# this step performs the desired warping operation
ds_list = warplib.memwarp_multi_fn([refdem, srcdem],
res=args.comparison_res,
t_srs=t_srs)
refma = iolib.ds_getma(ds_list[0])
srcma = iolib.ds_getma(ds_list[1])
init_diff = refma - srcma
init_stats = malib.get_stats_dict(init_diff)
print("Original descriptive statistics {}".format(init_stats))
init_diff_json_fn = os.path.join(
outfolder, '{}_precoreg_descriptive_stats.json'.format(header_str))
init_diff_json = json.dumps(init_stats)
with open(init_diff_json_fn, 'w') as f:
f.write(init_diff_json)
logging.info("Saved initial stats at {}".format(init_diff_json))
refslope = gdaldem(ds_list[0])
# stats for elevation difference vs reference DEM elevation
elev_bin, diff_mean, diff_median, diff_std, diff_perc = cummulative_profile(
refma, init_diff, args.elev_bin_width)
# stats for elevation difference vs reference DEM slope
slope_bin, diff_mean_s, diff_median_s, diff_std_s, diff_perc_s = cummulative_profile(
refslope, init_diff, args.slope_bin_width)
f, ax = plt.subplots(1, 2, figsize=(10, 4))
im = ax[0].scatter(elev_bin, diff_mean, c=diff_perc, cmap='inferno')
ax[0].set_xlabel('Elevation (m)')
divider = make_axes_locatable(ax[0])
cax = divider.append_axes('right', size='2.5%', pad=0.05)
plt.colorbar(im,
cax=cax,
orientation='vertical',
label='pixel count percentage')
im2 = ax[1].scatter(slope_bin, diff_mean_s, c=diff_perc_s, cmap='inferno')
ax[1].set_xlabel('Slope (degrees)')
divider = make_axes_locatable(ax[1])
cax = divider.append_axes('right', size='2.5%', pad=0.05)
plt.colorbar(im2,
cax=cax,
orientation='vertical',
label='pixel count percentage')
for axa in ax.ravel():
axa.axhline(y=0, c='k')
axa.set_ylabel('Elevation Difference (m)')
plt.tight_layout()
precoreg_plot = os.path.join(outfolder,
header_str + '_precoreg_binned_plot.png')
f.savefig(precoreg_plot, dpi=300, bbox_inches='tight', pad_inches=0.1)
logging.info("Saved binned plot at {}".format(precoreg_plot))
if args.coreg == 1:
logging.info("will attempt coregisteration")
if args.local_ortho == 1:
ref_local_ortho = os.path.splitext(refdem)[0] + '_local_ortho.tif'
src_local_ortho = os.path.splitext(srcdem)[0] + '_local_ortho.tif'
# coregisteration works best at mean resolution
# we will rewarp if the initial args.res was not mean
if args.comparison_res != 'mean':
ds_list = warplib.memwarp_multi_fn([refdem, srcdem],
res='mean',
t_srs=t_srs)
refma = iolib.ds_getma(ds_list[0])
srcma = iolib.ds_getma(ds_list[1])
iolib.writeGTiff(refma, ref_local_ortho, ds_list[0])
iolib.writeGTiff(srcma, src_local_ortho, ds_list[1])
coreg_ref = ref_local_ortho
src_ref = src_local_ortho
else:
coreg_ref = refdem
src_ref = srcdem
demcoreg_dir = os.path.join(outfolder, 'coreg_results')
align_opts = [
'-mode', 'nuth', '-max_iter', '12', '-max_offset', '400',
'-outdir', demcoreg_dir
]
align_args = [coreg_ref, src_ref]
align_cmd = ['dem_align.py'] + align_opts + align_args
subprocess.call(align_cmd)
#ah final round of warping and stats calculation
try:
srcdem_align = glob.glob(os.path.join(demcoreg_dir,
'*align.tif'))[0]
logging.info(
"Attempting stats calculation for aligned DEM {}".format(
srcdem_align))
ds_list = warplib.memwarp_multi_fn([args.refdem, srcdem_align],
res=args.comparison_res,
t_srs=t_srs)
refma = iolib.ds_getma(ds_list[0])
srcma = iolib.ds_getma(ds_list[1])
# this is creepy, but I am recycling variable names to save on memory
init_diff = refma - srcma
init_stats = malib.get_stats_dict(init_diff)
print("Final descriptive statistics {}".format(init_stats))
init_diff_json_fn = os.path.join(
outfolder,
'{}_postcoreg_descriptive_stats.json'.format(header_str))
init_diff_json = json.dumps(init_stats)
with open(init_diff_json_fn, 'w') as f:
f.write(init_diff_json)
logging.info("Saved final stats at {}".format(init_diff_json))
refslope = gdaldem(ds_list[0])
# stats for elevation difference vs reference DEM elevation
elev_bin, diff_mean, diff_median, diff_std, diff_perc = cummulative_profile(
refma, init_diff, args.elev_bin_width)
# stats for elevation difference vs reference DEM slope
slope_bin, diff_mean_s, diff_median_s, diff_std_s, diff_perc_s = cummulative_profile(
refslope, init_diff, args.slope_bin_width)
f, ax = plt.subplots(1, 2, figsize=(10, 4))
im = ax[0].scatter(elev_bin,
diff_mean,
c=diff_perc,
cmap='inferno')
ax[0].set_xlabel('Elevation (m)')
divider = make_axes_locatable(ax[0])
cax = divider.append_axes('right', size='2.5%', pad=0.05)
plt.colorbar(im,
cax=cax,
orientation='vertical',
label='pixel count percentage')
im2 = ax[1].scatter(slope_bin,
diff_mean_s,
c=diff_perc_s,
cmap='inferno')
ax[1].set_xlabel('Slope (degrees)')
divider = make_axes_locatable(ax[1])
cax = divider.append_axes('right', size='2.5%', pad=0.05)
plt.colorbar(im2,
cax=cax,
orientation='vertical',
label='pixel count percentage')
for axa in ax.ravel():
axa.axhline(y=0, c='k')
axa.set_ylabel('Elevation Difference (m)')
plt.tight_layout()
precoreg_plot = os.path.join(
outfolder, header_str + '_postcoreg_binned_plot.png')
f.savefig(precoreg_plot,
dpi=300,
bbox_inches='tight',
pad_inches=0.1)
except:
logging.info(
"Failed to compute post coreg stats, see corresponding job log"
)
logging.info("Script is complete !")
def warp_multi(src_ds_list, res='first', extent='intersection', t_srs='first', r='cubic', warptype=memwarp, outdir=None, dst_ndv=None, verbose=True, debug=False):
"""This parses and checks inputs, then calls desired warp function with appropriate arguments for each input ds
Parameters
----------
src_ds_list : list of gdal.Dataset objects
List of original datasets to be warped
res : arbitrary type
Desired output resolution
extent : arbitrary type
Desired output extent
t_srs : arbitrary type
Desired output spatial reference
r : str
Desired resampling algorithm
warptype : function
Desired warp type (write to memory or disk)
outdir : str
Desired output directory (for disk warp)
dst_ndv : float
Desired output NoData Value
verbose : bool
Print warp parameters
debug : bool
Print extra information for debugging purposes
Returns
-------
out_ds_list : list of gdal.Dataset objects
List of warped datasets (either in memory or on disk)
"""
#Type cast arguments as str for evaluation
#Avoid path errors
#res = str(res)
#extent = str(extent)
#t_srs = str(t_srs)
#Parse the input
t_srs = parse_srs(t_srs, src_ds_list)
res = parse_res(res, src_ds_list, t_srs)
extent = parse_extent(extent, src_ds_list, t_srs)
if verbose:
print("\nWarping all inputs to the following:")
print("Resolution: %s" % res)
print("Extent: %s" % str(extent))
print("Projection: '%s'" % t_srs.ExportToProj4())
print("Resampling alg: %s\n" % r)
out_ds_list = []
for i, ds in enumerate(src_ds_list):
fn_list = ds.GetFileList()
fn = '[memory]'
if fn_list is not None:
fn = fn_list[0]
if verbose:
print("%i of %i: %s" % (i+1, len(src_ds_list), fn))
#If input srs are different, must warp
ds_t_srs = geolib.get_ds_srs(ds)
srscheck = bool(t_srs.IsSame(ds_t_srs))
if debug:
print('\n%s' % ds_t_srs.ExportToWkt())
print('%s\n' % t_srs.ExportToWkt())
print('srscheck: %s\n' % srscheck)
rescheck = False
extentcheck = False
#if srscheck:
#Extract info from ds to see if warp is necessary
ds_res = geolib.get_res(ds, square=True)[0]
ds_extent = geolib.ds_extent(ds)
#Note: these checks necessary to handle rounding and precision issues
#Round extent and res to nearest mm
precision = 1E-3
#Or if t_srs has units of degrees
if ds_t_srs.IsGeographic():
precision = 1E-8
rescheck = (res is None) or geolib.res_compare(res, ds_res, precision=precision)
extentcheck = (extent is None) or geolib.extent_compare(extent, ds_extent, precision=precision)
if debug:
print('\n%s, %s\n' % (ds_res, res))
print('%s' % ds_extent)
print('%s\n' % extent)
print('rescheck: %s' % rescheck)
print('extentcheck: %s\n' % extentcheck)
#If the ds passes all three, it is identical to desired output, short circuit
if rescheck and extentcheck and srscheck:
out_ds_list.append(ds)
else:
dst_ds = warptype(ds, res, extent, t_srs, r, outdir, dst_ndv=dst_ndv, verbose=verbose)
out_ds_list.append(dst_ds)
return out_ds_list
('meltwater', 'total_mmSLEa')]].sum())
#Compile stats for each division
print("\nTotal Gt/a for each aggregation")
print('agg', 'total_Gta', 'mb_mwea')
for i in [glac_df_mb_basin, glac_df_mb_region, glac_df_mb_qdgc]:
print(i.df_name, i[('mb_mwea', 'total_Gta')].sum(), i['mb_mwea',
'mean'].mean())
if False:
fig, ax = plt.subplots()
ax.set_aspect('equal')
print("Loading shaded relief map")
hs_ds = gdal.Open(hs_fn)
hs = iolib.ds_getma(hs_ds)
hs_extent = geolib.ds_extent(hs_ds)
hs_extent_cartopy = cartopy_extent(hs_extent)
print("Plotting image")
ax.imshow(hs,
cmap='gray',
origin='upper',
extent=hs_extent_cartopy,
transform=crs,
alpha=0.6)
else:
hs = None
"""
#This is currently broken
import cartopy.feature as cfeature
borders = cfeature.NaturalEarthFeature(category='cultural', name='admin_1_states_provinces_lines', scale='50m', facecolor='none')
ax.add_feature(borders, edgecolor='k')
def main(args=None):
parser = getparser()
args = parser.parse_args()
# Should check that files exist
ref_dem_fn = args.ref_fn
src_dem_fn = args.src_fn
mode = args.mode
mask_list = args.mask_list
max_offset = args.max_offset
max_dz = args.max_dz
slope_lim = tuple(args.slope_lim)
tiltcorr = args.tiltcorr
polyorder = args.polyorder
res = args.res
# Maximum number of iterations
max_iter = args.max_iter
# These are tolerances (in meters) to stop iteration
tol = args.tol
min_dx = tol
min_dy = tol
min_dz = tol
outdir = args.outdir
if outdir is None:
outdir = os.path.splitext(src_dem_fn)[0] + '_dem_align'
if tiltcorr:
outdir += '_tiltcorr'
tiltcorr_done = False
# Relax tolerance for initial round of co-registration
# tiltcorr_tol = 0.1
# if tol < tiltcorr_tol:
# tol = tiltcorr_tol
if not os.path.exists(outdir):
os.makedirs(outdir)
outprefix = '%s_%s' % (os.path.splitext(os.path.split(src_dem_fn)[-1])[0],
os.path.splitext(os.path.split(ref_dem_fn)[-1])[0])
outprefix = os.path.join(outdir, outprefix)
print("\nReference: %s" % ref_dem_fn)
print("Source: %s" % src_dem_fn)
print("Mode: %s" % mode)
print("Output: %s\n" % outprefix)
src_dem_ds = gdal.Open(src_dem_fn)
ref_dem_ds = gdal.Open(ref_dem_fn)
# Get local cartesian coordinate system
# local_srs = geolib.localtmerc_ds(src_dem_ds)
# Use original source dataset coordinate system
# Potentially issues with distortion and xyz/tiltcorr offsets for DEM with large extent
local_srs = geolib.get_ds_srs(src_dem_ds)
# local_srs = geolib.get_ds_srs(ref_dem_ds)
# Resample to common grid
ref_dem_res = geolib.get_res(ref_dem_ds, t_srs=local_srs, square=False)
# Create a copy to be updated in place
src_dem_ds_align = iolib.mem_drv.CreateCopy('', src_dem_ds, 0)
src_dem_res = geolib.get_res(src_dem_ds, t_srs=local_srs, square=False)
src_dem_ds = None
# Resample to user-specified resolution
ref_dem_ds, src_dem_ds_align = warplib.memwarp_multi([ref_dem_ds, src_dem_ds_align],
extent='intersection', res=args.res, t_srs=local_srs,
r='cubic')
res = geolib.get_res(src_dem_ds_align, square=False)
print("\nReference DEM res: %0.2s" % ref_dem_res)
print("Source DEM res: %0.2s" % src_dem_res)
print("Resolution for coreg: %s (%0.2s m)\n" % (args.res, res))
# Iteration number
n = 1
# Cumulative offsets
dx_total = 0
dy_total = 0
dz_total = 0
# Now iteratively update geotransform and vertical shift
while True:
print("*** Iteration %i ***" % n)
dx, dy, dz, static_mask, fig = compute_offset(ref_dem_ds, src_dem_ds_align, src_dem_fn, mode, max_offset,
mask_list=mask_list, max_dz=max_dz, slope_lim=slope_lim,
plot=True)
xyz_shift_str_iter = "dx=%+0.2fm, dy=%+0.2fm, dz=%+0.2fm" % (dx, dy, dz)
print("Incremental offset: %s" % xyz_shift_str_iter)
dx_total += dx
dy_total += dy
dz_total += dz
xyz_shift_str_cum = "dx=%+0.2fm, dy=%+0.2fm, dz=%+0.2fm" % (dx_total, dy_total, dz_total)
print("Cumulative offset: %s" % xyz_shift_str_cum)
# String to append to output filenames
xyz_shift_str_cum_fn = '_%s_x%+0.2f_y%+0.2f_z%+0.2f' % (mode, dx_total, dy_total, dz_total)
# Should make an animation of this converging
if n == 1:
# static_mask_orig = static_mask
if fig is not None:
dst_fn = outprefix + '_%s_iter%02i_plot.png' % (mode, n)
print("Writing offset plot: %s" % dst_fn)
fig.gca().set_title("Incremental: %s\nCumulative: %s" % (xyz_shift_str_iter, xyz_shift_str_cum))
fig.savefig(dst_fn, dpi=300)
# Apply the horizontal shift to the original dataset
src_dem_ds_align = coreglib.apply_xy_shift(src_dem_ds_align, dx, dy, createcopy=False)
# Should
src_dem_ds_align = coreglib.apply_z_shift(src_dem_ds_align, dz, createcopy=False)
n += 1
print("\n")
# If magnitude of shift in all directions is less than tol
# if n > max_iter or (abs(dx) <= min_dx and abs(dy) <= min_dy and abs(dz) <= min_dz):
# If magnitude of shift is less than tol
dm = np.sqrt(dx ** 2 + dy ** 2 + dz ** 2)
dm_total = np.sqrt(dx_total ** 2 + dy_total ** 2 + dz_total ** 2)
if dm_total > max_offset:
sys.exit(
"Total offset exceeded specified max_offset (%0.2f m). Consider increasing -max_offset argument" %
max_offset)
# Stop iteration
if n > max_iter or dm < tol:
if fig is not None:
dst_fn = outprefix + '_%s_iter%02i_plot.png' % (mode, n)
print("Writing offset plot: %s" % dst_fn)
fig.gca().set_title("Incremental:%s\nCumulative:%s" % (xyz_shift_str_iter, xyz_shift_str_cum))
fig.savefig(dst_fn, dpi=300)
# Compute final elevation difference
if True:
ref_dem_clip_ds_align, src_dem_clip_ds_align = warplib.memwarp_multi([ref_dem_ds, src_dem_ds_align],
res=res, extent='intersection',
t_srs=local_srs, r='cubic')
ref_dem_align = iolib.ds_getma(ref_dem_clip_ds_align, 1)
src_dem_align = iolib.ds_getma(src_dem_clip_ds_align, 1)
# ref_dem_clip_ds_align = None
diff_align = src_dem_align - ref_dem_align
src_dem_align = None
ref_dem_align = None
# Get updated, final mask
mask_glac = get_mask(src_dem_clip_ds_align, mask_list, src_dem_fn, erode=False)
# mask_glac_erode = get_mask(src_dem_clip_ds_align, mask_list, src_dem_fn, erode=False)
mask_glac = np.logical_or(np.ma.getmaskarray(diff_align), mask_glac)
# Final stats, before outlier removal
diff_align_compressed = diff_align[~mask_glac]
diff_align_stats = malib.get_stats_dict(diff_align_compressed, full=True)
# Prepare filtered version for tiltcorr fit
# 冰川区内大坡度区域
slope = get_filtered_slope(src_dem_clip_ds_align, slope_lim=(0.01, 35))
mask_glac_outlier = np.logical_and(mask_glac, np.ma.getmaskarray(slope))
diff_glac_outlier = np.ma.array(diff_align, mask=~mask_glac_outlier)
if diff_glac_outlier.count() > 0:
diff_align_glac_outlier = outlier_filter(np.ma.array(diff_align, mask=~mask_glac_outlier), f=2,
max_dz=100)
diff_align_glac_outlier[mask_glac_outlier == False] = diff_align[mask_glac_outlier == False]
else:
diff_align_glac_outlier = np.ma.array(diff_align, mask=None)
diff_align_filt_nonglac = np.ma.array(diff_align_glac_outlier, mask=mask_glac)
diff_align_filt_compressed = diff_align[~mask_glac]
diff_align_filt_nonglac = outlier_filter(diff_align_filt_nonglac, f=3, max_dz=max_dz)
diff_align_filt_stats = malib.get_stats_dict(diff_align_filt_nonglac, full=True)
diff_align_filt = np.ma.array(diff_align_filt_nonglac, mask=None)
diff_align_filt_mask = np.ma.getmaskarray(diff_align_filt_nonglac)
diff_align_filt[mask_glac == True] = diff_align_glac_outlier[mask_glac == True]
# Fit 2D polynomial to residuals and remove
# To do: add support for along-track and cross-track artifacts
if tiltcorr and not tiltcorr_done:
print("\n************")
print("Calculating 'tiltcorr' 2D polynomial fit to residuals with order %i" % polyorder)
print("************\n")
gt = src_dem_clip_ds_align.GetGeoTransform()
# Need to apply the mask here, so we're only fitting over static surfaces
# Note that the origmask=False will compute vals for all x and y indices, which is what we want
vals, resid, coeff = geolib.ma_fitpoly(diff_align_filt_nonglac, order=polyorder, gt=gt, perc=(2, 98),
origmask=False)
# vals, resid, coeff = geolib.ma_fitplane(diff_align_filt, gt, perc=(12.5, 87.5), origmask=False)
# Should write out coeff or grid with correction
vals_stats = malib.get_stats_dict(vals)
# Want to have max_tilt check here
# max_tilt = 4.0 #m
# Should do percentage
# vals.ptp() > max_tilt
# Note: dimensions of ds and vals will be different as vals are computed for clipped intersection
# Need to recompute planar offset for full src_dem_ds_align extent and apply
xgrid, ygrid = geolib.get_xy_grids(src_dem_ds_align)
valgrid = geolib.polyval2d(xgrid, ygrid, coeff)
# For results of ma_fitplane
# valgrid = coeff[0]*xgrid + coeff[1]*ygrid + coeff[2]
src_dem_ds_align = coreglib.apply_z_shift(src_dem_ds_align, -valgrid, createcopy=False)
# if True:
# print("Creating plot of polynomial fit to residuals")
# fig, axa = plt.subplots(1,2, figsize=(8, 4))
# dz_clim = malib.calcperc_sym(vals, (2, 98))
# ax = pltlib.iv(diff_align_filt_nonglac, ax=axa[0], cmap='RdBu', clim=dz_clim, \
# label='Residual dz (m)', scalebar=False)
# ax = pltlib.iv(valgrid, ax=axa[1], cmap='RdBu', clim=dz_clim, \
# label='Polyfit dz (m)', ds=src_dem_ds_align)
# #if tiltcorr:
# #xyz_shift_str_cum_fn += "_tiltcorr"
# tiltcorr_fig_fn = outprefix + '%s_polyfit.png' % xyz_shift_str_cum_fn
# print("Writing out figure: %s\n" % tiltcorr_fig_fn)
# fig.savefig(tiltcorr_fig_fn, dpi=300)
print("Applying tilt correction to difference map")
diff_align -= vals
# Should iterate until tilts are below some threshold
# For now, only do one tiltcorr
tiltcorr_done = True
# Now use original tolerance, and number of iterations
tol = args.tol
max_iter = n + args.max_iter
else:
break
if True:
# Write out aligned difference map for clipped extent with vertial offset removed
align_diff_fn = outprefix + '%s_align_diff.tif' % xyz_shift_str_cum_fn
print("Writing out aligned difference map with median vertical offset removed")
iolib.writeGTiff(diff_align, align_diff_fn, src_dem_clip_ds_align)
if True:
# Write out fitered aligned difference map
align_diff_filt_fn = outprefix + '%s_align_diff_filt.tif' % xyz_shift_str_cum_fn
print("Writing out filtered aligned difference map with median vertical offset removed")
iolib.writeGTiff(diff_align_filt, align_diff_filt_fn, src_dem_clip_ds_align)
# Extract final center coordinates for intersection
center_coord_ll = geolib.get_center(src_dem_clip_ds_align, t_srs=geolib.wgs_srs)
center_coord_xy = geolib.get_center(src_dem_clip_ds_align)
src_dem_clip_ds_align = None
# Write out final aligned src_dem
align_fn = outprefix + '%s_align.tif' % xyz_shift_str_cum_fn
print("Writing out shifted src_dem with median vertical offset removed: %s" % align_fn)
# Open original uncorrected dataset at native resolution
src_dem_ds = gdal.Open(src_dem_fn)
src_dem_ds_align = iolib.mem_drv.CreateCopy('', src_dem_ds, 0)
# Apply final horizontal and vertial shift to the original dataset
# Note: potentially issues if we used a different projection during coregistration!
src_dem_ds_align = coreglib.apply_xy_shift(src_dem_ds_align, dx_total, dy_total, createcopy=False)
src_dem_ds_align = coreglib.apply_z_shift(src_dem_ds_align, dz_total, createcopy=False)
if tiltcorr:
xgrid, ygrid = geolib.get_xy_grids(src_dem_ds_align)
valgrid = geolib.polyval2d(xgrid, ygrid, coeff)
# For results of ma_fitplane
# valgrid = coeff[0]*xgrid + coeff[1]*ygrid + coeff[2]
src_dem_ds_align = coreglib.apply_z_shift(src_dem_ds_align, -valgrid, createcopy=False)
# Might be cleaner way to write out MEM ds directly to disk
src_dem_full_align = iolib.ds_getma(src_dem_ds_align)
iolib.writeGTiff(src_dem_full_align, align_fn, src_dem_ds_align)
if True:
# Output final aligned src_dem, masked so only best pixels are preserved
# Useful if creating a new reference product
# Can also use apply_mask.py
print("Applying filter to shifted src_dem")
align_diff_filt_full_ds = \
warplib.memwarp_multi_fn([align_diff_filt_fn, ], res=src_dem_ds_align, extent=src_dem_ds_align,
t_srs=src_dem_ds_align)[0]
align_diff_filt_full = iolib.ds_getma(align_diff_filt_full_ds)
align_diff_filt_full_ds = None
align_fn_masked = outprefix + '%s_align_filt.tif' % xyz_shift_str_cum_fn
iolib.writeGTiff(np.ma.array(src_dem_full_align, mask=np.ma.getmaskarray(align_diff_filt_full)),
align_fn_masked, src_dem_ds_align)
del src_dem_full_align
del src_dem_ds_align
# Compute original elevation difference
if True:
ref_dem_clip_ds, src_dem_clip_ds = warplib.memwarp_multi([ref_dem_ds, src_dem_ds],
res=res, extent='intersection', t_srs=local_srs,
r='cubic')
# src_dem_ds = None
ref_dem_ds = None
ref_dem_orig = iolib.ds_getma(ref_dem_clip_ds)
src_dem_orig = iolib.ds_getma(src_dem_clip_ds)
# Needed for plotting
ref_dem_hs = geolib.gdaldem_mem_ds(ref_dem_clip_ds, processing='hillshade', returnma=True, computeEdges=True)
src_dem_hs = geolib.gdaldem_mem_ds(src_dem_clip_ds, processing='hillshade', returnma=True, computeEdges=True)
diff_orig = src_dem_orig - ref_dem_orig
# Only compute stats over valid surfaces
static_mask_orig = get_mask(src_dem_clip_ds, mask_list, src_dem_fn)
# Note: this doesn't include outlier removal or slope mask!
static_mask_orig = np.logical_or(np.ma.getmaskarray(diff_orig), static_mask_orig)
# For some reason, ASTER DEM diff have a spike near the 0 bin, could be an issue with masking?
diff_orig_compressed = diff_orig[~static_mask_orig]
diff_orig_stats = malib.get_stats_dict(diff_orig_compressed, full=True)
# Prepare filtered version for comparison
diff_orig_filt = np.ma.array(diff_orig, mask=static_mask_orig)
diff_orig_filt = outlier_filter(diff_orig_filt, f=3, max_dz=max_dz)
# diff_orig_filt = outlier_filter(diff_orig_filt, perc=(12.5, 87.5), max_dz=max_dz)
slope = get_filtered_slope(src_dem_clip_ds)
diff_orig_filt = np.ma.array(diff_orig_filt, mask=np.ma.getmaskarray(slope))
diff_orig_filt_stats = malib.get_stats_dict(diff_orig_filt, full=True)
# Write out original difference map
print("Writing out original difference map for common intersection before alignment")
orig_diff_fn = outprefix + '_orig_diff.tif'
iolib.writeGTiff(diff_orig, orig_diff_fn, ref_dem_clip_ds)
# src_dem_clip_ds = None
ref_dem_clip_ds = None
if True:
align_stats_fn = outprefix + '%s_align_stats.json' % xyz_shift_str_cum_fn
align_stats = {}
align_stats['src_fn'] = src_dem_fn
align_stats['ref_fn'] = ref_dem_fn
align_stats['align_fn'] = align_fn
align_stats['res'] = {}
align_stats['res']['src'] = src_dem_res
align_stats['res']['ref'] = ref_dem_res
align_stats['res']['coreg'] = res
align_stats['center_coord'] = {'lon': center_coord_ll[0], 'lat': center_coord_ll[1],
'x': center_coord_xy[0], 'y': center_coord_xy[1]}
align_stats['shift'] = {'dx': dx_total, 'dy': dy_total, 'dz': dz_total, 'dm': dm_total}
# This tiltcorr flag gets set to false, need better flag
if tiltcorr:
align_stats['tiltcorr'] = {}
align_stats['tiltcorr']['coeff'] = coeff.tolist()
align_stats['tiltcorr']['val_stats'] = vals_stats
align_stats['before'] = diff_orig_stats
align_stats['before_filt'] = diff_orig_filt_stats
align_stats['after'] = diff_align_stats
align_stats['after_filt'] = diff_align_filt_stats
import json
with open(align_stats_fn, 'w') as f:
json.dump(align_stats, f)
# Create output plot
if True:
datadir = iolib.get_datadir()
shp_fn = os.path.join(datadir, 'gamdam/gamdam_merge_refine_line.shp')
shp_ds = ogr.Open(shp_fn)
lyr = shp_ds.GetLayer()
lyr_srs = lyr.GetSpatialRef()
shp_extent = geolib.lyr_extent(lyr)
ds_extent = geolib.ds_extent(src_dem_ds, t_srs=lyr_srs)
if geolib.extent_compare(shp_extent, ds_extent) is False:
ext = '_n' + str(int(center_coord_ll[0])) + '_n' + str(int(center_coord_ll[1])).zfill(3)
# ext = os.path.splitext(os.path.split(ref_dem_fn)[-1])[0][4:13]
out_fn = os.path.splitext(shp_fn)[0] + ext + '_clip.shp'
geolib.clip_shp(shp_fn, extent=ds_extent, out_fn=out_fn)
shp_fn = out_fn
print("Creating final plot")
# f, axa = plt.subplots(2, 4, figsize=(11, 8.5))
f, axa = plt.subplots(2, 4, figsize=(16, 8))
# for ax in axa.ravel()[:-1]:
# ax.set_facecolor('w')
# pltlib.hide_ticks(ax)
dem_clim = malib.calcperc(ref_dem_orig, (2, 98))
axa[0, 0].imshow(ref_dem_hs, cmap='gray')
im = axa[0, 0].imshow(ref_dem_orig, cmap='terrain', clim=dem_clim, alpha=0.6)
pltlib.add_cbar(axa[0, 0], im, arr=ref_dem_orig, clim=dem_clim, label=None)
pltlib.add_scalebar(axa[0, 0], res=res[0])
axa[0, 0].set_title('Reference DEM')
axa[0, 0].set_facecolor('w')
pltlib.hide_ticks(axa[0, 0])
# pltlib.shp_overlay(axa[0,0], src_dem_clip_ds, shp_fn, color='k')
axa[0, 1].imshow(src_dem_hs, cmap='gray')
im = axa[0, 1].imshow(src_dem_orig, cmap='terrain', clim=dem_clim, alpha=0.6)
pltlib.add_cbar(axa[0, 1], im, arr=src_dem_orig, clim=dem_clim, label=None)
axa[0, 1].set_title('Source DEM')
axa[0, 1].set_facecolor('w')
pltlib.hide_ticks(axa[0, 1])
# pltlib.shp_overlay(axa[0,1], src_dem_clip_ds, shp_fn, color='k')
# axa[0,2].imshow(~static_mask_orig, clim=(0,1), cmap='gray')
axa[0, 2].imshow(~mask_glac, clim=(0, 1), cmap='gray')
axa[0, 2].set_title('Surfaces for co-registration')
axa[0, 2].set_facecolor('w')
pltlib.hide_ticks(axa[0, 2])
dz_clim = malib.calcperc_sym(diff_align_filt[mask_glac], (1, 99))
dz_clim_noglac = malib.calcperc_sym(diff_orig_compressed, (1, 99))
# dz_clim = (-10, 10)
# dz_clim_noglac = (-10, 10)
# axa[0,3].imshow(~static_mask_gla, clim=(0,1), cmap='gray')
# axa[0,3].set_title('static_mask_gla2')
# # dz_clim = malib.calcperc_sym(diff_orig_compressed, (1, 99))
bins = np.linspace(dz_clim_noglac[0], dz_clim_noglac[1], 256)
# bins = np.linspace(-50, 50, 256)
axa[0, 3].hist(diff_orig_compressed, bins, color='b', label='Before', alpha=0.5)
# axa[1,3].hist(diff_align_compressed, bins, color='g', label='After', alpha=0.5)
axa[0, 3].hist(diff_align_filt_compressed, bins, color='g', label='Filter', alpha=0.5)
# axa[0, 3].set_xlim(*dz_clim_noglac)
axa[0, 3].set_xlim(-50, 50)
axa[0, 3].axvline(0, color='k', linewidth=0.5, linestyle=':')
axa[0, 3].set_xlabel('Elev. Diff. (m)')
axa[0, 3].set_ylabel('Count (px)')
axa[0, 3].set_title("Source - Reference")
before_str = 'Before\nmed: %0.2f\nnmad: %0.2f' % (diff_orig_stats['med'], diff_orig_stats['nmad'])
axa[0, 3].text(0.05, 0.95, before_str, va='top', color='b', transform=axa[0, 3].transAxes, fontsize=8)
# after_str = 'After\nmed: %0.2f\nnmad: %0.2f' % (diff_align_stats['med'], diff_align_stats['nmad'])
# axa[1,3].text(0.05, 0.65, after_str, va='top', color='g', transform=axa[1,3].transAxes, fontsize=8)
filt_str = 'Filter\nmed: %0.2f\nnmad: %0.2f' % (diff_align_filt_stats['med'], diff_align_filt_stats['nmad'])
axa[0, 3].text(0.65, 0.95, filt_str, va='top', color='g', transform=axa[0, 3].transAxes, fontsize=8)
axa[1, 0].imshow(ref_dem_hs, cmap='gray')
im = axa[1, 0].imshow(diff_orig, cmap='cpt_rainbow_r', clim=dz_clim, alpha=0.6)
pltlib.add_cbar(axa[1, 0], im, arr=diff_orig, clim=dz_clim, label=None)
axa[1, 0].set_title('Elev. Diff. Before (m)')
axa[1, 0].set_facecolor('w')
pltlib.hide_ticks(axa[1, 0])
# pltlib.shp_overlay(axa[1,0], src_dem_clip_ds, shp_fn, color='k')
axa[1, 1].imshow(ref_dem_hs, cmap='gray')
im = axa[1, 1].imshow(diff_align, cmap='cpt_rainbow_r', clim=dz_clim, alpha=0.6)
pltlib.add_cbar(axa[1, 1], im, arr=diff_align, clim=dz_clim, label=None)
axa[1, 1].set_title('Elev. Diff. After (m)')
axa[1, 1].set_facecolor('w')
pltlib.hide_ticks(axa[1, 1])
# pltlib.shp_overlay(axa[1,1], src_dem_clip_ds, shp_fn, color='k')
# tight_dz_clim = (-1.0, 1.0)
# tight_dz_clim = (-10.0, 10.0)
# tight_dz_clim = malib.calcperc_sym(diff_align_filt, (5, 95))
# im = axa[1,2].imshow(diff_align_filt, cmap='cpt_rainbow', clim=tight_dz_clim)
# pltlib.add_cbar(axa[1,2], im, arr=diff_align_filt, clim=tight_dz_clim, label=None)
# axa[1,2].set_title('Elev. Diff. Remove. Outliers (m)')
axa[1, 2].imshow(ref_dem_hs, cmap='gray')
im = axa[1, 2].imshow(diff_align_filt, cmap='cpt_rainbow_r', clim=dz_clim, alpha=0.6)
pltlib.add_cbar(axa[1, 2], im, arr=diff_align_filt, clim=dz_clim, label=None)
axa[1, 2].set_title('Elev. Diff. Remove. Outliers (m)')
axa[1, 2].set_facecolor('w')
pltlib.hide_ticks(axa[1, 2])
# pltlib.shp_overlay(axa[1,2], src_dem_clip_ds, shp_fn, color='k')
tight_dz_clim = (-10, 10)
axa[1, 3].imshow(ref_dem_hs, cmap='gray')
im = axa[1, 3].imshow(diff_align_filt_nonglac, cmap='cpt_rainbow_r', clim=tight_dz_clim, alpha=0.6)
pltlib.add_cbar(axa[1, 3], im, arr=diff_align_filt_nonglac, clim=tight_dz_clim, label=None)
axa[1, 3].set_title('Elev. Diff. NoGlac (m)')
axa[1, 3].set_facecolor('w')
pltlib.hide_ticks(axa[1, 3])
# Tried to insert Nuth fig here
# ax_nuth.change_geometry(1,2,1)
# f.axes.append(ax_nuth)
suptitle = '%s\nx: %+0.2fm, y: %+0.2fm, z: %+0.2fm' % (
os.path.split(outprefix)[-1], dx_total, dy_total, dz_total)
f.suptitle(suptitle)
f.tight_layout()
plt.subplots_adjust(top=0.90)
fig_fn = outprefix + '%s_align.png' % xyz_shift_str_cum_fn
print("Writing out figure: %s" % fig_fn)
f.savefig(fig_fn, dpi=450)
if True:
fig2 = plt.figure(0)
ax = fig2.add_subplot(1, 1, 1)
ax.imshow(ref_dem_hs, cmap='gray')
im = ax.imshow(diff_align_filt, cmap='cpt_rainbow_r', clim=dz_clim, alpha=0.6)
pltlib.add_cbar(ax, im, arr=diff_align_filt, clim=dz_clim, label=None)
ax.set_title('cLon: %0.1fE cLat: %0.1fN\n\nElev. Diff. After. Coreg. (m)' % (
center_coord_ll[1], center_coord_ll[0]))
ax.set_facecolor('w')
pltlib.hide_ticks(ax)
# pltlib.latlon_ticks(ax, lat_in=0.25, lon_in=0.25, in_crs=local_srs.ExportToProj4())
pltlib.shp_overlay(ax, src_dem_clip_ds, shp_fn, color='k')
fig2_fn = outprefix + '_align_diff.png'
fig2.savefig(fig2_fn, dpi=600, bbox_inches='tight', pad_inches=0.1)
glas_pts = np.loadtxt(glas_csv_fn, delimiter=',', skiprows=1, dtype=None)
print("Saving npz: %s" % glas_npz_fn)
np.savez_compressed(glas_npz_fn, glas_pts)
else:
#This takes ~5 seconds to load ~9M records with 8 fields
print("Loading npz: %s" % glas_npz_fn)
glas_pts = np.load(glas_npz_fn)['arr_0']
dem_fn_list = sys.argv[1:]
for n,dem_fn in enumerate(dem_fn_list):
print("%i of %i" % (n+1, len(dem_fn_list)))
#Lat/lon extent filter
print("Loading DEM: %s" % dem_fn)
dem_ds = gdal.Open(dem_fn)
dem_ma = iolib.ds_getma(dem_ds)
dem_extent_wgs84 = geolib.ds_extent(dem_ds, t_srs=pt_srs)
xmin, ymin, xmax, ymax = dem_extent_wgs84
print("Applying spatial filter")
x = glas_pts[:,xcol]
y = glas_pts[:,ycol]
idx = ((x >= xmin) & (x <= xmax) & (y >= ymin) & (y <= ymax))
if idx.nonzero()[0].size == 0:
print("No points after spatial filter")
continue
print("Sampling DEM at masked point locations")
glas_pts_fltr = glas_pts[idx]
print("Writing out %i points after spatial filter" % glas_pts_fltr.shape[0])
out_csv_fn = os.path.splitext(dem_fn)[0]+'_%s.csv' % ext
def crop_sim_res_extent(img_list, outfol, vrt=False, rpc=False):
"""
Warp images to common 'finest' resolution and intersecting extent
This is useful for stereo processing with mapprojected imagery with the skysat pairs
Parameters
----------
img_list: list
list containing two images
outfol: str
path to folder where warped images will be saved
vrt: bool
Produce warped VRT instead of geotiffs if True
rpc: bool
Copy RPC information to warped images if True
Returns
----------
out: list
list containing the two warped images, first entry (left image) is the image which was of finer resolution (more nadir) initially
If the images do not intersect, two None objects are returned in the list
"""
resample_alg = 'lanczos'
img1 = img_list[0]
img2 = img_list[1]
img1_ds = iolib.fn_getds(img1)
img2_ds = iolib.fn_getds(img2)
res1 = geolib.get_res(img1_ds, square=True)[0]
res2 = geolib.get_res(img2_ds, square=True)[0]
# set left image as higher resolution, this is repeated for video, but
# good for triplet with no gsd information
if res1 < res2:
l_img = img1
r_img = img2
res = res1
else:
l_img = img2
r_img = img1
res = res2
# ASP stereo command expects the input to be .tif/.tiff, complains for .vrt
# Try to save with vrt driver but a tif extension ?
l_img_warp = os.path.join(
outfol,
os.path.splitext(os.path.basename(l_img))[0] + '_warp.tif')
r_img_warp = os.path.join(
outfol,
os.path.splitext(os.path.basename(r_img))[0] + '_warp.tif')
if not (os.path.exists(l_img_warp)):
# can turn on verbose during qa/qc
# Better to turn off during large runs, writing takes time
verbose = False
if not os.path.exists(outfol):
os.makedirs(outfol)
try:
#this will simply break and continue if the images do not intersect
ds_list = warplib.memwarp_multi_fn([l_img, r_img],
r=resample_alg,
verbose=verbose,
res='min',
extent='intersection')
if vrt:
extent = geolib.ds_extent(ds_list[0])
res = geolib.get_res(ds_list[0], square=True)
vrt_options = gdal.BuildVRTOptions(resampleAlg='average',
resolution='user',
xRes=res[0],
yRes=res[1],
outputBounds=tuple(extent))
l_vrt = gdal.BuildVRT(l_img_warp, [
l_img,
],
options=vrt_options)
r_vrt = gdal.BuildVRT(r_img_warp, [
r_img,
],
options=vrt_options)
# close vrt to save to disk
l_vrt = None
r_vrt = None
out = [l_img_warp, r_img_warp]
else:
# I am opting out of writing out vrt, to prevent correlation
# artifacts. GeoTiffs will be written out in the meantime
l_img_ma = iolib.ds_getma(ds_list[0])
r_img_ma = iolib.ds_getma(ds_list[1])
iolib.writeGTiff(l_img_ma, l_img_warp, ds_list[0])
iolib.writeGTiff(r_img_ma, r_img_warp, ds_list[1])
out = [l_img_warp, r_img_warp]
del (ds_list)
if rpc:
copy_rpc(l_img, l_img_warp)
copy_rpc(r_img, r_img_warp)
except BaseException:
out = None
else:
out = [l_img_warp, r_img_warp]
return out
description=desc_str,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-t_srs',
type=str,
default=None,
help='Proj4 string for desired output coordinates')
parser.add_argument('-pad', type=float, default=None, \
help='Width of padding to be applied to extent (meters, or units of specified t_srs')
parser.add_argument('fn', type=str, help='Raster filename')
return parser
parser = getparser()
args = parser.parse_args()
ds = gdal.Open(args.fn)
t_srs = None
if args.t_srs is not None:
t_srs = warplib.parse_srs(args.t_srs)
extent = geolib.ds_extent(ds, t_srs=t_srs)
#extent = geolib.ds_geom_extent(ds)
#Pad by desired amount
if args.pad is not None:
extent = geolib.pad_extent(extent, width=args.pad)
#Print to stdout
print(' '.join(map(str, extent)))