def butter_low(dt_list, val, lowpass=1.0):
import scipy.signal
#val_mask = np.ma.getmaskarray(val)
#dt is 300 s, 5 min
dt_diff = np.diff(dt_list)
if isinstance(dt_diff[0], float):
dt_diff *= 86400.
else:
dt_diff = np.array([dt.total_seconds() for dt in dt_diff])
dt = malib.fast_median(dt_diff)
#f is 0.00333 Hz
#288 samples/day
fs = 1./dt
nyq = fs/2.
order = 3
f_max = (1./(86400*lowpass)) / nyq
b, a = scipy.signal.butter(order, f_max, btype='lowpass')
#b, a = sp.signal.butter(order, (f_min, f_max), btype='bandstop')
w, h = scipy.signal.freqz(b, a, worN=2000)
# w_f = (nyq/np.pi)*w
# w_f_days = 1/w_f/86400.
#plt.plot(w_f_days, np.abs(h))
val_f = scipy.signal.filtfilt(b, a, val)
return val_f
def butter(dt_list, val, lowpass=1.0):
"""This is framework for a butterworth bandpass for 1D data
Needs to be cleaned up and generalized
"""
import scipy.signal
import matplotlib.pyplot as plt
#dt is 300 s, 5 min
dt_diff = np.diff(dt_list)
dt_diff = np.array([dt.total_seconds() for dt in dt_diff])
dt = malib.fast_median(dt_diff)
#f is 0.00333 Hz
#288 samples/day
fs = 1./dt
nyq = fs/2.
if False:
#psd, f = psd(z_msl, fs)
sp_f, sp_psd = scipy.signal.periodogram(val, fs, detrend='linear')
#sp_f, sp_psd = scipy.signal.welch(z_msl, fs, nperseg=2048)
sp_f_days = 1./sp_f/86400.
plt.figure()
plt.plot(sp_f, sp_psd)
plt.plot(sp_f_days, sp_psd)
plt.semilogy(sp_f_days, sp_psd)
plt.xlabel('Frequency')
plt.ylabel('Power')
print("Filtering tidal signal")
#Define bandpass filter
#f_min = dt/(86400*0.25)
f_max = (1./(86400*0.1)) / nyq
f_min = (1./(86400*1.8)) / nyq
order = 6
b, a = scipy.signal.butter(order, f_min, btype='highpass')
#b, a = sp.signal.butter(order, (f_min, f_max), btype='bandpass')
w, h = scipy.signal.freqz(b, a, worN=2000)
w_f = (nyq/np.pi)*w
w_f_days = 1/w_f/86400.
#plt.figure()
#plt.plot(w_f_days, np.abs(h))
val_f_tide = scipy.signal.filtfilt(b, a, val)
b, a = scipy.signal.butter(order, f_max, btype='lowpass')
#b, a = sp.signal.butter(order, (f_min, f_max), btype='bandstop')
w, h = scipy.signal.freqz(b, a, worN=2000)
w_f = (nyq/np.pi)*w
w_f_days = 1/w_f/86400.
#plt.plot(w_f_days, np.abs(h))
val_f_tide_denoise = scipy.signal.filtfilt(b, a, val_f_tide)
#val_f_notide = sp.signal.filtfilt(b, a, val)
val_f_notide = val - val_f_tide
def compute_offset_nuth(dh, slope, aspect, min_count=100, remove_outliers=True, plot=True):
"""Compute horizontal offset between input rasters using Nuth and Kaab [2011] (nuth) method
"""
import scipy.optimize as optimization
if dh.count() < min_count:
sys.exit("Not enough dh samples")
if slope.count() < min_count:
sys.exit("Not enough slope/aspect samples")
#mean_dh = dh.mean()
#mean_slope = slope.mean()
#c_seed = (mean_dh/np.tan(np.deg2rad(mean_slope)))
med_dh = malib.fast_median(dh)
med_slope = malib.fast_median(slope)
c_seed = (med_dh/np.tan(np.deg2rad(med_slope)))
x0 = np.array([0.0, 0.0, c_seed])
print("Computing common mask")
common_mask = ~(malib.common_mask([dh, aspect, slope]))
#Prepare x and y data
xdata = aspect[common_mask].data
ydata = (dh[common_mask]/np.tan(np.deg2rad(slope[common_mask]))).data
print("Initial sample count:")
print(ydata.size)
if remove_outliers:
print("Removing outliers")
#print("Absolute dz filter: %0.2f" % max_dz)
#diff = np.ma.masked_greater(diff, max_dz)
#print(diff.count())
#Outlier dz filter
f = 3
sigma, u = (ydata.std(), ydata.mean())
#sigma, u = malib.mad(ydata, return_med=True)
rmin = u - f*sigma
rmax = u + f*sigma
print("3-sigma filter: %0.2f - %0.2f" % (rmin, rmax))
idx = (ydata >= rmin) & (ydata <= rmax)
xdata = xdata[idx]
ydata = ydata[idx]
print(ydata.size)
#Generate synthetic data to test curve_fit
#xdata = np.arange(0,360,0.01)
#ydata = f(xdata, 20.0, 130.0, -3.0) + 20*np.random.normal(size=len(xdata))
#Limit sample size
#n = 10000
#idx = random.sample(range(xdata.size), n)
#xdata = xdata[idx]
#ydata = ydata[idx]
#Compute robust statistics for 1-degree bins
nbins = 360
bin_range = (0., 360.)
bin_width = 1.0
bin_count, bin_edges, bin_centers = malib.bin_stats(xdata, ydata, stat='count', nbins=nbins, bin_range=bin_range)
bin_med, bin_edges, bin_centers = malib.bin_stats(xdata, ydata, stat='median', nbins=nbins, bin_range=bin_range)
#Needed to estimate sigma for weighted lsq
#bin_mad, bin_edges, bin_centers = malib.bin_stats(xdata, ydata, stat=malib.mad, nbins=nbins, bin_range=bin_range)
#Started implementing this for more generic binning, needs testing
#bin_count, x_bin_edges, y_bin_edges = malib.get_2dhist(xdata, ydata, \
# xlim=bin_range, nbins=(nbins, nbins), stat='count')
"""
#Mask bins in grid directions, can potentially contain biased stats
#Especially true for SGM algorithm
#badbins = [0, 90, 180, 270, 360]
badbins = [0, 45, 90, 135, 180, 225, 270, 315, 360]
bin_stat = np.ma.masked_where(np.around(bin_edges[:-1]) % 45 == 0, bin_stat)
bin_edges = np.ma.masked_where(np.around(bin_edges[:-1]) % 45 == 0, bin_edges)
"""
#Remove any bins with only a few points
min_bin_sample_count = 9
idx = (bin_count.filled(0) >= min_bin_sample_count)
bin_count = bin_count[idx].data
bin_med = bin_med[idx].data
#bin_mad = bin_mad[idx].data
bin_centers = bin_centers[idx]
fit = None
fit_fig = None
#Want a good distribution of bins, at least 1/4 to 1/2 of sinusoid, to ensure good fit
#Need at least 3 valid bins to fit 3 parameters in nuth_func
#min_bin_count = 3
min_bin_count = 90
#Not going to help if we have a step function between two plateaus, but better than nothing
#Calculate bin aspect spread
bin_ptp = np.cos(np.radians(bin_centers)).ptp()
min_bin_ptp = 1.0
#Should iterate here, if not enough bins, increase bin width
if len(bin_med) >= min_bin_count and bin_ptp >= min_bin_ptp:
print("Computing fit")
#Unweighted fit
fit = optimization.curve_fit(nuth_func, bin_centers, bin_med, x0)[0]
#Weight by observed spread in each bin
#sigma = bin_mad
#fit = optimization.curve_fit(nuth_func, bin_centers, bin_med, x0, sigma, absolute_sigma=True)[0]
#Weight by bin count
#sigma = bin_count.max()/bin_count
#fit = optimization.curve_fit(nuth_func, bin_centers, bin_med, x0, sigma, absolute_sigma=False)[0]
print(fit)
if plot:
print("Generating Nuth and Kaab plot")
bin_idx = np.digitize(xdata, bin_edges)
output = []
for i in np.arange(1, len(bin_edges)):
output.append(ydata[bin_idx==i])
#flierprops={'marker':'.'}
lw = 0.25
whiskerprops={'linewidth':lw}
capprops={'linewidth':lw}
boxprops={'facecolor':'k', 'linewidth':0}
medianprops={'marker':'o', 'ms':1, 'color':'r'}
fit_fig, ax = plt.subplots(figsize=(6,6))
#widths = (bin_width/2.0)
widths = 2.5*(bin_count/bin_count.max())
#widths = bin_count/np.percentile(bin_count, 50)
#Stride
s=3
#This is inefficient, but we have list of arrays with different length, need to filter
#Reduntant with earlier filter, should refactor
bp = ax.boxplot(np.array(output)[idx][::s], positions=bin_centers[::s], widths=widths[::s], showfliers=False, \
patch_artist=True, boxprops=boxprops, whiskerprops=whiskerprops, capprops=capprops, \
medianprops=medianprops)
bin_ticks = [0, 45, 90, 135, 180, 225, 270, 315, 360]
ax.set_xticks(bin_ticks)
ax.set_xticklabels(bin_ticks)
"""
#Can pull out medians from boxplot
#We are computing multiple times, inefficient
bp_bin_med = []
for medline in bp['medians']:
bp_bin_med.append(medline.get_ydata()[0])
"""
#Plot the fit
f_a = nuth_func(bin_centers, fit[0], fit[1], fit[2])
nuth_func_str = r'$y=%0.2f*cos(%0.2f-x)+%0.2f$' % tuple(fit)
ax.plot(bin_centers, f_a, 'b', label=nuth_func_str)
ax.set_xlabel('Aspect (deg)')
ax.set_ylabel('dh/tan(slope) (m)')
ax.axhline(color='gray', linewidth=0.5)
ax.set_xlim(*bin_range)
ylim = ax.get_ylim()
abs_ylim = np.max(np.abs(ylim))
#abs_ylim = np.max(np.abs([ydata.min(), ydata.max()]))
#pad = 0.2 * abs_ylim
pad = 0
ylim = (-abs_ylim - pad, abs_ylim + pad)
minylim = (-10,10)
if ylim[0] > minylim[0]:
ylim = minylim
ax.set_ylim(*ylim)
ax.legend(prop={'size':8})
return fit, fit_fig
def compute_offset(ref_dem_ds, src_dem_ds, src_dem_fn, mode='nuth', remove_outliers=True, max_offset=100, \
max_dz=100, slope_lim=(0.1, 40), mask_list=['glaciers',], plot=True):
#Make sure the input datasets have the same resolution/extent
#Use projection of source DEM
ref_dem_clip_ds, src_dem_clip_ds = warplib.memwarp_multi([ref_dem_ds, src_dem_ds], \
res='max', extent='intersection', t_srs=src_dem_ds, r='cubic')
#Compute size of NCC and SAD search window in pixels
res = float(geolib.get_res(ref_dem_clip_ds, square=True)[0])
max_offset_px = (max_offset/res) + 1
#print(max_offset_px)
pad = (int(max_offset_px), int(max_offset_px))
#This will be updated geotransform for src_dem
src_dem_gt = np.array(src_dem_clip_ds.GetGeoTransform())
#Load the arrays
ref_dem = iolib.ds_getma(ref_dem_clip_ds, 1)
src_dem = iolib.ds_getma(src_dem_clip_ds, 1)
print("Elevation difference stats for uncorrected input DEMs (src - ref)")
diff = src_dem - ref_dem
static_mask = get_mask(src_dem_clip_ds, mask_list, src_dem_fn)
diff = np.ma.array(diff, mask=static_mask)
if diff.count() == 0:
sys.exit("No overlapping, unmasked pixels shared between input DEMs")
if remove_outliers:
diff = outlier_filter(diff, f=3, max_dz=max_dz)
#Want to use higher quality DEM, should determine automatically from original res/count
#slope = get_filtered_slope(ref_dem_clip_ds, slope_lim=slope_lim)
slope = get_filtered_slope(src_dem_clip_ds, slope_lim=slope_lim)
print("Computing aspect")
#aspect = geolib.gdaldem_mem_ds(ref_dem_clip_ds, processing='aspect', returnma=True, computeEdges=False)
aspect = geolib.gdaldem_mem_ds(src_dem_clip_ds, processing='aspect', returnma=True, computeEdges=False)
ref_dem_clip_ds = None
src_dem_clip_ds = None
#Apply slope filter to diff
#Note that we combine masks from diff and slope in coreglib
diff = np.ma.array(diff, mask=np.ma.getmaskarray(slope))
#Get final mask after filtering
static_mask = np.ma.getmaskarray(diff)
#Compute stats for new masked difference map
print("Filtered difference map")
diff_stats = malib.print_stats(diff)
dz = diff_stats[5]
print("Computing sub-pixel offset between DEMs using mode: %s" % mode)
#By default, don't create output figure
fig = None
#Default horizntal shift is (0,0)
dx = 0
dy = 0
#Sum of absolute differences
if mode == "sad":
ref_dem = np.ma.array(ref_dem, mask=static_mask)
src_dem = np.ma.array(src_dem, mask=static_mask)
m, int_offset, sp_offset = coreglib.compute_offset_sad(ref_dem, src_dem, pad=pad)
#Geotransform has negative y resolution, so don't need negative sign
#np array is positive down
#GDAL coordinates are positive up
dx = sp_offset[1]*src_dem_gt[1]
dy = sp_offset[0]*src_dem_gt[5]
#Normalized cross-correlation of clipped, overlapping areas
elif mode == "ncc":
ref_dem = np.ma.array(ref_dem, mask=static_mask)
src_dem = np.ma.array(src_dem, mask=static_mask)
m, int_offset, sp_offset, fig = coreglib.compute_offset_ncc(ref_dem, src_dem, \
pad=pad, prefilter=False, plot=plot)
dx = sp_offset[1]*src_dem_gt[1]
dy = sp_offset[0]*src_dem_gt[5]
#Nuth and Kaab (2011)
elif mode == "nuth":
#Compute relationship between elevation difference, slope and aspect
fit_param, fig = coreglib.compute_offset_nuth(diff, slope, aspect, plot=plot)
if fit_param is None:
print("Failed to calculate horizontal shift")
else:
#fit_param[0] is magnitude of shift vector
#fit_param[1] is direction of shift vector
#fit_param[2] is mean bias divided by tangent of mean slope
#print(fit_param)
dx = fit_param[0]*np.sin(np.deg2rad(fit_param[1]))
dy = fit_param[0]*np.cos(np.deg2rad(fit_param[1]))
med_slope = malib.fast_median(slope)
nuth_dz = fit_param[2]*np.tan(np.deg2rad(med_slope))
print('Median dz: %0.2f\nNuth dz: %0.2f' % (dz, nuth_dz))
#dz = nuth_dz
elif mode == "all":
print("Not yet implemented")
#Want to compare all methods, average offsets
#m, int_offset, sp_offset = coreglib.compute_offset_sad(ref_dem, src_dem)
#m, int_offset, sp_offset = coreglib.compute_offset_ncc(ref_dem, src_dem)
elif mode == "none":
print("Skipping alignment, writing out DEM with median bias over static surfaces removed")
dst_fn = outprefix+'_med%0.1f.tif' % dz
iolib.writeGTiff(src_dem_orig + dz, dst_fn, src_dem_ds)
sys.exit()
#Note: minus signs here since we are computing dz=(src-ref), but adjusting src
return -dx, -dy, -dz, static_mask, fig
z1_bin_counts, z1_bin_edges = np.histogram(z1, bins=z_bin_edges)
z1_bin_areas = z1_bin_counts * ds_res[0] * ds_res[1] / 1E6
z2_bin_counts, z2_bin_edges = np.histogram(z2, bins=z_bin_edges)
z2_bin_areas = z2_bin_counts * ds_res[0] * ds_res[1] / 1E6
#dz_bin_edges, dz_bin_centers = get_bins(dz, 1.)
#dz_bin_counts, dz_bin_edges = np.histogram(dz, bins=dz_bin_edges)
#dz_bin_areas = dz_bin_counts * ds_res * ds_res / 1E6
dz_bin_med = np.ma.masked_all_like(z1_bin_areas)
dz_bin_mad = np.ma.masked_all_like(z1_bin_areas)
idx = np.digitize(z1, z_bin_edges)
for bin_n in range(z_bin_centers.size):
dz_bin_samp = mb[(idx == bin_n + 1)]
#dz_bin_samp = dhdt[(idx == n+1)]
if dz_bin_samp.count() > 0:
dz_bin_med[bin_n] = malib.fast_median(dz_bin_samp)
dz_bin_mad[bin_n] = malib.mad(dz_bin_samp)
dz_bin_med[bin_n] = dz_bin_samp.mean()
dz_bin_mad[bin_n] = dz_bin_samp.std()
print("Generating map plot")
f, axa = plt.subplots(1, 3, figsize=(10, 7.5))
f.suptitle(feat_fn)
alpha = 1.0
hs = True
if hs:
z1_hs = geolib.gdaldem_wrapper(out_z1_fn,
product='hs',
returnma=True)
z2_hs = geolib.gdaldem_wrapper(out_z2_fn,
product='hs',
def hist_plot(gf, outdir, bin_width=10.0):
#print("Generating histograms")
#Create bins for full range of input data and specified bin width
#NOTE: these counts/areas are for valid pixels only
#Not necessarily a true representation of actual glacier hypsometry
#Need a void-filled DEM for this
z_bin_edges, z_bin_centers = malib.get_bins(gf.z1, bin_width)
z1_bin_counts, z1_bin_edges = np.histogram(gf.z1, bins=z_bin_edges)
z1_bin_areas = z1_bin_counts * gf.res[0] * gf.res[1] / 1E6
#RGI standard is integer thousandths of glaciers total area
#Should check to make sure sum of bin areas equals total area
z1_bin_areas_perc = 100. * z1_bin_areas / np.sum(z1_bin_areas)
z2_bin_counts, z2_bin_edges = np.histogram(gf.z2, bins=z_bin_edges)
z2_bin_areas = z2_bin_counts * gf.res[0] * gf.res[1] / 1E6
z2_bin_areas_perc = 100. * z2_bin_areas / np.sum(z2_bin_areas)
#Create arrays to store output
mb_bin_med = np.ma.masked_all_like(z1_bin_areas)
mb_bin_mad = np.ma.masked_all_like(z1_bin_areas)
mb_bin_mean = np.ma.masked_all_like(z1_bin_areas)
mb_bin_std = np.ma.masked_all_like(z1_bin_areas)
dz_bin_med = np.ma.masked_all_like(z1_bin_areas)
dz_bin_mad = np.ma.masked_all_like(z1_bin_areas)
dz_bin_mean = np.ma.masked_all_like(z1_bin_areas)
dz_bin_std = np.ma.masked_all_like(z1_bin_areas)
if gf.debris_class is not None:
perc_clean = np.ma.masked_all_like(z1_bin_areas)
perc_debris = np.ma.masked_all_like(z1_bin_areas)
perc_pond = np.ma.masked_all_like(z1_bin_areas)
debris_thick_med = np.ma.masked_all_like(z1_bin_areas)
debris_thick_mad = np.ma.masked_all_like(z1_bin_areas)
#Loop through each bin and extract stats
idx = np.digitize(gf.z1, z_bin_edges)
for bin_n in range(z_bin_centers.size):
mb_bin_samp = gf.mb[(idx == bin_n + 1)]
if mb_bin_samp.count() > 0:
mb_bin_med[bin_n] = malib.fast_median(mb_bin_samp)
mb_bin_mad[bin_n] = malib.mad(mb_bin_samp)
mb_bin_mean[bin_n] = mb_bin_samp.mean()
mb_bin_std[bin_n] = mb_bin_samp.std()
dz_bin_samp = gf.dhdt[(idx == bin_n + 1)]
if dz_bin_samp.count() > 0:
dz_bin_med[bin_n] = malib.fast_median(dz_bin_samp)
dz_bin_mad[bin_n] = malib.mad(dz_bin_samp)
dz_bin_mean[bin_n] = dz_bin_samp.mean()
dz_bin_std[bin_n] = dz_bin_samp.std()
if gf.debris_class is not None:
debris_class_bin_samp = gf.debris_class[(idx == bin_n + 1)]
if debris_class_bin_samp.count() > 0:
perc_clean[bin_n] = 100. * (
debris_class_bin_samp
== 1).sum() / debris_class_bin_samp.count()
perc_debris[bin_n] = 100. * (
debris_class_bin_samp
== 2).sum() / debris_class_bin_samp.count()
perc_pond[bin_n] = 100. * (
debris_class_bin_samp
== 3).sum() / debris_class_bin_samp.count()
debris_thick_bin_samp = gf.debris_thick[(idx == bin_n + 1)]
debris_thick_med[bin_n] = malib.fast_median(debris_thick_bin_samp)
debris_thick_mad[bin_n] = malib.mad(debris_thick_bin_samp)
outbins_header = 'bin_center_elev_m, z1_bin_count_valid, z1_bin_area_valid_km2, z1_bin_area_perc, z2_bin_count_valid, z2_bin_area_valid_km2, z2_bin_area_perc, dhdt_bin_med_ma, dhdt_bin_mad_ma, dhdt_bin_mean_ma, dhdt_bin_std_ma, mb_bin_med_mwea, mb_bin_mad_mwea, mb_bin_mean_mwea, mb_bin_std_mwea'
fmt = '%0.1f, %i, %0.3f, %0.2f, %i, %0.3f, %0.2f, %0.2f, %0.2f, %0.2f, %0.2f, %0.2f, %0.2f, %0.2f, %0.2f'
outbins = [
z_bin_centers, z1_bin_counts, z1_bin_areas, z1_bin_areas_perc,
z2_bin_counts, z2_bin_areas, z2_bin_areas_perc, dz_bin_med, dz_bin_mad,
dz_bin_mean, dz_bin_std, mb_bin_med, mb_bin_mad, mb_bin_mean,
mb_bin_std
]
if gf.debris_class is not None:
outbins_header += ', debris_thick_med_m, debris_thick_mad_m, perc_debris, perc_pond, perc_clean'
fmt += ', %0.2f, %0.2f, %0.2f, %0.2f, %0.2f'
debris_thick_med[debris_thick_med == -(np.inf)] = 0.00
debris_thick_mad[debris_thick_mad == -(np.inf)] = 0.00
outbins.extend([
debris_thick_med, debris_thick_mad, perc_debris, perc_pond,
perc_clean
])
#print(len(outbins), len(fmt.split(',')), len(outbins_header.split(',')))
outbins = np.ma.array(outbins).T.astype('float32')
np.ma.set_fill_value(outbins, -9999.0)
outbins_fn = os.path.join(outdir, gf.feat_fn + '_mb_bins.csv')
#print(outbins.shape)
np.savetxt(outbins_fn,
outbins,
fmt=fmt,
delimiter=',',
header=outbins_header)
#print("Generating aed plot")
#f,axa = plt.subplots(1,2, figsize=(6, 6))
f, axa = plt.subplots(1, 3, figsize=(10, 7.5))
f.suptitle(gf.feat_fn)
axa[0].plot(z1_bin_areas, z_bin_centers, label='%0.2f' % gf.t1)
axa[0].plot(z2_bin_areas, z_bin_centers, label='%0.2f' % gf.t2)
axa[0].axhline(gf.z1_ela, ls=':', c='C0')
axa[0].axhline(gf.z2_ela, ls=':', c='C1')
axa[0].legend(prop={'size': 8}, loc='upper right')
axa[0].set_ylabel('Elevation (m WGS84)')
axa[0].set_xlabel('Area $\mathregular{km^2}$')
pltlib.minorticks_on(axa[0])
axa[1].axvline(0, lw=1.0, c='k')
axa[1].axvline(gf.mb_mean,
lw=0.5,
ls=':',
c='k',
label='%0.2f m w.e./yr' % gf.mb_mean)
axa[1].legend(prop={'size': 8}, loc='upper right')
axa[1].plot(mb_bin_med, z_bin_centers, color='k')
axa[1].fill_betweenx(z_bin_centers,
mb_bin_med - mb_bin_mad,
mb_bin_med + mb_bin_mad,
color='k',
alpha=0.1)
axa[1].fill_betweenx(z_bin_centers,
0,
mb_bin_med,
where=(mb_bin_med < 0),
color='r',
alpha=0.2)
axa[1].fill_betweenx(z_bin_centers,
0,
mb_bin_med,
where=(mb_bin_med > 0),
color='b',
alpha=0.2)
#axa[1].set_ylabel('Elevation (m WGS84)')
#axa[1].set_xlabel('dh/dt (m/yr)')
axa[1].set_xlabel('mb (m w.e./yr)')
pltlib.minorticks_on(axa[1])
#Hide y-axis labels
axa[1].axes.yaxis.set_ticklabels([])
#axa[1].set_xlim(-2.0, 2.0)
#axa[1].set_xlim(-8.0, 8.0)
axa[1].set_xlim(-3.0, 3.0)
if gf.debris_class is not None:
axa[2].errorbar(debris_thick_med * 100.,
z_bin_centers,
xerr=debris_thick_mad * 100,
color='k',
fmt='o',
ms=3,
label='Thickness',
alpha=0.6)
axa[2].plot(perc_debris,
z_bin_centers,
color='sienna',
label='Debris Coverage')
axa[2].plot(perc_pond,
z_bin_centers,
color='turquoise',
label='Pond Coverage')
axa[2].set_xlim(0, 100)
pltlib.minorticks_on(axa[2])
axa[2].legend(prop={'size': 8}, loc='upper right')
axa[2].set_xlabel('Debris thickness (cm), coverage (%)')
axa[2].yaxis.tick_right()
axa[2].yaxis.set_label_position("right")
plt.tight_layout()
#Make room for suptitle
plt.subplots_adjust(top=0.95)
#print("Saving aed plot")
fig_fn = os.path.join(outdir, gf.feat_fn + '_mb_aed.png')
plt.savefig(fig_fn, bbox_inches='tight', dpi=300)
plt.close(f)
return z_bin_edges
def sample_stack(ex, ey, geoid_offset=False, pad=3):
if ex > m.shape[2]-1 or ey > m.shape[1]-1:
print "Input coordinates are outside stack extent:"
print ex, ey
print m.shape
v = None
else:
print "Sampling with pad: %i" % pad
if pad == 0:
v = m[:,ey,ex]
else:
window_x = np.around(np.clip([ex-pad, ex+pad+1], 0, m.shape[2]-1)).astype(int)
window_y = np.around(np.clip([ey-pad, ey+pad+1], 0, m.shape[1]-1)).astype(int)
print window_x
print window_y
v = m[:,window_y[0]:window_y[1],window_x[0]:window_x[1]].reshape(m.shape[0], np.ptp(window_x)*np.ptp(window_y))
#v = v.mean(axis=1)
v = np.ma.median(v, axis=1)
if v.count() == 0:
print "No valid values"
else:
mx, my = geolib.pixelToMap(ex, ey, gt)
print ex, ey, mx, my
print "Count: %i" % v.count()
#Hack to get elevations relative to geoid
#Note: this can be added multiple times if clicked quickly
if geoid_offset:
#geoid_offset = geolib.sps2geoid(mx, my, 0.0)[2]
geoid_offset = geolib.nps2geoid(mx, my, 0.0)[2]
print "Removing geoid offset: %0.1f" % geoid_offset
v += geoid_offset
#Should filter here
#RS1 has some values that are many 1000s of m/yr below neighbors
if filter_outliers:
if True:
med = malib.fast_median(v)
mad = malib.mad(v)
min_v = med - mad*4
f_idx = (v < min_v).filled(False)
if np.any(f_idx):
print med, mad
print "Outliers removed by absolute filter: (val < %0.1f)" % min_v
print timelib.o2dt(d[f_idx])
print v[f_idx]
v[f_idx] = np.ma.masked
if True:
v_idx = (~np.ma.getmaskarray(v)).nonzero()[0]
#This tries to maintain fixed window in time
f = filtlib.rolling_fltr(v, size=7)
#This uses fixed number of neighbors
f = filtlib.rolling_fltr(v[v_idx], size=7)
#f_diff = np.abs(f - v)
#Note: the issue is usually that the velocity values are too low
#f_diff = f - v
f_diff = f - v[v_idx]
diff_thresh = 2000
#f_idx = (f_diff > diff_thresh).filled(False)
#f_idx = (f_diff < diff_thresh).filled(False)
f_idx = np.zeros_like(v.data).astype(bool)
f_idx[v_idx] = (f_diff > diff_thresh)
if np.any(f_idx):
print "Outliers removed by rolling median filter: (val < %0.1f)" % diff_thresh
print timelib.o2dt(d[f_idx])
print v[f_idx]
v[f_idx] = np.ma.masked
return v
def main():
parser = getparser()
args = parser.parse_args()
t_unit = args.dt
plot = args.plot
remove_offsets = args.remove_offsets
mask_fn = args.mask_fn
if mask_fn is not None:
remove_offsets = True
#Input is 3-band disparity map, extract bands directly
src_fn = args.disp_fn
if not iolib.fn_check(src_fn):
sys.exit("Unable to locate input file: %s" % src_fn)
src_ds = iolib.fn_getds(src_fn)
if src_ds.RasterCount != 3:
sys.exit("Input file must be ASP disparity map (3 bands: x, y, mask)")
#Extract pixel resolution
h_res, v_res = geolib.get_res(src_ds)
#Horizontal scale factor
#If running on disparity_view output (gdal_translate -outsize 5% 5% F.tif F_5.tif)
#h_res /= 20
#v_res /= 20
#Load horizontal and vertical disparities
h = iolib.ds_getma(src_ds, bnum=1)
v = iolib.ds_getma(src_ds, bnum=2)
#ASP output has northward motion as negative values in band 2
v *= -1
t1, t2 = timelib.fn_getdatetime_list(src_fn)
dt = t2 - t1
#Default t_factor is in 1/years
t_factor = timelib.get_t_factor(t1, t2)
#Input timestamp arrays if inputs are mosaics
if False:
t1_fn = ''
t2_fn = ''
if os.path.exists(t1_fn) and os.path.exists(t2_fn):
t_factor = timelib.get_t_factor_fn(t1_fn, t2_fn)
if t_factor is None:
sys.exit("Unable to determine input timestamps")
if t_unit == 'day':
t_factor *= 365.25
print("Input dates:")
print(t1)
print(t2)
print(dt)
print(t_factor, t_unit)
#Scale values for polar stereographic distortion
srs = geolib.get_ds_srs(src_ds)
proj_scale_factor = 1.0
#Want to scale to get correct distances for polar sterographic
if srs.IsSame(geolib.nps_srs) or srs.IsSame(geolib.sps_srs):
proj_scale_factor = geolib.scale_ps_ds(src_ds)
#Convert disparity values in pixels to m/t_unit
h_myr = h * h_res * proj_scale_factor / t_factor
h = None
v_myr = v * v_res * proj_scale_factor / t_factor
v = None
#Velocity Magnitude
m = np.ma.sqrt(h_myr**2 + v_myr**2)
print("Velocity Magnitude stats")
malib.print_stats(m)
#Remove x and y offsets over control surfaces
offset_str = ''
if remove_offsets:
if mask_fn is None:
from demcoreg.dem_mask import get_lulc_mask
print(
"\nUsing demcoreg to prepare mask of stable control surfaces\n"
)
mask = get_lulc_mask(src_ds,
mask_glaciers=True,
filter='rock+ice+water')
else:
print("\nWarping input raster mask")
#This can be from previous dem_mask.py run (e.g. *rockmask.tif)
mask_ds = warplib.memwarp_multi_fn([
mask_fn,
],
res=src_ds,
extent=src_ds,
t_srs=src_ds)[0]
mask = iolib.ds_getma(mask_ds)
#The default from ds_getma is a masked array, so need to isolate boolean mask
#Assume input is 0 for masked, 1 for unmasked (valid control surface)
mask = mask.filled().astype('bool')
#This should work, as the *rockmask.py is 1 for unmasked, 0 for masked, with ndv=0
#mask = np.ma.getmaskarray(mask)
#Vector mask - untested
if os.path.splitext(mask_fn)[1] == 'shp':
mask = geolib.shp2array(mask_fn, src_ds)
print("\nRemoving median x and y offset over static control surfaces")
h_myr_count = h_myr.count()
h_myr_static_count = h_myr[mask].count()
h_myr_med = malib.fast_median(h_myr[mask])
v_myr_med = malib.fast_median(v_myr[mask])
h_myr_mad = malib.mad(h_myr[mask])
v_myr_mad = malib.mad(v_myr[mask])
print("Static pixel count: %i (%0.1f%%)" %
(h_myr_static_count,
100 * float(h_myr_static_count) / h_myr_count))
print("median (+/-NMAD)")
print("x velocity offset: %0.2f (+/-%0.2f) m/%s" %
(h_myr_med, h_myr_mad, t_unit))
print("y velocity offset: %0.2f (+/-%0.2f) m/%s" %
(v_myr_med, v_myr_mad, t_unit))
h_myr -= h_myr_med
v_myr -= v_myr_med
offset_str = '_offsetcorr_h%0.2f_v%0.2f' % (h_myr_med, v_myr_med)
#Velocity Magnitude
m = np.ma.sqrt(h_myr**2 + v_myr**2)
print("Velocity Magnitude stats after correction")
malib.print_stats(m)
if plot:
fig_fn = os.path.splitext(src_fn)[0] + '.png'
label = 'Velocity (m/%s)' % t_unit
f, ax = make_plot(m, fig_fn, label)
plotvec(h_myr, v_myr)
plt.tight_layout()
plt.savefig(fig_fn,
dpi=300,
bbox_inches='tight',
pad_inches=0,
edgecolor='none')
print("Writing out files")
gt = src_ds.GetGeoTransform()
proj = src_ds.GetProjection()
dst_fn = os.path.splitext(src_fn)[0] + '_vm%s.tif' % offset_str
iolib.writeGTiff(m, dst_fn, create=True, gt=gt, proj=proj)
dst_fn = os.path.splitext(src_fn)[0] + '_vx%s.tif' % offset_str
iolib.writeGTiff(h_myr, dst_fn, create=True, gt=gt, proj=proj)
dst_fn = os.path.splitext(src_fn)[0] + '_vy%s.tif' % offset_str
iolib.writeGTiff(v_myr, dst_fn, create=True, gt=gt, proj=proj)
src_ds = None
def hist_plot(gf, outdir, bin_width=10.0):
#print("Generating histograms")
z_bin_edges, z_bin_centers = get_bins(gf.z1, bin_width)
z1_bin_counts, z1_bin_edges = np.histogram(gf.z1, bins=z_bin_edges)
z1_bin_areas = z1_bin_counts * gf.res[0] * gf.res[1] / 1E6
#RGI standard is integer thousandths of glaciers total area
#Should check to make sure sum of bin areas equals total area
z1_bin_areas_perc = 100. * z1_bin_areas / np.sum(z1_bin_areas)
z2_bin_counts, z2_bin_edges = np.histogram(gf.z2, bins=z_bin_edges)
z2_bin_areas = z2_bin_counts * gf.res[0] * gf.res[1] / 1E6
z2_bin_areas_perc = 100. * z2_bin_areas / np.sum(z2_bin_areas)
#dz_bin_edges, dz_bin_centers = get_bins(dz, 1.)
#dz_bin_counts, dz_bin_edges = np.histogram(dz, bins=dz_bin_edges)
#dz_bin_areas = dz_bin_counts * res * res / 1E6
mb_bin_med = np.ma.masked_all_like(z1_bin_areas)
mb_bin_mad = np.ma.masked_all_like(z1_bin_areas)
dz_bin_med = np.ma.masked_all_like(z1_bin_areas)
dz_bin_mad = np.ma.masked_all_like(z1_bin_areas)
idx = np.digitize(gf.z1, z_bin_edges)
for bin_n in range(z_bin_centers.size):
mb_bin_samp = gf.mb[(idx == bin_n + 1)]
if mb_bin_samp.count() > 0:
mb_bin_med[bin_n] = malib.fast_median(mb_bin_samp)
mb_bin_mad[bin_n] = malib.mad(mb_bin_samp)
mb_bin_med[bin_n] = mb_bin_samp.mean()
mb_bin_mad[bin_n] = mb_bin_samp.std()
dz_bin_samp = gf.dhdt[(idx == bin_n + 1)]
if dz_bin_samp.count() > 0:
dz_bin_med[bin_n] = malib.fast_median(dz_bin_samp)
dz_bin_mad[bin_n] = malib.mad(dz_bin_samp)
dz_bin_med[bin_n] = dz_bin_samp.mean()
dz_bin_mad[bin_n] = dz_bin_samp.std()
#Should also export original dh/dt numbers, not mb
#outbins_header = 'bin_center_elev, bin_count, dhdt_bin_med, dhdt_bin_mad, mb_bin_med, mb_bin_mad'
#outbins = np.ma.dstack([z_bin_centers, z1_bin_counts, dz_bin_med, dz_bin_mad, mb_bin_med, mb_bin_mad]).astype('float32')[0]
#fmt='%0.2f'
outbins_header = 'bin_center_elev_m, z1_bin_count_valid, z1_bin_area_valid_km2, z1_bin_area_perc, z2_bin_count_valid, z2_bin_area_valid_km2, z2_bin_area_perc, dhdt_bin_med_ma, dhdt_bin_mad_ma, mb_bin_med_mwe, mb_bin_mad_mwe'
fmt = '%0.1f, %i, %0.3f, %0.2f, %i, %0.3f, %0.2f, %0.2f, %0.2f, %0.2f, %0.2f'
outbins = np.ma.dstack([
z_bin_centers, z1_bin_counts, z1_bin_areas, z1_bin_areas_perc,
z2_bin_counts, z2_bin_areas, z2_bin_areas_perc, dz_bin_med, dz_bin_mad,
mb_bin_med, mb_bin_mad
]).astype('float32')[0]
np.ma.set_fill_value(outbins, -9999.0)
outbins_fn = os.path.join(outdir, gf.feat_fn + '_mb_bins.csv')
np.savetxt(outbins_fn,
outbins,
fmt=fmt,
delimiter=',',
header=outbins_header)
#print("Generating aed plot")
f, axa = plt.subplots(1, 2, figsize=(6, 6))
f.suptitle(gf.feat_fn)
axa[0].plot(z1_bin_areas, z_bin_centers, label='%0.2f' % gf.t1)
axa[0].plot(z2_bin_areas, z_bin_centers, label='%0.2f' % gf.t2)
axa[0].axhline(gf.z1_ela, ls=':', c='C0')
axa[0].axhline(gf.z2_ela, ls=':', c='C1')
axa[0].legend(prop={'size': 8}, loc='upper right')
axa[0].set_ylabel('Elevation (m WGS84)')
axa[0].set_xlabel('Area $\mathregular{km^2}$')
axa[0].minorticks_on()
axa[1].yaxis.tick_right()
axa[1].yaxis.set_label_position("right")
axa[1].axvline(0, lw=1.0, c='k')
axa[1].axvline(gf.mb_mean,
lw=0.5,
ls=':',
c='k',
label='%0.2f m w.e./yr' % gf.mb_mean)
axa[1].legend(prop={'size': 8}, loc='upper right')
axa[1].plot(mb_bin_med, z_bin_centers, color='k')
axa[1].fill_betweenx(z_bin_centers,
0,
mb_bin_med,
where=(mb_bin_med < 0),
color='r',
alpha=0.2)
axa[1].fill_betweenx(z_bin_centers,
0,
mb_bin_med,
where=(mb_bin_med > 0),
color='b',
alpha=0.2)
#axa[1].set_ylabel('Elevation (m WGS84)')
#axa[1].set_xlabel('dh/dt (m/yr)')
axa[1].set_xlabel('mb (m w.e./yr)')
axa[1].minorticks_on()
axa[1].set_xlim(-2.0, 2.0)
#axa[1].set_xlim(-8.0, 8.0)
plt.tight_layout()
#Make room for suptitle
plt.subplots_adjust(top=0.95)
#print("Saving aed plot")
fig_fn = os.path.join(outdir, gf.feat_fn + '_mb_aed.png')
plt.savefig(fig_fn, bbox_inches='tight', dpi=300)
return z_bin_edges
def compute_offset_nuth(dh, slope, aspect):
"""Compute horizontal offset between input rasters using Nuth and Kaab [2011] (nuth) method
"""
import scipy.optimize as optimization
#mean_dh = dh.mean()
#mean_slope = slope.mean()
#c_seed = (mean_dh/np.tan(np.deg2rad(mean_slope)))
med_dh = malib.fast_median(dh)
med_slope = malib.fast_median(slope)
c_seed = (med_dh/np.tan(np.deg2rad(med_slope)))
x0 = np.array([0.0, 0.0, c_seed])
print("Computing common mask")
common_mask = ~(malib.common_mask([dh, aspect, slope]))
xdata = aspect[common_mask]
ydata = dh[common_mask]/np.tan(np.deg2rad(slope[common_mask]))
#Generate synthetic data to test curve_fit
#xdata = np.arange(0,360,0.01)
#ydata = f(xdata, 20.0, 130.0, -3.0) + 20*np.random.normal(size=len(xdata))
#Limit sample size
#n = 10000
#idx = random.sample(range(xdata.size), n)
#xdata = xdata[idx]
#ydata = ydata[idx]
"""
#Fit to original, unfiltered data
fit = optimization.curve_fit(nuth_func, xdata, ydata, x0)[0]
print(fit)
genplot(xdata, ydata, fit)
"""
"""
#Filter to remove outliers
#Compute median absolute difference
y_med = np.median(ydata)
y_mad = malib.mad(ydata)
mad_factor = 3
y_perc = [y_med - y_mad*mad_factor, y_med + y_mad*mad_factor]
y_idx = ((ydata >= y_perc[0]) & (ydata <= y_perc[1]))
ydata_clip = ydata[y_idx]
xdata_clip = xdata[y_idx]
fit = optimization.curve_fit(nuth_func, xdata_clip, ydata_clip, x0)[0]
print(fit)
genplot(xdata_clip, ydata_clip, fit)
"""
#Compute robust statistics for 1-degree bins
nbins = 360
bin_range = (0., 360.)
bin_count, bin_edges, bin_centers = malib.bin_stats(xdata, ydata, stat='count', \
nbins=nbins, bin_range=bin_range)
bin_med, bin_edges, bin_centers = malib.bin_stats(xdata, ydata, stat='median', \
nbins=nbins, bin_range=bin_range)
"""
#Mask bins in grid directions, can potentially contain biased stats
badbins = [0, 45, 90, 180, 225, 270, 315]
bin_stat = np.ma.masked_where(np.around(bin_edges[:-1]) % 45 == 0, bin_stat)
bin_edges = np.ma.masked_where(np.around(bin_edges[:-1]) % 45 == 0, bin_edges)
"""
#Remove any empty bins
#idx = ~(np.ma.getmaskarray(bin_med))
#Remove any bins with only a few points
min_count = 9
idx = (bin_count.filled(0) >= min_count)
bin_med = bin_med[idx]
bin_centers = bin_centers[idx]
fit = optimization.curve_fit(nuth_func, bin_centers, bin_med, x0)[0]
f = genplot(bin_centers, bin_med, fit, xdata=xdata, ydata=ydata)
plt.show()
#genplot(xdata, ydata, fit)
print(fit)
return fit, f
