def plot_2dhist(ax, x, y, xlim, ylim, log=False):
bins = (100, 100)
common_mask = ~(malib.common_mask([x, y]))
x = x[common_mask]
y = y[common_mask]
H, xedges, yedges = np.histogram2d(x, y, range=[xlim, ylim], bins=bins)
H = np.rot90(H)
H = np.flipud(H)
Hmasked = np.ma.masked_where(H == 0, H)
Hmed_idx = np.ma.argmax(Hmasked, axis=0)
ymax = (yedges[:-1] + np.diff(yedges))[Hmed_idx]
#Hmasked = H
H_clim = malib.calcperc(Hmasked, (2, 98))
if log:
import matplotlib.colors as colors
ax.pcolormesh(xedges,
yedges,
Hmasked,
cmap='inferno',
norm=colors.LogNorm(vmin=H_clim[0], vmax=H_clim[1]))
else:
ax.pcolormesh(xedges,
yedges,
Hmasked,
cmap='inferno',
vmin=H_clim[0],
vmax=H_clim[1])
ax.plot(xedges[:-1] + np.diff(xedges), ymax, color='dodgerblue', lw=1.0)
def plot_2dhist(ax, x, y, xlim=None, ylim=None, xint=None, yint=None, nbins=(128,128), log=False, maxline=True, trendline=False):
from pygeotools.lib import malib
#Should compute number of bins automatically based on input values, xlim and ylim
common_mask = ~(malib.common_mask([x,y]))
x = x[common_mask]
y = y[common_mask]
if xlim is None:
#xlim = (x.min(), x.max())
xlim = malib.calcperc(x, (0.1, 99.9))
if ylim is None:
#ylim = (y.min(), y.max())
ylim = malib.calcperc(y, (0.1, 99.9))
#Note, round to nearest meter here
#xlim = np.rint(np.array(xlim))
xlim = np.array(xlim)
ylim = np.array(ylim)
if xint is not None:
xedges = np.arange(xlim[0], xlim[1]+xint, xint)
else:
xedges = nbins[0]
if yint is not None:
yedges = np.arange(ylim[0], ylim[1]+yint, yint)
else:
yedges = nbins[1]
H, xedges, yedges = np.histogram2d(x,y,range=[xlim,ylim],bins=[xedges, yedges])
#H, xedges, yedges = np.histogram2d(x,y,range=[xlim,ylim],bins=nbins)
#H = np.rot90(H)
#H = np.flipud(H)
H = H.T
#Mask any empty bins
Hmasked = np.ma.masked_where(H==0,H)
#Hmasked = H
H_clim = malib.calcperc(Hmasked, (2,98))
if log:
import matplotlib.colors as colors
ax.pcolormesh(xedges,yedges,Hmasked,cmap='inferno',norm=colors.LogNorm(vmin=H_clim[0],vmax=H_clim[1]))
else:
ax.pcolormesh(xedges,yedges,Hmasked,cmap='inferno',vmin=H_clim[0],vmax=H_clim[1])
if maxline:
#Add line for max values in each x bin
Hmed_idx = np.ma.argmax(Hmasked, axis=0)
ymax = (yedges[:-1]+np.diff(yedges))[Hmed_idx]
ax.plot(xedges[:-1]+np.diff(xedges), ymax, color='dodgerblue',lw=1.0)
if trendline:
#Add trendline
import scipy.stats
y_slope, y_intercept, r_value, p_value, std_err = scipy.stats.linregress(x, y)
y_f = y_slope * xlim + y_intercept
ax.plot(xlim, y_f, color='limegreen', ls='--', lw=0.5)
def cummulative_profile(xma, yma, xbin_width, limit_x_perc=(1, 99)):
"""
compute binned statistics for independent variable with respect to dependendent variable
Parameters
-----------
xma: masked array
independent variable (like slope)
yma: masked array
dependent variable (like elevation difference)
xbin_width: int
bin_width for independent variable
limit_x_perc: tuple
limit binning of independent variable to the given percentile (default: 1 to 99 %)
Returns
-----------
x_bins: np.array
bin locations
y_mean: np.array
binned mean value for dependent variable
y_meadian: np.array
binned median value for dependent variable
y_std: np.array
binned standard deviation value for dependent varuiable
y_perc: np.array
binned percentage of variables within the bin
"""
# xclim get rids of outliers in the independent variable
# we only look at the 1 to 99 percentile values by default
xclim = malib.calcperc(xma, limit_x_perc)
# this step computes common mask where pixels of both x and y variables are valid
xma_lim = np.ma.masked_outside(xma, xclim[0], xclim[1])
cmask = malib.common_mask([xma_lim, yma])
# the common mask is used to flatten the required points in a 1-D array
xma_c = np.ma.compressed(np.ma.array(xma_lim, mask=cmask))
yma_c = np.ma.compressed(np.ma.array(yma, mask=cmask))
# we then use pandas groupby to quickly compute binned statistics
df = pd.DataFrame({'x': xma_c, 'y': yma_c})
df['x_rounded'] = (df['x'] + (xbin_width - 1)) // (xbin_width) * xbin_width
grouped = df.groupby('x_rounded')
df2 = grouped['y'].agg([np.mean, np.count_nonzero, np.median, np.std])
df2.reset_index(inplace=True)
# variables are returned as numpy array
x_bins = df2['x_rounded'].values
y_mean = df2['mean'].values
y_median = df2['median'].values
y_std = df2['std'].values
y_perc = (df2['count_nonzero'].values /
np.sum(df2['count_nonzero'].values)) * 100
return x_bins, y_mean, y_median, y_std, y_perc
def compute_offset_nuth(dh, slope, aspect, min_count=100, remove_outliers=True, plot=True):
"""Compute horizontal offset between input rasters using Nuth and Kaab [2011] (nuth) method
"""
import scipy.optimize as optimization
if dh.count() < min_count:
sys.exit("Not enough dh samples")
if slope.count() < min_count:
sys.exit("Not enough slope/aspect samples")
#mean_dh = dh.mean()
#mean_slope = slope.mean()
#c_seed = (mean_dh/np.tan(np.deg2rad(mean_slope)))
med_dh = malib.fast_median(dh)
med_slope = malib.fast_median(slope)
c_seed = (med_dh/np.tan(np.deg2rad(med_slope)))
x0 = np.array([0.0, 0.0, c_seed])
print("Computing common mask")
common_mask = ~(malib.common_mask([dh, aspect, slope]))
#Prepare x and y data
xdata = aspect[common_mask].data
ydata = (dh[common_mask]/np.tan(np.deg2rad(slope[common_mask]))).data
print("Initial sample count:")
print(ydata.size)
if remove_outliers:
print("Removing outliers")
#print("Absolute dz filter: %0.2f" % max_dz)
#diff = np.ma.masked_greater(diff, max_dz)
#print(diff.count())
#Outlier dz filter
f = 3
sigma, u = (ydata.std(), ydata.mean())
#sigma, u = malib.mad(ydata, return_med=True)
rmin = u - f*sigma
rmax = u + f*sigma
print("3-sigma filter: %0.2f - %0.2f" % (rmin, rmax))
idx = (ydata >= rmin) & (ydata <= rmax)
xdata = xdata[idx]
ydata = ydata[idx]
print(ydata.size)
#Generate synthetic data to test curve_fit
#xdata = np.arange(0,360,0.01)
#ydata = f(xdata, 20.0, 130.0, -3.0) + 20*np.random.normal(size=len(xdata))
#Limit sample size
#n = 10000
#idx = random.sample(range(xdata.size), n)
#xdata = xdata[idx]
#ydata = ydata[idx]
#Compute robust statistics for 1-degree bins
nbins = 360
bin_range = (0., 360.)
bin_width = 1.0
bin_count, bin_edges, bin_centers = malib.bin_stats(xdata, ydata, stat='count', nbins=nbins, bin_range=bin_range)
bin_med, bin_edges, bin_centers = malib.bin_stats(xdata, ydata, stat='median', nbins=nbins, bin_range=bin_range)
#Needed to estimate sigma for weighted lsq
#bin_mad, bin_edges, bin_centers = malib.bin_stats(xdata, ydata, stat=malib.mad, nbins=nbins, bin_range=bin_range)
#Started implementing this for more generic binning, needs testing
#bin_count, x_bin_edges, y_bin_edges = malib.get_2dhist(xdata, ydata, \
# xlim=bin_range, nbins=(nbins, nbins), stat='count')
"""
#Mask bins in grid directions, can potentially contain biased stats
#Especially true for SGM algorithm
#badbins = [0, 90, 180, 270, 360]
badbins = [0, 45, 90, 135, 180, 225, 270, 315, 360]
bin_stat = np.ma.masked_where(np.around(bin_edges[:-1]) % 45 == 0, bin_stat)
bin_edges = np.ma.masked_where(np.around(bin_edges[:-1]) % 45 == 0, bin_edges)
"""
#Remove any bins with only a few points
min_bin_sample_count = 9
idx = (bin_count.filled(0) >= min_bin_sample_count)
bin_count = bin_count[idx].data
bin_med = bin_med[idx].data
#bin_mad = bin_mad[idx].data
bin_centers = bin_centers[idx]
fit = None
fit_fig = None
#Want a good distribution of bins, at least 1/4 to 1/2 of sinusoid, to ensure good fit
#Need at least 3 valid bins to fit 3 parameters in nuth_func
#min_bin_count = 3
min_bin_count = 90
#Not going to help if we have a step function between two plateaus, but better than nothing
#Calculate bin aspect spread
bin_ptp = np.cos(np.radians(bin_centers)).ptp()
min_bin_ptp = 1.0
#Should iterate here, if not enough bins, increase bin width
if len(bin_med) >= min_bin_count and bin_ptp >= min_bin_ptp:
print("Computing fit")
#Unweighted fit
fit = optimization.curve_fit(nuth_func, bin_centers, bin_med, x0)[0]
#Weight by observed spread in each bin
#sigma = bin_mad
#fit = optimization.curve_fit(nuth_func, bin_centers, bin_med, x0, sigma, absolute_sigma=True)[0]
#Weight by bin count
#sigma = bin_count.max()/bin_count
#fit = optimization.curve_fit(nuth_func, bin_centers, bin_med, x0, sigma, absolute_sigma=False)[0]
print(fit)
if plot:
print("Generating Nuth and Kaab plot")
bin_idx = np.digitize(xdata, bin_edges)
output = []
for i in np.arange(1, len(bin_edges)):
output.append(ydata[bin_idx==i])
#flierprops={'marker':'.'}
lw = 0.25
whiskerprops={'linewidth':lw}
capprops={'linewidth':lw}
boxprops={'facecolor':'k', 'linewidth':0}
medianprops={'marker':'o', 'ms':1, 'color':'r'}
fit_fig, ax = plt.subplots(figsize=(6,6))
#widths = (bin_width/2.0)
widths = 2.5*(bin_count/bin_count.max())
#widths = bin_count/np.percentile(bin_count, 50)
#Stride
s=3
#This is inefficient, but we have list of arrays with different length, need to filter
#Reduntant with earlier filter, should refactor
bp = ax.boxplot(np.array(output)[idx][::s], positions=bin_centers[::s], widths=widths[::s], showfliers=False, \
patch_artist=True, boxprops=boxprops, whiskerprops=whiskerprops, capprops=capprops, \
medianprops=medianprops)
bin_ticks = [0, 45, 90, 135, 180, 225, 270, 315, 360]
ax.set_xticks(bin_ticks)
ax.set_xticklabels(bin_ticks)
"""
#Can pull out medians from boxplot
#We are computing multiple times, inefficient
bp_bin_med = []
for medline in bp['medians']:
bp_bin_med.append(medline.get_ydata()[0])
"""
#Plot the fit
f_a = nuth_func(bin_centers, fit[0], fit[1], fit[2])
nuth_func_str = r'$y=%0.2f*cos(%0.2f-x)+%0.2f$' % tuple(fit)
ax.plot(bin_centers, f_a, 'b', label=nuth_func_str)
ax.set_xlabel('Aspect (deg)')
ax.set_ylabel('dh/tan(slope) (m)')
ax.axhline(color='gray', linewidth=0.5)
ax.set_xlim(*bin_range)
ylim = ax.get_ylim()
abs_ylim = np.max(np.abs(ylim))
#abs_ylim = np.max(np.abs([ydata.min(), ydata.max()]))
#pad = 0.2 * abs_ylim
pad = 0
ylim = (-abs_ylim - pad, abs_ylim + pad)
minylim = (-10,10)
if ylim[0] > minylim[0]:
ylim = minylim
ax.set_ylim(*ylim)
ax.legend(prop={'size':8})
return fit, fit_fig
dem2_zs = iolib.ds_getma(dem2_zs_ds)
zs_diff = dem2_zs - dem1_zs
dst_fn = os.path.join(outdir, outprefix + '_zs_diff.tif')
iolib.writeGTiff(zs_diff, dst_fn, dem1_ds, ndv=diffndv)
smb_diff = zs_diff - firnair_diff
#dst_fn = os.path.join(outdir, outprefix+'_smb_diff.tif')
#iolib.writeGTiff(smb_diff, dst_fn, dem1_ds, ndv=diffndv)
#Check to make sure inputs actually intersect
#Masked pixels are True
if not np.any(~dem1.mask * ~dem2.mask):
sys.exit("No valid overlap between input data")
#Compute common mask
print "Generating common mask"
common_mask = malib.common_mask([dem1, dem2])
#Compute relative elevation difference with Eulerian approach
print "Computing elevation difference with Eulerian approach"
diff_euler = np.ma.array(dem2 - dem1, mask=common_mask)
#Add output that has difference filter applied
#See dem_align
if True:
print "Eulerian elevation difference stats:"
diff_euler_stats = malib.print_stats(diff_euler)
diff_euler_med = diff_euler_stats[5]
if True:
print "Writing Eulerian elevation difference map"
plt.tight_layout()
if save:
fig_fn = '%s_WY%i_SWE_maps.png' % (outprefix, wy)
if prism is not None:
fig_fn = os.path.splitext(fig_fn)[0]+'_prism.png'
plt.savefig(fig_fn, dpi=300, bbox_inches='tight')
#Regression analysis, 2D Histogram plots
#Needs to be cleaned up and tested
if True:
#Compare SWE with slope and aspect, each should be optional and handled individually
slope = geolib.gdaldem_mem_ds(dem1_ds, processing='slope', returnma=True)
aspect = geolib.gdaldem_mem_ds(dem1_ds, processing='aspect', returnma=True)
#Mask to preserve one set of valid pixels from all datasets
mask = malib.common_mask([dem1, swe, slope, aspect])
dem1 = np.ma.array(dem1, mask=mask).compressed()
#dem_clim = malib.calcperc(dem1, (5,99.9))
swe = np.ma.array(swe, mask=mask).compressed()
slope = np.ma.array(slope, mask=mask).compressed()
slope_clim = malib.calcperc(slope, (0,99))
aspect = np.ma.array(aspect, mask=mask).compressed()
aspect_clim = (0., 360.)
if prism is not None:
prism = np.ma.array(prism, mask=mask).compressed()
#prism_clim = malib.calcperc(prism, (0,99))
prism_clim = swe_clim
from imview.lib import pltlib
f, axa = plt.subplots(1, nax+1, figsize=(10,2.5))
def compute_offset_nuth(dh, slope, aspect):
"""Compute horizontal offset between input rasters using Nuth and Kaab [2011] (nuth) method
"""
import scipy.optimize as optimization
#mean_dh = dh.mean()
#mean_slope = slope.mean()
#c_seed = (mean_dh/np.tan(np.deg2rad(mean_slope)))
med_dh = malib.fast_median(dh)
med_slope = malib.fast_median(slope)
c_seed = (med_dh/np.tan(np.deg2rad(med_slope)))
x0 = np.array([0.0, 0.0, c_seed])
print("Computing common mask")
common_mask = ~(malib.common_mask([dh, aspect, slope]))
xdata = aspect[common_mask]
ydata = dh[common_mask]/np.tan(np.deg2rad(slope[common_mask]))
#Generate synthetic data to test curve_fit
#xdata = np.arange(0,360,0.01)
#ydata = f(xdata, 20.0, 130.0, -3.0) + 20*np.random.normal(size=len(xdata))
#Limit sample size
#n = 10000
#idx = random.sample(range(xdata.size), n)
#xdata = xdata[idx]
#ydata = ydata[idx]
"""
#Fit to original, unfiltered data
fit = optimization.curve_fit(nuth_func, xdata, ydata, x0)[0]
print(fit)
genplot(xdata, ydata, fit)
"""
"""
#Filter to remove outliers
#Compute median absolute difference
y_med = np.median(ydata)
y_mad = malib.mad(ydata)
mad_factor = 3
y_perc = [y_med - y_mad*mad_factor, y_med + y_mad*mad_factor]
y_idx = ((ydata >= y_perc[0]) & (ydata <= y_perc[1]))
ydata_clip = ydata[y_idx]
xdata_clip = xdata[y_idx]
fit = optimization.curve_fit(nuth_func, xdata_clip, ydata_clip, x0)[0]
print(fit)
genplot(xdata_clip, ydata_clip, fit)
"""
#Compute robust statistics for 1-degree bins
nbins = 360
bin_range = (0., 360.)
bin_count, bin_edges, bin_centers = malib.bin_stats(xdata, ydata, stat='count', \
nbins=nbins, bin_range=bin_range)
bin_med, bin_edges, bin_centers = malib.bin_stats(xdata, ydata, stat='median', \
nbins=nbins, bin_range=bin_range)
"""
#Mask bins in grid directions, can potentially contain biased stats
badbins = [0, 45, 90, 180, 225, 270, 315]
bin_stat = np.ma.masked_where(np.around(bin_edges[:-1]) % 45 == 0, bin_stat)
bin_edges = np.ma.masked_where(np.around(bin_edges[:-1]) % 45 == 0, bin_edges)
"""
#Remove any empty bins
#idx = ~(np.ma.getmaskarray(bin_med))
#Remove any bins with only a few points
min_count = 9
idx = (bin_count.filled(0) >= min_count)
bin_med = bin_med[idx]
bin_centers = bin_centers[idx]
fit = optimization.curve_fit(nuth_func, bin_centers, bin_med, x0)[0]
f = genplot(bin_centers, bin_med, fit, xdata=xdata, ydata=ydata)
plt.show()
#genplot(xdata, ydata, fit)
print(fit)
return fit, f
def main():
parser = getparser()
args = parser.parse_args()
#This is output ndv, avoid using 0 for differences
diffndv = np.nan
dem1_fn = args.dem1_fn
dem2_fn = args.dem2_fn
if dem1_fn == dem2_fn:
sys.exit('Input filenames are identical')
fn_list = [dem1_fn, dem2_fn]
print("Warping DEMs to same res/extent/proj")
dem1_ds, dem2_ds = warplib.memwarp_multi_fn(fn_list,
extent=args.te,
res=args.tr,
t_srs=args.t_srs)
print("Loading input DEMs into masked arrays")
dem1 = iolib.ds_getma(dem1_ds)
dem2 = iolib.ds_getma(dem2_ds)
outdir = args.outdir
if outdir is None:
outdir = os.path.split(dem1_fn)[0]
outprefix = os.path.splitext(
os.path.split(dem1_fn)[1])[0] + '_' + os.path.splitext(
os.path.split(dem2_fn)[1])[0]
#Extract basename
adj = ''
if '-adj' in dem1_fn:
adj = '-adj'
dem1_fn_base = re.sub(adj, '', os.path.splitext(dem1_fn)[0])
dem2_fn_base = re.sub(adj, '', os.path.splitext(dem2_fn)[0])
#Check to make sure inputs actually intersect
#Masked pixels are True
if not np.any(~dem1.mask * ~dem2.mask):
sys.exit("No valid overlap between input data")
#Compute common mask
print("Generating common mask")
common_mask = malib.common_mask([dem1, dem2])
#Compute relative elevation difference with Eulerian approach
print("Computing elevation difference with Eulerian approach")
diff_euler = np.ma.array(dem2 - dem1, mask=common_mask)
#Absolute range filter on the difference
# removes differences outside of a range that likely arent related to canopy heights
diff_euler = range_fltr(diff_euler, (-3, 30))
if True:
print("Eulerian elevation difference stats:")
diff_euler_stats = malib.print_stats(diff_euler)
diff_euler_med = diff_euler_stats[5]
if True:
print("Writing Eulerian elevation difference map")
dst_fn = os.path.join(outdir, outprefix + '_dz_eul.tif')
print(dst_fn)
iolib.writeGTiff(diff_euler, dst_fn, dem1_ds, ndv=diffndv)
if False:
print("Writing Eulerian relative elevation difference map")
diff_euler_rel = diff_euler - diff_euler_med
dst_fn = os.path.join(outdir, outprefix + '_dz_eul_rel.tif')
print(dst_fn)
iolib.writeGTiff(diff_euler_rel, dst_fn, dem1_ds, ndv=diffndv)
if False:
print("Writing out DEM2 with median elevation difference removed")
dst_fn = os.path.splitext(
dem2_fn)[0] + '_med' + diff_euler_med + '.tif'
print(dst_fn)
iolib.writeGTiff(dem2 - diff_euler_med, dst_fn, dem1_ds, ndv=diffndv)
if False:
print("Writing Eulerian elevation difference percentage map")
diff_euler_perc = 100.0 * diff_euler / dem1
dst_fn = os.path.join(outdir, outprefix + '_dz_eul_perc.tif')
print(dst_fn)
iolib.writeGTiff(diff_euler_perc, dst_fn, dem1_ds, ndv=diffndv)
return dst_fn