def plot_2dhist(ax, x, y, xlim, ylim, log=False): bins = (100, 100) common_mask = ~(malib.common_mask([x, y])) x = x[common_mask] y = y[common_mask] H, xedges, yedges = np.histogram2d(x, y, range=[xlim, ylim], bins=bins) H = np.rot90(H) H = np.flipud(H) Hmasked = np.ma.masked_where(H == 0, H) Hmed_idx = np.ma.argmax(Hmasked, axis=0) ymax = (yedges[:-1] + np.diff(yedges))[Hmed_idx] #Hmasked = H H_clim = malib.calcperc(Hmasked, (2, 98)) if log: import matplotlib.colors as colors ax.pcolormesh(xedges, yedges, Hmasked, cmap='inferno', norm=colors.LogNorm(vmin=H_clim[0], vmax=H_clim[1])) else: ax.pcolormesh(xedges, yedges, Hmasked, cmap='inferno', vmin=H_clim[0], vmax=H_clim[1]) ax.plot(xedges[:-1] + np.diff(xedges), ymax, color='dodgerblue', lw=1.0)
def plot_2dhist(ax, x, y, xlim=None, ylim=None, xint=None, yint=None, nbins=(128,128), log=False, maxline=True, trendline=False): from pygeotools.lib import malib #Should compute number of bins automatically based on input values, xlim and ylim common_mask = ~(malib.common_mask([x,y])) x = x[common_mask] y = y[common_mask] if xlim is None: #xlim = (x.min(), x.max()) xlim = malib.calcperc(x, (0.1, 99.9)) if ylim is None: #ylim = (y.min(), y.max()) ylim = malib.calcperc(y, (0.1, 99.9)) #Note, round to nearest meter here #xlim = np.rint(np.array(xlim)) xlim = np.array(xlim) ylim = np.array(ylim) if xint is not None: xedges = np.arange(xlim[0], xlim[1]+xint, xint) else: xedges = nbins[0] if yint is not None: yedges = np.arange(ylim[0], ylim[1]+yint, yint) else: yedges = nbins[1] H, xedges, yedges = np.histogram2d(x,y,range=[xlim,ylim],bins=[xedges, yedges]) #H, xedges, yedges = np.histogram2d(x,y,range=[xlim,ylim],bins=nbins) #H = np.rot90(H) #H = np.flipud(H) H = H.T #Mask any empty bins Hmasked = np.ma.masked_where(H==0,H) #Hmasked = H H_clim = malib.calcperc(Hmasked, (2,98)) if log: import matplotlib.colors as colors ax.pcolormesh(xedges,yedges,Hmasked,cmap='inferno',norm=colors.LogNorm(vmin=H_clim[0],vmax=H_clim[1])) else: ax.pcolormesh(xedges,yedges,Hmasked,cmap='inferno',vmin=H_clim[0],vmax=H_clim[1]) if maxline: #Add line for max values in each x bin Hmed_idx = np.ma.argmax(Hmasked, axis=0) ymax = (yedges[:-1]+np.diff(yedges))[Hmed_idx] ax.plot(xedges[:-1]+np.diff(xedges), ymax, color='dodgerblue',lw=1.0) if trendline: #Add trendline import scipy.stats y_slope, y_intercept, r_value, p_value, std_err = scipy.stats.linregress(x, y) y_f = y_slope * xlim + y_intercept ax.plot(xlim, y_f, color='limegreen', ls='--', lw=0.5)
def cummulative_profile(xma, yma, xbin_width, limit_x_perc=(1, 99)): """ compute binned statistics for independent variable with respect to dependendent variable Parameters ----------- xma: masked array independent variable (like slope) yma: masked array dependent variable (like elevation difference) xbin_width: int bin_width for independent variable limit_x_perc: tuple limit binning of independent variable to the given percentile (default: 1 to 99 %) Returns ----------- x_bins: np.array bin locations y_mean: np.array binned mean value for dependent variable y_meadian: np.array binned median value for dependent variable y_std: np.array binned standard deviation value for dependent varuiable y_perc: np.array binned percentage of variables within the bin """ # xclim get rids of outliers in the independent variable # we only look at the 1 to 99 percentile values by default xclim = malib.calcperc(xma, limit_x_perc) # this step computes common mask where pixels of both x and y variables are valid xma_lim = np.ma.masked_outside(xma, xclim[0], xclim[1]) cmask = malib.common_mask([xma_lim, yma]) # the common mask is used to flatten the required points in a 1-D array xma_c = np.ma.compressed(np.ma.array(xma_lim, mask=cmask)) yma_c = np.ma.compressed(np.ma.array(yma, mask=cmask)) # we then use pandas groupby to quickly compute binned statistics df = pd.DataFrame({'x': xma_c, 'y': yma_c}) df['x_rounded'] = (df['x'] + (xbin_width - 1)) // (xbin_width) * xbin_width grouped = df.groupby('x_rounded') df2 = grouped['y'].agg([np.mean, np.count_nonzero, np.median, np.std]) df2.reset_index(inplace=True) # variables are returned as numpy array x_bins = df2['x_rounded'].values y_mean = df2['mean'].values y_median = df2['median'].values y_std = df2['std'].values y_perc = (df2['count_nonzero'].values / np.sum(df2['count_nonzero'].values)) * 100 return x_bins, y_mean, y_median, y_std, y_perc
def compute_offset_nuth(dh, slope, aspect, min_count=100, remove_outliers=True, plot=True): """Compute horizontal offset between input rasters using Nuth and Kaab [2011] (nuth) method """ import scipy.optimize as optimization if dh.count() < min_count: sys.exit("Not enough dh samples") if slope.count() < min_count: sys.exit("Not enough slope/aspect samples") #mean_dh = dh.mean() #mean_slope = slope.mean() #c_seed = (mean_dh/np.tan(np.deg2rad(mean_slope))) med_dh = malib.fast_median(dh) med_slope = malib.fast_median(slope) c_seed = (med_dh/np.tan(np.deg2rad(med_slope))) x0 = np.array([0.0, 0.0, c_seed]) print("Computing common mask") common_mask = ~(malib.common_mask([dh, aspect, slope])) #Prepare x and y data xdata = aspect[common_mask].data ydata = (dh[common_mask]/np.tan(np.deg2rad(slope[common_mask]))).data print("Initial sample count:") print(ydata.size) if remove_outliers: print("Removing outliers") #print("Absolute dz filter: %0.2f" % max_dz) #diff = np.ma.masked_greater(diff, max_dz) #print(diff.count()) #Outlier dz filter f = 3 sigma, u = (ydata.std(), ydata.mean()) #sigma, u = malib.mad(ydata, return_med=True) rmin = u - f*sigma rmax = u + f*sigma print("3-sigma filter: %0.2f - %0.2f" % (rmin, rmax)) idx = (ydata >= rmin) & (ydata <= rmax) xdata = xdata[idx] ydata = ydata[idx] print(ydata.size) #Generate synthetic data to test curve_fit #xdata = np.arange(0,360,0.01) #ydata = f(xdata, 20.0, 130.0, -3.0) + 20*np.random.normal(size=len(xdata)) #Limit sample size #n = 10000 #idx = random.sample(range(xdata.size), n) #xdata = xdata[idx] #ydata = ydata[idx] #Compute robust statistics for 1-degree bins nbins = 360 bin_range = (0., 360.) bin_width = 1.0 bin_count, bin_edges, bin_centers = malib.bin_stats(xdata, ydata, stat='count', nbins=nbins, bin_range=bin_range) bin_med, bin_edges, bin_centers = malib.bin_stats(xdata, ydata, stat='median', nbins=nbins, bin_range=bin_range) #Needed to estimate sigma for weighted lsq #bin_mad, bin_edges, bin_centers = malib.bin_stats(xdata, ydata, stat=malib.mad, nbins=nbins, bin_range=bin_range) #Started implementing this for more generic binning, needs testing #bin_count, x_bin_edges, y_bin_edges = malib.get_2dhist(xdata, ydata, \ # xlim=bin_range, nbins=(nbins, nbins), stat='count') """ #Mask bins in grid directions, can potentially contain biased stats #Especially true for SGM algorithm #badbins = [0, 90, 180, 270, 360] badbins = [0, 45, 90, 135, 180, 225, 270, 315, 360] bin_stat = np.ma.masked_where(np.around(bin_edges[:-1]) % 45 == 0, bin_stat) bin_edges = np.ma.masked_where(np.around(bin_edges[:-1]) % 45 == 0, bin_edges) """ #Remove any bins with only a few points min_bin_sample_count = 9 idx = (bin_count.filled(0) >= min_bin_sample_count) bin_count = bin_count[idx].data bin_med = bin_med[idx].data #bin_mad = bin_mad[idx].data bin_centers = bin_centers[idx] fit = None fit_fig = None #Want a good distribution of bins, at least 1/4 to 1/2 of sinusoid, to ensure good fit #Need at least 3 valid bins to fit 3 parameters in nuth_func #min_bin_count = 3 min_bin_count = 90 #Not going to help if we have a step function between two plateaus, but better than nothing #Calculate bin aspect spread bin_ptp = np.cos(np.radians(bin_centers)).ptp() min_bin_ptp = 1.0 #Should iterate here, if not enough bins, increase bin width if len(bin_med) >= min_bin_count and bin_ptp >= min_bin_ptp: print("Computing fit") #Unweighted fit fit = optimization.curve_fit(nuth_func, bin_centers, bin_med, x0)[0] #Weight by observed spread in each bin #sigma = bin_mad #fit = optimization.curve_fit(nuth_func, bin_centers, bin_med, x0, sigma, absolute_sigma=True)[0] #Weight by bin count #sigma = bin_count.max()/bin_count #fit = optimization.curve_fit(nuth_func, bin_centers, bin_med, x0, sigma, absolute_sigma=False)[0] print(fit) if plot: print("Generating Nuth and Kaab plot") bin_idx = np.digitize(xdata, bin_edges) output = [] for i in np.arange(1, len(bin_edges)): output.append(ydata[bin_idx==i]) #flierprops={'marker':'.'} lw = 0.25 whiskerprops={'linewidth':lw} capprops={'linewidth':lw} boxprops={'facecolor':'k', 'linewidth':0} medianprops={'marker':'o', 'ms':1, 'color':'r'} fit_fig, ax = plt.subplots(figsize=(6,6)) #widths = (bin_width/2.0) widths = 2.5*(bin_count/bin_count.max()) #widths = bin_count/np.percentile(bin_count, 50) #Stride s=3 #This is inefficient, but we have list of arrays with different length, need to filter #Reduntant with earlier filter, should refactor bp = ax.boxplot(np.array(output)[idx][::s], positions=bin_centers[::s], widths=widths[::s], showfliers=False, \ patch_artist=True, boxprops=boxprops, whiskerprops=whiskerprops, capprops=capprops, \ medianprops=medianprops) bin_ticks = [0, 45, 90, 135, 180, 225, 270, 315, 360] ax.set_xticks(bin_ticks) ax.set_xticklabels(bin_ticks) """ #Can pull out medians from boxplot #We are computing multiple times, inefficient bp_bin_med = [] for medline in bp['medians']: bp_bin_med.append(medline.get_ydata()[0]) """ #Plot the fit f_a = nuth_func(bin_centers, fit[0], fit[1], fit[2]) nuth_func_str = r'$y=%0.2f*cos(%0.2f-x)+%0.2f$' % tuple(fit) ax.plot(bin_centers, f_a, 'b', label=nuth_func_str) ax.set_xlabel('Aspect (deg)') ax.set_ylabel('dh/tan(slope) (m)') ax.axhline(color='gray', linewidth=0.5) ax.set_xlim(*bin_range) ylim = ax.get_ylim() abs_ylim = np.max(np.abs(ylim)) #abs_ylim = np.max(np.abs([ydata.min(), ydata.max()])) #pad = 0.2 * abs_ylim pad = 0 ylim = (-abs_ylim - pad, abs_ylim + pad) minylim = (-10,10) if ylim[0] > minylim[0]: ylim = minylim ax.set_ylim(*ylim) ax.legend(prop={'size':8}) return fit, fit_fig
dem2_zs = iolib.ds_getma(dem2_zs_ds) zs_diff = dem2_zs - dem1_zs dst_fn = os.path.join(outdir, outprefix + '_zs_diff.tif') iolib.writeGTiff(zs_diff, dst_fn, dem1_ds, ndv=diffndv) smb_diff = zs_diff - firnair_diff #dst_fn = os.path.join(outdir, outprefix+'_smb_diff.tif') #iolib.writeGTiff(smb_diff, dst_fn, dem1_ds, ndv=diffndv) #Check to make sure inputs actually intersect #Masked pixels are True if not np.any(~dem1.mask * ~dem2.mask): sys.exit("No valid overlap between input data") #Compute common mask print "Generating common mask" common_mask = malib.common_mask([dem1, dem2]) #Compute relative elevation difference with Eulerian approach print "Computing elevation difference with Eulerian approach" diff_euler = np.ma.array(dem2 - dem1, mask=common_mask) #Add output that has difference filter applied #See dem_align if True: print "Eulerian elevation difference stats:" diff_euler_stats = malib.print_stats(diff_euler) diff_euler_med = diff_euler_stats[5] if True: print "Writing Eulerian elevation difference map"
plt.tight_layout() if save: fig_fn = '%s_WY%i_SWE_maps.png' % (outprefix, wy) if prism is not None: fig_fn = os.path.splitext(fig_fn)[0]+'_prism.png' plt.savefig(fig_fn, dpi=300, bbox_inches='tight') #Regression analysis, 2D Histogram plots #Needs to be cleaned up and tested if True: #Compare SWE with slope and aspect, each should be optional and handled individually slope = geolib.gdaldem_mem_ds(dem1_ds, processing='slope', returnma=True) aspect = geolib.gdaldem_mem_ds(dem1_ds, processing='aspect', returnma=True) #Mask to preserve one set of valid pixels from all datasets mask = malib.common_mask([dem1, swe, slope, aspect]) dem1 = np.ma.array(dem1, mask=mask).compressed() #dem_clim = malib.calcperc(dem1, (5,99.9)) swe = np.ma.array(swe, mask=mask).compressed() slope = np.ma.array(slope, mask=mask).compressed() slope_clim = malib.calcperc(slope, (0,99)) aspect = np.ma.array(aspect, mask=mask).compressed() aspect_clim = (0., 360.) if prism is not None: prism = np.ma.array(prism, mask=mask).compressed() #prism_clim = malib.calcperc(prism, (0,99)) prism_clim = swe_clim from imview.lib import pltlib f, axa = plt.subplots(1, nax+1, figsize=(10,2.5))
def compute_offset_nuth(dh, slope, aspect): """Compute horizontal offset between input rasters using Nuth and Kaab [2011] (nuth) method """ import scipy.optimize as optimization #mean_dh = dh.mean() #mean_slope = slope.mean() #c_seed = (mean_dh/np.tan(np.deg2rad(mean_slope))) med_dh = malib.fast_median(dh) med_slope = malib.fast_median(slope) c_seed = (med_dh/np.tan(np.deg2rad(med_slope))) x0 = np.array([0.0, 0.0, c_seed]) print("Computing common mask") common_mask = ~(malib.common_mask([dh, aspect, slope])) xdata = aspect[common_mask] ydata = dh[common_mask]/np.tan(np.deg2rad(slope[common_mask])) #Generate synthetic data to test curve_fit #xdata = np.arange(0,360,0.01) #ydata = f(xdata, 20.0, 130.0, -3.0) + 20*np.random.normal(size=len(xdata)) #Limit sample size #n = 10000 #idx = random.sample(range(xdata.size), n) #xdata = xdata[idx] #ydata = ydata[idx] """ #Fit to original, unfiltered data fit = optimization.curve_fit(nuth_func, xdata, ydata, x0)[0] print(fit) genplot(xdata, ydata, fit) """ """ #Filter to remove outliers #Compute median absolute difference y_med = np.median(ydata) y_mad = malib.mad(ydata) mad_factor = 3 y_perc = [y_med - y_mad*mad_factor, y_med + y_mad*mad_factor] y_idx = ((ydata >= y_perc[0]) & (ydata <= y_perc[1])) ydata_clip = ydata[y_idx] xdata_clip = xdata[y_idx] fit = optimization.curve_fit(nuth_func, xdata_clip, ydata_clip, x0)[0] print(fit) genplot(xdata_clip, ydata_clip, fit) """ #Compute robust statistics for 1-degree bins nbins = 360 bin_range = (0., 360.) bin_count, bin_edges, bin_centers = malib.bin_stats(xdata, ydata, stat='count', \ nbins=nbins, bin_range=bin_range) bin_med, bin_edges, bin_centers = malib.bin_stats(xdata, ydata, stat='median', \ nbins=nbins, bin_range=bin_range) """ #Mask bins in grid directions, can potentially contain biased stats badbins = [0, 45, 90, 180, 225, 270, 315] bin_stat = np.ma.masked_where(np.around(bin_edges[:-1]) % 45 == 0, bin_stat) bin_edges = np.ma.masked_where(np.around(bin_edges[:-1]) % 45 == 0, bin_edges) """ #Remove any empty bins #idx = ~(np.ma.getmaskarray(bin_med)) #Remove any bins with only a few points min_count = 9 idx = (bin_count.filled(0) >= min_count) bin_med = bin_med[idx] bin_centers = bin_centers[idx] fit = optimization.curve_fit(nuth_func, bin_centers, bin_med, x0)[0] f = genplot(bin_centers, bin_med, fit, xdata=xdata, ydata=ydata) plt.show() #genplot(xdata, ydata, fit) print(fit) return fit, f
def main(): parser = getparser() args = parser.parse_args() #This is output ndv, avoid using 0 for differences diffndv = np.nan dem1_fn = args.dem1_fn dem2_fn = args.dem2_fn if dem1_fn == dem2_fn: sys.exit('Input filenames are identical') fn_list = [dem1_fn, dem2_fn] print("Warping DEMs to same res/extent/proj") dem1_ds, dem2_ds = warplib.memwarp_multi_fn(fn_list, extent=args.te, res=args.tr, t_srs=args.t_srs) print("Loading input DEMs into masked arrays") dem1 = iolib.ds_getma(dem1_ds) dem2 = iolib.ds_getma(dem2_ds) outdir = args.outdir if outdir is None: outdir = os.path.split(dem1_fn)[0] outprefix = os.path.splitext( os.path.split(dem1_fn)[1])[0] + '_' + os.path.splitext( os.path.split(dem2_fn)[1])[0] #Extract basename adj = '' if '-adj' in dem1_fn: adj = '-adj' dem1_fn_base = re.sub(adj, '', os.path.splitext(dem1_fn)[0]) dem2_fn_base = re.sub(adj, '', os.path.splitext(dem2_fn)[0]) #Check to make sure inputs actually intersect #Masked pixels are True if not np.any(~dem1.mask * ~dem2.mask): sys.exit("No valid overlap between input data") #Compute common mask print("Generating common mask") common_mask = malib.common_mask([dem1, dem2]) #Compute relative elevation difference with Eulerian approach print("Computing elevation difference with Eulerian approach") diff_euler = np.ma.array(dem2 - dem1, mask=common_mask) #Absolute range filter on the difference # removes differences outside of a range that likely arent related to canopy heights diff_euler = range_fltr(diff_euler, (-3, 30)) if True: print("Eulerian elevation difference stats:") diff_euler_stats = malib.print_stats(diff_euler) diff_euler_med = diff_euler_stats[5] if True: print("Writing Eulerian elevation difference map") dst_fn = os.path.join(outdir, outprefix + '_dz_eul.tif') print(dst_fn) iolib.writeGTiff(diff_euler, dst_fn, dem1_ds, ndv=diffndv) if False: print("Writing Eulerian relative elevation difference map") diff_euler_rel = diff_euler - diff_euler_med dst_fn = os.path.join(outdir, outprefix + '_dz_eul_rel.tif') print(dst_fn) iolib.writeGTiff(diff_euler_rel, dst_fn, dem1_ds, ndv=diffndv) if False: print("Writing out DEM2 with median elevation difference removed") dst_fn = os.path.splitext( dem2_fn)[0] + '_med' + diff_euler_med + '.tif' print(dst_fn) iolib.writeGTiff(dem2 - diff_euler_med, dst_fn, dem1_ds, ndv=diffndv) if False: print("Writing Eulerian elevation difference percentage map") diff_euler_perc = 100.0 * diff_euler / dem1 dst_fn = os.path.join(outdir, outprefix + '_dz_eul_perc.tif') print(dst_fn) iolib.writeGTiff(diff_euler_perc, dst_fn, dem1_ds, ndv=diffndv) return dst_fn