def plot_prediction_interpolated(lons, lats, ghfs): """ Plots predicted GHF values as an interpolated pseudocolor plot for Greenland. Args: lons: one-dimensional list (native, numpy, or pandas) of longitudes. lons: similar, latitudes. ghfs: similar, GHF values. """ m = Basemap(**GREENLAND_BASEMAP_ARGS) _mark_ice_cores(m, GREENLAND.lon.as_matrix(), GREENLAND.lat.as_matrix(), GREENLAND.ghf.as_matrix()) pcolor_args = {'cmap': SPECTRAL_CMAP} colorbar_args = {'location': 'right', 'pad': '5%'} plot_values_on_map_pcolormesh_interp(m, lons, lats, ghfs, parallel_step=5., meridian_step=10., colorbar_args=colorbar_args, pcolor_args=pcolor_args) m.drawparallels(np.arange(-80., 81., 5.), labels=[1, 0, 0, 0], fontsize=10, color='#c6c6c6') m.drawmeridians(np.arange(-180., 181., 10.), labels=[0, 0, 0, 1], fontsize=10, color='#c6c6c6') m.drawcoastlines(color='grey', linewidth=0.5) m.drawmapboundary(color='grey')
def exp_sensitivity(data): """ Evaluates sensitivity of GBRT and linear regression to perturbations in training GHF. Plot is saved to <OUT_DIR>/sensitivity.png. """ radius = GREENLAND_RADIUS roi_density = 11.3 # Greenland noise_amps = np.arange(0.025, .31, .025) ncenters = 50 dumpfile = 'sensitivity.txt' plotfile = 'sensitivity.png' plot_sensitivity_analysis(data, roi_density, radius, noise_amps, ncenters, dumpfile=dumpfile, replot=False) save_cur_fig(plotfile, title='GBRT prediction sensitivity to noise in training GHF', set_title_for=None)
def exp_error_by_density(data): """ Evaluates prediction error (normalized rmse and r2) for GBRT, linear regression and constant predictor by using increasingly large sample densities in ROIs, constrained to the specified region, with radius equal to that of Greenland. Plot is saved to <OUT_DIR>/error_by_density[<region>].png. """ densities = np.append(np.array([1]), np.arange(5, 51, 5)) radius = GREENLAND_RADIUS ncenters = 50 # region constraints: 'NA-WE', 'NA', 'WE', or None (i.e all) region = 'NA-WE' dumpfile = 'error_by_density[%s].txt' % region plotfile = 'error_by_density[%s].png' % region plot_error_by_density(data, densities, radius, ncenters, region=region, dumpfile=dumpfile, replot=False) save_cur_fig(plotfile)
def exp_error_by_radius(data): """ Evaluates prediction error (normalized rmse and r2) for GBRT, linear regression and constant predictor by using increasingly large radii for ROIs, constrained to the specified region, with sample density equal to that of Greenland. Plot is saved to <OUT_DIR>/error_by_radius[<region>].png. """ radius = GREENLAND_RADIUS roi_density = 11.3 # Greenland ncenters = 50 radii = np.arange(500, 4001, 500) region = 'NA-WE' dumpfile = 'error_by_radius[%s].txt' % region plotfile = 'error_by_radius[%s].png' % region sys.stderr.write('=> Experiment: Error by Radius (region: %s, no. centers: %d, no. radii: %d)\n' % (region, ncenters, len(radii))) plot_error_by_radius(data, roi_density, radii, ncenters, region=region, dumpfile=dumpfile, replot=False) save_cur_fig(plotfile)
def plot_error_by_density(data, roi_densities, radius, ncenters, region='NA-WE', replot=False, dumpfile=None, **gbrt_params): """ ncenters random centers are picked and over all given ROI densities. Cross-validation errors (normalized RMSE and r2) are averaged over ncenters. One standard deviation mark is shown by a shaded region. """ sys.stderr.write('=> Experiment: Error by Density (region: %s, no. centers: %d, no. densities: %d)\n' % (region, ncenters, len(roi_densities))) fig = plt.figure(figsize=(11,5)) ax_rmse, ax_r2 = fig.add_subplot(1, 2, 1), fig.add_subplot(1, 2, 2) if replot: results = pickle_load(dumpfile) else: centers = [ random_prediction_ctr(data, radius, region=region, min_density=max(roi_densities)) for _ in range(ncenters) ] shape = (ncenters, len(roi_densities)) # blank error matrix (keyed by center number and roi density index), # used to initialize multiple components of the results dictionary. blank = np.zeros(shape) results = { 'ncenters': ncenters, 'roi_densities': roi_densities, 'errors': { 'gbrt': {'rmse': blank.copy(), 'r2': blank.copy()}, 'linear': {'rmse': blank.copy(), 'r2': blank.copy()}, 'constant': {'rmse': blank.copy(), 'r2': blank.copy()}, }, } for idx_density, roi_density in enumerate(roi_densities): for idx_ctr, center in enumerate(centers): sys.stderr.write('# density = %.2f, center %d/%d ' % (roi_density, idx_ctr + 1, ncenters)) comp = compare_models(data, roi_density, radius, center, **gbrt_params) for k in results['errors'].keys(): # k is one of gbrt, linear, or constant results['errors'][k]['r2'][idx_ctr][idx_density] = comp[k][0] results['errors'][k]['rmse'][idx_ctr][idx_density] = comp[k][1] if dumpfile: pickle_dump(dumpfile, results, comment='GBRT performance results') errors = results['errors'] roi_densities = results['roi_densities'] ncenters = results['ncenters'] num_sigma = 1 # Plot GBRT results kw = {'alpha': .9, 'lw': 1, 'marker': 'o', 'markersize': 4, 'color': 'b'} mean_rmse = errors['gbrt']['rmse'].mean(axis=0) sd_rmse = np.sqrt(errors['gbrt']['rmse'].var(axis=0)) lower_rmse = mean_rmse - num_sigma * sd_rmse higher_rmse = mean_rmse + num_sigma * sd_rmse ax_rmse.plot(roi_densities, mean_rmse, label='GBRT', **kw) ax_rmse.fill_between(roi_densities, lower_rmse, higher_rmse, facecolor='b', edgecolor='b', alpha=.3) mean_r2 = errors['gbrt']['r2'].mean(axis=0) sd_r2 = np.sqrt(errors['gbrt']['r2'].var(axis=0)) lower_r2 = mean_r2 - num_sigma * sd_r2 higher_r2 = mean_r2 + num_sigma * sd_r2 ax_r2.plot(roi_densities, errors['gbrt']['r2'].mean(axis=0), **kw) ax_r2.fill_between(roi_densities, lower_r2, higher_r2, facecolor='b', edgecolor='b', alpha=.2) # Plot Linear Regression results kw = {'alpha': .7, 'lw': 1, 'marker': 'o', 'markersize': 4, 'markeredgecolor': 'r', 'color': 'r'} mean_rmse = errors['linear']['rmse'].mean(axis=0) sd_rmse = np.sqrt(errors['linear']['rmse'].var(axis=0)) lower_rmse = mean_rmse - num_sigma * sd_rmse higher_rmse = mean_rmse + num_sigma * sd_rmse ax_rmse.plot(roi_densities, mean_rmse, label='linear regression', **kw) ax_rmse.fill_between(roi_densities, lower_rmse, higher_rmse, facecolor='r', edgecolor='r', alpha=.3) mean_r2 = errors['linear']['r2'].mean(axis=0) sd_r2 = np.sqrt(errors['linear']['r2'].var(axis=0)) lower_r2 = mean_r2 - num_sigma * sd_r2 higher_r2 = mean_r2 + num_sigma * sd_r2 ax_r2.plot(roi_densities, errors['linear']['r2'].mean(axis=0), **kw) ax_r2.fill_between(roi_densities, lower_r2, higher_r2, facecolor='r', edgecolor='r', alpha=.2) # Plot constant predictor results kw = {'alpha': .7, 'lw': 1, 'ls': '--', 'marker': 'o', 'markersize': 4, 'color': 'k', 'markeredgecolor': 'k'} ax_rmse.plot(roi_densities, errors['constant']['rmse'].mean(axis=0), label='constant predictor', **kw) ax_r2.plot(roi_densities, errors['constant']['r2'].mean(axis=0), **kw) # Style plot ax_rmse.set_ylabel('Normalized RMSE', fontsize=14) ax_r2.set_ylabel('$r^2$', fontsize=16) ax_r2.set_ylim(-.05, 1) ax_r2.set_xlim(min(roi_densities) - 5, max(roi_densities) + 5) ax_r2.set_yticks(np.arange(0, 1.01, .1)) ax_rmse.set_ylim(0, .5) ax_rmse.set_yticks(np.arange(0, .51, .05)) ax_rmse.set_xlim(*ax_r2.get_xlim()) for ax in [ax_rmse, ax_r2]: # FIXME force xlims to be the same ax.set_xlabel('density of training points in ROI ($10^{-6}$ km $^{-2}$)', fontsize=14) ax.grid(True) ax_rmse.legend(prop={'size':15}, numpoints=1) fig.tight_layout()
def plot_generalization_analysis(data, roi_density, radius, ncenters, ns_estimators, replot=False, dumpfile=None): """ For all given values for n_estimators (number of trees) for GBRT, perform cross-validation over ncenters ROIs with given radius and sample density. The average training and validation error for each number of trees is plotted. This is the standard plot to detect overfitting defined as the turning point beyond which validation error starts increasing while training error is driven down to zero. As expected, GBRT does not overfit (validation error plateaus). One standard deviation is indicated by a shaded region. """ fig, ax = plt.subplots() if replot: res = pickle_load(dumpfile) for v in ['roi_density', 'radius', 'ns_estimators', 'train_rmses', 'test_rmses']: exec('%s = res["%s"]' % (v, v)) assert len(train_rmses) == len(test_rmses), \ 'array length (# of centers) should be the same for training and test' else: sys.stderr.write('=> Experiment: Generalization ' + \ '(roi_density: %.2f, radius: %.2f,' % (roi_density, radius) + ' no. centers: %d, no. of n_estimators: %d)\n' % (ncenters, len(ns_estimators))) centers = [random_prediction_ctr(data, radius, min_density=roi_density) for _ in range(ncenters)] train_rmses = np.zeros([ncenters, len(ns_estimators)]) test_rmses = np.zeros([ncenters, len(ns_estimators)]) for center_idx, center in enumerate(centers): sys.stderr.write('# center %d/%d\n' % (center_idx + 1, ncenters)) X_train, y_train, X_test, y_test = \ split_with_circle(data, center, roi_density=roi_density, radius=radius) X_train = X_train.drop(['lat', 'lon'], axis=1) X_test = X_test.drop(['lat', 'lon'], axis=1) assert not X_test.empty for n_idx, n in enumerate(ns_estimators): sys.stderr.write(' # n_estimators: %d ' % n) gbrt = train_gbrt(X_train, y_train, n_estimators=n) _, train_rmse = error_summary(y_train, gbrt.predict(X_train)) _, test_rmse = error_summary(y_test, gbrt.predict(X_test)) train_rmses[center_idx][n_idx] = train_rmse test_rmses[center_idx][n_idx] = test_rmse if dumpfile: res = {'roi_density': roi_density, 'radius': radius, 'ns_estimators': ns_estimators, 'train_rmses': train_rmses, 'test_rmses': test_rmses} pickle_dump(dumpfile, res, comment='generalization errors') num_sigma = 1 mean_rmse = test_rmses.mean(axis=0) sd_rmse = np.sqrt(test_rmses.var(axis=0)) lower_rmse = mean_rmse - num_sigma * sd_rmse higher_rmse = mean_rmse + num_sigma * sd_rmse ax.plot(ns_estimators, mean_rmse, 'r', marker='o', markersize=3, alpha=.9, label='validation') ax.fill_between(ns_estimators, lower_rmse, higher_rmse, facecolor='r', edgecolor='r', alpha=.3) mean_rmse = train_rmses.mean(axis=0) sd_rmse = np.sqrt(train_rmses.var(axis=0)) lower_rmse = mean_rmse - num_sigma * sd_rmse higher_rmse = mean_rmse + num_sigma * sd_rmse ax.plot(ns_estimators, mean_rmse, 'g', marker='o', markersize=3, alpha=.9, label='training') ax.fill_between(ns_estimators, lower_rmse, higher_rmse, facecolor='g', edgecolor='g', alpha=.3) ax.grid(True) ax.set_xlim(ns_estimators[0] - 100, ns_estimators[-1] + 100) ax.set_ylim(0, .3) ax.set_yticks(np.arange(0, .31, .05)) ax.set_xlabel('Number of trees') ax.set_ylabel('Normalized RMSE') ax.legend(prop={'size':12.5}) fig.tight_layout()
def plot_sensitivity_analysis(data, roi_density, radius, noise_amps, ncenters, replot=False, dumpfile=None): """ For each given noise amplitude, performs cross-validation on ncenters with given radius and density, the average over ncenters of normalized rmse between noise-free predictions and predictions based on noisy GHF is calculated. This perturbation in predictions is plotted against the expected absolute value of applied noise (amplitude). Both GBRT and linear regression are considered. One standard deviation is indicated by a shaded region. The case of Greenland is considered separately and overlayed. """ fig = plt.figure(figsize=(10, 5)) ax_gbrt = fig.add_subplot(1, 2, 1) ax_lin = fig.add_subplot(1, 2, 2) def _predict(X_train, y_train, X_test, noise_amp): # If noise ~ N(0, s^2), then mean(|noise|) = s * sqrt(2/pi), # cf. https://en.wikipedia.org/wiki/Half-normal_distribution # To get noise with mean(|noise|) / mean(y) = noise_ampl, we need to # have noise ~ N(0, s*^2) with s* = mean(y) * noise_ampl * sqrt(pi/2). noise = np.mean(y_train) * noise_amp * np.sqrt(np.pi/ 2) * np.random.randn(len(y_train)) gbrt = train_gbrt(X_train.drop(['lat', 'lon'], axis=1), y_train + noise) lin_reg = train_linear(X_train.drop(['lat', 'lon'], axis=1), y_train + noise) gbrt_pred = gbrt.predict(X_test.drop(['lat', 'lon'], axis=1)) lin_pred = lin_reg.predict(X_test.drop(['lat', 'lon'], axis=1)) return gbrt_pred, lin_pred if replot: res = pickle_load(dumpfile) rmses_gbrt, rmses_lin = res['rmses_gbrt'], res['rmses_lin'] noise_amps = res['noise_amps'] else: centers = [random_prediction_ctr(data, radius, min_density=roi_density) for _ in range(ncenters)] y0 = [] centers = [None] + centers # one extra "center" (Greenland) rmses_gbrt = np.zeros((len(centers), len(noise_amps))) rmses_lin = np.zeros((len(centers), len(noise_amps))) for idx_ctr, center in enumerate(centers): if center is None: # Greenland case X_train, y_train, X_test = greenland_train_test_sets() else: X_train, y_train, X_test, _ = \ split_with_circle(data, center, roi_density=roi_density, radius=radius) sys.stderr.write('(ctr %d) noise_amp = 0.00 ' % (idx_ctr + 1)) y0_gbrt, y0_lin = _predict(X_train, y_train, X_test, 0) for idx_noise, noise_amp in enumerate(noise_amps): sys.stderr.write('(ctr %d) noise_amp = %.2f ' % (idx_ctr + 1, noise_amp)) y_gbrt, y_lin = _predict(X_train, y_train, X_test, noise_amp) rmse_gbrt = sqrt(mean_squared_error(y0_gbrt, y_gbrt)) / np.mean(y0_gbrt) rmse_lin = sqrt(mean_squared_error(y0_lin, y_lin)) / np.mean(y0_lin) rmses_gbrt[idx_ctr][idx_noise] = rmse_gbrt rmses_lin[idx_ctr][idx_noise] = rmse_lin if dumpfile: res = {'rmses_lin': rmses_lin, 'rmses_gbrt': rmses_gbrt, 'noise_amps': noise_amps} pickle_dump(dumpfile, res, 'sensitivity analysis') kw = dict(alpha=.6, lw=2, marker='o', color='k', label='global average') noise_amps = np.append([0], noise_amps) num_sigma = 1 mean_rmse = rmses_lin[1:].mean(axis=0) sd_rmse = np.sqrt(rmses_lin[1:].var(axis=0)) lower_rmse = np.append([0], mean_rmse - num_sigma * sd_rmse) higher_rmse = np.append([0], mean_rmse + num_sigma * sd_rmse) mean_rmse = np.append([0], mean_rmse) ax_lin.plot(noise_amps, mean_rmse, **kw) ax_lin.fill_between(noise_amps, lower_rmse, higher_rmse, facecolor='k', edgecolor='k', alpha=.2) mean_rmse = rmses_gbrt[1:].mean(axis=0) sd_rmse = np.sqrt(rmses_gbrt[1:].var(axis=0)) lower_rmse = np.append([0], mean_rmse - num_sigma * sd_rmse) higher_rmse = np.append([0], mean_rmse + num_sigma * sd_rmse) mean_rmse = np.append([0], mean_rmse) ax_gbrt.plot(noise_amps, mean_rmse, **kw) ax_gbrt.fill_between(noise_amps, lower_rmse, higher_rmse, facecolor='k', edgecolor='k', alpha=.2) # Greenland case kw = dict(color='g', alpha=.5, lw=2.5, marker='o', markeredgewidth=0.0, label='Greenland') ax_lin.plot(noise_amps, np.append([0], rmses_lin[0]), **kw) ax_gbrt.plot(noise_amps, np.append([0], rmses_gbrt[0]), **kw) for ax in [ax_gbrt, ax_lin]: ax.set_xlabel('Relative magnitude of noise in training GHF', fontsize=12) ax.set_xlim(0, max(noise_amps) * 1.1) ax.set_aspect('equal') ax.grid(True) ax.set_xticks(np.arange(0, .35, .05)) ax.set_yticks(np.arange(0, .35, .05)) ax.set_xlim(-.025, .325) ax.set_ylim(-.025, .325) ax.legend(loc=1, fontsize=12) ax_gbrt.set_ylabel(r'Normalized RMSE difference in $\widehat{GHF}_{\mathrm{GBRT}}$', fontsize=12) ax_lin.set_ylabel(r'Normalized RMSE difference in $\widehat{GHF}_{\mathrm{lin}}$', fontsize=12) fig.tight_layout()
def plot_error_by_radius(data, roi_density, radii, ncenters, region='NA-WE', replot=False, dumpfile=None, **gbrt_params): """ ncenters random centers are picked and over all given radii. Cross-validation errors (normalized RMSE and r2) are averaged over ncenters. One standard deviation mark is shown by a shaded region. """ fig = plt.figure(figsize=(11,5)) ax_rmse, ax_r2 = fig.add_subplot(1, 2, 1), fig.add_subplot(1, 2, 2) if replot: results = pickle_load(dumpfile) else: centers = [ # HACK there's no easy way to check if for a given center the # demanded density is attainable for circles of all desired radii. # Ask for twice the density we need on the largest radius and hope # for the best! random_prediction_ctr(data, max(radii), region=region, min_density=2*roi_density) for _ in range(ncenters) ] shape = (ncenters, len(radii)) # blank error matrix (keyed by center number and roi density index), # used to initialize multiple components of the results dictionary. blank = np.zeros(shape) results = { 'ncenters': ncenters, 'radii': radii, 'errors': { 'gbrt': {'rmse': blank.copy(), 'r2': blank.copy()}, 'linear': {'rmse': blank.copy(), 'r2': blank.copy()}, 'constant': {'rmse': blank.copy(), 'r2': blank.copy()}, }, } for idx_radius, radius in enumerate(radii): for idx_ctr, center in enumerate(centers): sys.stderr.write('# radius = %.0f, center %d/%d ' % (radius, idx_ctr + 1, ncenters)) comp = compare_models(data, roi_density, radius, center, **gbrt_params) for k in results['errors'].keys(): # k is one of gbrt, linear, or constant results['errors'][k]['r2'][idx_ctr][idx_radius] = comp[k][0] results['errors'][k]['rmse'][idx_ctr][idx_radius] = comp[k][1] if dumpfile: pickle_dump(dumpfile, results, comment='GBRT performance results') errors = results['errors'] radii = results['radii'] ncenters = results['ncenters'] num_sigma = 1 # Plot GBRT results kw = {'alpha': .9, 'lw': 1, 'marker': 'o', 'markersize': 4, 'color': 'b'} mean_rmse = errors['gbrt']['rmse'].mean(axis=0) sd_rmse = np.sqrt(errors['gbrt']['rmse'].var(axis=0)) lower_rmse = mean_rmse - num_sigma * sd_rmse higher_rmse = mean_rmse + num_sigma * sd_rmse ax_rmse.plot(radii, mean_rmse, label='GBRT', **kw) ax_rmse.fill_between(radii, lower_rmse, higher_rmse, facecolor='b', edgecolor='b', alpha=.3) mean_r2 = errors['gbrt']['r2'].mean(axis=0) sd_r2 = np.sqrt(errors['gbrt']['r2'].var(axis=0)) lower_r2 = mean_r2 - num_sigma * sd_r2 higher_r2 = mean_r2 + num_sigma * sd_r2 ax_r2.plot(radii, errors['gbrt']['r2'].mean(axis=0), **kw) ax_r2.fill_between(radii, lower_r2, higher_r2, facecolor='b', edgecolor='b', alpha=.2) # Plot Linear Regression results kw = {'alpha': .7, 'lw': 1, 'marker': 'o', 'markersize': 4, 'markeredgecolor': 'r', 'color': 'r'} mean_rmse = errors['linear']['rmse'].mean(axis=0) sd_rmse = np.sqrt(errors['linear']['rmse'].var(axis=0)) lower_rmse = mean_rmse - num_sigma * sd_rmse higher_rmse = mean_rmse + num_sigma * sd_rmse ax_rmse.plot(radii, mean_rmse, label='linear regression', **kw) ax_rmse.fill_between(radii, lower_rmse, higher_rmse, facecolor='r', edgecolor='r', alpha=.3) mean_r2 = errors['linear']['r2'].mean(axis=0) sd_r2 = np.sqrt(errors['linear']['r2'].var(axis=0)) lower_r2 = mean_r2 - num_sigma * sd_r2 higher_r2 = mean_r2 + num_sigma * sd_r2 ax_r2.plot(radii, errors['linear']['r2'].mean(axis=0), **kw) ax_r2.fill_between(radii, lower_r2, higher_r2, facecolor='r', edgecolor='r', alpha=.2) # Plot constant predictor results kw = {'alpha': .7, 'lw': 1, 'ls': '--', 'marker': 'o', 'markersize': 4, 'color': 'k', 'markeredgecolor': 'k'} ax_rmse.plot(radii, errors['constant']['rmse'].mean(axis=0), label='constant predictor', **kw) ax_r2.plot(radii, errors['constant']['r2'].mean(axis=0), **kw) # Style plot ax_rmse.set_ylabel('Normalized RMSE', fontsize=14) ax_r2.set_ylabel('$r^2$', fontsize=16) ax_r2.set_ylim(-.05, 1) ax_r2.set_xlim(min(radii) - 100, max(radii) + 100) ax_r2.set_yticks(np.arange(0, 1.01, .1)) ax_rmse.set_ylim(0, .5) ax_rmse.set_yticks(np.arange(0, .51, .05)) ax_rmse.set_xlim(*ax_r2.get_xlim()) for ax in [ax_rmse, ax_r2]: # FIXME force xlims to be the same ax.set_xlabel('radius of ROI (km)', fontsize=14) ax.grid(True) ax_rmse.legend(prop={'size':15}, numpoints=1) fig.tight_layout()