def _predict(X_train, y_train, X_test, noise_amp):
# If noise ~ N(0, s^2), then mean(|noise|) = s * sqrt(2/pi),
# cf. https://en.wikipedia.org/wiki/Half-normal_distribution
# To get noise with mean(|noise|) / mean(y) = noise_ampl, we need to
# have noise ~ N(0, s*^2) with s* = mean(y) * noise_ampl * sqrt(pi/2).
noise = np.mean(y_train) * noise_amp * np.sqrt(np.pi/ 2) * np.random.randn(len(y_train))
gbrt = train_gbrt(X_train.drop(['lat', 'lon'], axis=1),
y_train + noise)
lin_reg = train_linear(X_train.drop(['lat', 'lon'], axis=1),
y_train + noise)
gbrt_pred = gbrt.predict(X_test.drop(['lat', 'lon'], axis=1))
lin_pred = lin_reg.predict(X_test.drop(['lat', 'lon'], axis=1))
return gbrt_pred, lin_pred
def plot_error_by_density(data, roi_densities, radius, ncenters, region='NA-WE',
replot=False, dumpfile=None, **gbrt_params):
""" ncenters random centers are picked and over all given ROI densities.
Cross-validation errors (normalized RMSE and r2) are averaged over
ncenters. One standard deviation mark is shown by a shaded region.
"""
sys.stderr.write('=> Experiment: Error by Density (region: %s, no. centers: %d, no. densities: %d)\n' %
(region, ncenters, len(roi_densities)))
fig = plt.figure(figsize=(11,5))
ax_rmse, ax_r2 = fig.add_subplot(1, 2, 1), fig.add_subplot(1, 2, 2)
if replot:
results = pickle_load(dumpfile)
else:
centers = [
random_prediction_ctr(data, radius, region=region, min_density=max(roi_densities))
for _ in range(ncenters)
]
shape = (ncenters, len(roi_densities))
# blank error matrix (keyed by center number and roi density index),
# used to initialize multiple components of the results dictionary.
blank = np.zeros(shape)
results = {
'ncenters': ncenters,
'roi_densities': roi_densities,
'errors': {
'gbrt': {'rmse': blank.copy(), 'r2': blank.copy()},
'linear': {'rmse': blank.copy(), 'r2': blank.copy()},
'constant': {'rmse': blank.copy(), 'r2': blank.copy()},
},
}
for idx_density, roi_density in enumerate(roi_densities):
for idx_ctr, center in enumerate(centers):
sys.stderr.write('# density = %.2f, center %d/%d ' % (roi_density, idx_ctr + 1, ncenters))
comp = compare_models(data, roi_density, radius, center, **gbrt_params)
for k in results['errors'].keys():
# k is one of gbrt, linear, or constant
results['errors'][k]['r2'][idx_ctr][idx_density] = comp[k][0]
results['errors'][k]['rmse'][idx_ctr][idx_density] = comp[k][1]
if dumpfile:
pickle_dump(dumpfile, results, comment='GBRT performance results')
errors = results['errors']
roi_densities = results['roi_densities']
ncenters = results['ncenters']
num_sigma = 1
# Plot GBRT results
kw = {'alpha': .9, 'lw': 1, 'marker': 'o', 'markersize': 4, 'color': 'b'}
mean_rmse = errors['gbrt']['rmse'].mean(axis=0)
sd_rmse = np.sqrt(errors['gbrt']['rmse'].var(axis=0))
lower_rmse = mean_rmse - num_sigma * sd_rmse
higher_rmse = mean_rmse + num_sigma * sd_rmse
ax_rmse.plot(roi_densities, mean_rmse, label='GBRT', **kw)
ax_rmse.fill_between(roi_densities, lower_rmse, higher_rmse, facecolor='b', edgecolor='b', alpha=.3)
mean_r2 = errors['gbrt']['r2'].mean(axis=0)
sd_r2 = np.sqrt(errors['gbrt']['r2'].var(axis=0))
lower_r2 = mean_r2 - num_sigma * sd_r2
higher_r2 = mean_r2 + num_sigma * sd_r2
ax_r2.plot(roi_densities, errors['gbrt']['r2'].mean(axis=0), **kw)
ax_r2.fill_between(roi_densities, lower_r2, higher_r2, facecolor='b', edgecolor='b', alpha=.2)
# Plot Linear Regression results
kw = {'alpha': .7, 'lw': 1, 'marker': 'o', 'markersize': 4, 'markeredgecolor': 'r', 'color': 'r'}
mean_rmse = errors['linear']['rmse'].mean(axis=0)
sd_rmse = np.sqrt(errors['linear']['rmse'].var(axis=0))
lower_rmse = mean_rmse - num_sigma * sd_rmse
higher_rmse = mean_rmse + num_sigma * sd_rmse
ax_rmse.plot(roi_densities, mean_rmse, label='linear regression', **kw)
ax_rmse.fill_between(roi_densities, lower_rmse, higher_rmse, facecolor='r', edgecolor='r', alpha=.3)
mean_r2 = errors['linear']['r2'].mean(axis=0)
sd_r2 = np.sqrt(errors['linear']['r2'].var(axis=0))
lower_r2 = mean_r2 - num_sigma * sd_r2
higher_r2 = mean_r2 + num_sigma * sd_r2
ax_r2.plot(roi_densities, errors['linear']['r2'].mean(axis=0), **kw)
ax_r2.fill_between(roi_densities, lower_r2, higher_r2, facecolor='r', edgecolor='r', alpha=.2)
# Plot constant predictor results
kw = {'alpha': .7, 'lw': 1, 'ls': '--', 'marker': 'o', 'markersize': 4, 'color': 'k', 'markeredgecolor': 'k'}
ax_rmse.plot(roi_densities, errors['constant']['rmse'].mean(axis=0), label='constant predictor', **kw)
ax_r2.plot(roi_densities, errors['constant']['r2'].mean(axis=0), **kw)
# Style plot
ax_rmse.set_ylabel('Normalized RMSE', fontsize=14)
ax_r2.set_ylabel('$r^2$', fontsize=16)
ax_r2.set_ylim(-.05, 1)
ax_r2.set_xlim(min(roi_densities) - 5, max(roi_densities) + 5)
ax_r2.set_yticks(np.arange(0, 1.01, .1))
ax_rmse.set_ylim(0, .5)
ax_rmse.set_yticks(np.arange(0, .51, .05))
ax_rmse.set_xlim(*ax_r2.get_xlim())
for ax in [ax_rmse, ax_r2]:
# FIXME force xlims to be the same
ax.set_xlabel('density of training points in ROI ($10^{-6}$ km $^{-2}$)',
fontsize=14)
ax.grid(True)
ax_rmse.legend(prop={'size':15}, numpoints=1)
fig.tight_layout()
def plot_feature_importance_analysis(data, roi_density, radius, ncenters,
dumpfile=None, replot=False, **gbrt_params):
""" Plots feature importance results (cf. Friedman 2001 or ESL) averaged
over ncenters rounds of cross validation for given ROI training density
and radius.
"""
raw_features = list(data)
for f in ['lat', 'lon', 'GHF']:
raw_features.pop(raw_features.index(f))
# a map to collapse categorical dummies for feature importances. The dict
# has keys in `raw_features` indices, and values in `features` indices.
decat_by_raw_idx = {}
features = []
for idx, f in enumerate(raw_features):
match = [c for c in CATEGORICAL_FEATURES if c == f[:len(c)]]
if match:
assert len(match) == 1
try:
i = features.index(match[0])
except ValueError:
features.append(match[0])
i = len(features) - 1
decat_by_raw_idx[idx] = i
continue
features.append(f)
decat_by_raw_idx[idx] = len(features) - 1
if replot:
res = pickle_load(dumpfile)
gbrt_importances = res['gbrt_importances']
else:
# at this point features contains original feature names and raw_features
# contains categorical dummies, in each round we map
# feature_importances_, which has the same size as raw_features, to feature
# importances for original features by adding the importances of each
# categorical dummy.
centers = [random_prediction_ctr(data, radius, min_density=roi_density) for _ in range(ncenters)]
gbrt_importances = np.zeros([ncenters, len(features)])
for center_idx, center in enumerate(centers):
sys.stderr.write('%d / %d ' % (center_idx + 1, ncenters))
X_train, y_train, X_test, y_test = \
split_with_circle(data, center, roi_density=roi_density, radius=radius)
X_train = X_train.drop(['lat', 'lon'], axis=1)
X_test = X_test.drop(['lat', 'lon'], axis=1)
assert not X_test.empty
gbrt = train_gbrt(X_train, y_train, **gbrt_params)
raw_importances = gbrt.feature_importances_
for idx, value in enumerate(raw_importances):
gbrt_importances[center_idx][decat_by_raw_idx[idx]] += value
if dumpfile:
res = {'gbrt_importances': gbrt_importances, 'features': features}
pickle_dump(dumpfile, res, 'feature importances')
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
means = gbrt_importances.mean(axis=0)
sds = np.sqrt(gbrt_importances.var(axis=0))
sort_order = list(np.argsort(means))
feature_names = [FEATURE_NAMES[features[i]] for i in sort_order]
means, sds = [means[i] for i in sort_order], [sds[i] for i in sort_order]
_yrange = [i-0.4 for i in range(len(features))] # labels in the middle of bars
ax.barh(_yrange, means, color='k', ecolor='k', alpha=.3, xerr=sds[::-1])
ax.set_ylim(-1, len(features))
ax.grid(True)
ax.set_yticks(range(len(features)))
ax.set_yticklabels(feature_names, rotation=0, fontsize=10)
ax.set_title('GBRT feature importances')
fig.subplots_adjust(left=0.3) # for vertical xtick labels
def plot_generalization_analysis(data, roi_density, radius, ncenters,
ns_estimators, replot=False, dumpfile=None):
""" For all given values for n_estimators (number of trees) for GBRT,
perform cross-validation over ncenters ROIs with given radius and
sample density. The average training and validation error for each
number of trees is plotted. This is the standard plot to detect
overfitting defined as the turning point beyond which validation error
starts increasing while training error is driven down to zero. As
expected, GBRT does not overfit (validation error plateaus).
One standard deviation is indicated by a shaded region.
"""
fig, ax = plt.subplots()
if replot:
res = pickle_load(dumpfile)
for v in ['roi_density', 'radius', 'ns_estimators', 'train_rmses', 'test_rmses']:
exec('%s = res["%s"]' % (v, v))
assert len(train_rmses) == len(test_rmses), \
'array length (# of centers) should be the same for training and test'
else:
sys.stderr.write('=> Experiment: Generalization ' + \
'(roi_density: %.2f, radius: %.2f,' % (roi_density, radius) +
' no. centers: %d, no. of n_estimators: %d)\n' % (ncenters, len(ns_estimators)))
centers = [random_prediction_ctr(data, radius, min_density=roi_density)
for _ in range(ncenters)]
train_rmses = np.zeros([ncenters, len(ns_estimators)])
test_rmses = np.zeros([ncenters, len(ns_estimators)])
for center_idx, center in enumerate(centers):
sys.stderr.write('# center %d/%d\n' % (center_idx + 1, ncenters))
X_train, y_train, X_test, y_test = \
split_with_circle(data, center, roi_density=roi_density, radius=radius)
X_train = X_train.drop(['lat', 'lon'], axis=1)
X_test = X_test.drop(['lat', 'lon'], axis=1)
assert not X_test.empty
for n_idx, n in enumerate(ns_estimators):
sys.stderr.write(' # n_estimators: %d ' % n)
gbrt = train_gbrt(X_train, y_train, n_estimators=n)
_, train_rmse = error_summary(y_train, gbrt.predict(X_train))
_, test_rmse = error_summary(y_test, gbrt.predict(X_test))
train_rmses[center_idx][n_idx] = train_rmse
test_rmses[center_idx][n_idx] = test_rmse
if dumpfile:
res = {'roi_density': roi_density,
'radius': radius,
'ns_estimators': ns_estimators,
'train_rmses': train_rmses,
'test_rmses': test_rmses}
pickle_dump(dumpfile, res, comment='generalization errors')
num_sigma = 1
mean_rmse = test_rmses.mean(axis=0)
sd_rmse = np.sqrt(test_rmses.var(axis=0))
lower_rmse = mean_rmse - num_sigma * sd_rmse
higher_rmse = mean_rmse + num_sigma * sd_rmse
ax.plot(ns_estimators, mean_rmse, 'r', marker='o', markersize=3, alpha=.9, label='validation')
ax.fill_between(ns_estimators, lower_rmse, higher_rmse, facecolor='r', edgecolor='r', alpha=.3)
mean_rmse = train_rmses.mean(axis=0)
sd_rmse = np.sqrt(train_rmses.var(axis=0))
lower_rmse = mean_rmse - num_sigma * sd_rmse
higher_rmse = mean_rmse + num_sigma * sd_rmse
ax.plot(ns_estimators, mean_rmse, 'g', marker='o', markersize=3, alpha=.9, label='training')
ax.fill_between(ns_estimators, lower_rmse, higher_rmse, facecolor='g', edgecolor='g', alpha=.3)
ax.grid(True)
ax.set_xlim(ns_estimators[0] - 100, ns_estimators[-1] + 100)
ax.set_ylim(0, .3)
ax.set_yticks(np.arange(0, .31, .05))
ax.set_xlabel('Number of trees')
ax.set_ylabel('Normalized RMSE')
ax.legend(prop={'size':12.5})
fig.tight_layout()
def plot_sensitivity_analysis(data, roi_density, radius, noise_amps, ncenters,
replot=False, dumpfile=None):
""" For each given noise amplitude, performs cross-validation on ncenters
with given radius and density, the average over ncenters of
normalized rmse between noise-free predictions and predictions based on
noisy GHF is calculated. This perturbation in predictions is plotted
against the expected absolute value of applied noise (amplitude).
Both GBRT and linear regression are considered.
One standard deviation is indicated by a shaded region.
The case of Greenland is considered separately and overlayed.
"""
fig = plt.figure(figsize=(10, 5))
ax_gbrt = fig.add_subplot(1, 2, 1)
ax_lin = fig.add_subplot(1, 2, 2)
def _predict(X_train, y_train, X_test, noise_amp):
# If noise ~ N(0, s^2), then mean(|noise|) = s * sqrt(2/pi),
# cf. https://en.wikipedia.org/wiki/Half-normal_distribution
# To get noise with mean(|noise|) / mean(y) = noise_ampl, we need to
# have noise ~ N(0, s*^2) with s* = mean(y) * noise_ampl * sqrt(pi/2).
noise = np.mean(y_train) * noise_amp * np.sqrt(np.pi/ 2) * np.random.randn(len(y_train))
gbrt = train_gbrt(X_train.drop(['lat', 'lon'], axis=1),
y_train + noise)
lin_reg = train_linear(X_train.drop(['lat', 'lon'], axis=1),
y_train + noise)
gbrt_pred = gbrt.predict(X_test.drop(['lat', 'lon'], axis=1))
lin_pred = lin_reg.predict(X_test.drop(['lat', 'lon'], axis=1))
return gbrt_pred, lin_pred
if replot:
res = pickle_load(dumpfile)
rmses_gbrt, rmses_lin = res['rmses_gbrt'], res['rmses_lin']
noise_amps = res['noise_amps']
else:
centers = [random_prediction_ctr(data, radius, min_density=roi_density)
for _ in range(ncenters)]
y0 = []
centers = [None] + centers # one extra "center" (Greenland)
rmses_gbrt = np.zeros((len(centers), len(noise_amps)))
rmses_lin = np.zeros((len(centers), len(noise_amps)))
for idx_ctr, center in enumerate(centers):
if center is None:
# Greenland case
X_train, y_train, X_test = greenland_train_test_sets()
else:
X_train, y_train, X_test, _ = \
split_with_circle(data, center, roi_density=roi_density, radius=radius)
sys.stderr.write('(ctr %d) noise_amp = 0.00 ' % (idx_ctr + 1))
y0_gbrt, y0_lin = _predict(X_train, y_train, X_test, 0)
for idx_noise, noise_amp in enumerate(noise_amps):
sys.stderr.write('(ctr %d) noise_amp = %.2f ' % (idx_ctr + 1, noise_amp))
y_gbrt, y_lin = _predict(X_train, y_train, X_test, noise_amp)
rmse_gbrt = sqrt(mean_squared_error(y0_gbrt, y_gbrt)) / np.mean(y0_gbrt)
rmse_lin = sqrt(mean_squared_error(y0_lin, y_lin)) / np.mean(y0_lin)
rmses_gbrt[idx_ctr][idx_noise] = rmse_gbrt
rmses_lin[idx_ctr][idx_noise] = rmse_lin
if dumpfile:
res = {'rmses_lin': rmses_lin, 'rmses_gbrt': rmses_gbrt, 'noise_amps': noise_amps}
pickle_dump(dumpfile, res, 'sensitivity analysis')
kw = dict(alpha=.6, lw=2, marker='o', color='k', label='global average')
noise_amps = np.append([0], noise_amps)
num_sigma = 1
mean_rmse = rmses_lin[1:].mean(axis=0)
sd_rmse = np.sqrt(rmses_lin[1:].var(axis=0))
lower_rmse = np.append([0], mean_rmse - num_sigma * sd_rmse)
higher_rmse = np.append([0], mean_rmse + num_sigma * sd_rmse)
mean_rmse = np.append([0], mean_rmse)
ax_lin.plot(noise_amps, mean_rmse, **kw)
ax_lin.fill_between(noise_amps, lower_rmse, higher_rmse, facecolor='k', edgecolor='k', alpha=.2)
mean_rmse = rmses_gbrt[1:].mean(axis=0)
sd_rmse = np.sqrt(rmses_gbrt[1:].var(axis=0))
lower_rmse = np.append([0], mean_rmse - num_sigma * sd_rmse)
higher_rmse = np.append([0], mean_rmse + num_sigma * sd_rmse)
mean_rmse = np.append([0], mean_rmse)
ax_gbrt.plot(noise_amps, mean_rmse, **kw)
ax_gbrt.fill_between(noise_amps, lower_rmse, higher_rmse, facecolor='k', edgecolor='k', alpha=.2)
# Greenland case
kw = dict(color='g', alpha=.5, lw=2.5, marker='o',
markeredgewidth=0.0, label='Greenland')
ax_lin.plot(noise_amps, np.append([0], rmses_lin[0]), **kw)
ax_gbrt.plot(noise_amps, np.append([0], rmses_gbrt[0]), **kw)
for ax in [ax_gbrt, ax_lin]:
ax.set_xlabel('Relative magnitude of noise in training GHF', fontsize=12)
ax.set_xlim(0, max(noise_amps) * 1.1)
ax.set_aspect('equal')
ax.grid(True)
ax.set_xticks(np.arange(0, .35, .05))
ax.set_yticks(np.arange(0, .35, .05))
ax.set_xlim(-.025, .325)
ax.set_ylim(-.025, .325)
ax.legend(loc=1, fontsize=12)
ax_gbrt.set_ylabel(r'Normalized RMSE difference in $\widehat{GHF}_{\mathrm{GBRT}}$', fontsize=12)
ax_lin.set_ylabel(r'Normalized RMSE difference in $\widehat{GHF}_{\mathrm{lin}}$', fontsize=12)
fig.tight_layout()
def plot_error_by_radius(data, roi_density, radii, ncenters, region='NA-WE',
replot=False, dumpfile=None, **gbrt_params):
""" ncenters random centers are picked and over all given radii.
Cross-validation errors (normalized RMSE and r2) are averaged over
ncenters. One standard deviation mark is shown by a shaded region.
"""
fig = plt.figure(figsize=(11,5))
ax_rmse, ax_r2 = fig.add_subplot(1, 2, 1), fig.add_subplot(1, 2, 2)
if replot:
results = pickle_load(dumpfile)
else:
centers = [
# HACK there's no easy way to check if for a given center the
# demanded density is attainable for circles of all desired radii.
# Ask for twice the density we need on the largest radius and hope
# for the best!
random_prediction_ctr(data, max(radii), region=region, min_density=2*roi_density)
for _ in range(ncenters)
]
shape = (ncenters, len(radii))
# blank error matrix (keyed by center number and roi density index),
# used to initialize multiple components of the results dictionary.
blank = np.zeros(shape)
results = {
'ncenters': ncenters,
'radii': radii,
'errors': {
'gbrt': {'rmse': blank.copy(), 'r2': blank.copy()},
'linear': {'rmse': blank.copy(), 'r2': blank.copy()},
'constant': {'rmse': blank.copy(), 'r2': blank.copy()},
},
}
for idx_radius, radius in enumerate(radii):
for idx_ctr, center in enumerate(centers):
sys.stderr.write('# radius = %.0f, center %d/%d ' % (radius, idx_ctr + 1, ncenters))
comp = compare_models(data, roi_density, radius, center, **gbrt_params)
for k in results['errors'].keys():
# k is one of gbrt, linear, or constant
results['errors'][k]['r2'][idx_ctr][idx_radius] = comp[k][0]
results['errors'][k]['rmse'][idx_ctr][idx_radius] = comp[k][1]
if dumpfile:
pickle_dump(dumpfile, results, comment='GBRT performance results')
errors = results['errors']
radii = results['radii']
ncenters = results['ncenters']
num_sigma = 1
# Plot GBRT results
kw = {'alpha': .9, 'lw': 1, 'marker': 'o', 'markersize': 4, 'color': 'b'}
mean_rmse = errors['gbrt']['rmse'].mean(axis=0)
sd_rmse = np.sqrt(errors['gbrt']['rmse'].var(axis=0))
lower_rmse = mean_rmse - num_sigma * sd_rmse
higher_rmse = mean_rmse + num_sigma * sd_rmse
ax_rmse.plot(radii, mean_rmse, label='GBRT', **kw)
ax_rmse.fill_between(radii, lower_rmse, higher_rmse, facecolor='b', edgecolor='b', alpha=.3)
mean_r2 = errors['gbrt']['r2'].mean(axis=0)
sd_r2 = np.sqrt(errors['gbrt']['r2'].var(axis=0))
lower_r2 = mean_r2 - num_sigma * sd_r2
higher_r2 = mean_r2 + num_sigma * sd_r2
ax_r2.plot(radii, errors['gbrt']['r2'].mean(axis=0), **kw)
ax_r2.fill_between(radii, lower_r2, higher_r2, facecolor='b', edgecolor='b', alpha=.2)
# Plot Linear Regression results
kw = {'alpha': .7, 'lw': 1, 'marker': 'o', 'markersize': 4, 'markeredgecolor': 'r', 'color': 'r'}
mean_rmse = errors['linear']['rmse'].mean(axis=0)
sd_rmse = np.sqrt(errors['linear']['rmse'].var(axis=0))
lower_rmse = mean_rmse - num_sigma * sd_rmse
higher_rmse = mean_rmse + num_sigma * sd_rmse
ax_rmse.plot(radii, mean_rmse, label='linear regression', **kw)
ax_rmse.fill_between(radii, lower_rmse, higher_rmse, facecolor='r', edgecolor='r', alpha=.3)
mean_r2 = errors['linear']['r2'].mean(axis=0)
sd_r2 = np.sqrt(errors['linear']['r2'].var(axis=0))
lower_r2 = mean_r2 - num_sigma * sd_r2
higher_r2 = mean_r2 + num_sigma * sd_r2
ax_r2.plot(radii, errors['linear']['r2'].mean(axis=0), **kw)
ax_r2.fill_between(radii, lower_r2, higher_r2, facecolor='r', edgecolor='r', alpha=.2)
# Plot constant predictor results
kw = {'alpha': .7, 'lw': 1, 'ls': '--', 'marker': 'o', 'markersize': 4, 'color': 'k', 'markeredgecolor': 'k'}
ax_rmse.plot(radii, errors['constant']['rmse'].mean(axis=0), label='constant predictor', **kw)
ax_r2.plot(radii, errors['constant']['r2'].mean(axis=0), **kw)
# Style plot
ax_rmse.set_ylabel('Normalized RMSE', fontsize=14)
ax_r2.set_ylabel('$r^2$', fontsize=16)
ax_r2.set_ylim(-.05, 1)
ax_r2.set_xlim(min(radii) - 100, max(radii) + 100)
ax_r2.set_yticks(np.arange(0, 1.01, .1))
ax_rmse.set_ylim(0, .5)
ax_rmse.set_yticks(np.arange(0, .51, .05))
ax_rmse.set_xlim(*ax_r2.get_xlim())
for ax in [ax_rmse, ax_r2]:
# FIXME force xlims to be the same
ax.set_xlabel('radius of ROI (km)', fontsize=14)
ax.grid(True)
ax_rmse.legend(prop={'size':15}, numpoints=1)
fig.tight_layout()