def test(self):
"""Tests out the models that validate and optimal_model_search has determined are best.
Uses the pandas.DataFrame optimal_model created by optimal_model_search method, and tests each of the models
on testing data. Plots the result side by side with the actual dependent variable.
"""
for (model_name, technique), (_lambda, poly, _, _) in self.optimal_model.iterrows():
for _model in self.models:
if _model.name == model_name:
model = _model
X = poly_design_matrix(int(poly), self.data['x_train'])
model.scaler.fit(X[:,1:])
X[:,1:] = model.scaler.transform(X[:,1:])
model.set_lambda(_lambda)
model.fit(X, self.data['y_train'])
X_test = poly_design_matrix(int(poly), self.data['x_test'])
X_test[:,1:] = model.scaler.transform(X_test[:,1:])
y_pred = model.predict(X_test)
mse = metrics.MSE(data['y_test'], y_pred)
self.optimal_model.loc[model_name, technique]['Test error'] = mse
if self.verbose >= 1:
print(f"MSE: {mse} for poly = {poly} and lambda = {_lambda} with model {model_name}.")
plotting.side_by_side(
['Predicted', self.data['x_test'], y_pred],
['Ground truth', self.data['x_test'], self.data['y_test']],
title=f"Test: {model.name}, $p$={poly}, $\\lambda$={_lambda}, best according to {technique}",
filename=f"{name}_test_{model.name}_p{poly}{[f'_lambda{_lambda}', ''][int(_lambda==0)]}")
def plot_data(*data_args, random=False):
"""Plots terrain data created by get_data function.
Parameters:
-----------
*data_args:
Passed on to get_data()
"""
data = get_data(*data_args, random=random)
plotting.side_by_side(
['Train', data['x_train'], data['y_train']],
['Validate', data['x_train'], data['y_train']],
['Test', data['x_test'], data['y_test']],
title=f"Training, validation and testing data for real terrain",
filename=f"real_terrain")
def plot_sgd_errors(sgd_errors_df, title, metric_string='MSE'):
"""Takes a dataframe of errors, and formats it in a way that plotting.side_by_side can handle.
Parameters:
-----------
errors_df: pd.DataFrame
Dataframe with titles, initial conditions and labels as multiindex, and epochs as column
"""
plots = []
for subplot_name, subplot_errors in sgd_errors_df.groupby(level=0):
y = []
labels = []
for j, (label_names,
label_errors) in enumerate(subplot_errors.groupby(level=1)):
labels.append(label_names)
min_errors = label_errors.min(axis=0).to_numpy()[np.newaxis, :]
max_errors = label_errors.max(axis=0).to_numpy()[np.newaxis, :]
best_index = label_errors.iloc[slice(None), -1].idxmin()
best_errors = label_errors.loc(axis=0)[slice(None),
slice(None),
best_index].to_numpy()
y_ = np.concatenate((min_errors, best_errors, max_errors), axis=0)
y.append((y_, {'color': 'C' + str(j + 1)}))
label_enum = np.arange(len(labels))
colors = [f"C{number}" for number in label_enum + 1]
handler_map = {
i: plotting.LegendObject(colors[i])
for i in range(len(colors))
}
legend = [label_enum, labels], {'handler_map': handler_map}
plots.append([subplot_name, subplot_errors.columns, y, legend])
side_by_side_parameters = {
'plotter': plotting.confidence_interval_plotter,
'axis_labels': ('Epoch', metric_string),
'title': title,
'yscale': 'log',
}
plotting.side_by_side(*plots, **side_by_side_parameters)
def plot_data(**data_args):
"""Plots Franke function data, side by side with Franke function + noise data.
Parameters:
-----------
*data_args:
Passed on to get_data()
"""
data = get_data(**data_args)
data_args['noise_std'] = 0
data_wo_noise = get_data(**data_args)
plotting.side_by_side([
"$f(x_1, x_2)$", data['x'], data_wo_noise['y'].reshape(-1)
], [
f"$f(x_1, x_2) + \\varepsilon$, where $\\varepsilon \\sim N(0,{0.2})$",
data['x'], data['y'].reshape(-1)
],
title="Franke function ($f(x_1, x_2)$)",
filename="franke")
def plot_data(*data_args, random=False):
"""Plots terrain data created by get_data function.
Parameters:
-----------
*data_args:
Passed on to get_data()
"""
data = get_data(*data_args, random=random)
plotting.side_by_side(
['Train', data['x_train'], data['y_train']],
['Validate', data['x_train'], data['y_train']],
['Test', data['x_test'], data['y_test']],
title=[
f"Training, validation and testing data for real terrain",
"real_terrain"
],
plotter=plotting.trisurface_plotter,
projection='3d',
view_angles=[15, 240])
def beta_variance(data, epochs=20000, mini_batch_sizes=[10, 20]):
"""Plots variance of beta parameters for different linear models on regression problem for real terrain"""
polynomials = 8
X_train = linear_models.poly_design_matrix(polynomials, data['x_train'])
X_validate = linear_models.poly_design_matrix(polynomials, data['x_validate'])
total_steps = epochs * len(data['x_train'])//mini_batch_sizes[0]
learning_rate_func = learning_rate.Learning_rate(base=5e-2, decay=1/100000).compile(total_steps)
common_ridge1_kwargs = {'_lambda': 0.01, 'name': 'Ridge', 'beta_func': linear_models.beta_ridge, 'momentum': 0.5, 'init_conds': 300, 'x_shape': X_train.shape[1]}
common_ridge2_kwargs = {'_lambda': 0.0001, 'name': 'Ridge', 'beta_func': linear_models.beta_ridge, 'momentum': 0.5, 'init_conds': 300, 'x_shape': X_train.shape[1]}
subplot_ridge_uniques = [{}]
subsubplot_ridge_linear_uniques = [{'learning_rate': learning_rate_func}]
common_ols_kwargs = {'momentum': 0.5, 'init_conds': 300, 'x_shape': X_train.shape[1]}
subplot_ols_uniques = [{}]
subsubplot_ols_linear_uniques = [{'learning_rate': learning_rate_func}]
unique_sgd_kwargs = [{'mini_batch_size': mini_batch_size} for mini_batch_size in mini_batch_sizes]
ridge_models1 = helpers.make_models(
linear_models.RegularisedLinearRegression,
common_ridge1_kwargs,
[{}],
len(unique_sgd_kwargs),
subsubplot_ridge_linear_uniques
)
ridge_models2 = helpers.make_models(
linear_models.RegularisedLinearRegression,
common_ridge2_kwargs,
[{}],
len(unique_sgd_kwargs),
subsubplot_ridge_linear_uniques
)
ols_models = helpers.make_models(
linear_models.LinearRegression,
common_ols_kwargs,
[{}],
len(unique_sgd_kwargs),
subsubplot_ols_linear_uniques
)
models_list = ridge_models1 + ridge_models2 + ols_models
subplots = [(models, sgd_kwargs) for models, sgd_kwargs in zip(models_list, unique_sgd_kwargs*(len(models_list)))]
errors, subtitle, subplots, metrics_string = sgd.sgd_on_models(X_train, X_validate, data['y_train'], data['y_validate'], *subplots, epochs=epochs, epochs_without_progress=200)
title = ['Convergence test', 'convergence', subtitle]
sgd.plot_sgd_errors(errors, title, metrics_string)
sgd_ridge1_betas = np.array([model.beta for models in ridge_models1 for model in models])
sgd_ridge1_betas = sgd_ridge1_betas.transpose(1, 2, 0).reshape((sgd_ridge1_betas.shape[1], -1)).T
sgd_ridge2_betas = np.array([model.beta for models in ridge_models2 for model in models])
sgd_ridge2_betas = sgd_ridge2_betas.transpose(1, 2, 0).reshape((sgd_ridge2_betas.shape[1], -1)).T
sgd_ols_betas = np.array([model.beta for models in ols_models for model in models])
sgd_ols_betas = sgd_ols_betas.transpose(1, 2, 0).reshape((sgd_ols_betas.shape[1], -1)).T
optimal_models = [linear_models.RegularisedLinearRegression("Ridge", linear_models.beta_ridge, _lambda=_lambda) for _lambda in (0.01, 0.)]
optimal_betas = np.array([model.fit(X_train, data['y_train']) for model in optimal_models])[:,:,0]
plots = [
['Ridge ($\\lambda 0.01$) trained by SGD', np.arange(1, optimal_betas.shape[1] + 1),
[[sgd_ridge1_betas, {'notch': False, 'sym': ''}],
[optimal_betas[0], plotting.scatter_plotter, {'label': 'Analytical $\\lambda$ 0.01'}],
]
],
['Ridge ($\\lambda 0.0001$) trained by SGD', np.arange(1, optimal_betas.shape[1] + 1),
[[sgd_ridge2_betas, {'notch': False, 'sym': ''}],
[optimal_betas[0], plotting.scatter_plotter, {'label': 'Analytical $\\lambda$ 0.01'}],
]
],
['OLS trained by SGD', np.arange(1, optimal_betas.shape[1] + 1),
[[sgd_ols_betas, {'notch': False, 'sym': ''}],
[optimal_betas[0], plotting.scatter_plotter, {'label': 'Analytical $\\lambda$ 0.01'}],
]
],
]
side_by_side_parameters = {
'plotter': plotting.box_plotter,
'axis_labels': ('Beta parameter #', 'Values'),
'title': ['$\\beta$ parameters', 'beta', subtitle],
}
plotting.side_by_side(*plots, **side_by_side_parameters)
def terrain_reggression(data, epochs=5000, epochs_without_progress=500, mini_batch_size=20):
"""Plots performance of different neural models on regression problem for real terrain"""
polynomials = 8
X_train = linear_models.poly_design_matrix(polynomials, data['x_train'])
X_validate = linear_models.poly_design_matrix(polynomials, data['x_validate'])
X_test = linear_models.poly_design_matrix(polynomials, data['x_test'])
total_steps = epochs * len(data['x_train'])//mini_batch_size
l_rate = learning_rate.Learning_rate(base=3e-2, decay=1/20000).ramp_up(10).compile(total_steps)
sigmoid = {'name': 'Sigmoid',
'layers': [{'height': 2},
{'height': 2},
{'height': 2},
{'height': 1, 'activations': activations.linears}],
'learning_rate': l_rate,
'momentum': 0.6}
leaky = {'name': 'Leaky ReLu',
'layers': [{'height': 2},
{'height': 8, 'activations': activations.leaky_relus},
{'height': 8, 'activations': activations.leaky_relus},
{'height': 1, 'activations': activations.linears}],
'learning_rate': l_rate,
'momentum': 0.6}
relu = {'name': 'ReLu',
'layers': [{'height': 2},
{'height': 8, 'activations': activations.relus},
{'height': 8, 'activations': activations.relus},
{'height': 1, 'activations': activations.linears}],
'learning_rate': l_rate,
'momentum': 0.6}
neural_models = []
for model in [sigmoid, leaky, relu]:
neural_models.append(neural_model.Network(**model))
ridge_model = [linear_models.RegularisedLinearRegression(
name = 'Ridge',
beta_func = linear_models.beta_ridge,
momentum = 0.6,
_lambda = 0.001,
x_shape = X_train.shape[1],
learning_rate = 0.01
), X_train, X_validate, data['y_train'], data['y_validate']]
subplots = [([*neural_models, ridge_model], {'mini_batch_size': mini_batch_size})]
errors, subtitle, subplots, metrics_string = sgd.sgd_on_models(data['x_train'], data['x_validate'], data['y_train'], data['y_validate'], *subplots, epochs=epochs, epochs_without_progress=epochs_without_progress)
title = ['Best models of each type, regression', 'regression_vs', subtitle]
sgd.plot_sgd_errors(errors, title, metrics_string)
print("Model performances on test data")
neural_subplots = []
for model in neural_models:
y_pred = model.predict(data['x_test'])
mse = metrics.MSE(data['y_test'], y_pred)
r2 = metrics.R_2(data['y_test'], y_pred)
neural_subplots.append((model.name, data['x_test'], y_pred[:,0]))
print(f"{model.name}: {mse} {r2}")
X_test = linear_models.poly_design_matrix(polynomials, data['x_test'])
ridge_pred = ridge_model[0].predict(X_test)
mse = metrics.MSE(data['y_test'], ridge_pred)
r2 = metrics.R_2(data['y_test'], ridge_pred)
print(f"{ridge_model[0].name}: {mse} {r2}")
plotting.side_by_side(
*neural_subplots,
['Ridge', data['x_test'], ridge_pred[:,0]],
['Ground truth', data['x_test'], data['y_test'][:,0]],
title = ["Terrain predictions", "terrain_pred"],
projection = '3d',
plotter = plotting.trisurface_plotter,
view_angles = [30, 230])
kfold_mse = resampling.kfold(model, X, y)
resampled_errors[f'{model.name} boot bias'][i] = bias
resampled_errors[f'{model.name} boot variance'][i] = var
resampled_errors[f'{model.name} boot MSE'][i] = boot_mse
resampled_errors[f'{model.name} kfold MSE'][i] = kfold_mse
plotting.side_by_side([
'OLS', poly_iter,
[[resampled_errors['OLS boot MSE'], 'MSE'],
[resampled_errors['OLS boot bias'], 'Bias'],
[resampled_errors['OLS boot variance'], 'Variance']]
], [
'Ridge ($\\lambda=0.001$)', poly_iter,
[[resampled_errors['Ridge boot MSE'], 'MSE'],
[resampled_errors['Ridge boot bias'], 'Bias'],
[resampled_errors['Ridge boot variance'], 'Variance']]
], [
'LASSO ($\\lambda=0.001$)', poly_iter,
[[resampled_errors['LASSO boot MSE'], 'MSE'],
[resampled_errors['LASSO boot bias'], 'Bias'],
[resampled_errors['LASSO boot variance'], 'Variance']]
],
axis_labels=['Complexity (maximum degree)', 'Error'],
filename="bias_variance_franke",
title="Bias variance tradeoff for different models",
_3d=False)
plotting.side_by_side(
[
'OLS', poly_iter,
[[resampled_errors['OLS boot MSE'], 'Bootstrap'],
[resampled_errors['OLS kfold MSE'], 'Kfold']]
