def test(self): """Tests out the models that validate and optimal_model_search has determined are best. Uses the pandas.DataFrame optimal_model created by optimal_model_search method, and tests each of the models on testing data. Plots the result side by side with the actual dependent variable. """ for (model_name, technique), (_lambda, poly, _, _) in self.optimal_model.iterrows(): for _model in self.models: if _model.name == model_name: model = _model X = poly_design_matrix(int(poly), self.data['x_train']) model.scaler.fit(X[:,1:]) X[:,1:] = model.scaler.transform(X[:,1:]) model.set_lambda(_lambda) model.fit(X, self.data['y_train']) X_test = poly_design_matrix(int(poly), self.data['x_test']) X_test[:,1:] = model.scaler.transform(X_test[:,1:]) y_pred = model.predict(X_test) mse = metrics.MSE(data['y_test'], y_pred) self.optimal_model.loc[model_name, technique]['Test error'] = mse if self.verbose >= 1: print(f"MSE: {mse} for poly = {poly} and lambda = {_lambda} with model {model_name}.") plotting.side_by_side( ['Predicted', self.data['x_test'], y_pred], ['Ground truth', self.data['x_test'], self.data['y_test']], title=f"Test: {model.name}, $p$={poly}, $\\lambda$={_lambda}, best according to {technique}", filename=f"{name}_test_{model.name}_p{poly}{[f'_lambda{_lambda}', ''][int(_lambda==0)]}")
def plot_data(*data_args, random=False): """Plots terrain data created by get_data function. Parameters: ----------- *data_args: Passed on to get_data() """ data = get_data(*data_args, random=random) plotting.side_by_side( ['Train', data['x_train'], data['y_train']], ['Validate', data['x_train'], data['y_train']], ['Test', data['x_test'], data['y_test']], title=f"Training, validation and testing data for real terrain", filename=f"real_terrain")
def plot_sgd_errors(sgd_errors_df, title, metric_string='MSE'): """Takes a dataframe of errors, and formats it in a way that plotting.side_by_side can handle. Parameters: ----------- errors_df: pd.DataFrame Dataframe with titles, initial conditions and labels as multiindex, and epochs as column """ plots = [] for subplot_name, subplot_errors in sgd_errors_df.groupby(level=0): y = [] labels = [] for j, (label_names, label_errors) in enumerate(subplot_errors.groupby(level=1)): labels.append(label_names) min_errors = label_errors.min(axis=0).to_numpy()[np.newaxis, :] max_errors = label_errors.max(axis=0).to_numpy()[np.newaxis, :] best_index = label_errors.iloc[slice(None), -1].idxmin() best_errors = label_errors.loc(axis=0)[slice(None), slice(None), best_index].to_numpy() y_ = np.concatenate((min_errors, best_errors, max_errors), axis=0) y.append((y_, {'color': 'C' + str(j + 1)})) label_enum = np.arange(len(labels)) colors = [f"C{number}" for number in label_enum + 1] handler_map = { i: plotting.LegendObject(colors[i]) for i in range(len(colors)) } legend = [label_enum, labels], {'handler_map': handler_map} plots.append([subplot_name, subplot_errors.columns, y, legend]) side_by_side_parameters = { 'plotter': plotting.confidence_interval_plotter, 'axis_labels': ('Epoch', metric_string), 'title': title, 'yscale': 'log', } plotting.side_by_side(*plots, **side_by_side_parameters)
def plot_data(**data_args): """Plots Franke function data, side by side with Franke function + noise data. Parameters: ----------- *data_args: Passed on to get_data() """ data = get_data(**data_args) data_args['noise_std'] = 0 data_wo_noise = get_data(**data_args) plotting.side_by_side([ "$f(x_1, x_2)$", data['x'], data_wo_noise['y'].reshape(-1) ], [ f"$f(x_1, x_2) + \\varepsilon$, where $\\varepsilon \\sim N(0,{0.2})$", data['x'], data['y'].reshape(-1) ], title="Franke function ($f(x_1, x_2)$)", filename="franke")
def plot_data(*data_args, random=False): """Plots terrain data created by get_data function. Parameters: ----------- *data_args: Passed on to get_data() """ data = get_data(*data_args, random=random) plotting.side_by_side( ['Train', data['x_train'], data['y_train']], ['Validate', data['x_train'], data['y_train']], ['Test', data['x_test'], data['y_test']], title=[ f"Training, validation and testing data for real terrain", "real_terrain" ], plotter=plotting.trisurface_plotter, projection='3d', view_angles=[15, 240])
def beta_variance(data, epochs=20000, mini_batch_sizes=[10, 20]): """Plots variance of beta parameters for different linear models on regression problem for real terrain""" polynomials = 8 X_train = linear_models.poly_design_matrix(polynomials, data['x_train']) X_validate = linear_models.poly_design_matrix(polynomials, data['x_validate']) total_steps = epochs * len(data['x_train'])//mini_batch_sizes[0] learning_rate_func = learning_rate.Learning_rate(base=5e-2, decay=1/100000).compile(total_steps) common_ridge1_kwargs = {'_lambda': 0.01, 'name': 'Ridge', 'beta_func': linear_models.beta_ridge, 'momentum': 0.5, 'init_conds': 300, 'x_shape': X_train.shape[1]} common_ridge2_kwargs = {'_lambda': 0.0001, 'name': 'Ridge', 'beta_func': linear_models.beta_ridge, 'momentum': 0.5, 'init_conds': 300, 'x_shape': X_train.shape[1]} subplot_ridge_uniques = [{}] subsubplot_ridge_linear_uniques = [{'learning_rate': learning_rate_func}] common_ols_kwargs = {'momentum': 0.5, 'init_conds': 300, 'x_shape': X_train.shape[1]} subplot_ols_uniques = [{}] subsubplot_ols_linear_uniques = [{'learning_rate': learning_rate_func}] unique_sgd_kwargs = [{'mini_batch_size': mini_batch_size} for mini_batch_size in mini_batch_sizes] ridge_models1 = helpers.make_models( linear_models.RegularisedLinearRegression, common_ridge1_kwargs, [{}], len(unique_sgd_kwargs), subsubplot_ridge_linear_uniques ) ridge_models2 = helpers.make_models( linear_models.RegularisedLinearRegression, common_ridge2_kwargs, [{}], len(unique_sgd_kwargs), subsubplot_ridge_linear_uniques ) ols_models = helpers.make_models( linear_models.LinearRegression, common_ols_kwargs, [{}], len(unique_sgd_kwargs), subsubplot_ols_linear_uniques ) models_list = ridge_models1 + ridge_models2 + ols_models subplots = [(models, sgd_kwargs) for models, sgd_kwargs in zip(models_list, unique_sgd_kwargs*(len(models_list)))] errors, subtitle, subplots, metrics_string = sgd.sgd_on_models(X_train, X_validate, data['y_train'], data['y_validate'], *subplots, epochs=epochs, epochs_without_progress=200) title = ['Convergence test', 'convergence', subtitle] sgd.plot_sgd_errors(errors, title, metrics_string) sgd_ridge1_betas = np.array([model.beta for models in ridge_models1 for model in models]) sgd_ridge1_betas = sgd_ridge1_betas.transpose(1, 2, 0).reshape((sgd_ridge1_betas.shape[1], -1)).T sgd_ridge2_betas = np.array([model.beta for models in ridge_models2 for model in models]) sgd_ridge2_betas = sgd_ridge2_betas.transpose(1, 2, 0).reshape((sgd_ridge2_betas.shape[1], -1)).T sgd_ols_betas = np.array([model.beta for models in ols_models for model in models]) sgd_ols_betas = sgd_ols_betas.transpose(1, 2, 0).reshape((sgd_ols_betas.shape[1], -1)).T optimal_models = [linear_models.RegularisedLinearRegression("Ridge", linear_models.beta_ridge, _lambda=_lambda) for _lambda in (0.01, 0.)] optimal_betas = np.array([model.fit(X_train, data['y_train']) for model in optimal_models])[:,:,0] plots = [ ['Ridge ($\\lambda 0.01$) trained by SGD', np.arange(1, optimal_betas.shape[1] + 1), [[sgd_ridge1_betas, {'notch': False, 'sym': ''}], [optimal_betas[0], plotting.scatter_plotter, {'label': 'Analytical $\\lambda$ 0.01'}], ] ], ['Ridge ($\\lambda 0.0001$) trained by SGD', np.arange(1, optimal_betas.shape[1] + 1), [[sgd_ridge2_betas, {'notch': False, 'sym': ''}], [optimal_betas[0], plotting.scatter_plotter, {'label': 'Analytical $\\lambda$ 0.01'}], ] ], ['OLS trained by SGD', np.arange(1, optimal_betas.shape[1] + 1), [[sgd_ols_betas, {'notch': False, 'sym': ''}], [optimal_betas[0], plotting.scatter_plotter, {'label': 'Analytical $\\lambda$ 0.01'}], ] ], ] side_by_side_parameters = { 'plotter': plotting.box_plotter, 'axis_labels': ('Beta parameter #', 'Values'), 'title': ['$\\beta$ parameters', 'beta', subtitle], } plotting.side_by_side(*plots, **side_by_side_parameters)
def terrain_reggression(data, epochs=5000, epochs_without_progress=500, mini_batch_size=20): """Plots performance of different neural models on regression problem for real terrain""" polynomials = 8 X_train = linear_models.poly_design_matrix(polynomials, data['x_train']) X_validate = linear_models.poly_design_matrix(polynomials, data['x_validate']) X_test = linear_models.poly_design_matrix(polynomials, data['x_test']) total_steps = epochs * len(data['x_train'])//mini_batch_size l_rate = learning_rate.Learning_rate(base=3e-2, decay=1/20000).ramp_up(10).compile(total_steps) sigmoid = {'name': 'Sigmoid', 'layers': [{'height': 2}, {'height': 2}, {'height': 2}, {'height': 1, 'activations': activations.linears}], 'learning_rate': l_rate, 'momentum': 0.6} leaky = {'name': 'Leaky ReLu', 'layers': [{'height': 2}, {'height': 8, 'activations': activations.leaky_relus}, {'height': 8, 'activations': activations.leaky_relus}, {'height': 1, 'activations': activations.linears}], 'learning_rate': l_rate, 'momentum': 0.6} relu = {'name': 'ReLu', 'layers': [{'height': 2}, {'height': 8, 'activations': activations.relus}, {'height': 8, 'activations': activations.relus}, {'height': 1, 'activations': activations.linears}], 'learning_rate': l_rate, 'momentum': 0.6} neural_models = [] for model in [sigmoid, leaky, relu]: neural_models.append(neural_model.Network(**model)) ridge_model = [linear_models.RegularisedLinearRegression( name = 'Ridge', beta_func = linear_models.beta_ridge, momentum = 0.6, _lambda = 0.001, x_shape = X_train.shape[1], learning_rate = 0.01 ), X_train, X_validate, data['y_train'], data['y_validate']] subplots = [([*neural_models, ridge_model], {'mini_batch_size': mini_batch_size})] errors, subtitle, subplots, metrics_string = sgd.sgd_on_models(data['x_train'], data['x_validate'], data['y_train'], data['y_validate'], *subplots, epochs=epochs, epochs_without_progress=epochs_without_progress) title = ['Best models of each type, regression', 'regression_vs', subtitle] sgd.plot_sgd_errors(errors, title, metrics_string) print("Model performances on test data") neural_subplots = [] for model in neural_models: y_pred = model.predict(data['x_test']) mse = metrics.MSE(data['y_test'], y_pred) r2 = metrics.R_2(data['y_test'], y_pred) neural_subplots.append((model.name, data['x_test'], y_pred[:,0])) print(f"{model.name}: {mse} {r2}") X_test = linear_models.poly_design_matrix(polynomials, data['x_test']) ridge_pred = ridge_model[0].predict(X_test) mse = metrics.MSE(data['y_test'], ridge_pred) r2 = metrics.R_2(data['y_test'], ridge_pred) print(f"{ridge_model[0].name}: {mse} {r2}") plotting.side_by_side( *neural_subplots, ['Ridge', data['x_test'], ridge_pred[:,0]], ['Ground truth', data['x_test'], data['y_test'][:,0]], title = ["Terrain predictions", "terrain_pred"], projection = '3d', plotter = plotting.trisurface_plotter, view_angles = [30, 230])
kfold_mse = resampling.kfold(model, X, y) resampled_errors[f'{model.name} boot bias'][i] = bias resampled_errors[f'{model.name} boot variance'][i] = var resampled_errors[f'{model.name} boot MSE'][i] = boot_mse resampled_errors[f'{model.name} kfold MSE'][i] = kfold_mse plotting.side_by_side([ 'OLS', poly_iter, [[resampled_errors['OLS boot MSE'], 'MSE'], [resampled_errors['OLS boot bias'], 'Bias'], [resampled_errors['OLS boot variance'], 'Variance']] ], [ 'Ridge ($\\lambda=0.001$)', poly_iter, [[resampled_errors['Ridge boot MSE'], 'MSE'], [resampled_errors['Ridge boot bias'], 'Bias'], [resampled_errors['Ridge boot variance'], 'Variance']] ], [ 'LASSO ($\\lambda=0.001$)', poly_iter, [[resampled_errors['LASSO boot MSE'], 'MSE'], [resampled_errors['LASSO boot bias'], 'Bias'], [resampled_errors['LASSO boot variance'], 'Variance']] ], axis_labels=['Complexity (maximum degree)', 'Error'], filename="bias_variance_franke", title="Bias variance tradeoff for different models", _3d=False) plotting.side_by_side( [ 'OLS', poly_iter, [[resampled_errors['OLS boot MSE'], 'Bootstrap'], [resampled_errors['OLS kfold MSE'], 'Kfold']]