Beispiel #1
0
    def test(self):
        """Tests out the models that validate and optimal_model_search has determined are best.

        Uses the pandas.DataFrame optimal_model created by optimal_model_search method, and tests each of the models
        on testing data. Plots the result side by side with the actual dependent variable.

        """
        for (model_name, technique), (_lambda, poly, _, _) in self.optimal_model.iterrows():

            for _model in self.models:
                if _model.name == model_name:
                    model = _model

            X = poly_design_matrix(int(poly), self.data['x_train'])

            model.scaler.fit(X[:,1:])
            X[:,1:] = model.scaler.transform(X[:,1:])
            model.set_lambda(_lambda)
            model.fit(X, self.data['y_train'])

            X_test = poly_design_matrix(int(poly), self.data['x_test'])
            X_test[:,1:] = model.scaler.transform(X_test[:,1:])
            y_pred = model.predict(X_test)

            mse = metrics.MSE(data['y_test'], y_pred)
            self.optimal_model.loc[model_name, technique]['Test error'] = mse

            if self.verbose >= 1:
                print(f"MSE: {mse} for poly = {poly} and lambda = {_lambda} with model {model_name}.")
            
            plotting.side_by_side(
                ['Predicted', self.data['x_test'], y_pred],
                ['Ground truth', self.data['x_test'], self.data['y_test']],
                title=f"Test: {model.name}, $p$={poly}, $\\lambda$={_lambda}, best according to {technique}",
                filename=f"{name}_test_{model.name}_p{poly}{[f'_lambda{_lambda}', ''][int(_lambda==0)]}")
def plot_data(*data_args, random=False):
    """Plots terrain data created by get_data function.

    Parameters:
    -----------
    *data_args:  
                Passed on to get_data()

    """
    data = get_data(*data_args, random=random)

    plotting.side_by_side(
        ['Train', data['x_train'], data['y_train']],
        ['Validate', data['x_train'], data['y_train']],
        ['Test', data['x_test'], data['y_test']],
        title=f"Training, validation and testing data for real terrain",
        filename=f"real_terrain")
Beispiel #3
0
def plot_sgd_errors(sgd_errors_df, title, metric_string='MSE'):
    """Takes a dataframe of errors, and formats it in a way that plotting.side_by_side can handle.

    Parameters:
    -----------
    errors_df:  pd.DataFrame
                Dataframe with titles, initial conditions and labels as multiindex, and epochs as column
    """
    plots = []
    for subplot_name, subplot_errors in sgd_errors_df.groupby(level=0):
        y = []
        labels = []
        for j, (label_names,
                label_errors) in enumerate(subplot_errors.groupby(level=1)):
            labels.append(label_names)
            min_errors = label_errors.min(axis=0).to_numpy()[np.newaxis, :]
            max_errors = label_errors.max(axis=0).to_numpy()[np.newaxis, :]
            best_index = label_errors.iloc[slice(None), -1].idxmin()
            best_errors = label_errors.loc(axis=0)[slice(None),
                                                   slice(None),
                                                   best_index].to_numpy()
            y_ = np.concatenate((min_errors, best_errors, max_errors), axis=0)
            y.append((y_, {'color': 'C' + str(j + 1)}))

            label_enum = np.arange(len(labels))
            colors = [f"C{number}" for number in label_enum + 1]
            handler_map = {
                i: plotting.LegendObject(colors[i])
                for i in range(len(colors))
            }
            legend = [label_enum, labels], {'handler_map': handler_map}

        plots.append([subplot_name, subplot_errors.columns, y, legend])

    side_by_side_parameters = {
        'plotter': plotting.confidence_interval_plotter,
        'axis_labels': ('Epoch', metric_string),
        'title': title,
        'yscale': 'log',
    }
    plotting.side_by_side(*plots, **side_by_side_parameters)
def plot_data(**data_args):
    """Plots Franke function data, side by side with Franke function + noise data.

    Parameters:
    -----------
    *data_args:  
                Passed on to get_data()

    """
    data = get_data(**data_args)
    data_args['noise_std'] = 0
    data_wo_noise = get_data(**data_args)

    plotting.side_by_side([
        "$f(x_1, x_2)$", data['x'], data_wo_noise['y'].reshape(-1)
    ], [
        f"$f(x_1, x_2) + \\varepsilon$, where $\\varepsilon \\sim N(0,{0.2})$",
        data['x'], data['y'].reshape(-1)
    ],
                          title="Franke function ($f(x_1, x_2)$)",
                          filename="franke")
Beispiel #5
0
def plot_data(*data_args, random=False):
    """Plots terrain data created by get_data function.

    Parameters:
    -----------
    *data_args:  
                Passed on to get_data()

    """
    data = get_data(*data_args, random=random)

    plotting.side_by_side(
        ['Train', data['x_train'], data['y_train']],
        ['Validate', data['x_train'], data['y_train']],
        ['Test', data['x_test'], data['y_test']],
        title=[
            f"Training, validation and testing data for real terrain",
            "real_terrain"
        ],
        plotter=plotting.trisurface_plotter,
        projection='3d',
        view_angles=[15, 240])
Beispiel #6
0
def beta_variance(data, epochs=20000, mini_batch_sizes=[10, 20]):
    """Plots variance of beta parameters for different linear models on regression problem for real terrain"""
    polynomials = 8
    X_train = linear_models.poly_design_matrix(polynomials, data['x_train'])
    X_validate = linear_models.poly_design_matrix(polynomials, data['x_validate'])

    total_steps = epochs * len(data['x_train'])//mini_batch_sizes[0]
    learning_rate_func = learning_rate.Learning_rate(base=5e-2, decay=1/100000).compile(total_steps)
    
    common_ridge1_kwargs = {'_lambda': 0.01, 'name': 'Ridge', 'beta_func': linear_models.beta_ridge, 'momentum': 0.5, 'init_conds': 300, 'x_shape': X_train.shape[1]}
    common_ridge2_kwargs = {'_lambda': 0.0001, 'name': 'Ridge', 'beta_func': linear_models.beta_ridge, 'momentum': 0.5, 'init_conds': 300, 'x_shape': X_train.shape[1]}
    subplot_ridge_uniques = [{}]
    subsubplot_ridge_linear_uniques = [{'learning_rate': learning_rate_func}]

    common_ols_kwargs = {'momentum': 0.5, 'init_conds': 300, 'x_shape': X_train.shape[1]}
    subplot_ols_uniques = [{}]
    subsubplot_ols_linear_uniques = [{'learning_rate': learning_rate_func}]

    unique_sgd_kwargs = [{'mini_batch_size': mini_batch_size} for mini_batch_size in mini_batch_sizes]

    ridge_models1 = helpers.make_models(
        linear_models.RegularisedLinearRegression,
        common_ridge1_kwargs,
        [{}],
        len(unique_sgd_kwargs),
        subsubplot_ridge_linear_uniques
        )

    ridge_models2 = helpers.make_models(
        linear_models.RegularisedLinearRegression,
        common_ridge2_kwargs,
        [{}],
        len(unique_sgd_kwargs),
        subsubplot_ridge_linear_uniques
        )

    ols_models = helpers.make_models(
        linear_models.LinearRegression,
        common_ols_kwargs,
        [{}],
        len(unique_sgd_kwargs),
        subsubplot_ols_linear_uniques
        )

    models_list = ridge_models1 + ridge_models2 + ols_models 

    subplots = [(models, sgd_kwargs) for models, sgd_kwargs in zip(models_list, unique_sgd_kwargs*(len(models_list)))]
    errors, subtitle, subplots, metrics_string = sgd.sgd_on_models(X_train, X_validate, data['y_train'], data['y_validate'], *subplots, epochs=epochs, epochs_without_progress=200)

    title = ['Convergence test', 'convergence', subtitle]
    sgd.plot_sgd_errors(errors, title, metrics_string)

    sgd_ridge1_betas = np.array([model.beta for models in ridge_models1 for model in models])
    sgd_ridge1_betas = sgd_ridge1_betas.transpose(1, 2, 0).reshape((sgd_ridge1_betas.shape[1], -1)).T

    sgd_ridge2_betas = np.array([model.beta for models in ridge_models2 for model in models])
    sgd_ridge2_betas = sgd_ridge2_betas.transpose(1, 2, 0).reshape((sgd_ridge2_betas.shape[1], -1)).T

    sgd_ols_betas = np.array([model.beta for models in ols_models for model in models])
    sgd_ols_betas = sgd_ols_betas.transpose(1, 2, 0).reshape((sgd_ols_betas.shape[1], -1)).T

    optimal_models = [linear_models.RegularisedLinearRegression("Ridge", linear_models.beta_ridge, _lambda=_lambda) for _lambda in (0.01, 0.)]
    optimal_betas = np.array([model.fit(X_train, data['y_train']) for model in optimal_models])[:,:,0]

    plots = [
        ['Ridge ($\\lambda 0.01$) trained by SGD', np.arange(1, optimal_betas.shape[1] + 1),
            [[sgd_ridge1_betas, {'notch': False, 'sym': ''}],
            [optimal_betas[0], plotting.scatter_plotter, {'label': 'Analytical $\\lambda$ 0.01'}],
            ]
        ],
        ['Ridge ($\\lambda 0.0001$) trained by SGD', np.arange(1, optimal_betas.shape[1] + 1),
            [[sgd_ridge2_betas, {'notch': False, 'sym': ''}],
            [optimal_betas[0], plotting.scatter_plotter, {'label': 'Analytical $\\lambda$ 0.01'}],
            ]
        ],
        ['OLS trained by SGD', np.arange(1, optimal_betas.shape[1] + 1),
            [[sgd_ols_betas, {'notch': False, 'sym': ''}],
            [optimal_betas[0], plotting.scatter_plotter, {'label': 'Analytical $\\lambda$ 0.01'}],
            ]
        ],
    ]

    side_by_side_parameters = {
        'plotter': plotting.box_plotter,
        'axis_labels': ('Beta parameter #', 'Values'),
        'title': ['$\\beta$ parameters', 'beta', subtitle],
    }
    plotting.side_by_side(*plots, **side_by_side_parameters)
Beispiel #7
0
def terrain_reggression(data, epochs=5000, epochs_without_progress=500, mini_batch_size=20):
    """Plots performance of different neural models on regression problem for real terrain"""
    polynomials = 8
    X_train = linear_models.poly_design_matrix(polynomials, data['x_train'])
    X_validate = linear_models.poly_design_matrix(polynomials, data['x_validate'])
    X_test = linear_models.poly_design_matrix(polynomials, data['x_test'])

    total_steps = epochs * len(data['x_train'])//mini_batch_size
    
    l_rate = learning_rate.Learning_rate(base=3e-2, decay=1/20000).ramp_up(10).compile(total_steps)

    sigmoid = {'name': 'Sigmoid',
                'layers': [{'height': 2},
                           {'height': 2},
                           {'height': 2},
                           {'height': 1, 'activations': activations.linears}],
                'learning_rate': l_rate,
                'momentum': 0.6}
    leaky   = {'name': 'Leaky ReLu',
                'layers': [{'height': 2},
                           {'height': 8, 'activations': activations.leaky_relus},
                           {'height': 8, 'activations': activations.leaky_relus},
                           {'height': 1, 'activations': activations.linears}],
                'learning_rate': l_rate,
                'momentum': 0.6}
    relu    = {'name': 'ReLu',
                'layers': [{'height': 2},
                           {'height': 8, 'activations': activations.relus},
                           {'height': 8, 'activations': activations.relus},
                           {'height': 1, 'activations': activations.linears}],
                'learning_rate': l_rate,
                'momentum': 0.6}

    neural_models = []
    for model in [sigmoid, leaky, relu]:
        neural_models.append(neural_model.Network(**model))

    ridge_model = [linear_models.RegularisedLinearRegression(
        name = 'Ridge',
        beta_func = linear_models.beta_ridge,
        momentum = 0.6,
        _lambda = 0.001,
        x_shape = X_train.shape[1],
        learning_rate = 0.01
        ), X_train, X_validate, data['y_train'], data['y_validate']]

    subplots = [([*neural_models, ridge_model], {'mini_batch_size': mini_batch_size})]

    errors, subtitle, subplots, metrics_string = sgd.sgd_on_models(data['x_train'], data['x_validate'], data['y_train'], data['y_validate'], *subplots, epochs=epochs, epochs_without_progress=epochs_without_progress)

    title = ['Best models of each type, regression', 'regression_vs', subtitle]

    sgd.plot_sgd_errors(errors, title, metrics_string)

    print("Model performances on test data")
    neural_subplots = []
    for model in neural_models:
        y_pred = model.predict(data['x_test'])
        mse = metrics.MSE(data['y_test'], y_pred)
        r2 = metrics.R_2(data['y_test'], y_pred)
        neural_subplots.append((model.name, data['x_test'], y_pred[:,0]))
        print(f"{model.name}: {mse} {r2}")

    X_test = linear_models.poly_design_matrix(polynomials, data['x_test'])
    ridge_pred = ridge_model[0].predict(X_test)
    mse = metrics.MSE(data['y_test'], ridge_pred)
    r2 = metrics.R_2(data['y_test'], ridge_pred)
    print(f"{ridge_model[0].name}: {mse} {r2}")

    plotting.side_by_side(
        *neural_subplots,
        ['Ridge', data['x_test'], ridge_pred[:,0]],
        ['Ground truth', data['x_test'], data['y_test'][:,0]],
        title = ["Terrain predictions", "terrain_pred"],
        projection = '3d',
        plotter = plotting.trisurface_plotter,
        view_angles = [30, 230])
Beispiel #8
0
        kfold_mse = resampling.kfold(model, X, y)
        resampled_errors[f'{model.name} boot bias'][i] = bias
        resampled_errors[f'{model.name} boot variance'][i] = var
        resampled_errors[f'{model.name} boot MSE'][i] = boot_mse
        resampled_errors[f'{model.name} kfold MSE'][i] = kfold_mse

plotting.side_by_side([
    'OLS', poly_iter,
    [[resampled_errors['OLS boot MSE'], 'MSE'],
     [resampled_errors['OLS boot bias'], 'Bias'],
     [resampled_errors['OLS boot variance'], 'Variance']]
], [
    'Ridge ($\\lambda=0.001$)', poly_iter,
    [[resampled_errors['Ridge boot MSE'], 'MSE'],
     [resampled_errors['Ridge boot bias'], 'Bias'],
     [resampled_errors['Ridge boot variance'], 'Variance']]
], [
    'LASSO ($\\lambda=0.001$)', poly_iter,
    [[resampled_errors['LASSO boot MSE'], 'MSE'],
     [resampled_errors['LASSO boot bias'], 'Bias'],
     [resampled_errors['LASSO boot variance'], 'Variance']]
],
                      axis_labels=['Complexity  (maximum degree)', 'Error'],
                      filename="bias_variance_franke",
                      title="Bias variance tradeoff for different models",
                      _3d=False)

plotting.side_by_side(
    [
        'OLS', poly_iter,
        [[resampled_errors['OLS boot MSE'], 'Bootstrap'],
         [resampled_errors['OLS kfold MSE'], 'Kfold']]