def evaluate_linear_approx(data_input, targets, test_fraction):
# the linear performance
train_error, test_error, residual_variance = simple_evaluation_linear_model(
data_input, targets, test_fraction=test_fraction)
print("Linear Regression without Regularisation:")
print("\t(train_error, test_error) = %r" % ((train_error, test_error), ))
reg_params = np.logspace(-15, -4, 11)
train_errors = []
test_errors = []
for reg_param in reg_params:
train_error, test_error, residual_variance = simple_evaluation_linear_model(
data_input,
targets,
test_fraction=test_fraction,
reg_param=reg_param)
train_errors.append(train_error)
test_errors.append(test_error)
fig, ax = plot_train_test_errors("$\lambda$", reg_params, train_errors,
test_errors)
ax.set_xscale('log')
return fig, ax, train_error, test_error
def evaluate_linear_approx(inputs, targets, test_fraction):
# the linear performance
train_error, test_error = simple_evaluation_linear_model(
inputs, targets, test_fraction=test_fraction)
print("Linear Regression:")
print("\t(train_error, test_error) = %r" % ((train_error, test_error), ))
return train_error, test_error
def simple_linear_without_regularisation(inputs,targets,folds,test_fraction):
# train_error - the training error for the approximation
# test_error - the test error for the approximation
train_error, test_error = simple_evaluation_linear_model(inputs, targets, test_fraction=test_fraction)
print("Linear Regression without Regularistaion:")
print("\t(train_error, test_error) = %r" % ((train_error, test_error),))
# cross-validation evaluation of linear model
train_errors, test_errors= cv_evaluation_linear_model(inputs, targets, folds)
print("Train Errors for Linear Regression without Regularisation:")
print("\t(train_error) = %r" % train_errors)
# output:
# train errors: [ 0.64904221, 0.63487906, 0.64919719, 0.64966031, 0.64325636]
print ("Test Errors for Linear Regression without Regularisation:")
print("\t(test_error) = %r" % test_errors)
# output:
# test errors: [ 0.63450161, 0.68883366, 0.63512469, 0.6360172 , 0.65815973]
print ("Average Mean Errors:")
print("\t(train_error,test_error)= %r" % (( np.mean(train_errors),np.mean(test_errors)),))
# output:
# average mean errors - train error: 0.64510190051379168
# average mean errors - test error: 0.651298048054331
return np.mean(train_errors),np.mean(test_errors)
def plot_with_regularisation(inputs,targets,folds):
"""
Linear regression does not use a feature mapping, typically with such a
simple model regularisation does not have much effect. The plot is the
same with and without regularisation. Regularisation has only a weak affect
on simple linear regression. Using regularisation on simple linear
regression may not be that effective. In simple linear regression,
regularisation will slightly penalise functions that are further from
constant (i.e. with larger gradients). So it will have an effect, but
only a small one and that will be to give slightly lower weights than if
you had used least squares.
"""
reg_params = np.logspace(-10,1)
train_errors = []
test_errors = []
for reg_param in reg_params:
print("Evaluating reg_params: " + str(reg_param))
old_train,old_test=simple_evaluation_linear_model(inputs, targets, test_fraction=0.2, reg_param=reg_param)
train_error, test_error = cv_evaluation_linear_model(inputs, targets, folds,reg_param=reg_param)
# collect the errors
train_errors.append(np.mean(train_error))
test_errors.append(np.mean(test_error))
# plot the results
fig, ax = plot_train_test_errors("$\lambda$", reg_params, train_errors, test_errors)
plt.title('Linear Regression Model')
ax.set_xscale('log')
def simple_linear_regression(inputs,targets,folds,test_fraction,test_inputs,test_targets):
train_error, test_error = simple_evaluation_linear_model(inputs, targets, test_fraction=test_fraction)
print("Linear Regression with no cross-validation and no regularisation: (train_error, test_error) = %r" % ((train_error, test_error),))
train_errors, test_errors,weights= cv_evaluation_linear_model(inputs, targets, folds)
print("The train errors for linear regression using cross validation are:= %r" % (train_errors),)
print ("The test error for linear regression using cross validation are:= %r" % (test_errors),)
print ("The average means of errors are the following: (train_error,test_error)= %r" % (( np.mean(train_errors),np.mean(test_errors) ) , ) )
final_error=root_mean_squared_error(test_targets,linear_model_predict(test_inputs,weights))
print("The final test error for linear regression is: %r" % final_error)
return np.mean(train_errors),np.mean(test_errors)
def evaluate_rbf_for_various_reg_params(inputs, targets, test_fraction,
test_error_linear):
# for rbf feature mappings
# for the centres of the basis functions choose 10% of the data
n = inputs.shape[0]
centres = inputs[
np.random.choice([False, True], size=n, p=[0.90, 0.10]), :]
print("centres shape = %r" % (centres.shape, ))
# the width (analogous to standard deviation) of the basis functions
scale = 8.5 # of the basis functions
print("centres = %r" % (centres, ))
print("scale = %r" % (scale, ))
feature_mapping = construct_rbf_feature_mapping(centres, scale)
design_matrix = feature_mapping(inputs)
train_part, test_part = train_and_test_split(n,
test_fraction=test_fraction)
train_design_matrix, train_targets, test_design_matrix, test_targets = \
train_and_test_partition(
design_matrix, targets, train_part, test_part)
# outputting the shapes of the train and test parts for debugging
print("training design matrix shape = %r" % (train_design_matrix.shape, ))
print("testing design matrix shape = %r" % (test_design_matrix.shape, ))
print("training targets shape = %r" % (train_targets.shape, ))
print("testing targets shape = %r" % (test_targets.shape, ) + "\n")
# the rbf feature mapping performance
reg_params = np.logspace(-15, 5, 20)
train_errors = []
test_errors = []
for reg_param in reg_params:
print("Evaluating reg. parameter " + str(reg_param))
train_error, test_error = simple_evaluation_linear_model(
design_matrix,
targets,
test_fraction=test_fraction,
reg_param=reg_param)
train_errors.append(train_error)
test_errors.append(test_error)
fig, ax = plot_train_test_errors("$\lambda$", reg_params, train_errors,
test_errors)
# plotting a straight line showing the linear performance
x_lim = ax.get_xlim()
ax.plot(x_lim, test_error_linear * np.ones(2), 'g:')
ax.set_xscale('log')
ax.set_title('Evaluating RBF Performance')
fig.savefig("../plots/rbf_vs_linear.pdf", fmt="pdf")
def regression_with_regularization(inputs,targets,folds):
reg_params = np.logspace(-10,1)
train_errors = []
test_errors = []
for reg_param in reg_params:
# evaluate the test and train error for this regularisation parameter
old_train,old_test=simple_evaluation_linear_model(inputs, targets, test_fraction=0.2, reg_param=reg_param)
train_error, test_error,weights = cv_evaluation_linear_model(inputs, targets, folds,reg_param=reg_param)
print(" (train_error_without_cross,test_error_without_cross,train_error_with_cross,test_error_with_cross)= %r" % ((old_train,old_test,np.mean(train_error),np.mean(test_error)),) )
# collect the errors
train_errors.append(np.mean(train_error))
test_errors.append(np.mean(test_error))
# plot the results
fig, ax = plot_train_test_errors("$\lambda$", reg_params, train_errors, test_errors)
ax.set_xscale('log')
def evaluate_rbf_for_various_reg_params(inputs, targets, test_fraction,
test_error_linear):
"""
"""
# for rbf feature mappings
# for the centres of the basis functions choose 10% of the data
N = inputs.shape[0]
centres = inputs[np.random.choice([False, True], size=N, p=[0.9, 0.1]), :]
print("centres.shape = %r" % (centres.shape, ))
scale = 10. # of the basis functions
feature_mapping = construct_rbf_feature_mapping(centres, scale)
designmtx = feature_mapping(inputs)
train_part, test_part = train_and_test_split(N,
test_fraction=test_fraction)
train_designmtx, train_targets, test_designmtx, test_targets = \
train_and_test_partition(
designmtx, targets, train_part, test_part)
# output the shapes of the train and test parts for debugging
print("train_designmtx.shape = %r" % (train_designmtx.shape, ))
print("test_designmtx.shape = %r" % (test_designmtx.shape, ))
print("train_targets.shape = %r" % (train_targets.shape, ))
print("test_targets.shape = %r" % (test_targets.shape, ))
# the rbf feature mapping performance
reg_params = np.logspace(-15, -4, 11)
train_errors = []
test_errors = []
for reg_param in reg_params:
print("Evaluating reg_para " + str(reg_param))
train_error, test_error = simple_evaluation_linear_model(
designmtx,
targets,
test_fraction=test_fraction,
reg_param=reg_param)
train_errors.append(train_error)
test_errors.append(test_error)
fig, ax = plot_train_test_errors("$\lambda$", reg_params, train_errors,
test_errors)
# we also want to plot a straight line showing the linear performance
xlim = ax.get_xlim()
ax.plot(xlim, test_error_linear * np.ones(2), 'g:')
ax.set_xscale('log')
