def ex_1_1_d(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 b)
Remember to set alpha to 0 when initializing the model
Use n_iterations = 10000 and tol=1e-8
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
hidden_neurons_list = [2, 8, 40]
n_iterations = 10000
train_mses = numpy.zeros((3, n_iterations))
test_mses = numpy.zeros((3, n_iterations))
r = 0
for n_hidden_neuron in hidden_neurons_list:
trained_regressor = MLPRegressor(
warm_start=True,
hidden_layer_sizes=(n_hidden_neuron, ),
activation='logistic',
solver='adam',
alpha=0,
tol=1e-8,
max_iter=1)
for i in range(n_iterations):
trained_regressor = trained_regressor.fit(x_train, y_train)
train_mses[r][i] = calculate_mse(trained_regressor, x_train,
y_train)
test_mses[r][i] = calculate_mse(trained_regressor, x_test, y_test)
r = r + 1
plot_mse_vs_iterations(train_mses, test_mses, n_iterations,
hidden_neurons_list)
pass
def ex_1_1_d(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 d)
Remember to set alpha to 0 when initializing the model.
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
nh = [2, 5, 50]
seed = 0
iterations = 5000
mse_all_train = np.zeros(shape=(3, iterations))
mse_all_test = np.zeros(shape=(3, iterations))
for j in range(0, 3):
nn = MLPRegressor(activation='logistic',
solver='lbfgs',
warm_start=True,
max_iter=1,
alpha=0,
hidden_layer_sizes=(nh[j], ),
random_state=seed)
for i in range(0, iterations):
nn.fit(x_train, y_train)
mse_train = calculate_mse(nn, x_train, y_train)
mse_test = calculate_mse(nn, x_test, y_test)
mse_all_train[j][i] = mse_train
mse_all_test[j][i] = mse_test
plot_mse_vs_iterations(mse_all_train, mse_all_test, iterations, nh)
## TODO
pass
def ex_1_1_d(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 d)
Remember to set alpha to 0 when initializing the model.
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
hiddenN = [2, 5, 50]
max_iter = 5000
MSE_train, MSE_test = np.ndarray((len(hiddenN), max_iter)), np.ndarray(
(len(hiddenN), max_iter))
for n in range(len(hiddenN)):
nn = MLPRegressor(activation='logistic',
solver='lbfgs',
max_iter=1,
warm_start=True,
hidden_layer_sizes=(hiddenN[n], ),
alpha=0,
random_state=0)
for i in range(max_iter):
nn.fit(x_train, y_train)
MSE_train[n][i] = calculate_mse(nn, x_train, y_train)
MSE_test[n][i] = calculate_mse(nn, x_test, y_test)
plot_mse_vs_iterations(MSE_train, MSE_test, max_iter, hiddenN)
pass
def ex_1_1_d(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 b)
Remember to set alpha to 0 when initializing the model
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
## TODO
neuron_numbers = [2, 8, 20]
mses_test = np.zeros((3, 1000))
mses_train = np.zeros((3, 1000))
for n in neuron_numbers:
regressor = MLPRegressor(hidden_layer_sizes=(n, ),
solver="adam",
activation="logistic",
alpha=0.0,
max_iter=1,
warm_start=True)
for j in range(0, 1000):
regressor.fit(x_train, y_train)
mses_train[neuron_numbers.index(n)][j] = calculate_mse(
regressor, x_train, y_train)
mses_test[neuron_numbers.index(n)][j] = calculate_mse(
regressor, x_test, y_test)
plot_mse_vs_iterations(mses_train, mses_test, 1000, neuron_numbers)
pass
def ex_1_1_d(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 b)
Remember to set alpha to 0 when initializing the model
Use n_iterations = 10000 and tol=1e-8
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
iter = 10000
hidden_neuron_list = [2, 8, 40]
mse = np.zeros((len(hidden_neuron_list), iter, 2))
for j in range(len(hidden_neuron_list)):
regressor = MLPRegressor(hidden_layer_sizes=(hidden_neuron_list[j], ),
activation='logistic',
solver='sgd',
alpha=0,
max_iter=1,
random_state=0,
warm_start=True)
for i in range(iter):
regressor.fit(x_train, y_train)
mse[j][i] = calculate_mse(regressor, [x_train, x_test],
[y_train, y_test])
plot_mse_vs_iterations(mse[:, :, 0], mse[:, :, 1], iter,
hidden_neuron_list)
## TODO
pass
def ex_1_1_d(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 d)
Remember to set alpha to 0 when initializing the model
Use n_iterations = 10000 and tol=1e-8
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
## TODO
hidden_neurons_totest = np.array([2, 8, 40])
hidden_neurons_list = [2, 8, 40]
dim1 = hidden_neurons_totest.shape[0]
mse_test_matrix = np.zeros((dim1, 10000))
mse_train_matrix = np.zeros((dim1, 10000))
k = 0
for i in hidden_neurons_totest:
n_hidden_neurons = i
nn = MLPRegressor(activation='logistic', solver='lbfgs', max_iter=1, tol=1e-8,
hidden_layer_sizes=(n_hidden_neurons,), alpha=0, random_state=0, warm_start=True)
# test again with solver='adam' and 'sgd'
for j in range(10000):
nn.fit(x_train, y_train)
mse_test_matrix[k, j] = calculate_mse(nn, x_test, y_test)
mse_train_matrix[k, j] = calculate_mse(nn, x_train, y_train)
k += 1
plot_mse_vs_iterations(mse_train_matrix, mse_test_matrix, 10000, hidden_neurons_totest)
plt.show()
def ex_1_1_d(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 b)
Remember to set alpha to 0 when initializing the model
Use n_iterations = 10000 and tol=1e-8
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
## TODO - done
n_h = [2, 8, 40]
iterations = 10000
train_array = np.zeros((3, iterations))
test_array = np.zeros((3, iterations))
#solver = 'adam'
#solver = 'lbfgs'
#solver = 'sgd'
for n in n_h:
nn = MLPRegressor(tol=1e-8, activation='logistic', solver='adam', alpha=0.0, hidden_layer_sizes=(n,),
max_iter=1, random_state=0, warm_start=True)
for i in range(0, iterations):
index = n_h.index(n)
nn.fit(x_train, y_train)
train_array[index][i] = calculate_mse(nn, x_train, y_train)
test_array[index][i] = calculate_mse(nn, x_test, y_test)
plot_mse_vs_iterations(train_array, test_array, iterations, n_h)
def ex_1_1_d(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 b)
Remember to set alpha to 0 when initializing the model
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
'''
• Write code to train a neural network with nh ∈ 2, 8, 20 hidden neurons on one layer and
calculate the MSE for the testing and training set at each training iteration for a single seed.
To be able to calculate the MSEs at each iteration, set warm_start to True and max_iter to
1 when initializing the network. The usage of warm_start always keeps the previously learnt
parameters instead of reinitializing them randomly when fit is called (see the documentation
of scikit learn for more information). Then, loop over iterations and successively call the fit
function and calculate the MSE on both datasets. Use the training solver ‘lbfgs’, for 1000
iterations. Stack the results in an array with where the first dimension correspond to the
number of hidden neurons and the second correspond to the number of iterations Use the
function plot_mse_vs_iterations in nn_regression_plot.py to plot the variation of MSE
with iterations.
• Replace the solver by ‘sgd’ or ‘adam’ and compute the MSE across iterations for the same
values of nh.
'''
params_n_h = np.array([2, 8, 20])
solvers = np.array(['lbfgs', 'sgd', 'adam'])
seed = np.random.seed(1)
max_iterations = 1000
train_mses = np.zeros((params_n_h.shape[0], max_iterations))
test_mses = np.zeros((params_n_h.shape[0], max_iterations))
for solver in solvers:
for index_n_h, n_h in np.ndenumerate(params_n_h):
nn = MLPRegressor(solver=solver,
max_iter=1,
warm_start=True,
activation='logistic',
hidden_layer_sizes=(n_h, ),
alpha=0,
random_state=seed)
for iteration in range(max_iterations):
nn.fit(x_train, y_train)
train_mses[index_n_h,
iteration] = calculate_mse(nn, x_train, y_train)
test_mses[index_n_h,
iteration] = calculate_mse(nn, x_test, y_test)
print("Using solver = " + solver)
plot_mse_vs_iterations(train_mses, test_mses, max_iterations,
params_n_h)
# plot_mse_vs_iterations(train_mses, test_mses, max_iterations, params_n_h, solver)
pass
def ex_1_1_d(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 b)
Remember to set alpha to 0 when initializing the model
Use n_iterations = 10000 and tol=1e-8
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
m = 0
n = 0
# declaring variables used in MLP-Regressor
hidden_layers = np.array([2, 8, 40])
activation_mode = 'logistic'
alpha = 0
max_iter = 1
tol = 1e-8
solver_mode = 'lbfgs'
#solver_mode = 'adam'
#solver_mode = 'sgd'
train_mse = np.zeros((hidden_layers.size, 10000))
test_mse = np.zeros((hidden_layers.size, 10000))
for m in range(hidden_layers.size):
nn = MLPRegressor(hidden_layer_sizes=(hidden_layers[m], ),
activation=activation_mode,
solver=solver_mode,
warm_start=True,
alpha=alpha,
max_iter=max_iter)
for n in range(10000):
nn.fit(x_train, y_train)
# calculate for every random seed train_mse and test_mse
train_mse[m][n] = calculate_mse(nn, x_train, y_train)
test_mse[m][n] = calculate_mse(nn, x_test, y_test)
plot_mse_vs_iterations(train_mse, test_mse, 10000, hidden_layers)
pass