def ex_1_1_d(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 b)
Remember to set alpha to 0 when initializing the model
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
## TODO
neuron_numbers = [2, 8, 20]
mses_test = np.zeros((3, 1000))
mses_train = np.zeros((3, 1000))
for n in neuron_numbers:
regressor = MLPRegressor(hidden_layer_sizes=(n, ),
solver="adam",
activation="logistic",
alpha=0.0,
max_iter=1,
warm_start=True)
for j in range(0, 1000):
regressor.fit(x_train, y_train)
mses_train[neuron_numbers.index(n)][j] = calculate_mse(
regressor, x_train, y_train)
mses_test[neuron_numbers.index(n)][j] = calculate_mse(
regressor, x_test, y_test)
plot_mse_vs_iterations(mses_train, mses_test, 1000, neuron_numbers)
pass
def ex_1_1_d(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 b)
Remember to set alpha to 0 when initializing the model
Use n_iterations = 10000 and tol=1e-8
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
## TODO - done
n_h = [2, 8, 40]
iterations = 10000
train_array = np.zeros((3, iterations))
test_array = np.zeros((3, iterations))
#solver = 'adam'
#solver = 'lbfgs'
#solver = 'sgd'
for n in n_h:
nn = MLPRegressor(tol=1e-8, activation='logistic', solver='adam', alpha=0.0, hidden_layer_sizes=(n,),
max_iter=1, random_state=0, warm_start=True)
for i in range(0, iterations):
index = n_h.index(n)
nn.fit(x_train, y_train)
train_array[index][i] = calculate_mse(nn, x_train, y_train)
test_array[index][i] = calculate_mse(nn, x_test, y_test)
plot_mse_vs_iterations(train_array, test_array, iterations, n_h)
def ex_1_1_c(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 c)
Remember to set alpha to 0 when initializing the model
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
params_n_h = np.array([1, 2, 3, 4, 6, 8, 12, 20, 40])
seeds = np.array(range(1, 11))
train_mses = np.zeros((params_n_h.shape[0], seeds.shape[0]))
test_mses = np.zeros((params_n_h.shape[0], seeds.shape[0]))
for index_seed, seed in np.ndenumerate(seeds):
for index_n_h, n_h in np.ndenumerate(params_n_h):
nn = MLPRegressor(solver='lbfgs',
max_iter=200,
activation='logistic',
hidden_layer_sizes=(n_h, ),
alpha=0,
random_state=seed)
nn.fit(x_train, y_train)
train_mses[index_n_h,
index_seed] = calculate_mse(nn, x_train, y_train)
test_mses[index_n_h,
index_seed] = calculate_mse(nn, x_test, y_test)
print("Min MSE ", np.min(train_mses))
plot_mse_vs_neurons(train_mses, test_mses, params_n_h)
pass
def ex_1_2(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.2 a)
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
## TODO#
nh = 50
alp = [0.000001, 0.00001, 0.0001, 0.001, 0.01, 0.1, 1, 10, 100]
seed = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
iterations = 5000
mse_all_train = np.zeros(shape=(9, 10))
mse_all_test = np.zeros(shape=(9, 10))
for j in range(0, 9):
for i in range(0, 10):
nn = MLPRegressor(
activation='logistic',
solver='lbfgs',
max_iter=5000,
alpha=alp[j],
hidden_layer_sizes=(50, ),
random_state=seed[i]) #for i in range(0,iterations):
nn.fit(x_train, y_train)
mse_train = calculate_mse(nn, x_train, y_train)
mse_test = calculate_mse(nn, x_test, y_test)
mse_all_train[j][i] = mse_train
mse_all_test[j][i] = mse_test
plot_mse_vs_alpha(mse_all_train, mse_all_test, alp)
pass
def ex_1_2_a(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.2 a)
Remember to set alpha to 0 when initializing the model
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
## TODO
hidden_neurons = 40
alphas = [1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1, 10, 100]
mses_train = np.zeros((len(alphas), 10))
mses_test = np.zeros((len(alphas), 10))
for alpha in alphas:
for i in range(10):
random_seed = randint(0, 1000)
regressor = MLPRegressor(hidden_layer_sizes=(hidden_neurons, ),
solver="lbfgs",
activation="logistic",
alpha=alpha,
max_iter=200,
random_state=random_seed)
regressor.fit(x_train, y_train)
mses_test[alphas.index(alpha)][i] = calculate_mse(
regressor, x_test, y_test)
mses_train[alphas.index(alpha)][i] = calculate_mse(
regressor, x_train, y_train)
plot_mse_vs_alpha(mses_train, mses_test, alphas)
def ex_1_1_d(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 d)
Remember to set alpha to 0 when initializing the model.
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
nh = [2, 5, 50]
seed = 0
iterations = 5000
mse_all_train = np.zeros(shape=(3, iterations))
mse_all_test = np.zeros(shape=(3, iterations))
for j in range(0, 3):
nn = MLPRegressor(activation='logistic',
solver='lbfgs',
warm_start=True,
max_iter=1,
alpha=0,
hidden_layer_sizes=(nh[j], ),
random_state=seed)
for i in range(0, iterations):
nn.fit(x_train, y_train)
mse_train = calculate_mse(nn, x_train, y_train)
mse_test = calculate_mse(nn, x_test, y_test)
mse_all_train[j][i] = mse_train
mse_all_test[j][i] = mse_test
plot_mse_vs_iterations(mse_all_train, mse_all_test, iterations, nh)
## TODO
pass
def ex_1_1_b(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 b)
Remember to set alpha to 0 when initializing the model
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
## TODO - done
n_h = 8
train_array = np.zeros(10)
test_array = np.zeros(10)
for i in range(0, 10):
nn = MLPRegressor(activation='logistic', solver='lbfgs', alpha=0.0, hidden_layer_sizes=(n_h,), max_iter=200,
random_state=i)
nn.fit(x_train, y_train)
test_array[i] = calculate_mse(nn, x_test, y_test)
train_array[i] = calculate_mse(nn, x_train, y_train)
print("MEAN:\n", np.mean(train_array))
print("Standard derivation:\n", np.std(train_array))
print("Train MSE:\n", train_array)
print("Test MSE:\n", test_array)
def ex_1_2_b(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.2 b)
Remember to set alpha and momentum to 0 when initializing the model
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
## TODO
indices = np.random.permutation(len(y_train))
new_size = int(len(y_train) / 2)
x_train_ = np.ndarray((new_size, 1))
y_train_ = np.ndarray(new_size)
indices_train = indices[:new_size]
indices_val = indices[new_size:]
x_val = np.ndarray((new_size, 1))
y_val = np.ndarray(new_size)
for i in range(new_size):
x_train_[i] = x_train[indices_train[i]]
y_train_[i] = y_train[indices_train[i]]
x_val[i] = x_train[indices_val[i]]
y_val[i] = y_train[indices_val[i]]
x_train = x_train_
y_train = y_train_
min_test_errors = np.zeros(10)
last_test_errors = np.zeros(10)
min_val_errors = np.zeros(10)
for i in range(10):
regressor = MLPRegressor(hidden_layer_sizes=(40, ),
solver="lbfgs",
activation="logistic",
alpha=10e-3,
max_iter=1,
warm_start=True,
random_state=randint(1, 1000))
val_errors = []
test_errors = []
for j in range(0, 2000):
regressor.fit(x_train, y_train)
if j % 20 == 0:
test_errors.append(calculate_mse(regressor, x_test, y_test))
val_errors.append(calculate_mse(regressor, x_val, y_val))
last_test_errors[i] = calculate_mse(regressor, x_test, y_test)
min_val_errors[i] = test_errors[np.argmin(val_errors)]
min_test_errors[i] = test_errors[np.argmin(test_errors)]
plot_bars_early_stopping_mse_comparison(last_test_errors, min_val_errors,
min_test_errors)
pass
def ex_1_1_d(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 d)
Remember to set alpha to 0 when initializing the model
Use n_iterations = 10000 and tol=1e-8
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
## TODO
hidden_neurons_totest = np.array([2, 8, 40])
hidden_neurons_list = [2, 8, 40]
dim1 = hidden_neurons_totest.shape[0]
mse_test_matrix = np.zeros((dim1, 10000))
mse_train_matrix = np.zeros((dim1, 10000))
k = 0
for i in hidden_neurons_totest:
n_hidden_neurons = i
nn = MLPRegressor(activation='logistic', solver='lbfgs', max_iter=1, tol=1e-8,
hidden_layer_sizes=(n_hidden_neurons,), alpha=0, random_state=0, warm_start=True)
# test again with solver='adam' and 'sgd'
for j in range(10000):
nn.fit(x_train, y_train)
mse_test_matrix[k, j] = calculate_mse(nn, x_test, y_test)
mse_train_matrix[k, j] = calculate_mse(nn, x_train, y_train)
k += 1
plot_mse_vs_iterations(mse_train_matrix, mse_test_matrix, 10000, hidden_neurons_totest)
plt.show()
def ex_1_2_a(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.2 a)
Remember to set alpha to 0 when initializing the model
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
## TODO - done
n_h = 40
test_array = np.zeros((11, 10))
train_array = np.zeros((11, 10))
alphas = [pow(10, i) for i in range(-8, 3)]
for alpha in alphas:
for j in range(0, 10):
index = alphas.index(alpha)
nn = MLPRegressor(activation='logistic', solver='lbfgs', alpha=alpha, hidden_layer_sizes=(n_h,),
max_iter=200, random_state=j)
nn.fit(x_train, y_train)
test_array[index][j] = calculate_mse(nn, x_test, y_test)
train_array[index][j] = calculate_mse(nn, x_train, y_train)
plot_mse_vs_alpha(train_array, test_array, alphas)
def ex_1_2_a(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.2 a)
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
## TODO
total_iter = 5000
n_neuro = 50
n_seeds = 10
alph = np.power(10, np.linspace(-8, 2, 11))
mse_train = np.zeros([np.size(alph), n_seeds])
mse_test = np.zeros([np.size(alph), n_seeds])
for j in range(np.size(alph)):
for s in range(n_seeds):
seed = np.random.randint(100)
reg = MLPRegressor(hidden_layer_sizes=(n_neuro, ),
activation='logistic',
solver='lbfgs',
alpha=alph[j],
random_state=seed,
max_iter=total_iter)
reg.fit(x_train, y_train)
mse_train[j, s] = calculate_mse(reg, x_train, y_train)
mse_test[j, s] = calculate_mse(reg, x_test, y_test)
plot_mse_vs_alpha(mse_train, mse_test, alph)
def ex_1_2_a(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.2 a)
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
## TODO
#pass
N = 5000
n_hidden = 50
alpha_ = np.power(10, np.linspace(-8, 2, 11))
mse_train = np.zeros([np.size(alpha_), N])
mse_test = np.zeros([np.size(alpha_), N])
for j in range(np.size(alpha_)):
reg = MLPRegressor(hidden_layer_sizes=(n_hidden, ),
activation='logistic',
solver='lbfgs',
alpha=alpha_[j],
random_state=0,
warm_start=True,
max_iter=1)
for r in range(N):
reg.fit(x_train, y_train)
mse_train[j, r] = calculate_mse(reg, x_train, y_train)
mse_test[j, r] = calculate_mse(reg, x_test, y_test)
plot_mse_vs_alpha(mse_train, mse_test, alpha_)
def ex_1_1_d(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 b)
Remember to set alpha to 0 when initializing the model
Use n_iterations = 10000 and tol=1e-8
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
iter = 10000
hidden_neuron_list = [2, 8, 40]
mse = np.zeros((len(hidden_neuron_list), iter, 2))
for j in range(len(hidden_neuron_list)):
regressor = MLPRegressor(hidden_layer_sizes=(hidden_neuron_list[j], ),
activation='logistic',
solver='sgd',
alpha=0,
max_iter=1,
random_state=0,
warm_start=True)
for i in range(iter):
regressor.fit(x_train, y_train)
mse[j][i] = calculate_mse(regressor, [x_train, x_test],
[y_train, y_test])
plot_mse_vs_iterations(mse[:, :, 0], mse[:, :, 1], iter,
hidden_neuron_list)
## TODO
pass
def ex_1_2_a(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.2 a)
Remember to set alpha to 0 when initializing the model
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
alphas = [1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1, 10, 100]
seeds = 10
mse = np.zeros((len(alphas), seeds, 2))
for i in range(len(alphas)):
for j in range(seeds):
regressor = MLPRegressor(hidden_layer_sizes=(40, ),
activation='logistic',
solver='lbfgs',
alpha=alphas[i],
max_iter=200,
random_state=j,
tol=1e-8)
regressor.fit(x_train, y_train)
# mse shape: [train_mses, test_mses]
mse[i][j] = calculate_mse(regressor, [x_train, x_test],
[y_train, y_test])
plot_mse_vs_alpha(mse[:, :, 0], mse[:, :, 1], alphas)
## TODO
pass
def ex_1_1_c(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 c)
Remember to set alpha to 0 when initializing the model
Use max_iter = 10000 and tol=1e-8
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
## TODO
hidden_neurons_totest = np.array([1, 2, 3, 4, 6, 8, 12, 20, 40])
# hidden_neurons_totest = np.array([20])
dim1 = hidden_neurons_totest.shape[0]
mse_test_matrix = np.zeros((dim1, 10))
mse_train_matrix = np.zeros((dim1, 10))
k = 0
for i in hidden_neurons_totest:
n_hidden_neurons = i
for j in range(10):
nn = MLPRegressor(activation='logistic', solver='lbfgs', max_iter=10000, tol=1e-8,
hidden_layer_sizes=(n_hidden_neurons,), alpha=0, random_state=j)
nn.fit(x_train, y_train)
predictions_test = nn.predict(x_test)
mse_test_matrix[k, j] = calculate_mse(nn, x_test, y_test)
mse_train_matrix[k, j] = calculate_mse(nn, x_train, y_train)
k += 1
plot_mse_vs_neurons(mse_train_matrix, mse_test_matrix, hidden_neurons_totest)
plt.show()
plot_learned_function(40, x_train, y_train, 0, x_test, y_test, predictions_test)
plt.show()
def ex_1_1_b(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 b)
Remember to set alpha to 0 when initializing the model
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
## TODO
mse_list_train = np.zeros(10,)
mse_list_test = np.zeros(10,)
for i in range(10):
n_hidden_neurons = 8
nn = MLPRegressor(activation='logistic', solver='lbfgs', max_iter=200, hidden_layer_sizes=(n_hidden_neurons,),
alpha=0, random_state=i)
nn.fit(x_train, y_train)
mse_list_train[i] = calculate_mse(nn, x_train, y_train)
mse_list_test[i] = calculate_mse(nn, x_test, y_test)
mse_test_std = np.std(mse_list_test)
mse_test_avg = np.average(mse_list_test)
mse_train_std = np.std(mse_list_train)
mse_train_avg = np.average(mse_list_train)
print('Train min:', mse_list_train.min(),'Train max:',mse_list_train.max(),'Train avg:',mse_train_avg,'Train STD:', mse_train_std)
print('Test min:', mse_list_test.min(),'Test max:',mse_list_test.max(), 'Test avg:',mse_test_avg,'Test STD:', mse_test_std)
print('Train errors:',mse_list_train)
print('Test errors:', mse_list_test)
def ex_1_2_a(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.2 a)
Remember to set alpha to 0 when initializing the model
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
## TODO
alpha_values = np.array([1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1, 10, 100])
dim1 = alpha_values.shape[0]
mse_test_matrix = np.zeros((dim1, 10))
mse_train_matrix = np.zeros((dim1, 10))
k = 0
for i in alpha_values:
alpha = i
for j in range(10):
nn = MLPRegressor(activation='logistic', solver='lbfgs', max_iter=200,
hidden_layer_sizes=(40,), alpha=i, random_state=j)
nn.fit(x_train, y_train)
predictions_test = nn.predict(x_test)
mse_test_matrix[k, j] = calculate_mse(nn, x_test, y_test)
mse_train_matrix[k, j] = calculate_mse(nn, x_train, y_train)
k += 1
plot_mse_vs_alpha(mse_train_matrix, mse_test_matrix, alpha_values)
plt.show()
def ex_1_1_d(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 b)
Remember to set alpha to 0 when initializing the model
Use n_iterations = 10000 and tol=1e-8
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
hidden_neurons_list = [2, 8, 40]
n_iterations = 10000
train_mses = numpy.zeros((3, n_iterations))
test_mses = numpy.zeros((3, n_iterations))
r = 0
for n_hidden_neuron in hidden_neurons_list:
trained_regressor = MLPRegressor(
warm_start=True,
hidden_layer_sizes=(n_hidden_neuron, ),
activation='logistic',
solver='adam',
alpha=0,
tol=1e-8,
max_iter=1)
for i in range(n_iterations):
trained_regressor = trained_regressor.fit(x_train, y_train)
train_mses[r][i] = calculate_mse(trained_regressor, x_train,
y_train)
test_mses[r][i] = calculate_mse(trained_regressor, x_test, y_test)
r = r + 1
plot_mse_vs_iterations(train_mses, test_mses, n_iterations,
hidden_neurons_list)
pass
def ex_1_1_c(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 c)
Remember to set alpha to 0 when initializing the model
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
## TODO
neuron_numbers = [1, 2, 3, 4, 6, 8, 12, 20, 40]
mses_test = np.zeros((9, 10))
mses_train = np.zeros((9, 10))
for n in neuron_numbers:
for i in range(0, 10):
random_seed = randint(1, 1000)
regressor = MLPRegressor(hidden_layer_sizes=(n, ),
solver="lbfgs",
activation="logistic",
alpha=0.0,
max_iter=200,
random_state=random_seed)
regressor.fit(x_train, y_train)
mses_train[neuron_numbers.index(n)][i] = calculate_mse(
regressor, x_train, y_train)
mses_test[neuron_numbers.index(n)][i] = calculate_mse(
regressor, x_test, y_test)
plot_mse_vs_neurons(mses_train, mses_test, neuron_numbers)
pass
class MLPRegressorImpl():
def __init__(self,
hidden_layer_sizes=(100, ),
activation='relu',
solver='adam',
alpha=0.0001,
batch_size='auto',
learning_rate='constant',
learning_rate_init=0.001,
power_t=0.5,
max_iter=200,
shuffle=True,
random_state=None,
tol=0.0001,
verbose=False,
warm_start=False,
momentum=0.9,
nesterovs_momentum=True,
early_stopping=False,
validation_fraction=0.1,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
n_iter_no_change=10):
self._hyperparams = {
'hidden_layer_sizes': hidden_layer_sizes,
'activation': activation,
'solver': solver,
'alpha': alpha,
'batch_size': batch_size,
'learning_rate': learning_rate,
'learning_rate_init': learning_rate_init,
'power_t': power_t,
'max_iter': max_iter,
'shuffle': shuffle,
'random_state': random_state,
'tol': tol,
'verbose': verbose,
'warm_start': warm_start,
'momentum': momentum,
'nesterovs_momentum': nesterovs_momentum,
'early_stopping': early_stopping,
'validation_fraction': validation_fraction,
'beta_1': beta_1,
'beta_2': beta_2,
'epsilon': epsilon,
'n_iter_no_change': n_iter_no_change
}
def fit(self, X, y=None):
self._sklearn_model = SKLModel(**self._hyperparams)
if (y is not None):
self._sklearn_model.fit(X, y)
else:
self._sklearn_model.fit(X)
return self
def predict(self, X):
return self._sklearn_model.predict(X)
def ex_1_1_b(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 b)
Remember to set alpha to 0 when initializing the model
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
## TODO
MSE_TEST = np.zeros(10, dtype=float)
MSE_TRAIN = np.zeros(10, dtype=float)
n_neur = 5
for j in range(10):
seed = np.random.randint(100)
reg = MLPRegressor(hidden_layer_sizes=(n_neur, ),
max_iter=5000,
activation='logistic',
solver='lbfgs',
alpha=0,
random_state=seed)
reg.fit(x_train, y_train)
mse = calculate_mse(reg, x_test, y_test)
MSE_TEST[j] = mse
mse = calculate_mse(reg, x_train, y_train)
MSE_TRAIN[j] = mse
mse_test_max = max(MSE_TEST)
mse_test_min = min(MSE_TEST)
mse_test_mean = np.mean(MSE_TEST)
mse_test_std = np.std(MSE_TEST)
print("##### Radom Seeds #####")
print("### Test Set ###")
print(" MAX / MIN / MEAN / STD")
print(round(mse_test_max, 3), "/", round(mse_test_min, 3), "/",
round(mse_test_mean, 3), "/", round(mse_test_std, 3))
print("\n", MSE_TEST, "n")
mse_train_max = max(MSE_TRAIN)
mse_train_min = min(MSE_TRAIN)
mse_train_mean = np.mean(MSE_TRAIN)
mse_train_std = np.std(MSE_TRAIN)
print("\n### Training Set ###")
print(" MAX / MIN / MEAN / STD")
print(round(mse_train_max, 3), "/", round(mse_train_min, 3), "/",
round(mse_train_mean, 3), "/", round(mse_train_std, 3))
print("\n", MSE_TRAIN, "n")
def ex_1_1_d(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 b)
Remember to set alpha to 0 when initializing the model
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
'''
• Write code to train a neural network with nh ∈ 2, 8, 20 hidden neurons on one layer and
calculate the MSE for the testing and training set at each training iteration for a single seed.
To be able to calculate the MSEs at each iteration, set warm_start to True and max_iter to
1 when initializing the network. The usage of warm_start always keeps the previously learnt
parameters instead of reinitializing them randomly when fit is called (see the documentation
of scikit learn for more information). Then, loop over iterations and successively call the fit
function and calculate the MSE on both datasets. Use the training solver ‘lbfgs’, for 1000
iterations. Stack the results in an array with where the first dimension correspond to the
number of hidden neurons and the second correspond to the number of iterations Use the
function plot_mse_vs_iterations in nn_regression_plot.py to plot the variation of MSE
with iterations.
• Replace the solver by ‘sgd’ or ‘adam’ and compute the MSE across iterations for the same
values of nh.
'''
params_n_h = np.array([2, 8, 20])
solvers = np.array(['lbfgs', 'sgd', 'adam'])
seed = np.random.seed(1)
max_iterations = 1000
train_mses = np.zeros((params_n_h.shape[0], max_iterations))
test_mses = np.zeros((params_n_h.shape[0], max_iterations))
for solver in solvers:
for index_n_h, n_h in np.ndenumerate(params_n_h):
nn = MLPRegressor(solver=solver,
max_iter=1,
warm_start=True,
activation='logistic',
hidden_layer_sizes=(n_h, ),
alpha=0,
random_state=seed)
for iteration in range(max_iterations):
nn.fit(x_train, y_train)
train_mses[index_n_h,
iteration] = calculate_mse(nn, x_train, y_train)
test_mses[index_n_h,
iteration] = calculate_mse(nn, x_test, y_test)
print("Using solver = " + solver)
plot_mse_vs_iterations(train_mses, test_mses, max_iterations,
params_n_h)
# plot_mse_vs_iterations(train_mses, test_mses, max_iterations, params_n_h, solver)
pass
def ex_1_2_b(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.2 b)
Remember to set alpha and momentum to 0 when initializing the model
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
seeds = 10
iter = 100
data = np.column_stack((x_train, y_train))
data_shuff = np.random.permutation(data)
train_data = data_shuff[0:30]
valid_data = data_shuff[30:60]
x_train = np.array([train_data[:, 0]]).transpose()
y_train = np.array([train_data[:, 1]]).transpose()
x_valid = np.array([valid_data[:, 0]]).transpose()
y_valid = np.array([valid_data[:, 1]]).transpose()
mse = np.zeros((seeds, iter, 2))
for j in range(seeds):
regressor = MLPRegressor(hidden_layer_sizes=(40, ),
activation='logistic',
solver='lbfgs',
alpha=1e-3,
max_iter=20,
random_state=j,
warm_start=True)
for i in range(iter):
regressor.fit(x_train, y_train)
mse[j][i] = calculate_mse(regressor, [x_valid, x_test],
[y_valid, y_test])
test_mse_end = mse[:, iter - 1, 1]
test_mse_early_stopping = np.zeros(seeds)
test_mse_ideal = np.zeros(seeds)
for j in range(seeds):
min_index = np.argmin(mse[j, :, 0])
print(min_index)
test_mse_early_stopping[j] = mse[j, min_index, 1]
test_mse_ideal[j] = np.min(mse[j, :, 1])
plot_bars_early_stopping_mse_comparison(test_mse_end,
test_mse_early_stopping,
test_mse_ideal)
pass
def train(train_X, train_y, test_X, param, n_weeks):
string = "bp_" + param + str(n_weeks) + '.m'
try:
bp_model = joblib.load(string)
print("训练好的bp", bp_model)
except:
bp_model = MLPRegressor(hidden_layer_sizes=(18),
max_iter=5000,
learning_rate_init=0.1)
bp_model.fit(train_X, train_y)
joblib.dump(bp_model, string)
return bp_model
def ex_1_1_b(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 b)
Remember to set alpha to 0 when initializing the model.
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
## TODO
nh = 5
max_it = 10
min_i_train = 11
min_i_test = 11
max_train = 0
min_train = 1
min_test = 11
mean_train = 1
std_train = 0
mse = np.zeros(shape=(10))
for i in range(0, max_it):
nn = MLPRegressor(activation='logistic',
solver='lbfgs',
max_iter=5000,
alpha=0,
hidden_layer_sizes=(nh, ),
random_state=i)
nn.fit(x_train, y_train)
mse_train = calculate_mse(nn, x_train, y_train)
mse_test = calculate_mse(nn, x_test, y_test)
mse[i] = mse_train
print(i, '.)MSE train ', mse_train)
# print(i,'.)MSE test ',mse_test)
if (mse_train < min_train):
min_train = mse_train
min_i_train = i
if (mse_test < min_test):
min_test = mse_test
min_i_test = i
if (mse_train > max_train):
max_train = mse_train
print("Minimum Train: ", min_train)
print("Maximum Train: ", max_train)
mean_train = mse.mean()
print("Mean Train: ", mean_train)
std_train = np.std(mse)
print("Standard Derivation Train: ", std_train)
print("Min Index train: ", min_i_train, " & test: ", min_i_test)
pass
def ex_1_2_b(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.2 b)
Remember to set alpha and momentum to 0 when initializing the model
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
## TODO
np.random.shuffle(x_train)
np.random.shuffle(y_train)
x_train_half_size = int(len(x_train) / 2)
y_train_half_size = int(len(y_train) / 2)
x_valid = x_train[x_train_half_size:]
x_train = x_train[:x_train_half_size]
y_valid = y_train[y_train_half_size:]
y_train = y_train[:y_train_half_size]
n = 40
alpha = pow(10, -3)
iteration = 2000
random_seed = 10
max_iter = 20
mse_test = np.zeros((iteration, random_seed))
mse_valid = np.zeros((iteration, random_seed))
mse_last_iter = []
mse_min_valid = []
mse_ideal_min_test = []
for seed in range(0, random_seed):
nn = MLPRegressor(alpha=alpha, activation=ACTIVATION, solver='lbfgs', hidden_layer_sizes=(n,),
max_iter=max_iter, random_state=seed, momentum=False)
mse = 0
for i in range(0, iteration):
nn.fit(x_train, y_train)
mse = calculate_mse(nn, x_test, y_test)
mse_test[i][seed] = mse
mse_valid[i][seed] = calculate_mse(nn, x_valid, y_valid)
mse_last_iter.append(mse)
mse_min_valid.append(np.min(mse_valid[seed]))
mse_ideal_min_test.append(np.min(mse_test[seed]))
plot_bars_early_stopping_mse_comparison(mse_last_iter, mse_min_valid, mse_ideal_min_test)
def ex_1_1_d(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 b)
Remember to set alpha to 0 when initializing the model
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
## TODO
#pass
N = 500
n_hidden = [2, 5, 50]
mse_train = np.zeros([np.size(n_hidden), N])
mse_test = np.zeros([np.size(n_hidden), N])
for j in range(np.size(n_hidden)):
reg = MLPRegressor(hidden_layer_sizes=(n_hidden[j], ),
activation='logistic',
solver='lbfgs',
alpha=0,
random_state=0,
warm_start=True,
max_iter=1)
for r in range(N):
reg.fit(x_train, y_train)
mse_train[j, r] = calculate_mse(reg, x_train, y_train)
mse_test[j, r] = calculate_mse(reg, x_test, y_test)
# PLOT
plot_mse_vs_neurons(mse_train, mse_test, n_hidden)
#mse_test_mean = np.mean(mse_test, axis=1)
#ind = np.argmin(mse_test_mean)
ind = np.unravel_index(np.argmin(mse_test), mse_test.shape)
# geht es auch ohne den MLPRegressor nochmal zu initialisieren? Keine Ahnung obs anders besser geht
reg = MLPRegressor(hidden_layer_sizes=(n_hidden[j], ),
activation='logistic',
solver='lbfgs',
alpha=0,
random_state=random.randint(0, 1000),
max_iter=500)
reg.fit(x_train, y_train)
y_pred_test = reg.predict(x_test)
y_pred_train = reg.predict(x_train)
plot_learned_function(n_hidden[ind[0]], x_train, y_train, y_pred_train,
x_test, y_test, y_pred_test)
def main(test_X, test_y):
m = MLPRegressor(hidden_layer_sizes=24,
learning_rate_init=0.1,
max_iter=500)
# 6. 预测
m.fit(train_X, train_y)
yhat = m.predict(test_X)
test_X = test_X.reshape((test_X.shape[0], TIME_STEPS * INPUT_DIMS))
inv_yhat = inverse_trans(yhat, test_X, scaler, N_TRAIN_WEEKS, INPUT_DIMS)
inv_y = inverse_trans(test_y, test_X, scaler, N_TRAIN_WEEKS, INPUT_DIMS)
mae_array.append(mean_absolute_error(inv_y, inv_yhat))
rmse_array.append(sqrt(mean_squared_error(inv_y, inv_yhat)))
def ex_1_1_c(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 c)
Remember to set alpha to 0 when initializing the model
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
## TODO
#pass
N = 10
n_hidden = [1, 2, 3, 4, 6, 8, 12, 20, 40]
mse_train = np.zeros([np.size(n_hidden), N])
mse_test = np.zeros([np.size(n_hidden), N])
for j in range(np.size(n_hidden)):
reg = MLPRegressor(hidden_layer_sizes=(1, n_hidden[j]),
activation='logistic',
solver='lbfgs',
alpha=0,
random_state=random.randint(0, 1000))
for r in range(N):
reg.fit(x_train, y_train)
mse_train[j, r] = calculate_mse(reg, x_train, y_train)
mse_test[j, r] = calculate_mse(reg, x_test, y_test)
# PLOT
plot_mse_vs_neurons(mse_train, mse_test, n_hidden)
"""
mse_test_mean = np.mean(mse_test, axis=1)
ind = np.argmin(mse_test_mean)
"""
ind = np.unravel_index(np.argmin(mse_test), mse_test.shape)
reg = MLPRegressor(hidden_layer_sizes=(n_hidden[ind[0]], ),
activation='logistic',
solver='lbfgs',
alpha=0,
random_state=random.randint(0, 1000))
reg.fit(x_train, y_train)
y_pred_test = reg.predict(x_test)
y_pred_train = reg.predict(x_train)
plot_learned_function(n_hidden[ind[0]], x_train, y_train, y_pred_train,
x_test, y_test, y_pred_test)
def ex_1_1_b(x_train, x_test, y_train, y_test):
"""
Solution for exercise 1.1 b)
Remember to set alpha to 0 when initializing the model
:param x_train: The training dataset
:param x_test: The testing dataset
:param y_train: The training targets
:param y_test: The testing targets
:return:
"""
n_iterations = 10
train_mses = numpy.zeros(n_iterations)
test_mses = numpy.zeros(n_iterations)
for i in range(n_iterations):
trained_regressor = MLPRegressor(warm_start=False,
hidden_layer_sizes=(8, ),
activation='logistic',
solver='lbfgs',
alpha=0,
max_iter=200,
random_state=random.randint(1, 100))
trained_regressor = trained_regressor.fit(x_train, y_train)
train_mses[i] = calculate_mse(trained_regressor, x_train, y_train)
test_mses[i] = calculate_mse(trained_regressor, x_test, y_test)
print("Train_mses")
print(train_mses)
print("Train_min")
print(train_mses.min())
print("Train_min_seed")
print(np.argmin(train_mses))
print("Train_max")
print(train_mses.max())
print("Train_std")
print(train_mses.std())
print("Test_mses")
print(test_mses)
print("Test_min")
print(test_mses.min())
print("Test_min_seed")
print(np.argmin(test_mses))
print("Test_max")
print(test_mses.max())
print("Test_std")
print(test_mses.std())
pass
