def ex_1_1_c(x_train, x_test, y_train, y_test): """ Solution for exercise 1.1 c) Remember to set alpha to 0 when initializing the model Use max_iter = 10000 and tol=1e-8 :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ ## TODO n_h = [1, 2, 3, 4, 6, 8, 12, 20, 40] train_array = np.zeros((9, 10)) test_array = np.zeros((9, 10)) for n in n_h: for i in range(0, 10): index = n_h.index(n) nn = MLPRegressor(tol=1e-8, activation='logistic', solver='lbfgs', alpha=0.0, hidden_layer_sizes=(n_h[index],), max_iter=10000, random_state=i) nn.fit(x_train, y_train) train_array[index][i] = calculate_mse(nn, x_train, y_train) test_array[index][i] = calculate_mse(nn, x_test, y_test) y_pred_train = nn.predict(x_train) y_pred_test = nn.predict(x_test) if n == 1: plot_learned_function(n, x_train, y_train, y_pred_train, x_test, y_test, y_pred_test) plot_mse_vs_neurons(np.array(train_array), np.array(test_array), n_h)
def ex_1_1_b(x_train, x_test, y_train, y_test): """ Solution for exercise 1.1 b) Remember to set alpha to 0 when initializing the model :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ ## TODO - done n_h = 8 train_array = np.zeros(10) test_array = np.zeros(10) for i in range(0, 10): nn = MLPRegressor(activation='logistic', solver='lbfgs', alpha=0.0, hidden_layer_sizes=(n_h,), max_iter=200, random_state=i) nn.fit(x_train, y_train) test_array[i] = calculate_mse(nn, x_test, y_test) train_array[i] = calculate_mse(nn, x_train, y_train) print("MEAN:\n", np.mean(train_array)) print("Standard derivation:\n", np.std(train_array)) print("Train MSE:\n", train_array) print("Test MSE:\n", test_array)
def ex_1_1_c(x_train, x_test, y_train, y_test): """ Solution for exercise 1.1 c) Remember to set alpha to 0 when initializing the model :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ params_n_h = np.array([1, 2, 3, 4, 6, 8, 12, 20, 40]) seeds = np.array(range(1, 11)) train_mses = np.zeros((params_n_h.shape[0], seeds.shape[0])) test_mses = np.zeros((params_n_h.shape[0], seeds.shape[0])) for index_seed, seed in np.ndenumerate(seeds): for index_n_h, n_h in np.ndenumerate(params_n_h): nn = MLPRegressor(solver='lbfgs', max_iter=200, activation='logistic', hidden_layer_sizes=(n_h, ), alpha=0, random_state=seed) nn.fit(x_train, y_train) train_mses[index_n_h, index_seed] = calculate_mse(nn, x_train, y_train) test_mses[index_n_h, index_seed] = calculate_mse(nn, x_test, y_test) print("Min MSE ", np.min(train_mses)) plot_mse_vs_neurons(train_mses, test_mses, params_n_h) pass
def ex_1_1_a(x_train, x_test, y_train, y_test): """ Solution for exercise 1.1 a) Remember to set alpha to 0 when initializing the model :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ ## TODO - done # using 8 hidden neurons n_h = [2, 8, 40] for i in n_h: # use random state for a fixed split - guarantees always same output (200 seems beautiful) nn = MLPRegressor(activation='logistic', solver='lbfgs', alpha=0.0, hidden_layer_sizes=(i,), max_iter=200, random_state=200) nn.fit(x_train, y_train) pred_train_y = nn.predict(x_train) pred_test_y = nn.predict(x_test) plot_learned_function(n_h, x_train, y_train, pred_train_y, x_test, y_test, pred_test_y)
def ex_1_1_c(x_train, x_test, y_train, y_test): """ Solution for exercise 1.1 c) Remember to set alpha to 0 when initializing the model :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ ## TODO neuron_numbers = [1, 2, 3, 4, 6, 8, 12, 20, 40] mses_test = np.zeros((9, 10)) mses_train = np.zeros((9, 10)) for n in neuron_numbers: for i in range(0, 10): random_seed = randint(1, 1000) regressor = MLPRegressor(hidden_layer_sizes=(n, ), solver="lbfgs", activation="logistic", alpha=0.0, max_iter=200, random_state=random_seed) regressor.fit(x_train, y_train) mses_train[neuron_numbers.index(n)][i] = calculate_mse( regressor, x_train, y_train) mses_test[neuron_numbers.index(n)][i] = calculate_mse( regressor, x_test, y_test) plot_mse_vs_neurons(mses_train, mses_test, neuron_numbers) pass
def ex_1_1_d(x_train, x_test, y_train, y_test): """ Solution for exercise 1.1 b) Remember to set alpha to 0 when initializing the model Use n_iterations = 10000 and tol=1e-8 :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ ## TODO - done n_h = [2, 8, 40] iterations = 10000 train_array = np.zeros((3, iterations)) test_array = np.zeros((3, iterations)) #solver = 'adam' #solver = 'lbfgs' #solver = 'sgd' for n in n_h: nn = MLPRegressor(tol=1e-8, activation='logistic', solver='adam', alpha=0.0, hidden_layer_sizes=(n,), max_iter=1, random_state=0, warm_start=True) for i in range(0, iterations): index = n_h.index(n) nn.fit(x_train, y_train) train_array[index][i] = calculate_mse(nn, x_train, y_train) test_array[index][i] = calculate_mse(nn, x_test, y_test) plot_mse_vs_iterations(train_array, test_array, iterations, n_h)
def ex_1_1_a(x_train, x_test, y_train, y_test): """ Solution for exercise 1.1 a) Remember to set alpha to 0 when initializing the model :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ ## TODO regressor = MLPRegressor( hidden_layer_sizes=(40, ), #8,40 solver="lbfgs", activation="logistic", alpha=0.0, max_iter=200, ) regressor.fit(x_train, y_train) #plot_learned_function(2,x_train, y_train,regressor.predict(x_train), x_test, y_test, regressor.predict(x_test)) #plot_learned_function(8, x_train, y_train, regressor.predict(x_train), x_test, y_test, regressor.predict(x_test)) plot_learned_function(40, x_train, y_train, regressor.predict(x_train), x_test, y_test, regressor.predict(x_test)) pass
def ex_1_2_a(x_train, x_test, y_train, y_test): """ Solution for exercise 1.2 a) Remember to set alpha to 0 when initializing the model :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ ## TODO hidden_neurons = 40 alphas = [1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1, 10, 100] mses_train = np.zeros((len(alphas), 10)) mses_test = np.zeros((len(alphas), 10)) for alpha in alphas: for i in range(10): random_seed = randint(0, 1000) regressor = MLPRegressor(hidden_layer_sizes=(hidden_neurons, ), solver="lbfgs", activation="logistic", alpha=alpha, max_iter=200, random_state=random_seed) regressor.fit(x_train, y_train) mses_test[alphas.index(alpha)][i] = calculate_mse( regressor, x_test, y_test) mses_train[alphas.index(alpha)][i] = calculate_mse( regressor, x_train, y_train) plot_mse_vs_alpha(mses_train, mses_test, alphas)
def ex_1_1_a(x_train, x_test, y_train, y_test): """ Solution for exercise 1.1 a) Remember to set alpha to 0 when initializing the model :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ params_n_h = [2, 8, 40, 100] for n_h in params_n_h: nn = MLPRegressor(solver='lbfgs', max_iter=200, activation='logistic', hidden_layer_sizes=(n_h, ), alpha=0, verbose=False, random_state=0) # zero randomness # verbose=True nn.fit(x_train, y_train) y_pred_train = nn.predict(x_train) y_pred_test = nn.predict(x_test) plot_learned_function(n_h, x_train, y_train, y_pred_train, x_test, y_test, y_pred_test) pass
def ex_1_2_a(x_train, x_test, y_train, y_test): """ Solution for exercise 1.2 a) Remember to set alpha to 0 when initializing the model :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ ## TODO - done n_h = 40 test_array = np.zeros((11, 10)) train_array = np.zeros((11, 10)) alphas = [pow(10, i) for i in range(-8, 3)] for alpha in alphas: for j in range(0, 10): index = alphas.index(alpha) nn = MLPRegressor(activation='logistic', solver='lbfgs', alpha=alpha, hidden_layer_sizes=(n_h,), max_iter=200, random_state=j) nn.fit(x_train, y_train) test_array[index][j] = calculate_mse(nn, x_test, y_test) train_array[index][j] = calculate_mse(nn, x_train, y_train) plot_mse_vs_alpha(train_array, test_array, alphas)
def ex_1_2(x_train, x_test, y_train, y_test): """ Solution for exercise 1.2 a) :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ ## TODO# nh = 50 alp = [0.000001, 0.00001, 0.0001, 0.001, 0.01, 0.1, 1, 10, 100] seed = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] iterations = 5000 mse_all_train = np.zeros(shape=(9, 10)) mse_all_test = np.zeros(shape=(9, 10)) for j in range(0, 9): for i in range(0, 10): nn = MLPRegressor( activation='logistic', solver='lbfgs', max_iter=5000, alpha=alp[j], hidden_layer_sizes=(50, ), random_state=seed[i]) #for i in range(0,iterations): nn.fit(x_train, y_train) mse_train = calculate_mse(nn, x_train, y_train) mse_test = calculate_mse(nn, x_test, y_test) mse_all_train[j][i] = mse_train mse_all_test[j][i] = mse_test plot_mse_vs_alpha(mse_all_train, mse_all_test, alp) pass
def ex_1_2_a(x_train, x_test, y_train, y_test): """ Solution for exercise 1.2 a) Remember to set alpha to 0 when initializing the model :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ alphas = [1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1, 10, 100] seeds = 10 mse = np.zeros((len(alphas), seeds, 2)) for i in range(len(alphas)): for j in range(seeds): regressor = MLPRegressor(hidden_layer_sizes=(40, ), activation='logistic', solver='lbfgs', alpha=alphas[i], max_iter=200, random_state=j, tol=1e-8) regressor.fit(x_train, y_train) # mse shape: [train_mses, test_mses] mse[i][j] = calculate_mse(regressor, [x_train, x_test], [y_train, y_test]) plot_mse_vs_alpha(mse[:, :, 0], mse[:, :, 1], alphas) ## TODO pass
def ex_1_1_a(x_train, x_test, y_train, y_test): """ Solution for exercise 1.1 a) Remember to set alpha to 0 when initializing the model :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ n_hidden = 40 regressor = MLPRegressor(hidden_layer_sizes=(n_hidden, ), activation='logistic', solver='lbfgs', alpha=0, max_iter=200) regressor.fit(x_train, y_train) y_pred_train = regressor.predict(x_train) y_pred_test = regressor.predict(x_test) plot_learned_function(n_hidden, x_train, y_train, y_pred_train, x_test, y_test, y_pred_test) # [train_mses, test_mses] = calculate_mse(regressor, [x_train, x_test], [y_train, y_test]) # plot_mse_vs_neurons(train_mses, test_mses, n_hidden_neurons_list) pass
def ex_1_1_c(x_train, x_test, y_train, y_test): """ Solution for exercise 1.1 c) Remember to set alpha to 0 when initializing the model Use max_iter = 10000 and tol=1e-8 :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ ## TODO hidden_neurons_totest = np.array([1, 2, 3, 4, 6, 8, 12, 20, 40]) # hidden_neurons_totest = np.array([20]) dim1 = hidden_neurons_totest.shape[0] mse_test_matrix = np.zeros((dim1, 10)) mse_train_matrix = np.zeros((dim1, 10)) k = 0 for i in hidden_neurons_totest: n_hidden_neurons = i for j in range(10): nn = MLPRegressor(activation='logistic', solver='lbfgs', max_iter=10000, tol=1e-8, hidden_layer_sizes=(n_hidden_neurons,), alpha=0, random_state=j) nn.fit(x_train, y_train) predictions_test = nn.predict(x_test) mse_test_matrix[k, j] = calculate_mse(nn, x_test, y_test) mse_train_matrix[k, j] = calculate_mse(nn, x_train, y_train) k += 1 plot_mse_vs_neurons(mse_train_matrix, mse_test_matrix, hidden_neurons_totest) plt.show() plot_learned_function(40, x_train, y_train, 0, x_test, y_test, predictions_test) plt.show()
def ex_1_1_d(x_train, x_test, y_train, y_test): """ Solution for exercise 1.1 b) Remember to set alpha to 0 when initializing the model Use n_iterations = 10000 and tol=1e-8 :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ iter = 10000 hidden_neuron_list = [2, 8, 40] mse = np.zeros((len(hidden_neuron_list), iter, 2)) for j in range(len(hidden_neuron_list)): regressor = MLPRegressor(hidden_layer_sizes=(hidden_neuron_list[j], ), activation='logistic', solver='sgd', alpha=0, max_iter=1, random_state=0, warm_start=True) for i in range(iter): regressor.fit(x_train, y_train) mse[j][i] = calculate_mse(regressor, [x_train, x_test], [y_train, y_test]) plot_mse_vs_iterations(mse[:, :, 0], mse[:, :, 1], iter, hidden_neuron_list) ## TODO pass
def ex_1_1_b(x_train, x_test, y_train, y_test): """ Solution for exercise 1.1 b) Remember to set alpha to 0 when initializing the model :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ ## TODO mse_list_train = np.zeros(10,) mse_list_test = np.zeros(10,) for i in range(10): n_hidden_neurons = 8 nn = MLPRegressor(activation='logistic', solver='lbfgs', max_iter=200, hidden_layer_sizes=(n_hidden_neurons,), alpha=0, random_state=i) nn.fit(x_train, y_train) mse_list_train[i] = calculate_mse(nn, x_train, y_train) mse_list_test[i] = calculate_mse(nn, x_test, y_test) mse_test_std = np.std(mse_list_test) mse_test_avg = np.average(mse_list_test) mse_train_std = np.std(mse_list_train) mse_train_avg = np.average(mse_list_train) print('Train min:', mse_list_train.min(),'Train max:',mse_list_train.max(),'Train avg:',mse_train_avg,'Train STD:', mse_train_std) print('Test min:', mse_list_test.min(),'Test max:',mse_list_test.max(), 'Test avg:',mse_test_avg,'Test STD:', mse_test_std) print('Train errors:',mse_list_train) print('Test errors:', mse_list_test)
def ex_1_2_a(x_train, x_test, y_train, y_test): """ Solution for exercise 1.2 a) Remember to set alpha to 0 when initializing the model :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ ## TODO alpha_values = np.array([1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1, 10, 100]) dim1 = alpha_values.shape[0] mse_test_matrix = np.zeros((dim1, 10)) mse_train_matrix = np.zeros((dim1, 10)) k = 0 for i in alpha_values: alpha = i for j in range(10): nn = MLPRegressor(activation='logistic', solver='lbfgs', max_iter=200, hidden_layer_sizes=(40,), alpha=i, random_state=j) nn.fit(x_train, y_train) predictions_test = nn.predict(x_test) mse_test_matrix[k, j] = calculate_mse(nn, x_test, y_test) mse_train_matrix[k, j] = calculate_mse(nn, x_train, y_train) k += 1 plot_mse_vs_alpha(mse_train_matrix, mse_test_matrix, alpha_values) plt.show()
def ex_1_1_d(x_train, x_test, y_train, y_test): """ Solution for exercise 1.1 d) Remember to set alpha to 0 when initializing the model Use n_iterations = 10000 and tol=1e-8 :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ ## TODO hidden_neurons_totest = np.array([2, 8, 40]) hidden_neurons_list = [2, 8, 40] dim1 = hidden_neurons_totest.shape[0] mse_test_matrix = np.zeros((dim1, 10000)) mse_train_matrix = np.zeros((dim1, 10000)) k = 0 for i in hidden_neurons_totest: n_hidden_neurons = i nn = MLPRegressor(activation='logistic', solver='lbfgs', max_iter=1, tol=1e-8, hidden_layer_sizes=(n_hidden_neurons,), alpha=0, random_state=0, warm_start=True) # test again with solver='adam' and 'sgd' for j in range(10000): nn.fit(x_train, y_train) mse_test_matrix[k, j] = calculate_mse(nn, x_test, y_test) mse_train_matrix[k, j] = calculate_mse(nn, x_train, y_train) k += 1 plot_mse_vs_iterations(mse_train_matrix, mse_test_matrix, 10000, hidden_neurons_totest) plt.show()
def ex_1_1_a(x_train, x_test, y_train, y_test): """ Solution for exercise 1.1 a) Remember to set alpha to 0 when initializing the model. :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ ## TODO nh = 50 nn = MLPRegressor(activation='logistic', solver='lbfgs', max_iter=5000, alpha=0, hidden_layer_sizes=(nh, )) nn.fit(x_train, y_train) y_pred_train = nn.predict(x_train) y_pred_test = nn.predict(x_test) plot_learned_function(nh, x_train, y_train, y_pred_train, x_test, y_test, y_pred_test) pass
def ex_1_1_d(x_train, x_test, y_train, y_test): """ Solution for exercise 1.1 b) Remember to set alpha to 0 when initializing the model :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ ## TODO neuron_numbers = [2, 8, 20] mses_test = np.zeros((3, 1000)) mses_train = np.zeros((3, 1000)) for n in neuron_numbers: regressor = MLPRegressor(hidden_layer_sizes=(n, ), solver="adam", activation="logistic", alpha=0.0, max_iter=1, warm_start=True) for j in range(0, 1000): regressor.fit(x_train, y_train) mses_train[neuron_numbers.index(n)][j] = calculate_mse( regressor, x_train, y_train) mses_test[neuron_numbers.index(n)][j] = calculate_mse( regressor, x_test, y_test) plot_mse_vs_iterations(mses_train, mses_test, 1000, neuron_numbers) pass
def ex_1_1_d(x_train, x_test, y_train, y_test): """ Solution for exercise 1.1 d) Remember to set alpha to 0 when initializing the model. :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ nh = [2, 5, 50] seed = 0 iterations = 5000 mse_all_train = np.zeros(shape=(3, iterations)) mse_all_test = np.zeros(shape=(3, iterations)) for j in range(0, 3): nn = MLPRegressor(activation='logistic', solver='lbfgs', warm_start=True, max_iter=1, alpha=0, hidden_layer_sizes=(nh[j], ), random_state=seed) for i in range(0, iterations): nn.fit(x_train, y_train) mse_train = calculate_mse(nn, x_train, y_train) mse_test = calculate_mse(nn, x_test, y_test) mse_all_train[j][i] = mse_train mse_all_test[j][i] = mse_test plot_mse_vs_iterations(mse_all_train, mse_all_test, iterations, nh) ## TODO pass
def ex_1_2_b(x_train, x_test, y_train, y_test): """ Solution for exercise 1.2 b) Remember to set alpha and momentum to 0 when initializing the model :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ ## TODO indices = np.random.permutation(len(y_train)) new_size = int(len(y_train) / 2) x_train_ = np.ndarray((new_size, 1)) y_train_ = np.ndarray(new_size) indices_train = indices[:new_size] indices_val = indices[new_size:] x_val = np.ndarray((new_size, 1)) y_val = np.ndarray(new_size) for i in range(new_size): x_train_[i] = x_train[indices_train[i]] y_train_[i] = y_train[indices_train[i]] x_val[i] = x_train[indices_val[i]] y_val[i] = y_train[indices_val[i]] x_train = x_train_ y_train = y_train_ min_test_errors = np.zeros(10) last_test_errors = np.zeros(10) min_val_errors = np.zeros(10) for i in range(10): regressor = MLPRegressor(hidden_layer_sizes=(40, ), solver="lbfgs", activation="logistic", alpha=10e-3, max_iter=1, warm_start=True, random_state=randint(1, 1000)) val_errors = [] test_errors = [] for j in range(0, 2000): regressor.fit(x_train, y_train) if j % 20 == 0: test_errors.append(calculate_mse(regressor, x_test, y_test)) val_errors.append(calculate_mse(regressor, x_val, y_val)) last_test_errors[i] = calculate_mse(regressor, x_test, y_test) min_val_errors[i] = test_errors[np.argmin(val_errors)] min_test_errors[i] = test_errors[np.argmin(test_errors)] plot_bars_early_stopping_mse_comparison(last_test_errors, min_val_errors, min_test_errors) pass
def ex_1_1_a(x_train, x_test, y_train, y_test): """ Solution for exercise 1.1 a) Remember to set alpha to 0 when initializing the model :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ ## TODO #pass n_hidden = 5 # 2, 5, 50 reg = MLPRegressor(hidden_layer_sizes=(n_hidden, 8), activation='logistic', solver='lbfgs', alpha=0) reg.fit(x_train, y_train) y_pred_test = reg.predict(x_test) y_pred_train = reg.predict(x_train) plot_learned_function(n_hidden, x_train, y_train, y_pred_train, x_test, y_test, y_pred_test)
def ex_1_2_a(x_train, x_test, y_train, y_test): """ Solution for exercise 1.2 a) :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ ## TODO #pass N = 5000 n_hidden = 50 alpha_ = np.power(10, np.linspace(-8, 2, 11)) mse_train = np.zeros([np.size(alpha_), N]) mse_test = np.zeros([np.size(alpha_), N]) for j in range(np.size(alpha_)): reg = MLPRegressor(hidden_layer_sizes=(n_hidden, ), activation='logistic', solver='lbfgs', alpha=alpha_[j], random_state=0, warm_start=True, max_iter=1) for r in range(N): reg.fit(x_train, y_train) mse_train[j, r] = calculate_mse(reg, x_train, y_train) mse_test[j, r] = calculate_mse(reg, x_test, y_test) plot_mse_vs_alpha(mse_train, mse_test, alpha_)
def ex_1_2_a(x_train, x_test, y_train, y_test): """ Solution for exercise 1.2 a) :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ ## TODO total_iter = 5000 n_neuro = 50 n_seeds = 10 alph = np.power(10, np.linspace(-8, 2, 11)) mse_train = np.zeros([np.size(alph), n_seeds]) mse_test = np.zeros([np.size(alph), n_seeds]) for j in range(np.size(alph)): for s in range(n_seeds): seed = np.random.randint(100) reg = MLPRegressor(hidden_layer_sizes=(n_neuro, ), activation='logistic', solver='lbfgs', alpha=alph[j], random_state=seed, max_iter=total_iter) reg.fit(x_train, y_train) mse_train[j, s] = calculate_mse(reg, x_train, y_train) mse_test[j, s] = calculate_mse(reg, x_test, y_test) plot_mse_vs_alpha(mse_train, mse_test, alph)
class MLPRegressorImpl(): def __init__(self, hidden_layer_sizes=(100, ), activation='relu', solver='adam', alpha=0.0001, batch_size='auto', learning_rate='constant', learning_rate_init=0.001, power_t=0.5, max_iter=200, shuffle=True, random_state=None, tol=0.0001, verbose=False, warm_start=False, momentum=0.9, nesterovs_momentum=True, early_stopping=False, validation_fraction=0.1, beta_1=0.9, beta_2=0.999, epsilon=1e-08, n_iter_no_change=10): self._hyperparams = { 'hidden_layer_sizes': hidden_layer_sizes, 'activation': activation, 'solver': solver, 'alpha': alpha, 'batch_size': batch_size, 'learning_rate': learning_rate, 'learning_rate_init': learning_rate_init, 'power_t': power_t, 'max_iter': max_iter, 'shuffle': shuffle, 'random_state': random_state, 'tol': tol, 'verbose': verbose, 'warm_start': warm_start, 'momentum': momentum, 'nesterovs_momentum': nesterovs_momentum, 'early_stopping': early_stopping, 'validation_fraction': validation_fraction, 'beta_1': beta_1, 'beta_2': beta_2, 'epsilon': epsilon, 'n_iter_no_change': n_iter_no_change } def fit(self, X, y=None): self._sklearn_model = SKLModel(**self._hyperparams) if (y is not None): self._sklearn_model.fit(X, y) else: self._sklearn_model.fit(X) return self def predict(self, X): return self._sklearn_model.predict(X)
def ex_1_1_b(x_train, x_test, y_train, y_test): """ Solution for exercise 1.1 b) Remember to set alpha to 0 when initializing the model :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ ## TODO MSE_TEST = np.zeros(10, dtype=float) MSE_TRAIN = np.zeros(10, dtype=float) n_neur = 5 for j in range(10): seed = np.random.randint(100) reg = MLPRegressor(hidden_layer_sizes=(n_neur, ), max_iter=5000, activation='logistic', solver='lbfgs', alpha=0, random_state=seed) reg.fit(x_train, y_train) mse = calculate_mse(reg, x_test, y_test) MSE_TEST[j] = mse mse = calculate_mse(reg, x_train, y_train) MSE_TRAIN[j] = mse mse_test_max = max(MSE_TEST) mse_test_min = min(MSE_TEST) mse_test_mean = np.mean(MSE_TEST) mse_test_std = np.std(MSE_TEST) print("##### Radom Seeds #####") print("### Test Set ###") print(" MAX / MIN / MEAN / STD") print(round(mse_test_max, 3), "/", round(mse_test_min, 3), "/", round(mse_test_mean, 3), "/", round(mse_test_std, 3)) print("\n", MSE_TEST, "n") mse_train_max = max(MSE_TRAIN) mse_train_min = min(MSE_TRAIN) mse_train_mean = np.mean(MSE_TRAIN) mse_train_std = np.std(MSE_TRAIN) print("\n### Training Set ###") print(" MAX / MIN / MEAN / STD") print(round(mse_train_max, 3), "/", round(mse_train_min, 3), "/", round(mse_train_mean, 3), "/", round(mse_train_std, 3)) print("\n", MSE_TRAIN, "n")
def ex_1_1_d(x_train, x_test, y_train, y_test): """ Solution for exercise 1.1 b) Remember to set alpha to 0 when initializing the model :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ ''' • Write code to train a neural network with nh ∈ 2, 8, 20 hidden neurons on one layer and calculate the MSE for the testing and training set at each training iteration for a single seed. To be able to calculate the MSEs at each iteration, set warm_start to True and max_iter to 1 when initializing the network. The usage of warm_start always keeps the previously learnt parameters instead of reinitializing them randomly when fit is called (see the documentation of scikit learn for more information). Then, loop over iterations and successively call the fit function and calculate the MSE on both datasets. Use the training solver ‘lbfgs’, for 1000 iterations. Stack the results in an array with where the first dimension correspond to the number of hidden neurons and the second correspond to the number of iterations Use the function plot_mse_vs_iterations in nn_regression_plot.py to plot the variation of MSE with iterations. • Replace the solver by ‘sgd’ or ‘adam’ and compute the MSE across iterations for the same values of nh. ''' params_n_h = np.array([2, 8, 20]) solvers = np.array(['lbfgs', 'sgd', 'adam']) seed = np.random.seed(1) max_iterations = 1000 train_mses = np.zeros((params_n_h.shape[0], max_iterations)) test_mses = np.zeros((params_n_h.shape[0], max_iterations)) for solver in solvers: for index_n_h, n_h in np.ndenumerate(params_n_h): nn = MLPRegressor(solver=solver, max_iter=1, warm_start=True, activation='logistic', hidden_layer_sizes=(n_h, ), alpha=0, random_state=seed) for iteration in range(max_iterations): nn.fit(x_train, y_train) train_mses[index_n_h, iteration] = calculate_mse(nn, x_train, y_train) test_mses[index_n_h, iteration] = calculate_mse(nn, x_test, y_test) print("Using solver = " + solver) plot_mse_vs_iterations(train_mses, test_mses, max_iterations, params_n_h) # plot_mse_vs_iterations(train_mses, test_mses, max_iterations, params_n_h, solver) pass
def ex_1_1_b(x_train, x_test, y_train, y_test): """ Solution for exercise 1.1 b) Remember to set alpha to 0 when initializing the model. :param x_train: The training dataset :param x_test: The testing dataset :param y_train: The training targets :param y_test: The testing targets :return: """ ## TODO nh = 5 max_it = 10 min_i_train = 11 min_i_test = 11 max_train = 0 min_train = 1 min_test = 11 mean_train = 1 std_train = 0 mse = np.zeros(shape=(10)) for i in range(0, max_it): nn = MLPRegressor(activation='logistic', solver='lbfgs', max_iter=5000, alpha=0, hidden_layer_sizes=(nh, ), random_state=i) nn.fit(x_train, y_train) mse_train = calculate_mse(nn, x_train, y_train) mse_test = calculate_mse(nn, x_test, y_test) mse[i] = mse_train print(i, '.)MSE train ', mse_train) # print(i,'.)MSE test ',mse_test) if (mse_train < min_train): min_train = mse_train min_i_train = i if (mse_test < min_test): min_test = mse_test min_i_test = i if (mse_train > max_train): max_train = mse_train print("Minimum Train: ", min_train) print("Maximum Train: ", max_train) mean_train = mse.mean() print("Mean Train: ", mean_train) std_train = np.std(mse) print("Standard Derivation Train: ", std_train) print("Min Index train: ", min_i_train, " & test: ", min_i_test) pass
def train(train_X, train_y, test_X, param, n_weeks): string = "bp_" + param + str(n_weeks) + '.m' try: bp_model = joblib.load(string) print("训练好的bp", bp_model) except: bp_model = MLPRegressor(hidden_layer_sizes=(18), max_iter=5000, learning_rate_init=0.1) bp_model.fit(train_X, train_y) joblib.dump(bp_model, string) return bp_model