def Franke_plot_overfitting(X_train,
X_test,
eta=0,
lmbd=0,
batch_size=0,
n_hidden_neurons=0,
epochs=0):
eta = 3.16227766e-01
lmbd = 2.68269580e-08
batch_size = 1
n_hidden_neurons = 57
epochs = 1000
n_categories = 1
np.random.seed(seed)
dnn = NN(X_train,
z_train,
eta=eta,
lmbd=lmbd,
epochs=int(epochs),
batch_size=batch_size,
n_hidden_neurons=n_hidden_neurons,
n_categories=n_categories,
cost_grad='MSE',
activation='sigmoid',
activation_out='ELU')
dnn.train_and_validate(X_val, z_val, MSE_store=True, validate=False)
MSE_val, MSE_train = dnn.MSE_epoch()
epo = np.arange(len(MSE_train))
plt.plot(epo, MSE_val, label='MSE val')
plt.plot(epo, MSE_train, label='MSE train')
plt.xlabel("Number of epochs")
plt.ylabel("MSE")
plt.title("MSE vs epochs")
plt.legend()
plt.show()
def hypertuning(iterations, cols, etamax, etamin, lmbdmax, lmbdmin,
batch_sizemax, batch_sizemin, hiddenmax, hiddenmin):
start_time = time.time()
if (cols < iterations):
cols = iterations
print(
"cols must be larger than 'iterations. Cols is set equal to iterations"
)
sig = signature(hypertuning)
rows = int(len(sig.parameters) / 2)
hyper = np.zeros((rows, cols))
MSE_array = np.zeros(iterations)
hyper[0] = np.logspace(etamin, etamax, cols)
hyper[1] = np.logspace(lmbdmin, lmbdmax, cols)
hyper[2] = np.round(
np.linspace(batch_sizemin, batch_sizemax, cols, dtype='int'))
hyper[3] = np.round(np.linspace(hiddenmin, hiddenmax, cols))
hyper[4] = np.zeros((cols))
#print(hyper)
for i in range(rows - 1):
np.random.shuffle(hyper[i])
#print(np.apply_along_axis(np.random.shuffle, 1, hyper[i]))
for it in range(iterations):
hyper_choice = hyper[:, it]
eta = hyper_choice[0]
lmbd = hyper_choice[1]
batch_size = hyper_choice[2]
n_hidden_neurons = hyper_choice[3]
dnn = NN(X_train,
z_train,
eta=eta,
lmbd=lmbd,
epochs=epochs,
batch_size=batch_size,
n_hidden_neurons=n_hidden_neurons,
n_categories=n_categories,
cost_grad='MSE',
activation='sigmoid',
activation_out='ELU')
dnn.train(X_val)
MSE_val = dnn.MSE_epoch(z_val)
best_pred_epoch = np.argmin(MSE_val)
dnn_test = NN(X_train,
z_train,
eta=eta,
lmbd=lmbd,
epochs=best_pred_epoch + 1,
batch_size=batch_size,
n_hidden_neurons=n_hidden_neurons,
n_categories=n_categories,
cost_grad='MSE',
activation='sigmoid',
activation_out='ELU')
dnn_test.train(X_test)
# kan jo bare bruke predict probabilities på siste her
z_pred = dnn_test.y_predict_epoch[best_pred_epoch]
MSE_array[it] = mean_squared_error(z_test, z_pred)
hyper[4][it] = best_pred_epoch
print(it)
if (it % m.ceil((iterations / 40)) == 0):
t = round((time.time() - start_time))
if t >= 60:
sec = t % 60
print("--- %s min," % int(t / 60), "%s sec ---" % sec)
else:
print("--- %s sec ---" % int(t))
MSE_best_index = np.argmin(MSE_array)
MSE_best = np.min(MSE_array)
print("MSE array: ", MSE_array)
print("best index: ", MSE_best_index)
print("best MSE: ", MSE_best)
return hyper[:, MSE_best_index]
def hypertuning_franke(z, x, y, iterations, cols, etamin, etamax, lmbdmin, lmbdmax, batch_sizemin, batch_sizemax, hiddenmin,hiddenmax, polymin, polymax, epochs = 1000, plot_MSE = False, validate = True):
start_time = time.time()
if (cols < iterations):
cols = iterations
print("cols must be larger than 'iterations. Cols is set equal to iterations")
rows = 6
hyper = np.zeros((rows, cols))
MSE_array = np.zeros(iterations)
#Making the matrix of parameters
hyper[0] = np.logspace(etamin, etamax, cols)
hyper[1] = np.logspace(lmbdmin, lmbdmax, cols)
hyper[2] = np.round(np.linspace(batch_sizemin, batch_sizemax, cols, dtype='int'))
hyper[3] = np.round(np.linspace(hiddenmin, hiddenmax, cols))
hyper[4] = np.random.randint(polymin, polymax, size=cols, dtype='int')
hyper[5] = np.zeros((cols))
for i in range(rows-1):
np.random.shuffle(hyper[i])
n_categories = 1
#iterating over all parameters
for it in range(iterations):
hyper_choice = hyper[:,it]
eta = hyper_choice[0]
lmbd = hyper_choice[1]
batch_size = hyper_choice[2]
n_hidden_neurons = hyper_choice[3]
X, X_train, X_test, X_val, z_train, z_test, z_val, indicies = CreateDesignMatrix_X(z, x,y, int(hyper[4][it]))
np.random.seed(seed)
dnn = NN(X_train, z_train, eta=eta, lmbd=lmbd, epochs=epochs, batch_size=batch_size, n_hidden_neurons=n_hidden_neurons, n_categories=n_categories,
cost_grad='MSE', activation='sigmoid', activation_out='ELU')
dnn.train_and_validate(X_val, z_val, MSE_store = plot_MSE, validate=validate)
z_pred = dnn.predict_probabilities(X_val)
MSE_array[it] = mean_squared_error(z_val,z_pred)
hyper[5][it] = dnn.epoch +1
#Optional: If one wishes to see how the parameter combination is doing, pass plot_MSE = True:
if(plot_MSE):
print("parameters: eta, lmbd, batch, hidden, poly, epochs \n", hyper[:,it:it+1])
MSE_val, MSE_train = dnn.MSE_epoch()
epo = np.arange(len(MSE_val))
plt.plot(epo, MSE_val, label='MSE val')
plt.plot(epo, MSE_train, label='MSE train')
plt.xlabel("Number of epochs")
plt.ylabel("MSE")
plt.title("MSE vs epochs")
plt.legend()
plt.show()
#Estimating the time the iteration takes
if (it%m.ceil((iterations/60))==0):
print('Iteration: ', it)
t = round((time.time() - start_time))
if (t >= 60) and (it > 0):
sec = t % 60
print("--- %s min," % int(t/60),"%s sec ---" % sec)
print("Estimated minutes left: ", int((t/it)*(iterations-it)/60))
else:
print("--- %s sec ---" %int(t))
# Finding the best parameters:
MSE_best_index = np.argmin(MSE_array)
MSE_best = np.min(MSE_array)
print("MSE array: ", MSE_array)
print("best index: ",MSE_best_index)
print("best MSE: ", MSE_best)
final_hyper = hyper[:,MSE_best_index]
print("parameters: eta, lmbd, batch, hidden, poly, epochs ", final_hyper)
eta_best = final_hyper[0]
lmbd_best = final_hyper[1]
batch_size_best = final_hyper[2]
n_hidden_neurons_best = final_hyper[3]
poly_best = final_hyper[4]
epochs_best = final_hyper[5]
return hyper[:,MSE_best_index]
