def ANNClassificationBase(Dtrain, Dtest, X, y, hidden_units):
# Parameters for neural network classifier
n_replicates = 1 # number of networks trained in each k-fold
max_iter = 10000
#max_iter = 1000
N, M = X.shape
X_train = X[Dtrain, :]
y_train = y[Dtrain]
X_test = X[Dtest, :]
y_test = y[Dtest]
mu = np.mean(X_train, 0)
sigma = np.std(X_train, 0)
X_train = (X_train - mu) / sigma
X_test = (X_test - mu) / sigma
X_train = torch.Tensor(X_train)
y_train = torch.Tensor(y_train)
X_test = torch.Tensor(X_test)
y_test = torch.Tensor(y_test)
N, M = X_train.shape
# The lambda-syntax defines an anonymous function, which is used here to
# make it easy to make new networks within each cross validation fold
model = lambda: torch.nn.Sequential(
torch.nn.Linear(M, hidden_units), # M features to H hiden units
# 1st transfer function, either Tanh or ReLU:
torch.nn.Tanh(), # torch.nn.ReLU(),
torch.nn.Linear(hidden_units, 1), # H hidden units to 1 output neuron
torch.nn.Sigmoid() # final tranfer function
)
loss_fn = torch.nn.BCELoss()
errors = [] # make a list for storing generalizaition error in each loop
net, final_loss, learning_curve = train_neural_net(
model,
loss_fn,
X=X_train,
y=y_train,
n_replicates=n_replicates,
max_iter=max_iter)
# Determine estimated class labels for test set
y_sigmoid = net(
X_test) # activation of final note, i.e. prediction of network
y_test_est = (y_sigmoid > .5).type(
dtype=torch.uint8) # threshold output of sigmoidal function
y_test = y_test.type(dtype=torch.uint8)
# Determine errors and error rate
e = (y_test_est != y_test)
error_rate = (sum(e).type(torch.float) / len(y_test)).data.numpy()
errors.append(error_rate) # store error rate for current CV fold
return error_rate
def ANN_regression_tester(Dpar, Dtest, X, y, n_hidden_units):
# Parameters for neural network classifier
#n_hidden_units = 4 # number of hidden units
n_replicates = 1 # number of networks trained in each k-fold
max_iter = 10000
N, M = X.shape
errors = [] # make a list for storing generalizaition error in each loop
X_train = X[Dpar, :]
y_train = y[Dpar]
X_test = X[Dtest, :]
y_test = y[Dtest]
mu = np.mean(X_train, 0)
sigma = np.std(X_train, 0)
X_train = (X_train - mu) / sigma
X_test = (X_test - mu) / sigma
X_train = torch.Tensor(X_train)
y_train = torch.Tensor(y_train)
X_test = torch.Tensor(X_test)
y_test = torch.Tensor(y_test)
N, M = X_train.shape
# Define the model
model = lambda: torch.nn.Sequential(
torch.nn.Linear(M, n_hidden_units), # M features to n_hidden_units
torch.nn.Tanh(), # 1st transfer function,
torch.nn.Linear(n_hidden_units, 1
), # n_hidden_units to 1 output neuron
# no final tranfer function, i.e. "linear output"
)
loss_fn = torch.nn.MSELoss(
) # notice how this is now a mean-squared-error loss
# Train the net on training data
net, final_loss, learning_curve = train_neural_net(
model,
loss_fn,
X=X_train,
y=y_train,
n_replicates=n_replicates,
max_iter=max_iter)
# Determine estimated class labels for test set
y_test_est = net(X_test)
# Determine errors and errors
se = (y_test_est.float() - y_test.float())**2 # squared error
mse = (sum(se).type(torch.float) / len(y_test)).data.numpy() #mean
errors.append(mse) # store error rate for current CV fold
return mse
def ANNCFN(X, y, Dtrain, Dtest, hidden_units):
n_replicates = 1 # number of networks trained in each k-fold
max_iter = 1000
N, M = X.shape
X_train = X[Dtrain, :]
y_train = y[Dtrain]
X_test = X[Dtest, :]
y_test = y[Dtest]
mu = np.mean(X_train, 0)
sigma = np.std(X_train, 0)
X_train = (X_train - mu) / sigma
X_test = (X_test - mu) / sigma
X_train = torch.Tensor(X_train)
y_train = torch.Tensor(y_train)
X_test = torch.Tensor(X_test)
y_test = torch.Tensor(y_test)
model = lambda: torch.nn.Sequential(
torch.nn.Linear(M, hidden_units), # M features to H hiden units
# 1st transfer function, either Tanh or ReLU:
torch.nn.Tanh(), # torch.nn.ReLU(),
torch.nn.Linear(hidden_units, 1), # H hidden units to 1 output neuron
torch.nn.Sigmoid() # final tranfer function
)
loss_fn = torch.nn.BCELoss()
errors = []
net, final_loss, learning_curve = train_neural_net(
model,
loss_fn,
X=X_train,
y=y_train,
n_replicates=n_replicates,
max_iter=max_iter)
# Determine estimated class labels for test set
y_sigmoid = net(
X_test) # activation of final note, i.e. prediction of network
y_test_est = (y_sigmoid > .5).type(
dtype=torch.uint8) # threshold output of sigmoidal function
y_test = y_test.type(dtype=torch.uint8)
# Determine errors and error rate
e = (y_test_est != y_test)
error_rate = (sum(e).type(torch.float) / len(y_test)).data.numpy()
errors.append(error_rate) # store error rate for current CV fold
return y_test_est
def neural_network_train(x_train, y_train, x_test, y_test, param):
x_train = torch.Tensor(x_train)
y_train = torch.Tensor(y_train)
x_test = torch.Tensor(x_test)
y_test = torch.Tensor(y_test)
global model, loss_fn
model = lambda: torch.nn.Sequential(
torch.nn.Linear(9, param), # M features to H hiden units
# 1st transfer function, either Tanh or ReLU:
torch.nn.Tanh(), # torch.nn.ReLU(),
torch.nn.Linear(param, 1), # H hidden units to 1 output neuron
# torch.nn.Sigmoid() # final tranfer function
)
loss_fn = torch.nn.MSELoss()
global net, final_loss, learning_curve
max_iter = 10000
y_train = y_train.unsqueeze(1)
net, final_loss, learning_curve = train_neural_net(model,
loss_fn,
X=x_train,
y=y_train,
n_replicates=1,
max_iter=max_iter)
y_test_est = net(
x_test) # activation of final note, i.e. prediction of network
# y_test_est = (y_sigmoid > .5)#._cast_uint8_t() # threshold output of sigmoidal function
# Determine errors and error rate
# e = (y_test_est != Y_test)
# error_rate = (sum(e).type(torch.float)/len(Y_test)).data.numpy()
y_test_est_n = y_test_est.detach().numpy()
y_test = y_test.unsqueeze(1)
loss = loss_fn(y_test, y_test_est)
# plt.figure()
# plt.plot(y_test, 'ok')
# plt.plot(y_test_est_n, 'or')
# plt.show()
return loss
def ann_validate(X, y, hidden_units, cvf, n_replicates, max_iter):
CV = model_selection.KFold(cvf, shuffle=True)
M = X.shape[1]
test_error = np.empty((cvf, len(hidden_units)))
f = 0
for train_index, test_index in CV.split(X, y):
X_train = X[train_index]
y_train = y[train_index]
X_test = X[test_index]
y_test = y[test_index]
for l in range(0, len(hidden_units)):
model = lambda: torch.nn.Sequential(
torch.nn.Linear(M, hidden_units[l]
), #M features to n_hidden_units
torch.nn.Tanh(), # 1st transfer function,
torch.nn.Linear(hidden_units[l], 1),
# no final tranfer function, i.e. "linear output"
)
loss_fn = torch.nn.MSELoss(
) # notice how this is now a mean-squared-error loss
# Train the net on training data
net, final_loss, learning_curve = train_neural_net(
model,
loss_fn,
X=X_train,
y=y_train,
n_replicates=n_replicates,
max_iter=max_iter)
y_test_est = net(X_test)
#print("y estimated",y_test_est)
#print("y test",y_test)
# Evaluate test performance
se = (y_test_est.float() - y_test.float())**2 # squared error
err = (sum(se).type(torch.float) / len(y_test)).data.numpy()
test_error[f, l] = err
optimal_hidden = hidden_units[np.argmin(np.mean(test_error,
axis=0))]
f = f + 1
return optimal_hidden, test_error
def ANNClassification(K, X, y, Dpar, s, vec_hidden_units):
ANN_val_error = {} # To store the validation error of each model
# Setup a dict to store Error values for each hidden unit (key:map)
for i in range(len(vec_hidden_units)):
ANN_val_error[i] = [
] # ANN_val_error = [[1, [error1, error2, error3]], [2 [error1, error2, error3]], ..., n]
# Parameters for neural network classifier
n_replicates = 1 # number of networks trained in each k-fold
max_iter = 10000
N, M = X.shape
iCV = model_selection.KFold(K, shuffle=True)
# Inner fold
for (k, (Dtrain, Dval)) in enumerate(iCV.split(X[Dpar, :], y[Dpar])):
X_train = torch.Tensor(stats.zscore(X[Dtrain, :]))
y_train = torch.Tensor(stats.zscore(y[Dtrain]))
X_test = torch.Tensor(stats.zscore(X[Dval, :]))
y_test = torch.Tensor(stats.zscore(y[Dval]))
# Define the model structure
# n_hidden_units = 1 # number of hidden units in the signle hidden layer
for i in range(s):
print(f"Inner: {k}, Model: {i}")
# The lambda-syntax defines an anonymous function, which is used here to
# make it easy to make new networks within each cross validation fold
model = lambda: torch.nn.Sequential(
torch.nn.Linear(M, vec_hidden_units[i]
), # M features to H hiden units
# 1st transfer function, either Tanh or ReLU:
torch.nn.Tanh(), # torch.nn.ReLU(),
torch.nn.Linear(vec_hidden_units[i], 1
), # H hidden units to 1 output neuron
torch.nn.Sigmoid() # final tranfer function
)
loss_fn = torch.nn.BCELoss()
errors = [
] # make a list for storing generalizaition error in each loop
net, final_loss, learning_curve = train_neural_net(
model,
loss_fn,
X=X_train,
y=y_train,
n_replicates=n_replicates,
max_iter=max_iter)
# Determine estimated class labels for test set
y_sigmoid = net(
X_test) # activation of final note, i.e. prediction of network
y_test_est = (y_sigmoid > .5).type(
dtype=torch.uint8) # threshold output of sigmoidal function
y_test = y_test.type(dtype=torch.uint8)
# Determine errors and error rate
e = (y_test_est != y_test)
error_rate = (sum(e).type(torch.float) / len(y_test)).data.numpy()
errors.append(error_rate) # store error rate for current CV fold
print('\n\tBest rate: {}\n'.format(error_rate))
# Only store the new error if its better than the previous
prevError = getVal(ANN_val_error, i)
# print(f"Prev: {prevError}, MSE: {mse}")
if not prevError: # Check whether there is an error
addToDict(ANN_val_error, i, error_rate)
elif error_rate < prevError:
removeFromDict(ANN_val_error, i, prevError)
addToDict(ANN_val_error, i, error_rate)
# Find the best model from the CV folds
# We do this by finding the model with lowest MSE
bestError = getVal(ANN_val_error, 0)
for i in range(len(ANN_val_error) - 1):
if (getVal(ANN_val_error, i + 1) < bestError):
# print(f"Best Error: {bestError}")
removeFromDict(ANN_val_error, i, bestError)
bestError = getVal(ANN_val_error, i + 1)
elif getVal(ANN_val_error, i + 1):
removeFromDict(ANN_val_error, i + 1, getVal(ANN_val_error, i + 1))
else:
addToDict(ANN_val_error, i, getVal(ANN_val_error, i))
for i in range(len(ANN_val_error)):
val = getVal(ANN_val_error, i)
if val:
return [i + 1,
val] # Plus one because the n-hidden-units starts with 1
def crossValidationANN():
# Create crossvalidation partition for evaluation
K_o_splits = 10
outer_it = 0
K_i_splits = 10
model_count = 10
summed_eval_i = np.zeros((model_count))
eval_i = np.zeros((model_count))
eval_o = np.zeros((K_o_splits))
optimal_hidden_units = np.zeros((K_o_splits))
CV1 = model_selection.KFold(n_splits=K_o_splits, shuffle=True)
CV2 = model_selection.KFold(n_splits=K_i_splits, shuffle=True)
#Outer k-fold split
for train_index_o, test_index_o in CV1.split(X, y):
print('Outer CV1-fold {0} of {1}'.format(outer_it + 1, K_o_splits))
# Extract training and test set for current CV fold, convert to tensors
X_train_o = torch.tensor(X[train_index_o, :], dtype=torch.float)
y_train_o = torch.tensor(y[train_index_o], dtype=torch.float)
X_test_o = torch.tensor(X[test_index_o, :], dtype=torch.float)
y_test_o = torch.tensor(y[test_index_o], dtype=torch.uint8)
#Inner validation loop
inner_it = 0
for train_index_i, test_index_i in CV2.split(X_train_o, y_train_o):
print('Inner CV2-fold {0} of {1}'.format(inner_it + 1, K_i_splits))
# Extract training and test set for current CV fold, convert to tensors
X_train_i = torch.tensor(X[train_index_i, :], dtype=torch.float)
y_train_i = torch.tensor(y[train_index_i], dtype=torch.float)
X_test_i = torch.tensor(X[test_index_i, :], dtype=torch.float)
y_test_i = torch.tensor(y[test_index_i], dtype=torch.uint8)
for idx in range(model_count):
if idx == 0:
n_hidden_units = idx + 1
else:
n_hidden_units = idx * 5
model = lambda: torch.nn.Sequential(
torch.nn.Linear(M, n_hidden_units),
torch.nn.ReLU(),
torch.nn.Linear(n_hidden_units, 1),
)
loss_fn = torch.nn.MSELoss()
net, final_loss, learning_curve = train_neural_net(
model,
loss_fn,
X=X_train_i,
y=y_train_i,
n_replicates=n_replicates,
max_iter=max_iter)
y_test_est = net(X_test_i)
se = (y_test_est.float() - y_test_i.float())**2
mse = (sum(se).type(torch.float) / len(y_test_i)).data.numpy()
errors.append(mse)
summed_eval_i[idx] += mse
inner_it += 1
eval_i = summed_eval_i * (len(X_test_i) / len(X_train_o))
print(eval_i)
idx = np.argmin(eval_i)
print(idx)
n_hidden_units = idx + 1
model = lambda: torch.nn.Sequential(
torch.nn.Linear(M, n_hidden_units),
torch.nn.ReLU(),
torch.nn.Linear(n_hidden_units, 1),
)
net, final_loss, learning_curve = train_neural_net(
model,
loss_fn,
X=X_train_o,
y=y_train_o,
n_replicates=n_replicates,
max_iter=max_iter)
y_test_est = net(X_test_o)
# Determine errors and errors
se = (y_test_est.float() - y_test_o.float())**2 # squared error
mse = (sum(se).type(torch.float) / len(y_test_o)).data.numpy() #mean
errors.append(mse) # store error rate for current CV fold
eval_o[outer_it] = mse
optimal_hidden_units[outer_it] = n_hidden_units
outer_it += 1
mode_reg, _ = np.unique(optimal_hidden_units, return_counts=True)
e_gen = np.sum(eval_o) * (len(X_test_o) / len(X))
print("ANN reggresion with variable hidden units and one layer")
print("ANN generalization error: %f with %s and %i" %
(e_gen, 'hidden units', mode_reg[0]))
print(eval_o)
print(optimal_hidden_units)
#ARTIFICIAL NEURAL NETWORK:
for h in range(0, n_hidden_units):
# Define the model
model_int = lambda: torch.nn.Sequential(
torch.nn.Linear(M2, h+1), #M features to n_hidden_units
torch.nn.Tanh(), # 1st transfer function,
torch.nn.Linear(h+1, 1), # n_hidden_units to 1 output neuron
# no final tranfer function, i.e. "linear output"
)
loss_fn = torch.nn.MSELoss() # notice how this is now a mean-squared-error loss
# Train the net on training data (inner loop):
net_int, final_loss_int, learning_curve_int = train_neural_net(model_int,
loss_fn,
X=X_train_int,
y=y_train_int,
n_replicates=n_replicates,
max_iter=max_iter)
# Determine estimated class labels for test set
y_test_est_int = net_int(X_test_int)
# Determine errors and errors
se_int = (y_test_est_int.float()-y_test_int.float())**2 # squared error
mse_int = (sum(se_int).type(torch.float)/len(y_test_int)).data.numpy() #mean
errors_int[k_int,h] = mse_int # store error rate for current CV fold
#Define cross validation results -> rlr:
opt_val_err = np.min(np.mean(test_error,axis=0))
opt_lambda = lambdas[np.argmin(np.mean(test_error,axis=0))]
train_err_vs_lambda = np.mean(train_error,axis=0)
test_err_vs_lambda = np.mean(test_error,axis=0)
print('Training model of type:\n\n{}\n'.format(str(model2())))
errors = [] # make a list for storing generalizaition error in each loop
for (j, (trainj_index, testj_index)) in enumerate(CV2.split(X_train,y_train)):
print("\n\tInternal cross")
# Extract training and test set for current CV fold, convert to tensors
Xj_train = torch.tensor(X[trainj_index,:], dtype=torch.float)
yj_train = torch.tensor(y[trainj_index], dtype=torch.float)
Xj_test = torch.tensor(X[testj_index,:], dtype=torch.float)
yj_test = torch.tensor(y[testj_index], dtype=torch.float)
# Train the net on training data
net, final_loss, learning_curve = train_neural_net(model2,
loss_fn2,
X=Xj_train,
y=yj_train,
n_replicates=n_replicates,
max_iter=max_iter)
print('\n\tBest loss: {}\n'.format(final_loss))
# Determine estimated class labels for test set
yj_test_est = net(Xj_test)
# Determine errors and errors
se = (yj_test_est.float()-yj_test.float())**2 # squared error
mse = (sum(se).type(torch.float)/len(yj_test)).data.numpy() #mean
errors.append(mse) # store error rate for current CV fold
generror1 = (len(yj_test)/len(y_train))*mse
if (j == 0):
generror = generror1
elif (generror1 < generror):
model = lambda: torch.nn.Sequential(
torch.nn.Linear(M - 1, n_hidden_units
), #M features to H hiden units
# 1st transfer function, either Tanh or ReLU:
#torch.nn.ReLU(),
torch.nn.Tanh(),
torch.nn.Linear(n_hidden_units, 1
), # H hidden units to 1 output neuron
)
loss_fn = torch.nn.MSELoss()
net, final_loss, learning_curve = train_neural_net(
model,
loss_fn,
X=X_train2,
y=y_train2,
n_replicates=1,
max_iter=max_iter,
tolerance=1e-12)
y_test_est = net(X_test2)
y_test_est = y_test_est.reshape(-1)
se = (y_test_est.float() - y_test2.float())**2 # squared error
mse = (sum(se).type(torch.float) /
len(y_test2)).data.numpy() #mean
# Set error matrix
errors2_ANN[i, h - 1] = mse
X_train_lin_in = X[train_index2, :]
# Output layer:
# H hidden units to C classes
# the nodes and their activation before the transfer
# function is often referred to as logits/logit output
torch.nn.Linear(n_hidden_units, C), # C logits
# To obtain normalised "probabilities" of each class
# we use the softmax-funtion along the "class" dimension
# (i.e. not the dimension describing observations)
torch.nn.Softmax(dim=1) # final tranfer function, normalisation of logit output
)
# Since we're training a multiclass problem, we cannot use binary cross entropy,
# but instead use the general cross entropy loss:
loss_fn = torch.nn.CrossEntropyLoss()
# Train the network:
net, _, _ = train_neural_net(model, loss_fn,
X=torch.tensor(X_train_inside_ANN, dtype=torch.float),
y=torch.tensor(y_train_inside_ANN, dtype=torch.long),
n_replicates=3,max_iter=5000)
# Determine probability of each class using trained network
softmax_logits = net(torch.tensor(X_test_inside_ANN, dtype=torch.float))
# Get the estimated class as the class with highest probability (argmax on softmax_logits)
y_test_est_ANN = (torch.max(softmax_logits, dim=1)[1]).data.numpy()
print(y_test_est_ANN.shape)
print(y_test_inside_ANN.shape)
# Determine errors
e = (y_test_est_ANN != y_test_inside_ANN)
train_error_rate1_ANN[i] = train_error_rate1_ANN[i]+ np.sum(y_test_est_ANN !=y_test_inside_ANN)/ len(y_test_inside_ANN)
# print('Number of miss-classifications for ANN:\n\t {0} out of {1}'.format(sum(e),len(e)))
predict = lambda x: (torch.max(net(torch.tensor(x, dtype=torch.float)), dim=1)[1]).data.numpy()
#figure(1,figsize=(9,9))
X_train_inner, y_train_inner, lambdas, internal_cross_validation)
# Extract training and test set for current CV fold, convert to tensors
X_train_ANN = torch.Tensor(X_train_inner) # X[train_index,:])
y_train_ANN = torch.Tensor(y_train_inner) # y[train_index])
X_test_ANN = torch.Tensor(X_test_inner) # X[test_index,:])
y_test_ANN = torch.Tensor(y_test_inner) # y[test_index])
###################
### ANN model 1 ###
###################
# Train the net on training data
net_m1, final_loss, learning_curve = train_neural_net(
ANN_model_1,
loss_fn_1,
X=X_train_ANN,
y=y_train_ANN,
n_replicates=n_replicates,
max_iter=max_iter)
print('\n\t M1 Best loss: {}\n'.format(final_loss))
# Determine estimated class labels for test set
y_test_est_m1 = net_m1(X_test_ANN).detach().numpy() ###
# Determine Mean square error
mse = np.square(y_test_inner - np.squeeze(y_test_est_m1)).sum(
axis=0) / y_test_inner.shape[0]
# Save it
ANN_error[k, 0] = mse # np.asarray(mse)
### ANN model 2
def ANNRegression(K, X, y, Dpar, s, vec_hidden_units):
ANN_val_error = {} # To store the validation error of each model
# Setup a dict to store Error values for each hidden unit (key:map)
for i in range(len(vec_hidden_units)):
ANN_val_error[i] = [
] # ANN_val_error = [[1, [error1, error2, error3]], [2 [error1, error2, error3]], ..., n]
# Parameters for neural network classifier
n_replicates = 1 # number of networks trained in each k-fold
max_iter = 5000 # Change to 1000 for faster computation
N, M = X.shape
# K-fold crossvalidation
iCV = model_selection.KFold(K, shuffle=True)
# Inner fold
for (k, (Dtrain, Dval)) in enumerate(iCV.split(X[Dpar, :], y[Dpar])):
X_train = X[Dtrain, :]
y_train = y[Dtrain]
X_test = X[Dval, :]
y_test = y[Dval]
mu = np.mean(X_train, 0)
sigma = np.std(X_train, 0)
X_train = (X_train - mu) / sigma
X_test = (X_test - mu) / sigma
X_train = torch.Tensor(X_train)
y_train = torch.Tensor(y_train)
X_test = torch.Tensor(X_test)
y_test = torch.Tensor(y_test)
N, M = X_train.shape
for i in range(s):
print(f"Inner: {k}, Model: {i}")
# Define the model
model = lambda: torch.nn.Sequential(
torch.nn.Linear(M, vec_hidden_units[i]
), # M features to n_hidden_units
torch.nn.Tanh(), # 1st transfer function,
torch.nn.Linear(vec_hidden_units[i], 1
), # n_hidden_units to 1 output neuron
# no final tranfer function, i.e. "linear output"
)
loss_fn = torch.nn.MSELoss(
) # notice how this is now a mean-squared-error loss
errors = [
] # make a list for storing generalizaition error in each loop
# Train the net on training data
net, final_loss, learning_curve = train_neural_net(
model,
loss_fn,
X=X_train,
y=y_train,
n_replicates=n_replicates,
max_iter=max_iter)
# Determine estimated class labels for test set
y_test_est = net(X_test)
# Determine errors and errors
se = (y_test_est.float() - y_test.float())**2 # squared error
mse = (sum(se).type(torch.float) /
len(y_test)).data.numpy() # mean
errors.append(mse) # store error rate for current CV fold
# Only store the new error if its better than the previous
prevError = getVal(ANN_val_error, i)
if not prevError: # Check whether there is an error
addToDict(ANN_val_error, i, mse)
elif mse < prevError:
removeFromDict(ANN_val_error, i, prevError)
addToDict(ANN_val_error, i, mse)
# Find the best model from the CV folds
# We do this by finding the model with lowest MSE
bestError = getVal(ANN_val_error, 0)
for i in range(len(ANN_val_error) - 1):
if (getVal(ANN_val_error, i + 1) < bestError):
removeFromDict(ANN_val_error, i, bestError)
bestError = getVal(ANN_val_error, i + 1)
elif getVal(ANN_val_error, i + 1):
removeFromDict(ANN_val_error, i + 1, getVal(ANN_val_error, i + 1))
else:
addToDict(ANN_val_error, i, getVal(ANN_val_error, i))
for i in range(len(ANN_val_error)):
val = getVal(ANN_val_error, i)
if val:
return [i + 1,
val] # Plus one because the n-hidden-units starts with 1
print('\n\t\tNumber of layers: {0}'.format(n_hidden_units))
# Define the model
model = lambda: torch.nn.Sequential(
torch.nn.Linear(M - 1, n_hidden_units
), #M features to n_hidden_units
torch.nn.Tanh(), # 1st transfer function,
torch.nn.Linear(n_hidden_units, 1
), # n_hidden_units to 1 output neuron
# no final tranfer function, i.e. "linear output"
)
# Train the net on training data
net, final_loss, learning_curve = train_neural_net(
model,
loss_fn,
X=X_train_ANN,
y=np.reshape(y_train_ANN, (-1, 1)),
n_replicates=1,
max_iter=max_iter)
# Determine estimated value for test set
y_val_est_ANN = net(X_val_ANN)
# Determine errors and errors
se = (y_val_est_ANN.float().view(-1) -
y_val_ANN.float())**2 # squared error
mse = (sum(se).type(torch.float) /
len(y_val_ANN)).data.numpy() #mean
val_errors[i, n] = mse # store error rate for current CV fold
print('\t\tTaining loss: {:.6f}'.format(final_loss))
print('\t\tValidation loss: {:.6f}'.format(mse))
def crossValidationANN():
# Create crossvalidation partition for evaluation
K_o_splits = 10
outer_it = 0
K_i_splits = 10
model_count = 10
summed_eval_i = np.zeros((model_count))
eval_i = np.zeros((model_count))
eval_o = np.zeros((model_count))
optimal_hidden_units = np.zeros((K_o_splits))
CV1 = model_selection.StratifiedKFold(n_splits=K_o_splits, shuffle=True)
CV2 = model_selection.StratifiedKFold(n_splits=K_i_splits, shuffle=True)
#Outer k-fold split
for train_index_o, test_index_o in CV1.split(X, y):
print('Outer CV1-fold {0} of {1}'.format(outer_it + 1, K_o_splits))
# Extract training and test set for current CV fold, convert to tensors
X_train_o = torch.tensor(X[train_index_o, :], dtype=torch.float)
y_train_o = torch.tensor(y[train_index_o], dtype=torch.float)
X_test_o = torch.tensor(X[test_index_o, :], dtype=torch.float)
y_test_o = torch.tensor(y[test_index_o], dtype=torch.uint8)
#Inner validation loop
inner_it = 0
for train_index_i, test_index_i in CV2.split(X_train_o, y_train_o):
print('Inner CV2-fold {0} of {1}'.format(inner_it + 1, K_i_splits))
# Extract training and test set for current CV fold, convert to tensors
X_train_i = torch.tensor(X[train_index_i, :], dtype=torch.float)
y_train_i = torch.tensor(y[train_index_i], dtype=torch.float)
X_test_i = torch.tensor(X[test_index_i, :], dtype=torch.float)
y_test_i = torch.tensor(y[test_index_i], dtype=torch.uint8)
#C specifies the inverse of regularization strength. Small C means high regularization
for idx in range(model_count):
n_hidden_units = idx + 1
model = lambda: torch.nn.Sequential(
torch.nn.Linear(M, n_hidden_units
), #M features to n_hidden_units
torch.nn.ReLU(), # 1st transfer function,
torch.nn.Linear(n_hidden_units, 1),
torch.nn.Sigmoid() # n_hidden_units to 1 output neuron
# no final tranfer function, i.e. "linear output"
)
# Train the net on training data
net, final_loss, learning_curve = train_neural_net(
model,
loss_fn,
X=X_train_i,
y=y_train_i,
n_replicates=n_replicates,
max_iter=max_iter)
y_sigmoid = net(X_test_i)
y_test_est = y_sigmoid > .5
e = y_test_est != y_test_i
current_err = 100 * (
(sum(e).type(torch.float) / len(y_test_i)).data.numpy())
a = 100 * ((sum(y_test_est == y_test_i).type(torch.float) /
len(y_test_i)).data.numpy())
errors.append(
current_err) # store error rate for current CV fold
acc_inner.append(a)
summed_eval_i[idx] += current_err
inner_it += 1
eval_i = summed_eval_i * (len(X_test_i) / len(X_train_o))
idx = np.argmin(eval_i)
n_hidden_units = idx + 1
model = lambda: torch.nn.Sequential(
torch.nn.Linear(M, n_hidden_units), torch.nn.ReLU(),
torch.nn.Linear(n_hidden_units, 1), torch.nn.Sigmoid())
# Train the net on training data
net, final_loss, learning_curve = train_neural_net(
model,
loss_fn,
X=X_train_o,
y=y_train_o,
n_replicates=n_replicates,
max_iter=max_iter)
y_sigmoid = net(X_test_o)
y_test_est = y_sigmoid > .5
# Determine errors and errors
e = y_test_est != y_test_o
a = 100 * ((sum(y_test_est == y_test_o).type(torch.float) /
len(y_test_o)).data.numpy())
current_err = 100 * (
(sum(e).type(torch.float) / len(y_test_o)).data.numpy())
eval_o[outer_it] = current_err
acc_outer.append(a)
errors.append(current_err) # store error rate for current CV fold
optimal_hidden_units[outer_it] = n_hidden_units
outer_it += 1
mode_reg, _ = np.unique(optimal_hidden_units, return_counts=True)
e_gen = np.sum(eval_o) * (len(X_test_o) / len(X))
#
print("ANN with variable hidden units and one layer")
print("ANN accuracy: %f with %s and %i" %
(np.max(a), 'number of layers', mode_reg[0]))
print("ANN generalization error: %f with %s and %i" %
(e_gen, 'number of layers', mode_reg[0]))
print(optimal_hidden_units)
print(acc_outer)
print(eval_o)
def network_validate_regression(X, y, h_interval):
''' Validate neural network model for regression using 'cvf'-fold cross validation.
Finds the optimal hidden units from the hidden unit list
Function returns: optimal test error value, optimal number of hidden units, optimal model after each fold
Parameters:
X training data set
y vector of values
h_interval the interval of hidden units. Should start consecutively. Eg: [1,2,3..]
Returns:
opt_val_err validation error for optimum lambda
opt_n_h_units optimal number of hidden units
'''
cvf = 10
n_replicates = 1 #due to computation time
max_iter = 10000 # this has to be adjusted. Has to be set to a high no if we want a good prediction
M = X.shape[1]
error_rate_matrix = np.empty((cvf, len(h_interval)))
for k, (train_index, test_index) in enumerate(CV3.split(X, y)):
print('\nCrossvalidation inner fold: {0}/{1}'.format(k + 1, 10))
for h in range(0, len(h_interval)):
print("NO OF HIDDEN UNITS:", h + 1)
model = lambda: torch.nn.Sequential(
torch.nn.Linear(
M, h + 1
), # M features to H hiden units - h+1 since h starts with 0
torch.nn.Tanh(), # 1st transfer function,
torch.nn.Linear(h + 1, 1), # H hidden units to 1 output neuron
# no final tranfer function
)
loss_fn = torch.nn.MSELoss()
# Extract training and test set for current CV fold, convert to tensors
X_train = torch.Tensor(X[train_index, :])
y_train = torch.Tensor(y[train_index])
X_test = torch.Tensor(X[test_index, :])
y_test = torch.Tensor(y[test_index])
# Train the net on training data
net, final_loss, learning_curve = train_neural_net(
model,
loss_fn,
X=X_train,
y=y_train,
n_replicates=n_replicates,
max_iter=max_iter)
# Determine estimated class labels for test set
y_test_est = net(X_test)
# y_test = stats.zscore(y_test.data.numpy())
# Determine errors and errors
se = (y_test_est.float() - y_test.float())**2 # squared error
mse = (sum(se).type(torch.float) /
len(y_test)).data.numpy() # mean
error_rate_matrix[k, h] = mse
print('\n\tBest loss for h= {}: {}\n'.format(h, final_loss))
opt_index = np.argmin(np.mean(error_rate_matrix, axis=0))
opt_val_err = np.min(np.mean(error_rate_matrix, axis=0))
opt_n_h_units = h_interval[opt_index]
return opt_val_err, opt_n_h_units
def network_validate_classification(X, y, h_interval):
''' Validate neural network model for classification using 'cvf'-fold cross validation.
Finds the optimal hidden units from the hidden unit list
Function returns: optimal test error value, optimal number of hidden units, optimal model after each fold
Parameters:
X training data set
y vector of values
h_interval the interval of hidden units. Should start consecutively. Eg: [1,2,3..]
Returns:
opt_val_err validation error for optimum lambda
opt_n_h_units optimal number of hidden units
'''
cvf = 10
n_replicates = 1 #due to computation time
max_iter = 10000
M = X.shape[1]
error_rate_matrix = np.empty((cvf, len(h_interval)))
for k, (train_index, test_index) in enumerate(CV1.split(X, y)):
print('\nCrossvalidation inner fold: {0}/{1}'.format(k + 1, 10))
for h in range(0, len(h_interval)):
print("NO OF HIDDEN UNITS:", h + 1)
model = lambda: torch.nn.Sequential(
torch.nn.Linear(
M, h + 1
), #M features to H hiden units - h+1 since h starts with 0
torch.nn.Tanh(), # 1st transfer function,
torch.nn.Linear(h + 1, 1), # H hidden units to 1 output neuron
torch.nn.Sigmoid() # final tranfer function
)
loss_fn = torch.nn.BCELoss()
# Extract training and test set for current CV fold, convert to tensors
X_train = torch.Tensor(X[train_index, :])
y_train = torch.Tensor(y[train_index])
X_test = torch.Tensor(X[test_index, :])
y_test = torch.Tensor(y[test_index])
# Train the net on training data
net, final_loss, learning_curve = train_neural_net(
model,
loss_fn,
X=X_train,
y=y_train,
n_replicates=n_replicates,
max_iter=max_iter)
# Determine estimated class labels for test set
#y predicted
y_sigmoid = net(X_test)
y_test_est = (y_sigmoid > .5).type(
dtype=torch.uint8) #set tershold to classify as 0 or 1
# Determine errors and errors
y_test = y_test.type(dtype=torch.uint8)
e = y_test_est != y_test
error_rate = (sum(e).type(torch.float) / len(y_test)).data.numpy()
error_rate_matrix[k, h] = error_rate
print('\n\tBest loss for h= {}: {}\n'.format(h, final_loss))
opt_index = np.argmin(np.mean(error_rate_matrix, axis=0))
opt_val_err = np.min(np.mean(error_rate_matrix, axis=0))
opt_n_h_units = h_interval[opt_index]
print("errors are", np.mean(error_rate_matrix, axis=0))
print("hidden unit index is", np.argmin(np.mean(error_rate_matrix,
axis=0)))
return opt_val_err, opt_n_h_units
model = lambda: torch.nn.Sequential(
torch.nn.Linear(M, h), #M features to n_hidden_units
torch.nn.ReLU(), # 1st transfer function,
torch.nn.Linear(h, 1), # n_hidden_units to 1 output neuron
# no final tranfer function, i.e. "linear output"
)
loss_fn = torch.nn.MSELoss(
) # notice how this is now a mean-squared-error loss
print('Training model of type:\n\n{}\n'.format(str(model())))
# Train the net on training data
net, final_loss, learning_curve = train_neural_net(
model,
loss_fn,
X=X_inner_train,
y=y_inner_train,
n_replicates=n_replicates,
max_iter=max_iter)
print('\n\tBest loss: {}\n'.format(final_loss))
# Determine estimated class labels for test set
y_test_est = net(X_inner_test) #not X_outer_test
# Determine errors and errors
se = (y_test_est.float() -
y_inner_test.float())**2 # squared error
MSE = (sum(se).type(torch.float) /
len(y_inner_test)).data.numpy() #mean
#errors.append(mse) # store error rate for current CV fold
errors_inner_ANN[j, k2] = MSE
def crossValidationANN():
# Create crossvalidation partition for evaluation
K_o_splits = 10
outer_it = 0
K_i_splits = 10
model_count = 5
n_hidden_units = 10
summed_eval_i = np.zeros((model_count))
eval_i = np.zeros((model_count))
eval_o = np.zeros((K_o_splits))
optimal_n_layers = np.zeros((K_o_splits))
CV1 = model_selection.StratifiedKFold(n_splits=K_o_splits, shuffle = True)
CV2 = model_selection.StratifiedKFold(n_splits=K_i_splits, shuffle = True)
#Outer k-fold split
for train_index_o, test_index_o in CV1.split(X,y):
print('Outer CV1-fold {0} of {1}'.format(outer_it+1,K_o_splits))
# Extract training and test set for current CV fold, convert to tensors
X_train_o = torch.tensor(X[train_index_o,:], dtype=torch.float)
y_train_o = torch.tensor(y[train_index_o], dtype=torch.float)
X_test_o = torch.tensor(X[test_index_o,:], dtype=torch.float)
y_test_o = torch.tensor(y[test_index_o], dtype=torch.uint8)
#Inner validation loop
inner_it = 0
for train_index_i, test_index_i in CV2.split(X_train_o,y_train_o):
print('Inner CV2-fold {0} of {1}'.format(inner_it+1,K_i_splits))
# Extract training and test set for current CV fold, convert to tensors
X_train_i = torch.tensor(X[train_index_i,:], dtype=torch.float)
y_train_i = torch.tensor(y[train_index_i], dtype=torch.float)
X_test_i = torch.tensor(X[test_index_i,:], dtype=torch.float)
y_test_i = torch.tensor(y[test_index_i], dtype=torch.uint8)
for idx in range(model_count):
n_layers = idx+1
model = NN_model(n_hidden_units,n_layers)
net, final_loss, learning_curve = train_neural_net(model,
loss_fn,
X=X_train_i,
y=y_train_i,
n_replicates=n_replicates,
max_iter=max_iter)
y_sigmoid = net(X_test_i)
y_test_est = y_sigmoid>.5
e = y_test_est != y_test_i
current_err = 100*( (sum(e).type(torch.float)/len(y_test_i)).data.numpy())
a = 100*( (sum(y_test_est == y_test_i).type(torch.float)/len(y_test_i)).data.numpy())
acc_inner.append(a)
errors.append(current_err)
summed_eval_i[idx] += current_err
inner_it += 1
eval_i = summed_eval_i * (len(X_test_i)/len(X_train_o))
idx = np.argmin(eval_i)
n_layers = idx+1
model = NN_model(n_hidden_units,n_layers)
net, final_loss, learning_curve = train_neural_net(model,
loss_fn,
X=X_train_o,
y=y_train_o,
n_replicates=n_replicates,
max_iter=max_iter)
y_sigmoid = net(X_test_o)
y_test_est = y_sigmoid>.5
e = y_test_est != y_test_o
a = 100*( (sum(y_test_est == y_test_o).type(torch.float)/len(y_test_o)).data.numpy())
current_err = 100*( (sum(e).type(torch.float)/len(y_test_o)).data.numpy())
eval_o[outer_it] = current_err
acc_outer.append(a)
errors.append(current_err) # store error rate for current CV fold
optimal_n_layers[outer_it] = n_layers
outer_it+=1
mode_reg, _= np.unique(optimal_n_layers, return_counts=True)
e_gen = np.sum(eval_o) * (len(X_test_o)/ len(X))
#
print("ANN classification with variable hidden layers and 10 hidden units")
print("ANN accuracy: %f with %s and %i" % (np.max(a),'number of layers',mode_reg[0]))
print("ANN generalization error: %f with %s and %i" % (e_gen,'number of layers',mode_reg[0]))
print(optimal_n_layers)
print(acc_outer)
print(eval_o)
print('Training model of type:\n\n{}\n'.format(str(model())))
errors = [] # make a list for storing generalizaition error in each loop
for k, (train_index, test_index) in enumerate(CV.split(X, y)):
print('\nCrossvalidation fold: {0}/{1}'.format(k + 1, K))
# Extract training and test set for current CV fold, convert to tensors
X_train = torch.Tensor(X[train_index, :])
y_train = torch.Tensor(y[train_index])
X_test = torch.Tensor(X[test_index, :])
y_test = torch.Tensor(y[test_index])
# Train the net on training data
net, final_loss, learning_curve = train_neural_net(
model,
loss_fn,
X=X_train,
y=y_train,
n_replicates=n_replicates,
max_iter=max_iter)
print('\n\tBest loss: {}\n'.format(final_loss))
# Determine estimated class labels for test set
#y predicted
y_sigmoid = net(X_test)
y_test_est = (y_sigmoid > .5).type(
dtype=torch.uint8) #set tershold to classify as 0 or 1
# Determine errors and errors
y_test = y_test.type(dtype=torch.uint8)
for n_hidden_units in hidden_array:
print("hidden unit = ", n_hidden_units)
model = lambda: torch.nn.Sequential(
torch.nn.Linear(M_ANN, n_hidden_units
), #M features to n_hidden_units
torch.nn.ReLU(), # 1st transfer function,
torch.nn.Linear(n_hidden_units, 1
), # n_hidden_units to 1 output neuron
)
loss_fn = torch.nn.MSELoss()
net, final_loss, learning_curve = train_neural_net(
model,
loss_fn,
X=X_inner_train,
y=y_inner_train,
n_replicates=n_replicates,
max_iter=max_iter)
test_est = net(X_inner_test)
# Determine errors and errors
se = (test_est.float() - y_inner_test.float())**2 # squared error
mse = (sum(se).type(torch.float) /
len(y_inner_test)).data.numpy() #mean
Error_ANN[j] += mse[0]
j += 1
Hidden_ANN[k] = hidden_array[int(
np.where(Error_ANN == np.amin(Error_ANN))[0])]
