def ANNClassificationBase(Dtrain, Dtest, X, y, hidden_units): # Parameters for neural network classifier n_replicates = 1 # number of networks trained in each k-fold max_iter = 10000 #max_iter = 1000 N, M = X.shape X_train = X[Dtrain, :] y_train = y[Dtrain] X_test = X[Dtest, :] y_test = y[Dtest] mu = np.mean(X_train, 0) sigma = np.std(X_train, 0) X_train = (X_train - mu) / sigma X_test = (X_test - mu) / sigma X_train = torch.Tensor(X_train) y_train = torch.Tensor(y_train) X_test = torch.Tensor(X_test) y_test = torch.Tensor(y_test) N, M = X_train.shape # The lambda-syntax defines an anonymous function, which is used here to # make it easy to make new networks within each cross validation fold model = lambda: torch.nn.Sequential( torch.nn.Linear(M, hidden_units), # M features to H hiden units # 1st transfer function, either Tanh or ReLU: torch.nn.Tanh(), # torch.nn.ReLU(), torch.nn.Linear(hidden_units, 1), # H hidden units to 1 output neuron torch.nn.Sigmoid() # final tranfer function ) loss_fn = torch.nn.BCELoss() errors = [] # make a list for storing generalizaition error in each loop net, final_loss, learning_curve = train_neural_net( model, loss_fn, X=X_train, y=y_train, n_replicates=n_replicates, max_iter=max_iter) # Determine estimated class labels for test set y_sigmoid = net( X_test) # activation of final note, i.e. prediction of network y_test_est = (y_sigmoid > .5).type( dtype=torch.uint8) # threshold output of sigmoidal function y_test = y_test.type(dtype=torch.uint8) # Determine errors and error rate e = (y_test_est != y_test) error_rate = (sum(e).type(torch.float) / len(y_test)).data.numpy() errors.append(error_rate) # store error rate for current CV fold return error_rate
def ANN_regression_tester(Dpar, Dtest, X, y, n_hidden_units): # Parameters for neural network classifier #n_hidden_units = 4 # number of hidden units n_replicates = 1 # number of networks trained in each k-fold max_iter = 10000 N, M = X.shape errors = [] # make a list for storing generalizaition error in each loop X_train = X[Dpar, :] y_train = y[Dpar] X_test = X[Dtest, :] y_test = y[Dtest] mu = np.mean(X_train, 0) sigma = np.std(X_train, 0) X_train = (X_train - mu) / sigma X_test = (X_test - mu) / sigma X_train = torch.Tensor(X_train) y_train = torch.Tensor(y_train) X_test = torch.Tensor(X_test) y_test = torch.Tensor(y_test) N, M = X_train.shape # Define the model model = lambda: torch.nn.Sequential( torch.nn.Linear(M, n_hidden_units), # M features to n_hidden_units torch.nn.Tanh(), # 1st transfer function, torch.nn.Linear(n_hidden_units, 1 ), # n_hidden_units to 1 output neuron # no final tranfer function, i.e. "linear output" ) loss_fn = torch.nn.MSELoss( ) # notice how this is now a mean-squared-error loss # Train the net on training data net, final_loss, learning_curve = train_neural_net( model, loss_fn, X=X_train, y=y_train, n_replicates=n_replicates, max_iter=max_iter) # Determine estimated class labels for test set y_test_est = net(X_test) # Determine errors and errors se = (y_test_est.float() - y_test.float())**2 # squared error mse = (sum(se).type(torch.float) / len(y_test)).data.numpy() #mean errors.append(mse) # store error rate for current CV fold return mse
def ANNCFN(X, y, Dtrain, Dtest, hidden_units): n_replicates = 1 # number of networks trained in each k-fold max_iter = 1000 N, M = X.shape X_train = X[Dtrain, :] y_train = y[Dtrain] X_test = X[Dtest, :] y_test = y[Dtest] mu = np.mean(X_train, 0) sigma = np.std(X_train, 0) X_train = (X_train - mu) / sigma X_test = (X_test - mu) / sigma X_train = torch.Tensor(X_train) y_train = torch.Tensor(y_train) X_test = torch.Tensor(X_test) y_test = torch.Tensor(y_test) model = lambda: torch.nn.Sequential( torch.nn.Linear(M, hidden_units), # M features to H hiden units # 1st transfer function, either Tanh or ReLU: torch.nn.Tanh(), # torch.nn.ReLU(), torch.nn.Linear(hidden_units, 1), # H hidden units to 1 output neuron torch.nn.Sigmoid() # final tranfer function ) loss_fn = torch.nn.BCELoss() errors = [] net, final_loss, learning_curve = train_neural_net( model, loss_fn, X=X_train, y=y_train, n_replicates=n_replicates, max_iter=max_iter) # Determine estimated class labels for test set y_sigmoid = net( X_test) # activation of final note, i.e. prediction of network y_test_est = (y_sigmoid > .5).type( dtype=torch.uint8) # threshold output of sigmoidal function y_test = y_test.type(dtype=torch.uint8) # Determine errors and error rate e = (y_test_est != y_test) error_rate = (sum(e).type(torch.float) / len(y_test)).data.numpy() errors.append(error_rate) # store error rate for current CV fold return y_test_est
def neural_network_train(x_train, y_train, x_test, y_test, param): x_train = torch.Tensor(x_train) y_train = torch.Tensor(y_train) x_test = torch.Tensor(x_test) y_test = torch.Tensor(y_test) global model, loss_fn model = lambda: torch.nn.Sequential( torch.nn.Linear(9, param), # M features to H hiden units # 1st transfer function, either Tanh or ReLU: torch.nn.Tanh(), # torch.nn.ReLU(), torch.nn.Linear(param, 1), # H hidden units to 1 output neuron # torch.nn.Sigmoid() # final tranfer function ) loss_fn = torch.nn.MSELoss() global net, final_loss, learning_curve max_iter = 10000 y_train = y_train.unsqueeze(1) net, final_loss, learning_curve = train_neural_net(model, loss_fn, X=x_train, y=y_train, n_replicates=1, max_iter=max_iter) y_test_est = net( x_test) # activation of final note, i.e. prediction of network # y_test_est = (y_sigmoid > .5)#._cast_uint8_t() # threshold output of sigmoidal function # Determine errors and error rate # e = (y_test_est != Y_test) # error_rate = (sum(e).type(torch.float)/len(Y_test)).data.numpy() y_test_est_n = y_test_est.detach().numpy() y_test = y_test.unsqueeze(1) loss = loss_fn(y_test, y_test_est) # plt.figure() # plt.plot(y_test, 'ok') # plt.plot(y_test_est_n, 'or') # plt.show() return loss
def ann_validate(X, y, hidden_units, cvf, n_replicates, max_iter): CV = model_selection.KFold(cvf, shuffle=True) M = X.shape[1] test_error = np.empty((cvf, len(hidden_units))) f = 0 for train_index, test_index in CV.split(X, y): X_train = X[train_index] y_train = y[train_index] X_test = X[test_index] y_test = y[test_index] for l in range(0, len(hidden_units)): model = lambda: torch.nn.Sequential( torch.nn.Linear(M, hidden_units[l] ), #M features to n_hidden_units torch.nn.Tanh(), # 1st transfer function, torch.nn.Linear(hidden_units[l], 1), # no final tranfer function, i.e. "linear output" ) loss_fn = torch.nn.MSELoss( ) # notice how this is now a mean-squared-error loss # Train the net on training data net, final_loss, learning_curve = train_neural_net( model, loss_fn, X=X_train, y=y_train, n_replicates=n_replicates, max_iter=max_iter) y_test_est = net(X_test) #print("y estimated",y_test_est) #print("y test",y_test) # Evaluate test performance se = (y_test_est.float() - y_test.float())**2 # squared error err = (sum(se).type(torch.float) / len(y_test)).data.numpy() test_error[f, l] = err optimal_hidden = hidden_units[np.argmin(np.mean(test_error, axis=0))] f = f + 1 return optimal_hidden, test_error
def ANNClassification(K, X, y, Dpar, s, vec_hidden_units): ANN_val_error = {} # To store the validation error of each model # Setup a dict to store Error values for each hidden unit (key:map) for i in range(len(vec_hidden_units)): ANN_val_error[i] = [ ] # ANN_val_error = [[1, [error1, error2, error3]], [2 [error1, error2, error3]], ..., n] # Parameters for neural network classifier n_replicates = 1 # number of networks trained in each k-fold max_iter = 10000 N, M = X.shape iCV = model_selection.KFold(K, shuffle=True) # Inner fold for (k, (Dtrain, Dval)) in enumerate(iCV.split(X[Dpar, :], y[Dpar])): X_train = torch.Tensor(stats.zscore(X[Dtrain, :])) y_train = torch.Tensor(stats.zscore(y[Dtrain])) X_test = torch.Tensor(stats.zscore(X[Dval, :])) y_test = torch.Tensor(stats.zscore(y[Dval])) # Define the model structure # n_hidden_units = 1 # number of hidden units in the signle hidden layer for i in range(s): print(f"Inner: {k}, Model: {i}") # The lambda-syntax defines an anonymous function, which is used here to # make it easy to make new networks within each cross validation fold model = lambda: torch.nn.Sequential( torch.nn.Linear(M, vec_hidden_units[i] ), # M features to H hiden units # 1st transfer function, either Tanh or ReLU: torch.nn.Tanh(), # torch.nn.ReLU(), torch.nn.Linear(vec_hidden_units[i], 1 ), # H hidden units to 1 output neuron torch.nn.Sigmoid() # final tranfer function ) loss_fn = torch.nn.BCELoss() errors = [ ] # make a list for storing generalizaition error in each loop net, final_loss, learning_curve = train_neural_net( model, loss_fn, X=X_train, y=y_train, n_replicates=n_replicates, max_iter=max_iter) # Determine estimated class labels for test set y_sigmoid = net( X_test) # activation of final note, i.e. prediction of network y_test_est = (y_sigmoid > .5).type( dtype=torch.uint8) # threshold output of sigmoidal function y_test = y_test.type(dtype=torch.uint8) # Determine errors and error rate e = (y_test_est != y_test) error_rate = (sum(e).type(torch.float) / len(y_test)).data.numpy() errors.append(error_rate) # store error rate for current CV fold print('\n\tBest rate: {}\n'.format(error_rate)) # Only store the new error if its better than the previous prevError = getVal(ANN_val_error, i) # print(f"Prev: {prevError}, MSE: {mse}") if not prevError: # Check whether there is an error addToDict(ANN_val_error, i, error_rate) elif error_rate < prevError: removeFromDict(ANN_val_error, i, prevError) addToDict(ANN_val_error, i, error_rate) # Find the best model from the CV folds # We do this by finding the model with lowest MSE bestError = getVal(ANN_val_error, 0) for i in range(len(ANN_val_error) - 1): if (getVal(ANN_val_error, i + 1) < bestError): # print(f"Best Error: {bestError}") removeFromDict(ANN_val_error, i, bestError) bestError = getVal(ANN_val_error, i + 1) elif getVal(ANN_val_error, i + 1): removeFromDict(ANN_val_error, i + 1, getVal(ANN_val_error, i + 1)) else: addToDict(ANN_val_error, i, getVal(ANN_val_error, i)) for i in range(len(ANN_val_error)): val = getVal(ANN_val_error, i) if val: return [i + 1, val] # Plus one because the n-hidden-units starts with 1
def crossValidationANN(): # Create crossvalidation partition for evaluation K_o_splits = 10 outer_it = 0 K_i_splits = 10 model_count = 10 summed_eval_i = np.zeros((model_count)) eval_i = np.zeros((model_count)) eval_o = np.zeros((K_o_splits)) optimal_hidden_units = np.zeros((K_o_splits)) CV1 = model_selection.KFold(n_splits=K_o_splits, shuffle=True) CV2 = model_selection.KFold(n_splits=K_i_splits, shuffle=True) #Outer k-fold split for train_index_o, test_index_o in CV1.split(X, y): print('Outer CV1-fold {0} of {1}'.format(outer_it + 1, K_o_splits)) # Extract training and test set for current CV fold, convert to tensors X_train_o = torch.tensor(X[train_index_o, :], dtype=torch.float) y_train_o = torch.tensor(y[train_index_o], dtype=torch.float) X_test_o = torch.tensor(X[test_index_o, :], dtype=torch.float) y_test_o = torch.tensor(y[test_index_o], dtype=torch.uint8) #Inner validation loop inner_it = 0 for train_index_i, test_index_i in CV2.split(X_train_o, y_train_o): print('Inner CV2-fold {0} of {1}'.format(inner_it + 1, K_i_splits)) # Extract training and test set for current CV fold, convert to tensors X_train_i = torch.tensor(X[train_index_i, :], dtype=torch.float) y_train_i = torch.tensor(y[train_index_i], dtype=torch.float) X_test_i = torch.tensor(X[test_index_i, :], dtype=torch.float) y_test_i = torch.tensor(y[test_index_i], dtype=torch.uint8) for idx in range(model_count): if idx == 0: n_hidden_units = idx + 1 else: n_hidden_units = idx * 5 model = lambda: torch.nn.Sequential( torch.nn.Linear(M, n_hidden_units), torch.nn.ReLU(), torch.nn.Linear(n_hidden_units, 1), ) loss_fn = torch.nn.MSELoss() net, final_loss, learning_curve = train_neural_net( model, loss_fn, X=X_train_i, y=y_train_i, n_replicates=n_replicates, max_iter=max_iter) y_test_est = net(X_test_i) se = (y_test_est.float() - y_test_i.float())**2 mse = (sum(se).type(torch.float) / len(y_test_i)).data.numpy() errors.append(mse) summed_eval_i[idx] += mse inner_it += 1 eval_i = summed_eval_i * (len(X_test_i) / len(X_train_o)) print(eval_i) idx = np.argmin(eval_i) print(idx) n_hidden_units = idx + 1 model = lambda: torch.nn.Sequential( torch.nn.Linear(M, n_hidden_units), torch.nn.ReLU(), torch.nn.Linear(n_hidden_units, 1), ) net, final_loss, learning_curve = train_neural_net( model, loss_fn, X=X_train_o, y=y_train_o, n_replicates=n_replicates, max_iter=max_iter) y_test_est = net(X_test_o) # Determine errors and errors se = (y_test_est.float() - y_test_o.float())**2 # squared error mse = (sum(se).type(torch.float) / len(y_test_o)).data.numpy() #mean errors.append(mse) # store error rate for current CV fold eval_o[outer_it] = mse optimal_hidden_units[outer_it] = n_hidden_units outer_it += 1 mode_reg, _ = np.unique(optimal_hidden_units, return_counts=True) e_gen = np.sum(eval_o) * (len(X_test_o) / len(X)) print("ANN reggresion with variable hidden units and one layer") print("ANN generalization error: %f with %s and %i" % (e_gen, 'hidden units', mode_reg[0])) print(eval_o) print(optimal_hidden_units)
#ARTIFICIAL NEURAL NETWORK: for h in range(0, n_hidden_units): # Define the model model_int = lambda: torch.nn.Sequential( torch.nn.Linear(M2, h+1), #M features to n_hidden_units torch.nn.Tanh(), # 1st transfer function, torch.nn.Linear(h+1, 1), # n_hidden_units to 1 output neuron # no final tranfer function, i.e. "linear output" ) loss_fn = torch.nn.MSELoss() # notice how this is now a mean-squared-error loss # Train the net on training data (inner loop): net_int, final_loss_int, learning_curve_int = train_neural_net(model_int, loss_fn, X=X_train_int, y=y_train_int, n_replicates=n_replicates, max_iter=max_iter) # Determine estimated class labels for test set y_test_est_int = net_int(X_test_int) # Determine errors and errors se_int = (y_test_est_int.float()-y_test_int.float())**2 # squared error mse_int = (sum(se_int).type(torch.float)/len(y_test_int)).data.numpy() #mean errors_int[k_int,h] = mse_int # store error rate for current CV fold #Define cross validation results -> rlr: opt_val_err = np.min(np.mean(test_error,axis=0)) opt_lambda = lambdas[np.argmin(np.mean(test_error,axis=0))] train_err_vs_lambda = np.mean(train_error,axis=0) test_err_vs_lambda = np.mean(test_error,axis=0)
print('Training model of type:\n\n{}\n'.format(str(model2()))) errors = [] # make a list for storing generalizaition error in each loop for (j, (trainj_index, testj_index)) in enumerate(CV2.split(X_train,y_train)): print("\n\tInternal cross") # Extract training and test set for current CV fold, convert to tensors Xj_train = torch.tensor(X[trainj_index,:], dtype=torch.float) yj_train = torch.tensor(y[trainj_index], dtype=torch.float) Xj_test = torch.tensor(X[testj_index,:], dtype=torch.float) yj_test = torch.tensor(y[testj_index], dtype=torch.float) # Train the net on training data net, final_loss, learning_curve = train_neural_net(model2, loss_fn2, X=Xj_train, y=yj_train, n_replicates=n_replicates, max_iter=max_iter) print('\n\tBest loss: {}\n'.format(final_loss)) # Determine estimated class labels for test set yj_test_est = net(Xj_test) # Determine errors and errors se = (yj_test_est.float()-yj_test.float())**2 # squared error mse = (sum(se).type(torch.float)/len(yj_test)).data.numpy() #mean errors.append(mse) # store error rate for current CV fold generror1 = (len(yj_test)/len(y_train))*mse if (j == 0): generror = generror1 elif (generror1 < generror):
model = lambda: torch.nn.Sequential( torch.nn.Linear(M - 1, n_hidden_units ), #M features to H hiden units # 1st transfer function, either Tanh or ReLU: #torch.nn.ReLU(), torch.nn.Tanh(), torch.nn.Linear(n_hidden_units, 1 ), # H hidden units to 1 output neuron ) loss_fn = torch.nn.MSELoss() net, final_loss, learning_curve = train_neural_net( model, loss_fn, X=X_train2, y=y_train2, n_replicates=1, max_iter=max_iter, tolerance=1e-12) y_test_est = net(X_test2) y_test_est = y_test_est.reshape(-1) se = (y_test_est.float() - y_test2.float())**2 # squared error mse = (sum(se).type(torch.float) / len(y_test2)).data.numpy() #mean # Set error matrix errors2_ANN[i, h - 1] = mse X_train_lin_in = X[train_index2, :]
# Output layer: # H hidden units to C classes # the nodes and their activation before the transfer # function is often referred to as logits/logit output torch.nn.Linear(n_hidden_units, C), # C logits # To obtain normalised "probabilities" of each class # we use the softmax-funtion along the "class" dimension # (i.e. not the dimension describing observations) torch.nn.Softmax(dim=1) # final tranfer function, normalisation of logit output ) # Since we're training a multiclass problem, we cannot use binary cross entropy, # but instead use the general cross entropy loss: loss_fn = torch.nn.CrossEntropyLoss() # Train the network: net, _, _ = train_neural_net(model, loss_fn, X=torch.tensor(X_train_inside_ANN, dtype=torch.float), y=torch.tensor(y_train_inside_ANN, dtype=torch.long), n_replicates=3,max_iter=5000) # Determine probability of each class using trained network softmax_logits = net(torch.tensor(X_test_inside_ANN, dtype=torch.float)) # Get the estimated class as the class with highest probability (argmax on softmax_logits) y_test_est_ANN = (torch.max(softmax_logits, dim=1)[1]).data.numpy() print(y_test_est_ANN.shape) print(y_test_inside_ANN.shape) # Determine errors e = (y_test_est_ANN != y_test_inside_ANN) train_error_rate1_ANN[i] = train_error_rate1_ANN[i]+ np.sum(y_test_est_ANN !=y_test_inside_ANN)/ len(y_test_inside_ANN) # print('Number of miss-classifications for ANN:\n\t {0} out of {1}'.format(sum(e),len(e))) predict = lambda x: (torch.max(net(torch.tensor(x, dtype=torch.float)), dim=1)[1]).data.numpy() #figure(1,figsize=(9,9))
X_train_inner, y_train_inner, lambdas, internal_cross_validation) # Extract training and test set for current CV fold, convert to tensors X_train_ANN = torch.Tensor(X_train_inner) # X[train_index,:]) y_train_ANN = torch.Tensor(y_train_inner) # y[train_index]) X_test_ANN = torch.Tensor(X_test_inner) # X[test_index,:]) y_test_ANN = torch.Tensor(y_test_inner) # y[test_index]) ################### ### ANN model 1 ### ################### # Train the net on training data net_m1, final_loss, learning_curve = train_neural_net( ANN_model_1, loss_fn_1, X=X_train_ANN, y=y_train_ANN, n_replicates=n_replicates, max_iter=max_iter) print('\n\t M1 Best loss: {}\n'.format(final_loss)) # Determine estimated class labels for test set y_test_est_m1 = net_m1(X_test_ANN).detach().numpy() ### # Determine Mean square error mse = np.square(y_test_inner - np.squeeze(y_test_est_m1)).sum( axis=0) / y_test_inner.shape[0] # Save it ANN_error[k, 0] = mse # np.asarray(mse) ### ANN model 2
def ANNRegression(K, X, y, Dpar, s, vec_hidden_units): ANN_val_error = {} # To store the validation error of each model # Setup a dict to store Error values for each hidden unit (key:map) for i in range(len(vec_hidden_units)): ANN_val_error[i] = [ ] # ANN_val_error = [[1, [error1, error2, error3]], [2 [error1, error2, error3]], ..., n] # Parameters for neural network classifier n_replicates = 1 # number of networks trained in each k-fold max_iter = 5000 # Change to 1000 for faster computation N, M = X.shape # K-fold crossvalidation iCV = model_selection.KFold(K, shuffle=True) # Inner fold for (k, (Dtrain, Dval)) in enumerate(iCV.split(X[Dpar, :], y[Dpar])): X_train = X[Dtrain, :] y_train = y[Dtrain] X_test = X[Dval, :] y_test = y[Dval] mu = np.mean(X_train, 0) sigma = np.std(X_train, 0) X_train = (X_train - mu) / sigma X_test = (X_test - mu) / sigma X_train = torch.Tensor(X_train) y_train = torch.Tensor(y_train) X_test = torch.Tensor(X_test) y_test = torch.Tensor(y_test) N, M = X_train.shape for i in range(s): print(f"Inner: {k}, Model: {i}") # Define the model model = lambda: torch.nn.Sequential( torch.nn.Linear(M, vec_hidden_units[i] ), # M features to n_hidden_units torch.nn.Tanh(), # 1st transfer function, torch.nn.Linear(vec_hidden_units[i], 1 ), # n_hidden_units to 1 output neuron # no final tranfer function, i.e. "linear output" ) loss_fn = torch.nn.MSELoss( ) # notice how this is now a mean-squared-error loss errors = [ ] # make a list for storing generalizaition error in each loop # Train the net on training data net, final_loss, learning_curve = train_neural_net( model, loss_fn, X=X_train, y=y_train, n_replicates=n_replicates, max_iter=max_iter) # Determine estimated class labels for test set y_test_est = net(X_test) # Determine errors and errors se = (y_test_est.float() - y_test.float())**2 # squared error mse = (sum(se).type(torch.float) / len(y_test)).data.numpy() # mean errors.append(mse) # store error rate for current CV fold # Only store the new error if its better than the previous prevError = getVal(ANN_val_error, i) if not prevError: # Check whether there is an error addToDict(ANN_val_error, i, mse) elif mse < prevError: removeFromDict(ANN_val_error, i, prevError) addToDict(ANN_val_error, i, mse) # Find the best model from the CV folds # We do this by finding the model with lowest MSE bestError = getVal(ANN_val_error, 0) for i in range(len(ANN_val_error) - 1): if (getVal(ANN_val_error, i + 1) < bestError): removeFromDict(ANN_val_error, i, bestError) bestError = getVal(ANN_val_error, i + 1) elif getVal(ANN_val_error, i + 1): removeFromDict(ANN_val_error, i + 1, getVal(ANN_val_error, i + 1)) else: addToDict(ANN_val_error, i, getVal(ANN_val_error, i)) for i in range(len(ANN_val_error)): val = getVal(ANN_val_error, i) if val: return [i + 1, val] # Plus one because the n-hidden-units starts with 1
print('\n\t\tNumber of layers: {0}'.format(n_hidden_units)) # Define the model model = lambda: torch.nn.Sequential( torch.nn.Linear(M - 1, n_hidden_units ), #M features to n_hidden_units torch.nn.Tanh(), # 1st transfer function, torch.nn.Linear(n_hidden_units, 1 ), # n_hidden_units to 1 output neuron # no final tranfer function, i.e. "linear output" ) # Train the net on training data net, final_loss, learning_curve = train_neural_net( model, loss_fn, X=X_train_ANN, y=np.reshape(y_train_ANN, (-1, 1)), n_replicates=1, max_iter=max_iter) # Determine estimated value for test set y_val_est_ANN = net(X_val_ANN) # Determine errors and errors se = (y_val_est_ANN.float().view(-1) - y_val_ANN.float())**2 # squared error mse = (sum(se).type(torch.float) / len(y_val_ANN)).data.numpy() #mean val_errors[i, n] = mse # store error rate for current CV fold print('\t\tTaining loss: {:.6f}'.format(final_loss)) print('\t\tValidation loss: {:.6f}'.format(mse))
def crossValidationANN(): # Create crossvalidation partition for evaluation K_o_splits = 10 outer_it = 0 K_i_splits = 10 model_count = 10 summed_eval_i = np.zeros((model_count)) eval_i = np.zeros((model_count)) eval_o = np.zeros((model_count)) optimal_hidden_units = np.zeros((K_o_splits)) CV1 = model_selection.StratifiedKFold(n_splits=K_o_splits, shuffle=True) CV2 = model_selection.StratifiedKFold(n_splits=K_i_splits, shuffle=True) #Outer k-fold split for train_index_o, test_index_o in CV1.split(X, y): print('Outer CV1-fold {0} of {1}'.format(outer_it + 1, K_o_splits)) # Extract training and test set for current CV fold, convert to tensors X_train_o = torch.tensor(X[train_index_o, :], dtype=torch.float) y_train_o = torch.tensor(y[train_index_o], dtype=torch.float) X_test_o = torch.tensor(X[test_index_o, :], dtype=torch.float) y_test_o = torch.tensor(y[test_index_o], dtype=torch.uint8) #Inner validation loop inner_it = 0 for train_index_i, test_index_i in CV2.split(X_train_o, y_train_o): print('Inner CV2-fold {0} of {1}'.format(inner_it + 1, K_i_splits)) # Extract training and test set for current CV fold, convert to tensors X_train_i = torch.tensor(X[train_index_i, :], dtype=torch.float) y_train_i = torch.tensor(y[train_index_i], dtype=torch.float) X_test_i = torch.tensor(X[test_index_i, :], dtype=torch.float) y_test_i = torch.tensor(y[test_index_i], dtype=torch.uint8) #C specifies the inverse of regularization strength. Small C means high regularization for idx in range(model_count): n_hidden_units = idx + 1 model = lambda: torch.nn.Sequential( torch.nn.Linear(M, n_hidden_units ), #M features to n_hidden_units torch.nn.ReLU(), # 1st transfer function, torch.nn.Linear(n_hidden_units, 1), torch.nn.Sigmoid() # n_hidden_units to 1 output neuron # no final tranfer function, i.e. "linear output" ) # Train the net on training data net, final_loss, learning_curve = train_neural_net( model, loss_fn, X=X_train_i, y=y_train_i, n_replicates=n_replicates, max_iter=max_iter) y_sigmoid = net(X_test_i) y_test_est = y_sigmoid > .5 e = y_test_est != y_test_i current_err = 100 * ( (sum(e).type(torch.float) / len(y_test_i)).data.numpy()) a = 100 * ((sum(y_test_est == y_test_i).type(torch.float) / len(y_test_i)).data.numpy()) errors.append( current_err) # store error rate for current CV fold acc_inner.append(a) summed_eval_i[idx] += current_err inner_it += 1 eval_i = summed_eval_i * (len(X_test_i) / len(X_train_o)) idx = np.argmin(eval_i) n_hidden_units = idx + 1 model = lambda: torch.nn.Sequential( torch.nn.Linear(M, n_hidden_units), torch.nn.ReLU(), torch.nn.Linear(n_hidden_units, 1), torch.nn.Sigmoid()) # Train the net on training data net, final_loss, learning_curve = train_neural_net( model, loss_fn, X=X_train_o, y=y_train_o, n_replicates=n_replicates, max_iter=max_iter) y_sigmoid = net(X_test_o) y_test_est = y_sigmoid > .5 # Determine errors and errors e = y_test_est != y_test_o a = 100 * ((sum(y_test_est == y_test_o).type(torch.float) / len(y_test_o)).data.numpy()) current_err = 100 * ( (sum(e).type(torch.float) / len(y_test_o)).data.numpy()) eval_o[outer_it] = current_err acc_outer.append(a) errors.append(current_err) # store error rate for current CV fold optimal_hidden_units[outer_it] = n_hidden_units outer_it += 1 mode_reg, _ = np.unique(optimal_hidden_units, return_counts=True) e_gen = np.sum(eval_o) * (len(X_test_o) / len(X)) # print("ANN with variable hidden units and one layer") print("ANN accuracy: %f with %s and %i" % (np.max(a), 'number of layers', mode_reg[0])) print("ANN generalization error: %f with %s and %i" % (e_gen, 'number of layers', mode_reg[0])) print(optimal_hidden_units) print(acc_outer) print(eval_o)
def network_validate_regression(X, y, h_interval): ''' Validate neural network model for regression using 'cvf'-fold cross validation. Finds the optimal hidden units from the hidden unit list Function returns: optimal test error value, optimal number of hidden units, optimal model after each fold Parameters: X training data set y vector of values h_interval the interval of hidden units. Should start consecutively. Eg: [1,2,3..] Returns: opt_val_err validation error for optimum lambda opt_n_h_units optimal number of hidden units ''' cvf = 10 n_replicates = 1 #due to computation time max_iter = 10000 # this has to be adjusted. Has to be set to a high no if we want a good prediction M = X.shape[1] error_rate_matrix = np.empty((cvf, len(h_interval))) for k, (train_index, test_index) in enumerate(CV3.split(X, y)): print('\nCrossvalidation inner fold: {0}/{1}'.format(k + 1, 10)) for h in range(0, len(h_interval)): print("NO OF HIDDEN UNITS:", h + 1) model = lambda: torch.nn.Sequential( torch.nn.Linear( M, h + 1 ), # M features to H hiden units - h+1 since h starts with 0 torch.nn.Tanh(), # 1st transfer function, torch.nn.Linear(h + 1, 1), # H hidden units to 1 output neuron # no final tranfer function ) loss_fn = torch.nn.MSELoss() # Extract training and test set for current CV fold, convert to tensors X_train = torch.Tensor(X[train_index, :]) y_train = torch.Tensor(y[train_index]) X_test = torch.Tensor(X[test_index, :]) y_test = torch.Tensor(y[test_index]) # Train the net on training data net, final_loss, learning_curve = train_neural_net( model, loss_fn, X=X_train, y=y_train, n_replicates=n_replicates, max_iter=max_iter) # Determine estimated class labels for test set y_test_est = net(X_test) # y_test = stats.zscore(y_test.data.numpy()) # Determine errors and errors se = (y_test_est.float() - y_test.float())**2 # squared error mse = (sum(se).type(torch.float) / len(y_test)).data.numpy() # mean error_rate_matrix[k, h] = mse print('\n\tBest loss for h= {}: {}\n'.format(h, final_loss)) opt_index = np.argmin(np.mean(error_rate_matrix, axis=0)) opt_val_err = np.min(np.mean(error_rate_matrix, axis=0)) opt_n_h_units = h_interval[opt_index] return opt_val_err, opt_n_h_units
def network_validate_classification(X, y, h_interval): ''' Validate neural network model for classification using 'cvf'-fold cross validation. Finds the optimal hidden units from the hidden unit list Function returns: optimal test error value, optimal number of hidden units, optimal model after each fold Parameters: X training data set y vector of values h_interval the interval of hidden units. Should start consecutively. Eg: [1,2,3..] Returns: opt_val_err validation error for optimum lambda opt_n_h_units optimal number of hidden units ''' cvf = 10 n_replicates = 1 #due to computation time max_iter = 10000 M = X.shape[1] error_rate_matrix = np.empty((cvf, len(h_interval))) for k, (train_index, test_index) in enumerate(CV1.split(X, y)): print('\nCrossvalidation inner fold: {0}/{1}'.format(k + 1, 10)) for h in range(0, len(h_interval)): print("NO OF HIDDEN UNITS:", h + 1) model = lambda: torch.nn.Sequential( torch.nn.Linear( M, h + 1 ), #M features to H hiden units - h+1 since h starts with 0 torch.nn.Tanh(), # 1st transfer function, torch.nn.Linear(h + 1, 1), # H hidden units to 1 output neuron torch.nn.Sigmoid() # final tranfer function ) loss_fn = torch.nn.BCELoss() # Extract training and test set for current CV fold, convert to tensors X_train = torch.Tensor(X[train_index, :]) y_train = torch.Tensor(y[train_index]) X_test = torch.Tensor(X[test_index, :]) y_test = torch.Tensor(y[test_index]) # Train the net on training data net, final_loss, learning_curve = train_neural_net( model, loss_fn, X=X_train, y=y_train, n_replicates=n_replicates, max_iter=max_iter) # Determine estimated class labels for test set #y predicted y_sigmoid = net(X_test) y_test_est = (y_sigmoid > .5).type( dtype=torch.uint8) #set tershold to classify as 0 or 1 # Determine errors and errors y_test = y_test.type(dtype=torch.uint8) e = y_test_est != y_test error_rate = (sum(e).type(torch.float) / len(y_test)).data.numpy() error_rate_matrix[k, h] = error_rate print('\n\tBest loss for h= {}: {}\n'.format(h, final_loss)) opt_index = np.argmin(np.mean(error_rate_matrix, axis=0)) opt_val_err = np.min(np.mean(error_rate_matrix, axis=0)) opt_n_h_units = h_interval[opt_index] print("errors are", np.mean(error_rate_matrix, axis=0)) print("hidden unit index is", np.argmin(np.mean(error_rate_matrix, axis=0))) return opt_val_err, opt_n_h_units
model = lambda: torch.nn.Sequential( torch.nn.Linear(M, h), #M features to n_hidden_units torch.nn.ReLU(), # 1st transfer function, torch.nn.Linear(h, 1), # n_hidden_units to 1 output neuron # no final tranfer function, i.e. "linear output" ) loss_fn = torch.nn.MSELoss( ) # notice how this is now a mean-squared-error loss print('Training model of type:\n\n{}\n'.format(str(model()))) # Train the net on training data net, final_loss, learning_curve = train_neural_net( model, loss_fn, X=X_inner_train, y=y_inner_train, n_replicates=n_replicates, max_iter=max_iter) print('\n\tBest loss: {}\n'.format(final_loss)) # Determine estimated class labels for test set y_test_est = net(X_inner_test) #not X_outer_test # Determine errors and errors se = (y_test_est.float() - y_inner_test.float())**2 # squared error MSE = (sum(se).type(torch.float) / len(y_inner_test)).data.numpy() #mean #errors.append(mse) # store error rate for current CV fold errors_inner_ANN[j, k2] = MSE
def crossValidationANN(): # Create crossvalidation partition for evaluation K_o_splits = 10 outer_it = 0 K_i_splits = 10 model_count = 5 n_hidden_units = 10 summed_eval_i = np.zeros((model_count)) eval_i = np.zeros((model_count)) eval_o = np.zeros((K_o_splits)) optimal_n_layers = np.zeros((K_o_splits)) CV1 = model_selection.StratifiedKFold(n_splits=K_o_splits, shuffle = True) CV2 = model_selection.StratifiedKFold(n_splits=K_i_splits, shuffle = True) #Outer k-fold split for train_index_o, test_index_o in CV1.split(X,y): print('Outer CV1-fold {0} of {1}'.format(outer_it+1,K_o_splits)) # Extract training and test set for current CV fold, convert to tensors X_train_o = torch.tensor(X[train_index_o,:], dtype=torch.float) y_train_o = torch.tensor(y[train_index_o], dtype=torch.float) X_test_o = torch.tensor(X[test_index_o,:], dtype=torch.float) y_test_o = torch.tensor(y[test_index_o], dtype=torch.uint8) #Inner validation loop inner_it = 0 for train_index_i, test_index_i in CV2.split(X_train_o,y_train_o): print('Inner CV2-fold {0} of {1}'.format(inner_it+1,K_i_splits)) # Extract training and test set for current CV fold, convert to tensors X_train_i = torch.tensor(X[train_index_i,:], dtype=torch.float) y_train_i = torch.tensor(y[train_index_i], dtype=torch.float) X_test_i = torch.tensor(X[test_index_i,:], dtype=torch.float) y_test_i = torch.tensor(y[test_index_i], dtype=torch.uint8) for idx in range(model_count): n_layers = idx+1 model = NN_model(n_hidden_units,n_layers) net, final_loss, learning_curve = train_neural_net(model, loss_fn, X=X_train_i, y=y_train_i, n_replicates=n_replicates, max_iter=max_iter) y_sigmoid = net(X_test_i) y_test_est = y_sigmoid>.5 e = y_test_est != y_test_i current_err = 100*( (sum(e).type(torch.float)/len(y_test_i)).data.numpy()) a = 100*( (sum(y_test_est == y_test_i).type(torch.float)/len(y_test_i)).data.numpy()) acc_inner.append(a) errors.append(current_err) summed_eval_i[idx] += current_err inner_it += 1 eval_i = summed_eval_i * (len(X_test_i)/len(X_train_o)) idx = np.argmin(eval_i) n_layers = idx+1 model = NN_model(n_hidden_units,n_layers) net, final_loss, learning_curve = train_neural_net(model, loss_fn, X=X_train_o, y=y_train_o, n_replicates=n_replicates, max_iter=max_iter) y_sigmoid = net(X_test_o) y_test_est = y_sigmoid>.5 e = y_test_est != y_test_o a = 100*( (sum(y_test_est == y_test_o).type(torch.float)/len(y_test_o)).data.numpy()) current_err = 100*( (sum(e).type(torch.float)/len(y_test_o)).data.numpy()) eval_o[outer_it] = current_err acc_outer.append(a) errors.append(current_err) # store error rate for current CV fold optimal_n_layers[outer_it] = n_layers outer_it+=1 mode_reg, _= np.unique(optimal_n_layers, return_counts=True) e_gen = np.sum(eval_o) * (len(X_test_o)/ len(X)) # print("ANN classification with variable hidden layers and 10 hidden units") print("ANN accuracy: %f with %s and %i" % (np.max(a),'number of layers',mode_reg[0])) print("ANN generalization error: %f with %s and %i" % (e_gen,'number of layers',mode_reg[0])) print(optimal_n_layers) print(acc_outer) print(eval_o)
print('Training model of type:\n\n{}\n'.format(str(model()))) errors = [] # make a list for storing generalizaition error in each loop for k, (train_index, test_index) in enumerate(CV.split(X, y)): print('\nCrossvalidation fold: {0}/{1}'.format(k + 1, K)) # Extract training and test set for current CV fold, convert to tensors X_train = torch.Tensor(X[train_index, :]) y_train = torch.Tensor(y[train_index]) X_test = torch.Tensor(X[test_index, :]) y_test = torch.Tensor(y[test_index]) # Train the net on training data net, final_loss, learning_curve = train_neural_net( model, loss_fn, X=X_train, y=y_train, n_replicates=n_replicates, max_iter=max_iter) print('\n\tBest loss: {}\n'.format(final_loss)) # Determine estimated class labels for test set #y predicted y_sigmoid = net(X_test) y_test_est = (y_sigmoid > .5).type( dtype=torch.uint8) #set tershold to classify as 0 or 1 # Determine errors and errors y_test = y_test.type(dtype=torch.uint8)
for n_hidden_units in hidden_array: print("hidden unit = ", n_hidden_units) model = lambda: torch.nn.Sequential( torch.nn.Linear(M_ANN, n_hidden_units ), #M features to n_hidden_units torch.nn.ReLU(), # 1st transfer function, torch.nn.Linear(n_hidden_units, 1 ), # n_hidden_units to 1 output neuron ) loss_fn = torch.nn.MSELoss() net, final_loss, learning_curve = train_neural_net( model, loss_fn, X=X_inner_train, y=y_inner_train, n_replicates=n_replicates, max_iter=max_iter) test_est = net(X_inner_test) # Determine errors and errors se = (test_est.float() - y_inner_test.float())**2 # squared error mse = (sum(se).type(torch.float) / len(y_inner_test)).data.numpy() #mean Error_ANN[j] += mse[0] j += 1 Hidden_ANN[k] = hidden_array[int( np.where(Error_ANN == np.amin(Error_ANN))[0])]