"""
Neural network calcualting logistic regression
"""
import sys
sys.path.append('network')
sys.path.append('../')
sys.path.append('../../')
import numpy as np
from fetch_2D_data import fetch_data
from NN import NeuralNet
X, Y, X_crit, Y_crit = fetch_data()
nn = NeuralNet(X,
Y,
nodes=[X.shape[1], 2],
activations=[None],
cost_func='log')
nn.TrainNN(epochs=200, batchSize=130, eta0=0.001, n_print=20)
ypred_crit = nn.feed_forward(X_crit, isTraining=False)
critError = nn.cost_function(Y_crit, ypred_crit)
critAcc = nn.accuracy(Y_crit, ypred_crit)
print("Critical error: %g, Critical accuracy: %g" % (critError, critAcc))
def plot_regularization():
"""
Ploting different models as a function of regularization strength
"""
#######################################################
###############Defining parameters#####################
#######################################################
lambs = np.logspace(-4, 0, 5)
# n_samples =[100, 1000, 4000, 10000]
n_samples = [3000]
# nodes = [10]
nodes=[50]
#######################################################
###############Fetching Data###########################
#######################################################
X, Y, X_critical, Y_critical = fetch_data()
X, Y = shuffle(X, Y)
X_critical, Y_critical = shuffle(X_critical, Y_critical)
for node in nodes:
for sample in n_samples:
X_train = X[:sample]; Y_train = Y[:sample]
X_test = X[sample:2*sample]; Y_test = Y[sample:2*sample]
X_crit = X_critical[:sample]; Y_crit = Y_critical[:sample]
#######################################################
###########Dictionaries for ploting####################
#######################################################
errors = {'ols': {'Train': [], 'Test': [], 'Crit': []},
'l1': {'Train': [], 'Test': [], 'Crit': []},
'l2': {'Train': [], 'Test': [], 'Crit': []}}
accuracies = {'ols': {'Train': [], 'Test': [], 'Crit': []},
'l1': {'Train': [], 'Test': [], 'Crit': []},
'l2': {'Train': [], 'Test': [], 'Crit': []}}
for lamb in lambs:
#######################################################
###########Initializing networks#######################
#######################################################
nn = NeuralNet( X_train, Y_train, nodes = [X.shape[1], node, 2],
activations = ['sigmoid', None], cost_func='log')
nn_l1 = NeuralNet( X_train, Y_train, nodes = [X.shape[1], node, 2],
activations = ['sigmoid', None], cost_func='log', regularization='l1', lamb=lamb)
nn_l2 = NeuralNet( X_train, Y_train, nodes = [X.shape[1], node, 2],
activations = ['sigmoid', None], cost_func='log', regularization='l2', lamb=lamb)
#######################################################
###########Spliting data#######################
#######################################################
nn.split_data(frac=0.5, shuffle=True)
nn_l1.split_data(frac=0.5, shuffle=True)
nn_l2.split_data(frac=0.5, shuffle=True)
#######################################################
###########Training network#######################
#######################################################
nn.TrainNN(epochs = 250, eta0 = 0.05, n_print=250)
nn_l1.TrainNN(epochs = 250, eta0 = 0.05, n_print=250)
nn_l2.TrainNN(epochs = 250, eta0 = 0.05, n_print=250)
#######################################################
###########Error and accuracies ols#######################
#######################################################
ypred_train = nn.feed_forward(X_train, isTraining=False)
ypred_test = nn.feed_forward(X_test, isTraining=False)
ypred_crit = nn.feed_forward(X_crit, isTraining=False)
errors['ols']['Train'].append(nn.cost_function(Y_train, ypred_train))
errors['ols']['Test'].append(nn.cost_function(Y_test, ypred_test))
errors['ols']['Crit'].append(nn.cost_function(Y_crit, ypred_crit))
accuracies['ols']['Train'].append(nn.accuracy(Y_train, ypred_train))
accuracies['ols']['Test'].append(nn.accuracy(Y_test, ypred_test))
accuracies['ols']['Crit'].append(nn.accuracy(Y_crit, ypred_crit))
#######################################################
###########Error and accuracies l1#######################
#######################################################
ypred_train = nn_l1.feed_forward(X_train, isTraining=False)
ypred_test = nn_l1.feed_forward(X_test, isTraining=False)
ypred_crit = nn_l1.feed_forward(X_crit, isTraining=False)
errors['l1']['Train'].append(nn_l1.cost_function(Y_train, ypred_train))
errors['l1']['Test'].append(nn_l1.cost_function(Y_test, ypred_test))
errors['l1']['Crit'].append(nn_l1.cost_function(Y_crit, ypred_crit))
accuracies['l1']['Train'].append(nn_l1.accuracy(Y_train, ypred_train))
accuracies['l1']['Test'].append(nn_l1.accuracy(Y_test, ypred_test))
accuracies['l1']['Crit'].append(nn_l1.accuracy(Y_crit, ypred_crit))
#######################################################
###########Error and accuracies l2#######################
#######################################################
ypred_train = nn_l2.feed_forward(X_train, isTraining=False)
ypred_test = nn_l2.feed_forward(X_test, isTraining=False)
ypred_crit = nn_l2.feed_forward(X_crit, isTraining=False)
errors['l2']['Train'].append(nn_l2.cost_function(Y_train, ypred_train))
errors['l2']['Test'].append(nn_l2.cost_function(Y_test, ypred_test))
errors['l2']['Crit'].append(nn_l2.cost_function(Y_crit, ypred_crit))
accuracies['l2']['Train'].append(nn_l2.accuracy(Y_train, ypred_train))
accuracies['l2']['Test'].append(nn_l2.accuracy(Y_test, ypred_test))
accuracies['l2']['Crit'].append(nn_l2.accuracy(Y_crit, ypred_crit))
datasetnames = ['Train', 'Test']
errfig, errax = plt.subplots()
accfig, accax = plt.subplots()
colors = ['#1f77b4', '#ff7f0e', '#2ca02c']
linestyles = ['-', '--']
for i, key in enumerate(accuracies):
for j, name in enumerate(datasetnames):
errax.set_xlabel(r'$\lambda$')
errax.set_ylabel('Error')
errax.semilogx(lambs[:-2], errors[str(key)][str(name)][:-2],
color=colors[i], linestyle=linestyles[j], label=str(key).capitalize()+'_'+str(name))
errax.legend()
accax.set_xlabel(r'$\lambda$')
accax.set_ylabel('Accuracy')
accax.semilogx(lambs, accuracies[str(key)][str(name)],
color=colors[i], linestyle=linestyles[j], label=str(key).capitalize()+'_'+str(name))
accax.legend()
critfig, critax = plt.subplots()
critax.set_xlabel(r'$\lambda$')
critax.set_ylabel('Accuracy')
for i, key in enumerate(accuracies):
critax.semilogx(lambs, accuracies[str(key)]['Crit'],
label=str(key).capitalize()+'_Crit')
critax.legend()
plt.show()
def best_architecture():
"""
Finding best architectures for neural network
"""
#######################################################
###############Defining parameters#####################
#######################################################
# lambdas = np.logspace(-4, 0, 5)
lambdas = [0.01]
nodes = [5]
regularizations = [None]
n_samples = [10, 20]
activation_functions = [['relu', None]]
df = pd.DataFrame()
########################################################
################Fetching Data###########################
########################################################
X, Y, X_critical, Y_critical = fetch_data()
X, Y = shuffle(X, Y)
X_critical, Y_critical = shuffle(X_critical, Y_critical)
for sample in n_samples:
print(sample)
X_train = X[:sample]; Y_train = Y[:sample]
X_test = X[sample:2*sample]; Y_test = Y[sample:2*sample]
X_crit = X_critical[:sample]; Y_crit = Y_critical[:sample]
for reg in regularizations:
print(reg)
for activation in activation_functions:
print(activation)
for n in nodes:
print(n)
node = [X.shape[1], 2]
node[1:1] = [n]*(len(activation)-1)
print(node)
for lamb in lambdas:
print(lamb)
nn = NeuralNet(
X_train,
Y_train,
nodes = node,
activations = activation,
cost_func='log',
regularization=reg,
lamb=lamb)
nn.TrainNN(epochs = 100)
ypred_train = nn.feed_forward(X_train, isTraining=False)
ypred_test = nn.feed_forward(X_test, isTraining=False)
ypred_crit = nn.feed_forward(X_crit, isTraining=False)
df = df.append({
'Sample size': sample,
'Lambda': lamb,
'Regularization': reg,
'Nodes': n,
'Activation': (len(activation)-1)*activation[0],
'Train error': nn.cost_function(Y_train, ypred_train),
'Test error': nn.cost_function(Y_test, ypred_test),
'Critical error': nn.cost_function(Y_crit, ypred_crit),
'Train accuracy':nn.accuracy(Y_train, ypred_train),
'Test accuracy': nn.accuracy(Y_test, ypred_test),
'Critical accuracy': nn.accuracy(Y_crit, ypred_crit)
}, ignore_index=True)
df.to_csv('best_architecture.csv', index_label='Index')
import sys
sys.path.append("network/")
from NN import NeuralNet
from metrics import gain_chart, prob_acc
#X, Y = get_data(normalized = False, standardized = False)
X, Y = get_data()
#with open("nn_arch.txt", "w+") as f:
f.write("Activation | Hidden layers | Nodes | Area Test | R2 Test | Error rate \n")
f.write("-"*70)
for act in ['tanh', 'sigmoid', 'relu']:
for size in [5, 10, 20, 50, 100]:
for n_lay in [1, 2, 3]:
nn = NeuralNet(X, Y.flatten(), nodes = [23] + [size]*n_lay + [2], \
activations = [act]*n_lay + [None], cost_func = 'log')
nn.split_data(frac = 0.5, shuffle = True)
nn.TrainNN(epochs = 2000, batchSize = 200, eta0 = 0.01, n_print = 100)
ypred_test = nn.feed_forward(nn.xTest, isTraining=False)
acc = nn.accuracy(nn.yTest, ypred_test)
err_rate = 1 - acc/100
area = gain_chart(nn.yTest, ypred_test, plot=False)
R2 = prob_acc(nn.yTest, ypred_test, plot=False)
f.write("\n %s | %i | %i | %.5f | %.5f | %.3f "\
%(act, n_lay, size, area, R2, err_rate))
f.write("\n" + "-"*70)