def plot():
"""
Ploting the matrix of the coupling constants
"""
n = [400, 10000]
fig, axarr = plt.subplots(nrows=2, ncols=3)
cmap_args=dict(vmin=-1., vmax=1., cmap='seismic')
for i in range(len(n)):
n_samples = n[i]
X = Data[0][:n_samples]
Y = Data[1][:n_samples]
regs = [None, 'l2','l1']
Js = np.zeros((3, L**2))
for j in range(3):
nn = NeuralNet(X,Y, nodes=[X.shape[1], 1], activations=[None], regularization=regs[j], lamb=0.1)
nn.TrainNN(epochs = 500, batchSize = 200, eta0 = 0.01, n_print = 50)
Js[j] = nn.Weights['W1'].flatten()
J_ols = Js[0].reshape(L,L)
J_ridge = Js[1].reshape(L,L)
J_lasso = Js[2].reshape(L,L)
axarr[i][0].imshow(J_ols,**cmap_args)
axarr[i][0].set_title('$\\mathrm{OLS}$',fontsize=16)
axarr[i][0].tick_params(labelsize=16)
axarr[i][1].imshow(J_ridge,**cmap_args)
axarr[i][1].set_title('$\\mathrm{Ridge},\ \\lambda=%.4f$' %(0.1),fontsize=16)
axarr[i][1].tick_params(labelsize=16)
im=axarr[i][2].imshow(J_lasso,**cmap_args)
axarr[i][2].set_title('$\\mathrm{LASSO},\ \\lambda=%.4f$' %(0.1),fontsize=16)
axarr[i][2].tick_params(labelsize=16)
divider = make_axes_locatable(axarr[i][2])
cax = divider.append_axes("right", size="5%", pad=0.05)
cbar=fig.colorbar(im, cax=cax)
cbar.ax.set_yticklabels(np.arange(-1.0, 1.0+0.25, 0.25),fontsize=14)
cbar.set_label('$J_{i,j}$',labelpad=-40, y=1.12,fontsize=16,rotation=0)
#fig.subplots_adjust(right=2.0)
plt.show()
def nn_resample(Pca: bool = 0,
smote: bool = 0,
over: bool = 0,
under: bool = 0,
X: np.ndarray = X,
Y: np.ndarray = Y) -> None:
np.random.seed(42)
if Pca:
pca = PCA(n_components=14)
X = pca.fit_transform(X)
##Initialize network###
node1 = X.shape[1]
nn = NeuralNet(X,
Y.flatten(),
nodes=[node1, 3, 2],
activations=['tanh', None],
cost_func='log',
regularization='l2',
lamb=0.001)
###Split Data###
nn.split_data(frac=0.5, shuffle=True)
##Choose resampling teqnique##
if smote:
# sm = SMOTE(random_state=42, sampling_strategy=1.0)
sm = SMOTE()
nn.xTrain, nn.yTrain = sm.fit_resample(nn.xTrain, nn.yTrain)
elif under:
resample = rs(nn.xTrain, nn.yTrain)
nn.xTrain, nn.yTrain = resample.Under()
elif over:
resample = rs(nn.xTrain, nn.yTrain)
nn.xTrain, nn.yTrain = resample.Over()
##Train network##
nn.TrainNN(epochs=100, batchSize=200, eta0=0.01, n_print=50)
def reg():
"""
Doing the linear regression using
a neural network
"""
lambdas = np.logspace(-4, 2, 7)
#lambdas = [0.01, 0.1]
n = len(lambdas)
train_mse = np.zeros((3,n))
test_mse = np.zeros((3,n))
regs = [None, 'l1', 'l2']
for j in range(3):
for i in range(n):
nn = NeuralNet(X,Y, nodes=[X.shape[1], 1], activations=[None], regularization=regs[j], lamb=lambdas[i])
nn.TrainNN(epochs = 1000, batchSize = 200, eta0 = 0.01, n_print = 10)
ypred_test = nn.feed_forward(nn.xTest, isTraining=False)
mse_test = nn.cost_function(nn.yTest, ypred_test)
test_mse[j,i] = mse_test
if j == 0:
test_mse[0].fill(test_mse[0,0])
break
plt.semilogx(lambdas, test_mse.T)
plt.xlabel(r'$\lambda$')
plt.ylabel('MSE')
plt.title("MSE on Test Set, %i samples" %(n_samples))
plt.legend(['No Reg', 'L1', 'L2'])
plt.grid(True)
plt.ylim([0, 25])
plt.show()
import numpy as np
from read_data import get_data
import sys
sys.path.append("network/")
from NN import NeuralNet
X, Y = get_data(normalized=False, standardized=True, file='droppedX6-X11.csv')
nn = NeuralNet(X,
Y.flatten(),
nodes=[18, 50, 50, 2],
activations=['tanh', 'tanh', None],
cost_func='log',
regularization='l2',
lamb=0.001)
nn.split_data(frac=0.5, shuffle=True, resample=True)
nn.TrainNN(epochs=1000, batchSize=200, eta0=0.01, n_print=10)
"""
Neural network calcualting logistic regression
"""
import sys
sys.path.append('network')
sys.path.append('../')
sys.path.append('../../')
import numpy as np
from fetch_2D_data import fetch_data
from NN import NeuralNet
X, Y, X_crit, Y_crit = fetch_data()
nn = NeuralNet(X,
Y,
nodes=[X.shape[1], 2],
activations=[None],
cost_func='log')
nn.TrainNN(epochs=200, batchSize=130, eta0=0.001, n_print=20)
ypred_crit = nn.feed_forward(X_crit, isTraining=False)
critError = nn.cost_function(Y_crit, ypred_crit)
critAcc = nn.accuracy(Y_crit, ypred_crit)
print("Critical error: %g, Critical accuracy: %g" % (critError, critAcc))
run on all the data available in
network/fetch_2D_data"""
import numpy as np
import sys
sys.path.append('network')
sys.path.append('../')
sys.path.append('../../')
from fetch_2D_data import fetch_data
from NN import NeuralNet
X, Y, X_crit, Y_crit = fetch_data()
### LOG-REG CASE
# nn = NeuralNet(X,Y, nodes = [X.shape[1], 2], activations = [None], cost_func='log')
# nn.split_data(frac=0.5, shuffle=True)
# nn.feed_forward(nn.xTrain)
# nn.backpropagation()
# nn.TrainNN(epochs = 100, eta = 0.001, n_print=5)
# 100% ACCURACY MADDAFAKKA
nn = NeuralNet(X,Y, nodes = [X.shape[1],10,2], activations = ['tanh',None],\
cost_func='log', regularization='l2', lamb=0.01)
nn.split_data(frac=0.5, shuffle=True)
nn.TrainNN(epochs=200, eta0=0.01, n_print=5)
ypred_crit = nn.feed_forward(X_crit, isTraining=False)
critError = nn.cost_function(Y_crit, ypred_crit)
critAcc = nn.accuracy(Y_crit, ypred_crit)
print("Critical error: %g, Critical accuracy: %g" % (critError, critAcc))
def best_architecture():
"""
Finding best architectures for neural network
"""
#######################################################
###############Defining parameters#####################
#######################################################
# lambdas = np.logspace(-4, 0, 5)
lambdas = [0.01]
nodes = [5]
regularizations = [None]
n_samples = [10, 20]
activation_functions = [['relu', None]]
df = pd.DataFrame()
########################################################
################Fetching Data###########################
########################################################
X, Y, X_critical, Y_critical = fetch_data()
X, Y = shuffle(X, Y)
X_critical, Y_critical = shuffle(X_critical, Y_critical)
for sample in n_samples:
print(sample)
X_train = X[:sample]; Y_train = Y[:sample]
X_test = X[sample:2*sample]; Y_test = Y[sample:2*sample]
X_crit = X_critical[:sample]; Y_crit = Y_critical[:sample]
for reg in regularizations:
print(reg)
for activation in activation_functions:
print(activation)
for n in nodes:
print(n)
node = [X.shape[1], 2]
node[1:1] = [n]*(len(activation)-1)
print(node)
for lamb in lambdas:
print(lamb)
nn = NeuralNet(
X_train,
Y_train,
nodes = node,
activations = activation,
cost_func='log',
regularization=reg,
lamb=lamb)
nn.TrainNN(epochs = 100)
ypred_train = nn.feed_forward(X_train, isTraining=False)
ypred_test = nn.feed_forward(X_test, isTraining=False)
ypred_crit = nn.feed_forward(X_crit, isTraining=False)
df = df.append({
'Sample size': sample,
'Lambda': lamb,
'Regularization': reg,
'Nodes': n,
'Activation': (len(activation)-1)*activation[0],
'Train error': nn.cost_function(Y_train, ypred_train),
'Test error': nn.cost_function(Y_test, ypred_test),
'Critical error': nn.cost_function(Y_crit, ypred_crit),
'Train accuracy':nn.accuracy(Y_train, ypred_train),
'Test accuracy': nn.accuracy(Y_test, ypred_test),
'Critical accuracy': nn.accuracy(Y_crit, ypred_crit)
}, ignore_index=True)
df.to_csv('best_architecture.csv', index_label='Index')
def plot_convergence():
"""
Ploting the convergence rate of the 6 best models
calculated in best_architectures.
"""
########################################################
################Fetching Data###########################
########################################################
sample = 10000
X, Y, X_critical, Y_critical = fetch_data()
X, Y = shuffle(X, Y)
X_critical, Y_critical = shuffle(X_critical, Y_critical)
X_train = X[:sample]; Y_train = Y[:sample]
X_test = X[sample:2*sample]; Y_test = Y[sample:2*sample]
X_crit = X_critical[:sample]; Y_crit = Y_critical[:sample]
#######################################################
###############Defining parameters#####################
#######################################################
best_architectures = [['tanh', None], ['tanh', None], ['tanh', None],
['tanh','tanh', None], ['tanh', 'tanh', 'tanh', None],
['sigmoid', 'sigmoid', None]]
nodes = [[X.shape[1], 100, 2], [X.shape[1], 50, 2], [X.shape[1], 10, 2],
[X.shape[1], 100, 100, 2], [X.shape[1], 100, 100, 100, 2],
[X.shape[1], 100, 100, 2]]
regularizations = ['l2', 'l2', 'l2', 'l2', 'l1', None]
lambdas = [0.1, 0.1, 0.1, 0.1, 0.001, 0.01]
df = pd.DataFrame()
convergence_dict = {}
walltime_list = []
i = 0
for activation, lamb, node, reg in zip(best_architectures, lambdas, nodes, regularizations):
nn = NeuralNet(
X_train,
Y_train,
nodes = node,
activations = activation,
cost_func='log',
regularization=reg,
lamb=lamb)
start = time()
nn.TrainNN(epochs = 100, n_print=1)
stop = time()
convergence = nn.convergence_rate
walltime = (stop - start)
convergence_dict[activation[0]*(len(activation)-1)+str(node[1])] = convergence['Test Accuracy']
walltime_list.append(walltime)
i += 1
fig, ax = plt.subplots()
for key in convergence_dict:
ax.plot(np.arange(101), convergence_dict[key], label=key)
ax.set_xlabel('Epochs')
ax.set_ylabel('Accuracy')
plt.legend()
# plt.savefig('../plots/convergence_rate.png')
figb, bx = plt.subplots()
names = list(convergence_dict.keys())
bx.bar(np.arange(len(walltime_list)), walltime_list, tick_label=names)
bx.set_ylabel('Walltime [s]')
plt.xticks(rotation=-20)
plt.savefig('../plots/walltime.png')
plt.tight_layout()
plt.show()
def plot_regularization():
"""
Ploting different models as a function of regularization strength
"""
#######################################################
###############Defining parameters#####################
#######################################################
lambs = np.logspace(-4, 0, 5)
# n_samples =[100, 1000, 4000, 10000]
n_samples = [3000]
# nodes = [10]
nodes=[50]
#######################################################
###############Fetching Data###########################
#######################################################
X, Y, X_critical, Y_critical = fetch_data()
X, Y = shuffle(X, Y)
X_critical, Y_critical = shuffle(X_critical, Y_critical)
for node in nodes:
for sample in n_samples:
X_train = X[:sample]; Y_train = Y[:sample]
X_test = X[sample:2*sample]; Y_test = Y[sample:2*sample]
X_crit = X_critical[:sample]; Y_crit = Y_critical[:sample]
#######################################################
###########Dictionaries for ploting####################
#######################################################
errors = {'ols': {'Train': [], 'Test': [], 'Crit': []},
'l1': {'Train': [], 'Test': [], 'Crit': []},
'l2': {'Train': [], 'Test': [], 'Crit': []}}
accuracies = {'ols': {'Train': [], 'Test': [], 'Crit': []},
'l1': {'Train': [], 'Test': [], 'Crit': []},
'l2': {'Train': [], 'Test': [], 'Crit': []}}
for lamb in lambs:
#######################################################
###########Initializing networks#######################
#######################################################
nn = NeuralNet( X_train, Y_train, nodes = [X.shape[1], node, 2],
activations = ['sigmoid', None], cost_func='log')
nn_l1 = NeuralNet( X_train, Y_train, nodes = [X.shape[1], node, 2],
activations = ['sigmoid', None], cost_func='log', regularization='l1', lamb=lamb)
nn_l2 = NeuralNet( X_train, Y_train, nodes = [X.shape[1], node, 2],
activations = ['sigmoid', None], cost_func='log', regularization='l2', lamb=lamb)
#######################################################
###########Spliting data#######################
#######################################################
nn.split_data(frac=0.5, shuffle=True)
nn_l1.split_data(frac=0.5, shuffle=True)
nn_l2.split_data(frac=0.5, shuffle=True)
#######################################################
###########Training network#######################
#######################################################
nn.TrainNN(epochs = 250, eta0 = 0.05, n_print=250)
nn_l1.TrainNN(epochs = 250, eta0 = 0.05, n_print=250)
nn_l2.TrainNN(epochs = 250, eta0 = 0.05, n_print=250)
#######################################################
###########Error and accuracies ols#######################
#######################################################
ypred_train = nn.feed_forward(X_train, isTraining=False)
ypred_test = nn.feed_forward(X_test, isTraining=False)
ypred_crit = nn.feed_forward(X_crit, isTraining=False)
errors['ols']['Train'].append(nn.cost_function(Y_train, ypred_train))
errors['ols']['Test'].append(nn.cost_function(Y_test, ypred_test))
errors['ols']['Crit'].append(nn.cost_function(Y_crit, ypred_crit))
accuracies['ols']['Train'].append(nn.accuracy(Y_train, ypred_train))
accuracies['ols']['Test'].append(nn.accuracy(Y_test, ypred_test))
accuracies['ols']['Crit'].append(nn.accuracy(Y_crit, ypred_crit))
#######################################################
###########Error and accuracies l1#######################
#######################################################
ypred_train = nn_l1.feed_forward(X_train, isTraining=False)
ypred_test = nn_l1.feed_forward(X_test, isTraining=False)
ypred_crit = nn_l1.feed_forward(X_crit, isTraining=False)
errors['l1']['Train'].append(nn_l1.cost_function(Y_train, ypred_train))
errors['l1']['Test'].append(nn_l1.cost_function(Y_test, ypred_test))
errors['l1']['Crit'].append(nn_l1.cost_function(Y_crit, ypred_crit))
accuracies['l1']['Train'].append(nn_l1.accuracy(Y_train, ypred_train))
accuracies['l1']['Test'].append(nn_l1.accuracy(Y_test, ypred_test))
accuracies['l1']['Crit'].append(nn_l1.accuracy(Y_crit, ypred_crit))
#######################################################
###########Error and accuracies l2#######################
#######################################################
ypred_train = nn_l2.feed_forward(X_train, isTraining=False)
ypred_test = nn_l2.feed_forward(X_test, isTraining=False)
ypred_crit = nn_l2.feed_forward(X_crit, isTraining=False)
errors['l2']['Train'].append(nn_l2.cost_function(Y_train, ypred_train))
errors['l2']['Test'].append(nn_l2.cost_function(Y_test, ypred_test))
errors['l2']['Crit'].append(nn_l2.cost_function(Y_crit, ypred_crit))
accuracies['l2']['Train'].append(nn_l2.accuracy(Y_train, ypred_train))
accuracies['l2']['Test'].append(nn_l2.accuracy(Y_test, ypred_test))
accuracies['l2']['Crit'].append(nn_l2.accuracy(Y_crit, ypred_crit))
datasetnames = ['Train', 'Test']
errfig, errax = plt.subplots()
accfig, accax = plt.subplots()
colors = ['#1f77b4', '#ff7f0e', '#2ca02c']
linestyles = ['-', '--']
for i, key in enumerate(accuracies):
for j, name in enumerate(datasetnames):
errax.set_xlabel(r'$\lambda$')
errax.set_ylabel('Error')
errax.semilogx(lambs[:-2], errors[str(key)][str(name)][:-2],
color=colors[i], linestyle=linestyles[j], label=str(key).capitalize()+'_'+str(name))
errax.legend()
accax.set_xlabel(r'$\lambda$')
accax.set_ylabel('Accuracy')
accax.semilogx(lambs, accuracies[str(key)][str(name)],
color=colors[i], linestyle=linestyles[j], label=str(key).capitalize()+'_'+str(name))
accax.legend()
critfig, critax = plt.subplots()
critax.set_xlabel(r'$\lambda$')
critax.set_ylabel('Accuracy')
for i, key in enumerate(accuracies):
critax.semilogx(lambs, accuracies[str(key)]['Crit'],
label=str(key).capitalize()+'_Crit')
critax.legend()
plt.show()