def activation_functions_vs_errors():
# load dataset
train_x, train_t, val_x, val_t, test_x, test_t = load_dataset()
# create l-dim network by just adding num of neurons in layer_dim
# first and last elements represent input and output layers dim
layer_dim = [1,50,50,50,50,50,1]
# add activation functions name here.
# input layer activation function is None
activations = [[None, 'sigmoid','sigmoid','sigmoid','sigmoid','sigmoid','identity'],
[None, 'tanh','tanh','tanh','tanh','tanh','identity'],
[None, 'relu','relu','relu','relu','relu','identity'],
[None, 'lrelu','lrelu','lrelu','lrelu','lrelu','identity']]
assert len(layer_dim) == len(activations[0]), "layer dim or activation is missing.."
# hyper parameters of neural network
learning_rate = 1e-3
num_epochs = 100
mini_batch_size = 25
train_loss = []
test_loss = []
for l in range(4):
nn = NeuralNetwork(layer_dim, activations[l], learning_rate, num_epochs, mini_batch_size)
# train neural network
nn.train(train_x, train_t, val_x, val_t)
# test neural network
trainloss, _ = nn.test(train_x, train_t)
train_loss.append(trainloss)
testloss, test_output = nn.test(test_x, test_t)
test_loss.append(testloss)
def activation_functions_vs_hidden_layers():
# load dataset
train_x, train_t, val_x, val_t, test_x, test_t = load_dataset()
# create l-dim network by just adding num of neurons in layer_dim
# first and last elements represent input and output layers dim
layer_dim = [[1,50,1],[1,50,50,1],[1,50,50,50,1],[1,50,50,50,50,50,1],[1,50,50,50,50,50,50,50,50,50,50,1]]
# add activation functions name here.
# input layer activation function is None
activations = [[[None, 'sigmoid','identity'],
[None, 'tanh','identity'],
[None, 'relu','identity'],
[None, 'lrelu','identity']],
[[None, 'sigmoid','sigmoid','identity'],
[None, 'tanh','tanh','identity'],
[None, 'relu','relu','identity'],
[None, 'lrelu','lrelu','identity']],
[[None, 'sigmoid','sigmoid','sigmoid','identity'],
[None, 'tanh','tanh','tanh','identity'],
[None, 'relu','relu','relu','identity'],
[None, 'lrelu','lrelu','lrelu','identity']],
[[None, 'sigmoid','sigmoid','sigmoid','sigmoid','sigmoid','identity'],
[None, 'tanh','tanh','tanh','tanh','tanh','identity'],
[None, 'relu','relu','relu','relu','relu','identity'],
[None, 'lrelu','lrelu','lrelu','lrelu','lrelu','identity']],
[[None, 'sigmoid','sigmoid','sigmoid','sigmoid','sigmoid','sigmoid','sigmoid','sigmoid','sigmoid','sigmoid','identity'],
[None, 'tanh','tanh','tanh','tanh','tanh','tanh','tanh','tanh','tanh','tanh','identity'],
[None, 'relu','relu','relu','relu','relu','relu','relu','relu','relu','relu','identity'],
[None, 'lrelu','lrelu','lrelu','lrelu','lrelu','lrelu','lrelu','lrelu','lrelu','lrelu','identity']]]
#hyper parameters of neural network
learning_rate = 1e-3
epochs = 100
mini_batch_size = 10
for i in range(5):
for j in range(4):
nn = NeuralNetwork(layer_dim[i], activations[i][j], learning_rate, num_epochs, mini_batch_size)
# train neural network
nn.train(train_x, train_t, val_x, val_t)
trainloss, _ = nn.test(train_x, train_t)
train_loss.append(trainloss)
testloss, test_output = nn.test(test_x, test_t)
# test neural network
if(j == 0):
test_error_sigmoid.append(testloss)
elif(j == 1):
test_error_tanh.append(testloss)
elif(j == 2):
test_error_relu.append(testloss)
else:
test_error_lrelu.append(testloss)
import numpy as np
import matplotlib.pyplot as plt
from neural_network import *
from f_load_dataset import load_dataset
# load dataset
train_x, train_t, test_x, test_t = load_dataset()
#idx = 0
#minibatch_input = train_x[idx:idx + 32,:]
#minibatch_target = train_t[idx:idx + 32,:]
# create l-dim network by just adding num of neurons in layer_dim
# first and last elements represent input and output layers dim
layer_dim = [784, 5, 5, 2]
# add activation functions name here.
# input layer activation function is None
activations = [None, 'sigmoid', 'sigmoid', 'softmax']
assert len(layer_dim) == len(
activations), "layer dim or activation is missing.."
# hyper parameters of neural network
learning_rate = 1e-3
num_epochs = 100
mini_batch_size = 32
nn = NeuralNetwork(layer_dim, activations, learning_rate, num_epochs,
mini_batch_size)
# train neural network
nn.train(train_x, train_t)
import numpy as np
import matplotlib.pyplot as plt
from neural_network import *
from f_load_dataset import load_dataset
# load dataset
batch_size = 100
train_loader, test_loader = load_dataset(batch_size)
# create l-dim network by just adding num of neurons in layer_dim
# first and last elements represent input and output layers dim
layer_dim = [784, 50, 50, 10]
# add activation functions name here.
# input layer activation function is None
activations = [None, 'relu', 'relu', 'identity']
assert len(layer_dim) == len(
activations), "layer dim or activation is missing.."
# hyper parameters of neural network
learning_rate = 1e-1
num_epochs = 15
#mini_batch_size = 10
# train neural network
train(train_loader, learning_rate, num_epochs, layer_dim, activations)
# test neural network
test(test_loader)