def test(): #np.random.seed(42) #X = np.matrix(data[X_names]) #y = np.matrix(data[y_names]) nn = NeuralNetwork(learning_rate=0.4) nn.add_layer(Layer(30)) nn.fit(X, matrix, epochs=10000, batch_size=32)
def create_model(hyperparams):
lr = hyperparams['eta']
l2 = hyperparams['lambda']
dim_hid = int(hyperparams['hidden_nodes'])
add_layer=hyperparams['add_layer']
activation_function1=hyperparams['activation_function1']
activation_function2=hyperparams['activation_function2']
optimizer=hyperparams['optimizer']
module = __import__('kernel_initialization')
KernelInit = getattr(module, hyperparams['init_weights'])
model = NeuralNetwork(loss='mse', metric='mee')
model.add_layer(dim_hid, input_dim=INPUT_DIM, activation=activation_function1, kernel_initialization=KernelInit())
if(add_layer==True):
dim_hid_2=int(hyperparams['hidden_nodes2'])
activation_function2=hyperparams['activation_function2']
model.add_layer(dim_hid_2,activation=activation_function2,kernel_initialization=KernelInit())
model.add_layer(OUTPUT_DIM, activation='linear', kernel_initialization=KernelInit())
if(optimizer=='SGD'):
mom = hyperparams['alpha']
model.compile(optimizer=SGD(lr=lr, mom=mom,l2=l2, nesterov=True))
if (optimizer == 'RMSprop'):
rho = hyperparams['rho']
model.compile(optimizer=RMSprop(lr=lr, moving_average=rho, l2=l2))
if (optimizer == 'Adam'):
beta_1 = hyperparams['beta_1']
beta_2 = hyperparams['beta_2']
model.compile(optimizer=Adam(lr=lr, beta_1=beta_1,beta_2=beta_2, l2=l2))
return model
def create_model(hyperparams):
lr = hyperparams['eta']
mom = hyperparams['alpha']
l2 = hyperparams['lambda']
dim_hid = int(hyperparams['hidden_nodes'])
_fun = hyperparams['functions']
kernel_init = hyperparams['kernel_init']
module = __import__('kernel_initialization')
KernelInit = getattr(module, kernel_init)
model = NeuralNetwork(loss='mse', metric='mee')
model.add_layer(dim_hid,
input_dim=INPUT_DIM,
activation=_fun,
kernel_initialization=KernelInit())
model.add_output_layer(2,
activation=_fun,
kernel_initialization=KernelInit())
model.compile(lr=lr, momentum=mom, l2=l2)
return model
def perform_two_layers_nn(train_input, train_output, test_input, test_output):
print(
'Perform two layers neural network [784 x 300 x 10] with sigmoid at start and softmax at the end'
)
nn = NeuralNetwork()
nn.add_layer(
Layer(shape=(784, 300),
activation_function=ActivationFunction.SIGMOID))
nn.add_layer(
Layer(shape=(300, 10), activation_function=ActivationFunction.SOFTMAX))
start = time()
print('Start pre-training')
nn.learn(train_input[:100], train_output[:100], 0.01, 0.49, 100)
print('Start main training')
nn.learn(train_input, train_output, 0.1, 0.7, 100)
nn.learn(train_input, train_output, 0.01, 0.8, 100)
val = time() - start
print("Time:", val)
print("Mse: ", nn.mse(test_input, test_output))
print("Accuracy: ", nn.accuracy(test_input, test_output))
return val
def create_model(hyperparams):
_module_kernelinit = __import__('kernel_initialization')
lr = hyperparams['eta']
mom = hyperparams['momentum']
l2 = hyperparams['lambda']
beta_1 = hyperparams['beta_1']
beta_2 = hyperparams['beta_2']
act_fun_1 = hyperparams['activation_function1']
hidden_nodes1 = hyperparams['hidden_nodes']
kernel_init = getattr(_module_kernelinit, hyperparams['init_weights'])
optimizer_str = hyperparams['optimizer']
model = NeuralNetwork(LOSS, METRIC)
model.add_layer(hidden_nodes1, input_dim=INPUT_DIM, activation=act_fun_1, kernel_initialization=kernel_init())
if hyperparams['add_layer']:
act_fun_2 = hyperparams['activation_function2']
hidden_nodes2 = hyperparams['hidden_nodes2']
model.add_layer(hidden_nodes2, activation=act_fun_2, kernel_initialization=kernel_init())
model.add_layer(OUTPUT_DIM, activation='linear', kernel_initialization=kernel_init())
if optimizer_str == "SGD":
if hyperparams['lr_linear_decay']:
decay_lr_par = {
"epoch_tau": hyperparams['epoch_tau'],
"epsilon_tau_perc": hyperparams['epsilon_tau_perc'],
"init_lr": lr
}
optimizer = SGD(lr=lr, mom=mom, l2=l2, decay_lr=decay_lr_par)
else:
optimizer = SGD(lr=lr, mom=mom, l2=l2)
elif optimizer_str == "RMSprop":
optimizer = RMSprop(lr=lr, moving_average=beta_1, l2=l2)
elif optimizer_str == "Adam":
optimizer = Adam(lr=lr, beta_1=beta_1, beta_2=beta_2, l2=l2)
else:
raise ValueError("You have to choose an existing optimizer")
model.compile(optimizer=optimizer)
return model
dim_in = one_hot
dim_hid = 3
n_attemps = 5
res_losses = np.zeros(n_attemps)
res_metrics = np.zeros(n_attemps)
res_losses_ts = np.zeros(n_attemps)
res_metrics_ts = np.zeros(n_attemps)
for i in range(n_attemps):
model = NeuralNetwork('mse', 'accuracy1-1')
model.add_layer(dim_hid,
input_dim=dim_in,
activation='sigmoid',
kernel_initialization=RandomNormalInitialization())
model.add_layer(dim_out,
activation='tanh',
kernel_initialization=RandomNormalInitialization())
model.compile(optimizer=SGD(lr=0.5, mom=0.8, l2=0.01))
history = model.fit(X_train,
Y_train,
200,
X_train.shape[0],
ts=(X_test, Y_test),
verbose=True)
if i == 0:
write_results(res=None,
# This is the configuration of the present experiment.
X, y, X_test, y_test = load_dataset("CasosArtrite2.csv",
input_dimension=17,
test_ratio=0.3)
epochs = 10000
model_data = np.vstack([np.arange(0, epochs), np.zeros([4, epochs])])
# ============== end config ==============
nn = NeuralNetwork(learning_rate=0.2)
nn.add_layer(Layer(20))
nn.initialize(X.shape[1], y.shape[1])
plt.ion()
for k in model_data[0, :]:
k = int(k)
# Train for a single epoch
for epoch, train_loss, train_acc in nn.fit(X, y, epochs=1):
model_data[1, k] = train_loss
model_data[2, k] = train_acc
# Then make a validation test
error, acc, out = nn.evaluate(X_test, y_test)
model_data[3, k] = error
PERC_VL = 0.25
PERC_TS = 0.25
set_style_plot()
parser = Cup_parser(DIR_CUP + PATH_TR)
data, targets = parser.parse(INPUT_DIM, OUTPUT_DIM)
parser_ts = Cup_parser(DIR_CUP + PATH_TS)
data_ts, _ = parser_ts.parse(INPUT_DIM, 0)
X_train, Y_train = data, targets
X_test = data_ts
model = NeuralNetwork('mse', 'mee')
model.add_layer(30,
input_dim=X_train.shape[1],
activation='tanh',
kernel_initialization=XavierNormalInitialization())
model.add_layer(30,
activation='tanh',
kernel_initialization=XavierNormalInitialization())
model.add_layer(Y_train.shape[1],
activation='linear',
kernel_initialization=XavierNormalInitialization())
model.compile(optimizer=RMSprop(lr=0.0001, moving_average=0.9, l2=0.0005))
model.fit(X_train,
Y_train,
500,
batch_size=64,
shuffle=True,
OUTPUT_DIM = 2
PERC_TS = 0.25
PERC_VL = 0.25
set_style_plot()
parser = Cup_parser(DIR_CUP + PATH_TR)
data, targets = parser.parse(INPUT_DIM, OUTPUT_DIM)
X_train, Y_train, X_val, Y_val, X_test, Y_test = train_val_test_split(
data, targets, val_size=PERC_VL, test_size=PERC_TS, shuffle=True)
model = NeuralNetwork('mse', 'mee')
model.add_layer(40,
input_dim=X_train.shape[1],
activation='tanh',
kernel_initialization=RandomNormalInitialization())
model.add_layer(30,
activation='sigmoid',
kernel_initialization=RandomNormalInitialization())
model.add_layer(Y_train.shape[1],
activation='linear',
kernel_initialization=RandomNormalInitialization())
model.compile(optimizer=RMSprop(lr=0.0001, moving_average=0.9, l2=0.0004))
model.fit(X_train,
Y_train,
500,
batch_size=64,
vl=(X_val, Y_val),
def single_run(hidden_neurons, learning_rate, hidden_activation,
output_activation):
model_data = np.vstack([np.arange(0, epochs), np.zeros([4, epochs])])
nn = NeuralNetwork(learning_rate=learning_rate)
nn.add_layer(Layer(hidden_neurons, activation=hidden_activation))
nn.initialize(X.shape[1], y.shape[1], output_activation=output_activation)
epoch_values = model_data[0, :]
start = time.time()
for k in epoch_values:
k = int(k)
# Train for a single epoch
for epoch, train_loss, train_acc in nn.fit(X, y, epochs=1):
model_data[1, k] = train_loss
model_data[2, k] = train_acc
# Then make a validation test
error, acc, out = nn.evaluate(X_test, y_test)
model_data[3, k] = error
model_data[4, k] = acc
#print("Validation: Loss {0}, accuracy {1}".format(error, acc))
elapsed = time.time() - start
plt.plot(model_data[0, :], model_data[1, :], label="Erro (treinamento)")
plt.plot(model_data[0, :], model_data[3, :], label="Erro (validação)")
plt.legend(loc="lower right")
plt.xlabel("Época")
plt.title("Erro ao longo do tempo")
plt.show()
plt.clf()
plt.plot(model_data[0, :],
model_data[2, :],
label="Acurácia (treinamento)")
plt.plot(model_data[0, :], model_data[4, :], label="Acurácia (validação)")
plt.legend(loc="lower right")
plt.xlabel("Época")
plt.title("Acurácia ao longo do tempo")
plt.show()
results = {
'best_training_loss': np.min(model_data[1, :]),
'best_training_loss_epoch': np.argmin(model_data[1, :]),
'best_validation_loss': np.min(model_data[3, :]),
'best_validation_loss_epoch': np.argmin(model_data[3, :]),
'best_training_acc': np.max(model_data[2, :]),
'best_training_acc_epoch': np.argmax(model_data[2, :]),
'best_validation_acc': np.max(model_data[4, :]),
'best_validation_acc_epoch': np.argmax(model_data[4, :]),
'final_validation_loss': model_data[3, -1],
'final_validation_acc': model_data[4, -1],
'final_training_loss': model_data[1, -1],
'final_training_acc': model_data[2, -1],
'learning_rate': learning_rate,
'hidden_neurons': hidden_neurons,
'hidden_activation': hidden_activation,
'output_activation': output_activation,
'elapsed_seconds': elapsed
}
return results
DIR_CUP = './data/cup/'
PATH_TR = 'ML-CUP19-TR.csv'
PATH_TS = 'ML-CUP19-TS.csv'
INPUT_DIM = 20
OUTPUT_DIM = 2
results = list()
parser = Cup_parser(DIR_CUP + PATH_TR)
data, targets = parser.parse(INPUT_DIM, OUTPUT_DIM)
X_train, Y_train, X_val, Y_val, X_test, Y_test = train_val_test_split(
data, targets, val_size=0.25, test_size=0.25, shuffle=True)
model_1 = NeuralNetwork('mse', 'mee')
model_1.add_layer(30,
input_dim=20,
activation='sigm',
kernel_initialization=RandomInitialization())
model_1.add_layer(40,
activation="tanh",
kernel_initialization=RandomInitialization())
model_1.add_layer(2,
activation='linear',
kernel_initialization=RandomInitialization())
optimizer = RMSprop(lr=0.001, moving_average=0.099, l2=0.0005)
model_1.compile(optimizer=optimizer)
model_1.fit(X_train,
Y_train,
early_stopping=None,
epochs=500,
batch_size=32,
vl=(X_val, Y_val),
dim_in = one_hot
dim_hid = 4
n_attemps = 5
res_losses = np.zeros(5)
res_metrics = np.zeros(5)
res_losses_ts = np.zeros(n_attemps)
res_metrics_ts = np.zeros(n_attemps)
for i in range(n_attemps):
model = NeuralNetwork('mse', 'accuracy')
model.add_layer(dim_hid,
input_dim=dim_in,
activation='relu',
kernel_initialization=HeInitialization())
model.add_layer(dim_out,
activation='sigmoid',
kernel_initialization=RandomNormalInitialization(0.5, 0.1))
model.compile(optimizer=SGD(lr=0.6, mom=0.8))
history = model.fit(X_train,
Y_train,
350,
X_train.shape[0],
ts=(X_test, Y_test),
verbose=True)
if i == 0:
write_results(res=None,
model=model,
def test_network_lesson09(self):
"""
Test initializing a neural NeuralNetwork
"""
input_shape = (28, 28, 1)
output_count = 10
network = NeuralNetwork('convnet for MNIST', input_shape, output_count)
# normal distribution parameters for random weights
mean = 0.0
stddev = 0.1
# General convolution shapes and parameters common to all convolutional layers
conv_stride_shape = (1, 1)
conv_pad_shape = (0, 0)
conv_pad_type = 'SAME'
pool_stride_shape = (2, 2)
pool_shape = (2, 2)
pool_pad_type = 'SAME'
activation = 'relu'
# Kernel depths and sizes for each convolution layer
depths = [32, 64, 128]
kernel_shapes = [(5, 5, depths[0]), (5, 5, depths[1]), (5, 5, depths[2])]
conv_layer_count = len(depths)
# Expected values for assertions
after_conv_output_shapes = [(28, 28, depths[0]), (14, 14, depths[1]), (7, 7, depths[2])]
after_pool_output_shapes = [(14, 14, depths[0]), (7, 7, depths[1]), (4, 4, depths[2])]
# Create convolutional layers
conv = None
for i in range(conv_layer_count):
name = 'l{:d}'.format(i)
if i > 0:
input_shape = conv.output_shape
conv = ConvolutionalLayer(name, input_shape, kernel_shapes[i], conv_stride_shape, \
conv_pad_shape, conv_pad_type, activation)
self.assertEqual(after_conv_output_shapes[i], conv.output_shape)
conv.add_pooling('max', pool_shape, pool_stride_shape, pool_pad_type)
self.assertEqual(after_pool_output_shapes[i], conv.output_shape)
network.add_layer(conv, mean, stddev)
# Create linear layers
# Output sizes for linear layers
linear_input_sizes = [4 * 4 * 128, 512]
linear_output_sizes = [512, 10]
linear_activations = ['tanh', None]
for i, input_size in enumerate(linear_input_sizes):
layer_index = i + conv_layer_count
name = 'l{:d}'.format(layer_index)
linear = LinearLayer(name, input_size, linear_output_sizes[i], linear_activations[i])
network.add_layer(linear, mean, stddev)
# MNIST classify 10 digits
network.define_network()
learning_rate = 0.001
network.define_operations(learning_rate, 'gradient_descent')
epochs = 10
batch_size = 128
saver = tf.train.Saver()
(train_inputs, train_labels, valid_inputs, valid_labels, test_inputs, test_labels) = \
get_mnist_data()
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
network.train_with_validate(sess, train_inputs, train_labels, valid_inputs, \
valid_labels, epochs, batch_size)
test_accuracy = network.evaluate_in_batches(sess, test_inputs, test_labels, batch_size)
print("Test accuracy:", test_accuracy)
saver.save(sess, 'convnet')
print("Model saved")
def test_network_lenet(self):
"""
Test using the lenet5 architecture
"""
input_shape = (32, 32, 1)
output_count = 10
network = NeuralNetwork('lenet5 for MNIST', input_shape, output_count)
# normal distribution parameters for random weights
mean = 0.0
stddev = 0.1
# General convolution shapes and parameters common to all convolutional layers
conv_stride_shape = (1, 1)
conv_pad_shape = (0, 0)
conv_pad_type = 'VALID'
pool_stride_shape = (2, 2)
pool_shape = (2, 2)
pool_pad_type = 'VALID'
activation = 'relu'
# Kernel depths and sizes for each convolution layer
depths = [6, 16]
kernel_shapes = [(5, 5, depths[0]), (5, 5, depths[1])]
conv_layer_count = len(depths)
# Create convolutional layers
conv = None
for i in range(conv_layer_count):
name = 'l{:d}'.format(i)
if i > 0:
input_shape = conv.output_shape
conv = ConvolutionalLayer(name, input_shape, kernel_shapes[i], conv_stride_shape, \
conv_pad_shape, conv_pad_type, activation)
conv.add_pooling('max', pool_shape, pool_stride_shape, pool_pad_type)
network.add_layer(conv, mean, stddev)
# Linear layer dimensions
linear_input_sizes = [400, 120, 84]
linear_output_sizes = [120, 84, 10]
linear_activations = ['relu', 'relu', None]
# Create linear layers
for i, input_size in enumerate(linear_input_sizes):
layer_index = i + conv_layer_count
name = 'l{:d}'.format(layer_index)
linear = LinearLayer(name, input_size, linear_output_sizes[i], linear_activations[i])
linear.init_weights_and_biases(mean, stddev)
network.add_layer(linear, mean, stddev)
network.define_network()
learning_rate = 0.001
network.define_operations(learning_rate, 'adam')
# Prepare data
(train_inputs, train_labels, valid_inputs, valid_labels, test_inputs, test_labels) = \
get_mnist_data(padding=(2, 2))
epochs = 10
batch_size = 128
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
network.train_with_validate(sess, train_inputs, train_labels, valid_inputs, \
valid_labels, epochs, batch_size)
test_accuracy = network.evaluate_in_batches(sess, test_inputs, test_labels, batch_size)
print("Test accuracy:", test_accuracy)
saver.save(sess, 'lenet')
print("Model saved")
X = data
print(X.shape)
## Encoding des labels en onehot
y_onehot.zero_()
y_onehot.scatter_(1, target.view(-1, 1), 1)
print(y_onehot.shape)
break
X = X.view(batch_size, 28 * 28)
y = y_onehot
def one_vs_one(X, y, c1, c2, test_size=0.3):
X = torch.cat((X[y[:, c1] == 1], X[y[:, c2] == 1]))
y = torch.cat((y[y[:, c1] == 1], y[y[:, c2] == 1]))
inds = list(range(len(X)))
np.random.shuffle(inds)
sep = int(test_size * len(X))
return X[inds][:sep], X[inds][sep:], y[inds][:sep], y[inds][sep:]
X_train, X_test, y_train, y_test = one_vs_one(X, y, 3, 5)
nn = NeuralNetwork(loss=MSE())
nn.add_layer([ModuleLinear(len(X[0]), 10), Activation(sigmoid, sigmoid_g)])
costs = nn.fit(X_train, y_train, mode='mini_batch')
plt.plot(range(len(costs)), costs)
plt.show()
#Loop through the etas and lambdas
for i, eta in enumerate(eta_vals):
for j, lmbd in enumerate(lmbd_vals):
#Make neural network
ffnn = NeuralNetwork(X_train,
y_train,
batch_size=int(n_samples/m),
n_categories = 1,
epochs = 100,
n_hidden_neurons = 20,
eta = eta,
lmbd = lmbd,
input_activation = 'sigmoid',
output_activation = 'linear',
cost_function = 'MSE')
ffnn.add_layer(20, activation_method = 'sigmoid')
ffnn.add_layer(20, activation_method = 'sigmoid')
#Train network
ffnn.train()
#Save predictions
y_tilde_train = ffnn.predict(X_train)
y_tilde_test = ffnn.predict(X_test)
#Save metrics into exportable matrices
train_mse[i][j], train_R2[i][j] = statistics.calc_statistics(y_train, y_tilde_train)
test_mse[i][j], test_R2[i][j] = statistics.calc_statistics(y_test, y_tilde_test)
if best_train_mse > train_mse[i][j]:
best_train_mse = train_mse[i][j]
parser = Monks_parser(path_tr, path_ts)
X_train, Y_train, X_test, Y_test = parser.parse(dim_in, dim_out, one_hot)
#Y_train = change_output_value(Y_train, 0, -1)
#Y_test = change_output_value(Y_test, 0, -1)
#X_train, Y_train, X_val, Y_val = train_test_split(X_train, Y_train, test_size=0.25)
dim_in = one_hot
dim_hid = 4
model = NeuralNetwork('mse', 'accuracy')
model.add_layer(dim_hid, input_dim=dim_in, activation='relu', kernel_initialization=RandomUniformInitialization(-0.5, 0.5))
model.add_output_layer(dim_out, activation='sigmoid', kernel_initialization=RandomUniformInitialization(-0.5, 0.5))
model.compile(0.8, 0.7, 0.0)
model.fit(
X_train, Y_train, 500, X_train.shape[0], ts=(X_test, Y_test),
verbose=True, tol=1e-2
)
err_tr, acc_tr = model.evaluate(X_train, Y_train)
err_ts, acc_ts = model.evaluate(X_test, Y_test)
acc_tr = acc_tr*100
acc_ts = acc_ts*100
errors = [err_tr, err_ts]
accuracy = [acc_tr, acc_ts]