def test(): #np.random.seed(42) #X = np.matrix(data[X_names]) #y = np.matrix(data[y_names]) nn = NeuralNetwork(learning_rate=0.4) nn.add_layer(Layer(30)) nn.fit(X, matrix, epochs=10000, batch_size=32)
def create_model(hyperparams): lr = hyperparams['eta'] l2 = hyperparams['lambda'] dim_hid = int(hyperparams['hidden_nodes']) add_layer=hyperparams['add_layer'] activation_function1=hyperparams['activation_function1'] activation_function2=hyperparams['activation_function2'] optimizer=hyperparams['optimizer'] module = __import__('kernel_initialization') KernelInit = getattr(module, hyperparams['init_weights']) model = NeuralNetwork(loss='mse', metric='mee') model.add_layer(dim_hid, input_dim=INPUT_DIM, activation=activation_function1, kernel_initialization=KernelInit()) if(add_layer==True): dim_hid_2=int(hyperparams['hidden_nodes2']) activation_function2=hyperparams['activation_function2'] model.add_layer(dim_hid_2,activation=activation_function2,kernel_initialization=KernelInit()) model.add_layer(OUTPUT_DIM, activation='linear', kernel_initialization=KernelInit()) if(optimizer=='SGD'): mom = hyperparams['alpha'] model.compile(optimizer=SGD(lr=lr, mom=mom,l2=l2, nesterov=True)) if (optimizer == 'RMSprop'): rho = hyperparams['rho'] model.compile(optimizer=RMSprop(lr=lr, moving_average=rho, l2=l2)) if (optimizer == 'Adam'): beta_1 = hyperparams['beta_1'] beta_2 = hyperparams['beta_2'] model.compile(optimizer=Adam(lr=lr, beta_1=beta_1,beta_2=beta_2, l2=l2)) return model
def create_model(hyperparams): lr = hyperparams['eta'] mom = hyperparams['alpha'] l2 = hyperparams['lambda'] dim_hid = int(hyperparams['hidden_nodes']) _fun = hyperparams['functions'] kernel_init = hyperparams['kernel_init'] module = __import__('kernel_initialization') KernelInit = getattr(module, kernel_init) model = NeuralNetwork(loss='mse', metric='mee') model.add_layer(dim_hid, input_dim=INPUT_DIM, activation=_fun, kernel_initialization=KernelInit()) model.add_output_layer(2, activation=_fun, kernel_initialization=KernelInit()) model.compile(lr=lr, momentum=mom, l2=l2) return model
def perform_two_layers_nn(train_input, train_output, test_input, test_output): print( 'Perform two layers neural network [784 x 300 x 10] with sigmoid at start and softmax at the end' ) nn = NeuralNetwork() nn.add_layer( Layer(shape=(784, 300), activation_function=ActivationFunction.SIGMOID)) nn.add_layer( Layer(shape=(300, 10), activation_function=ActivationFunction.SOFTMAX)) start = time() print('Start pre-training') nn.learn(train_input[:100], train_output[:100], 0.01, 0.49, 100) print('Start main training') nn.learn(train_input, train_output, 0.1, 0.7, 100) nn.learn(train_input, train_output, 0.01, 0.8, 100) val = time() - start print("Time:", val) print("Mse: ", nn.mse(test_input, test_output)) print("Accuracy: ", nn.accuracy(test_input, test_output)) return val
def create_model(hyperparams): _module_kernelinit = __import__('kernel_initialization') lr = hyperparams['eta'] mom = hyperparams['momentum'] l2 = hyperparams['lambda'] beta_1 = hyperparams['beta_1'] beta_2 = hyperparams['beta_2'] act_fun_1 = hyperparams['activation_function1'] hidden_nodes1 = hyperparams['hidden_nodes'] kernel_init = getattr(_module_kernelinit, hyperparams['init_weights']) optimizer_str = hyperparams['optimizer'] model = NeuralNetwork(LOSS, METRIC) model.add_layer(hidden_nodes1, input_dim=INPUT_DIM, activation=act_fun_1, kernel_initialization=kernel_init()) if hyperparams['add_layer']: act_fun_2 = hyperparams['activation_function2'] hidden_nodes2 = hyperparams['hidden_nodes2'] model.add_layer(hidden_nodes2, activation=act_fun_2, kernel_initialization=kernel_init()) model.add_layer(OUTPUT_DIM, activation='linear', kernel_initialization=kernel_init()) if optimizer_str == "SGD": if hyperparams['lr_linear_decay']: decay_lr_par = { "epoch_tau": hyperparams['epoch_tau'], "epsilon_tau_perc": hyperparams['epsilon_tau_perc'], "init_lr": lr } optimizer = SGD(lr=lr, mom=mom, l2=l2, decay_lr=decay_lr_par) else: optimizer = SGD(lr=lr, mom=mom, l2=l2) elif optimizer_str == "RMSprop": optimizer = RMSprop(lr=lr, moving_average=beta_1, l2=l2) elif optimizer_str == "Adam": optimizer = Adam(lr=lr, beta_1=beta_1, beta_2=beta_2, l2=l2) else: raise ValueError("You have to choose an existing optimizer") model.compile(optimizer=optimizer) return model
dim_in = one_hot dim_hid = 3 n_attemps = 5 res_losses = np.zeros(n_attemps) res_metrics = np.zeros(n_attemps) res_losses_ts = np.zeros(n_attemps) res_metrics_ts = np.zeros(n_attemps) for i in range(n_attemps): model = NeuralNetwork('mse', 'accuracy1-1') model.add_layer(dim_hid, input_dim=dim_in, activation='sigmoid', kernel_initialization=RandomNormalInitialization()) model.add_layer(dim_out, activation='tanh', kernel_initialization=RandomNormalInitialization()) model.compile(optimizer=SGD(lr=0.5, mom=0.8, l2=0.01)) history = model.fit(X_train, Y_train, 200, X_train.shape[0], ts=(X_test, Y_test), verbose=True) if i == 0: write_results(res=None,
# This is the configuration of the present experiment. X, y, X_test, y_test = load_dataset("CasosArtrite2.csv", input_dimension=17, test_ratio=0.3) epochs = 10000 model_data = np.vstack([np.arange(0, epochs), np.zeros([4, epochs])]) # ============== end config ============== nn = NeuralNetwork(learning_rate=0.2) nn.add_layer(Layer(20)) nn.initialize(X.shape[1], y.shape[1]) plt.ion() for k in model_data[0, :]: k = int(k) # Train for a single epoch for epoch, train_loss, train_acc in nn.fit(X, y, epochs=1): model_data[1, k] = train_loss model_data[2, k] = train_acc # Then make a validation test error, acc, out = nn.evaluate(X_test, y_test) model_data[3, k] = error
PERC_VL = 0.25 PERC_TS = 0.25 set_style_plot() parser = Cup_parser(DIR_CUP + PATH_TR) data, targets = parser.parse(INPUT_DIM, OUTPUT_DIM) parser_ts = Cup_parser(DIR_CUP + PATH_TS) data_ts, _ = parser_ts.parse(INPUT_DIM, 0) X_train, Y_train = data, targets X_test = data_ts model = NeuralNetwork('mse', 'mee') model.add_layer(30, input_dim=X_train.shape[1], activation='tanh', kernel_initialization=XavierNormalInitialization()) model.add_layer(30, activation='tanh', kernel_initialization=XavierNormalInitialization()) model.add_layer(Y_train.shape[1], activation='linear', kernel_initialization=XavierNormalInitialization()) model.compile(optimizer=RMSprop(lr=0.0001, moving_average=0.9, l2=0.0005)) model.fit(X_train, Y_train, 500, batch_size=64, shuffle=True,
OUTPUT_DIM = 2 PERC_TS = 0.25 PERC_VL = 0.25 set_style_plot() parser = Cup_parser(DIR_CUP + PATH_TR) data, targets = parser.parse(INPUT_DIM, OUTPUT_DIM) X_train, Y_train, X_val, Y_val, X_test, Y_test = train_val_test_split( data, targets, val_size=PERC_VL, test_size=PERC_TS, shuffle=True) model = NeuralNetwork('mse', 'mee') model.add_layer(40, input_dim=X_train.shape[1], activation='tanh', kernel_initialization=RandomNormalInitialization()) model.add_layer(30, activation='sigmoid', kernel_initialization=RandomNormalInitialization()) model.add_layer(Y_train.shape[1], activation='linear', kernel_initialization=RandomNormalInitialization()) model.compile(optimizer=RMSprop(lr=0.0001, moving_average=0.9, l2=0.0004)) model.fit(X_train, Y_train, 500, batch_size=64, vl=(X_val, Y_val),
def single_run(hidden_neurons, learning_rate, hidden_activation, output_activation): model_data = np.vstack([np.arange(0, epochs), np.zeros([4, epochs])]) nn = NeuralNetwork(learning_rate=learning_rate) nn.add_layer(Layer(hidden_neurons, activation=hidden_activation)) nn.initialize(X.shape[1], y.shape[1], output_activation=output_activation) epoch_values = model_data[0, :] start = time.time() for k in epoch_values: k = int(k) # Train for a single epoch for epoch, train_loss, train_acc in nn.fit(X, y, epochs=1): model_data[1, k] = train_loss model_data[2, k] = train_acc # Then make a validation test error, acc, out = nn.evaluate(X_test, y_test) model_data[3, k] = error model_data[4, k] = acc #print("Validation: Loss {0}, accuracy {1}".format(error, acc)) elapsed = time.time() - start plt.plot(model_data[0, :], model_data[1, :], label="Erro (treinamento)") plt.plot(model_data[0, :], model_data[3, :], label="Erro (validação)") plt.legend(loc="lower right") plt.xlabel("Época") plt.title("Erro ao longo do tempo") plt.show() plt.clf() plt.plot(model_data[0, :], model_data[2, :], label="Acurácia (treinamento)") plt.plot(model_data[0, :], model_data[4, :], label="Acurácia (validação)") plt.legend(loc="lower right") plt.xlabel("Época") plt.title("Acurácia ao longo do tempo") plt.show() results = { 'best_training_loss': np.min(model_data[1, :]), 'best_training_loss_epoch': np.argmin(model_data[1, :]), 'best_validation_loss': np.min(model_data[3, :]), 'best_validation_loss_epoch': np.argmin(model_data[3, :]), 'best_training_acc': np.max(model_data[2, :]), 'best_training_acc_epoch': np.argmax(model_data[2, :]), 'best_validation_acc': np.max(model_data[4, :]), 'best_validation_acc_epoch': np.argmax(model_data[4, :]), 'final_validation_loss': model_data[3, -1], 'final_validation_acc': model_data[4, -1], 'final_training_loss': model_data[1, -1], 'final_training_acc': model_data[2, -1], 'learning_rate': learning_rate, 'hidden_neurons': hidden_neurons, 'hidden_activation': hidden_activation, 'output_activation': output_activation, 'elapsed_seconds': elapsed } return results
DIR_CUP = './data/cup/' PATH_TR = 'ML-CUP19-TR.csv' PATH_TS = 'ML-CUP19-TS.csv' INPUT_DIM = 20 OUTPUT_DIM = 2 results = list() parser = Cup_parser(DIR_CUP + PATH_TR) data, targets = parser.parse(INPUT_DIM, OUTPUT_DIM) X_train, Y_train, X_val, Y_val, X_test, Y_test = train_val_test_split( data, targets, val_size=0.25, test_size=0.25, shuffle=True) model_1 = NeuralNetwork('mse', 'mee') model_1.add_layer(30, input_dim=20, activation='sigm', kernel_initialization=RandomInitialization()) model_1.add_layer(40, activation="tanh", kernel_initialization=RandomInitialization()) model_1.add_layer(2, activation='linear', kernel_initialization=RandomInitialization()) optimizer = RMSprop(lr=0.001, moving_average=0.099, l2=0.0005) model_1.compile(optimizer=optimizer) model_1.fit(X_train, Y_train, early_stopping=None, epochs=500, batch_size=32, vl=(X_val, Y_val),
dim_in = one_hot dim_hid = 4 n_attemps = 5 res_losses = np.zeros(5) res_metrics = np.zeros(5) res_losses_ts = np.zeros(n_attemps) res_metrics_ts = np.zeros(n_attemps) for i in range(n_attemps): model = NeuralNetwork('mse', 'accuracy') model.add_layer(dim_hid, input_dim=dim_in, activation='relu', kernel_initialization=HeInitialization()) model.add_layer(dim_out, activation='sigmoid', kernel_initialization=RandomNormalInitialization(0.5, 0.1)) model.compile(optimizer=SGD(lr=0.6, mom=0.8)) history = model.fit(X_train, Y_train, 350, X_train.shape[0], ts=(X_test, Y_test), verbose=True) if i == 0: write_results(res=None, model=model,
def test_network_lesson09(self): """ Test initializing a neural NeuralNetwork """ input_shape = (28, 28, 1) output_count = 10 network = NeuralNetwork('convnet for MNIST', input_shape, output_count) # normal distribution parameters for random weights mean = 0.0 stddev = 0.1 # General convolution shapes and parameters common to all convolutional layers conv_stride_shape = (1, 1) conv_pad_shape = (0, 0) conv_pad_type = 'SAME' pool_stride_shape = (2, 2) pool_shape = (2, 2) pool_pad_type = 'SAME' activation = 'relu' # Kernel depths and sizes for each convolution layer depths = [32, 64, 128] kernel_shapes = [(5, 5, depths[0]), (5, 5, depths[1]), (5, 5, depths[2])] conv_layer_count = len(depths) # Expected values for assertions after_conv_output_shapes = [(28, 28, depths[0]), (14, 14, depths[1]), (7, 7, depths[2])] after_pool_output_shapes = [(14, 14, depths[0]), (7, 7, depths[1]), (4, 4, depths[2])] # Create convolutional layers conv = None for i in range(conv_layer_count): name = 'l{:d}'.format(i) if i > 0: input_shape = conv.output_shape conv = ConvolutionalLayer(name, input_shape, kernel_shapes[i], conv_stride_shape, \ conv_pad_shape, conv_pad_type, activation) self.assertEqual(after_conv_output_shapes[i], conv.output_shape) conv.add_pooling('max', pool_shape, pool_stride_shape, pool_pad_type) self.assertEqual(after_pool_output_shapes[i], conv.output_shape) network.add_layer(conv, mean, stddev) # Create linear layers # Output sizes for linear layers linear_input_sizes = [4 * 4 * 128, 512] linear_output_sizes = [512, 10] linear_activations = ['tanh', None] for i, input_size in enumerate(linear_input_sizes): layer_index = i + conv_layer_count name = 'l{:d}'.format(layer_index) linear = LinearLayer(name, input_size, linear_output_sizes[i], linear_activations[i]) network.add_layer(linear, mean, stddev) # MNIST classify 10 digits network.define_network() learning_rate = 0.001 network.define_operations(learning_rate, 'gradient_descent') epochs = 10 batch_size = 128 saver = tf.train.Saver() (train_inputs, train_labels, valid_inputs, valid_labels, test_inputs, test_labels) = \ get_mnist_data() with tf.Session() as sess: sess.run(tf.initialize_all_variables()) network.train_with_validate(sess, train_inputs, train_labels, valid_inputs, \ valid_labels, epochs, batch_size) test_accuracy = network.evaluate_in_batches(sess, test_inputs, test_labels, batch_size) print("Test accuracy:", test_accuracy) saver.save(sess, 'convnet') print("Model saved")
def test_network_lenet(self): """ Test using the lenet5 architecture """ input_shape = (32, 32, 1) output_count = 10 network = NeuralNetwork('lenet5 for MNIST', input_shape, output_count) # normal distribution parameters for random weights mean = 0.0 stddev = 0.1 # General convolution shapes and parameters common to all convolutional layers conv_stride_shape = (1, 1) conv_pad_shape = (0, 0) conv_pad_type = 'VALID' pool_stride_shape = (2, 2) pool_shape = (2, 2) pool_pad_type = 'VALID' activation = 'relu' # Kernel depths and sizes for each convolution layer depths = [6, 16] kernel_shapes = [(5, 5, depths[0]), (5, 5, depths[1])] conv_layer_count = len(depths) # Create convolutional layers conv = None for i in range(conv_layer_count): name = 'l{:d}'.format(i) if i > 0: input_shape = conv.output_shape conv = ConvolutionalLayer(name, input_shape, kernel_shapes[i], conv_stride_shape, \ conv_pad_shape, conv_pad_type, activation) conv.add_pooling('max', pool_shape, pool_stride_shape, pool_pad_type) network.add_layer(conv, mean, stddev) # Linear layer dimensions linear_input_sizes = [400, 120, 84] linear_output_sizes = [120, 84, 10] linear_activations = ['relu', 'relu', None] # Create linear layers for i, input_size in enumerate(linear_input_sizes): layer_index = i + conv_layer_count name = 'l{:d}'.format(layer_index) linear = LinearLayer(name, input_size, linear_output_sizes[i], linear_activations[i]) linear.init_weights_and_biases(mean, stddev) network.add_layer(linear, mean, stddev) network.define_network() learning_rate = 0.001 network.define_operations(learning_rate, 'adam') # Prepare data (train_inputs, train_labels, valid_inputs, valid_labels, test_inputs, test_labels) = \ get_mnist_data(padding=(2, 2)) epochs = 10 batch_size = 128 saver = tf.train.Saver() with tf.Session() as sess: sess.run(tf.initialize_all_variables()) network.train_with_validate(sess, train_inputs, train_labels, valid_inputs, \ valid_labels, epochs, batch_size) test_accuracy = network.evaluate_in_batches(sess, test_inputs, test_labels, batch_size) print("Test accuracy:", test_accuracy) saver.save(sess, 'lenet') print("Model saved")
X = data print(X.shape) ## Encoding des labels en onehot y_onehot.zero_() y_onehot.scatter_(1, target.view(-1, 1), 1) print(y_onehot.shape) break X = X.view(batch_size, 28 * 28) y = y_onehot def one_vs_one(X, y, c1, c2, test_size=0.3): X = torch.cat((X[y[:, c1] == 1], X[y[:, c2] == 1])) y = torch.cat((y[y[:, c1] == 1], y[y[:, c2] == 1])) inds = list(range(len(X))) np.random.shuffle(inds) sep = int(test_size * len(X)) return X[inds][:sep], X[inds][sep:], y[inds][:sep], y[inds][sep:] X_train, X_test, y_train, y_test = one_vs_one(X, y, 3, 5) nn = NeuralNetwork(loss=MSE()) nn.add_layer([ModuleLinear(len(X[0]), 10), Activation(sigmoid, sigmoid_g)]) costs = nn.fit(X_train, y_train, mode='mini_batch') plt.plot(range(len(costs)), costs) plt.show()
#Loop through the etas and lambdas for i, eta in enumerate(eta_vals): for j, lmbd in enumerate(lmbd_vals): #Make neural network ffnn = NeuralNetwork(X_train, y_train, batch_size=int(n_samples/m), n_categories = 1, epochs = 100, n_hidden_neurons = 20, eta = eta, lmbd = lmbd, input_activation = 'sigmoid', output_activation = 'linear', cost_function = 'MSE') ffnn.add_layer(20, activation_method = 'sigmoid') ffnn.add_layer(20, activation_method = 'sigmoid') #Train network ffnn.train() #Save predictions y_tilde_train = ffnn.predict(X_train) y_tilde_test = ffnn.predict(X_test) #Save metrics into exportable matrices train_mse[i][j], train_R2[i][j] = statistics.calc_statistics(y_train, y_tilde_train) test_mse[i][j], test_R2[i][j] = statistics.calc_statistics(y_test, y_tilde_test) if best_train_mse > train_mse[i][j]: best_train_mse = train_mse[i][j]
parser = Monks_parser(path_tr, path_ts) X_train, Y_train, X_test, Y_test = parser.parse(dim_in, dim_out, one_hot) #Y_train = change_output_value(Y_train, 0, -1) #Y_test = change_output_value(Y_test, 0, -1) #X_train, Y_train, X_val, Y_val = train_test_split(X_train, Y_train, test_size=0.25) dim_in = one_hot dim_hid = 4 model = NeuralNetwork('mse', 'accuracy') model.add_layer(dim_hid, input_dim=dim_in, activation='relu', kernel_initialization=RandomUniformInitialization(-0.5, 0.5)) model.add_output_layer(dim_out, activation='sigmoid', kernel_initialization=RandomUniformInitialization(-0.5, 0.5)) model.compile(0.8, 0.7, 0.0) model.fit( X_train, Y_train, 500, X_train.shape[0], ts=(X_test, Y_test), verbose=True, tol=1e-2 ) err_tr, acc_tr = model.evaluate(X_train, Y_train) err_ts, acc_ts = model.evaluate(X_test, Y_test) acc_tr = acc_tr*100 acc_ts = acc_ts*100 errors = [err_tr, err_ts] accuracy = [acc_tr, acc_ts]