def _get_predictions(neural_network: NeuralNetwork, nr_of_elements, nr_of_dimensions): """ Precict ``nr_of_elements`` random elements by ``neural_network`` :param neural_network: The NeuralNetwork to predict with :param nr_of_elements: Amount of elements to predict :param nr_of_dimensions: Number of dimensions to reduce preditions to :return: """ should_reduce = nr_of_dimensions < neural_network.classcount if should_reduce: neural_network.save_dimension_reducer(nr_of_dimensions) data = { "x": [], "y": [], "true_label": [], "filename": [], } if nr_of_dimensions == 3: data["z"] = [] with open(os.devnull, "w") as f, contextlib.redirect_stdout(f): generator = ImageDataGenerator(rescale=1 / 255).flow_from_directory( neural_network.directory, color_mode="rgb", batch_size=32, shuffle=True, target_size=(neural_network.img_width, neural_network.img_height), follow_links=True, class_mode='categorical', ) class_indices = {v: k for k, v in generator.class_indices.items()} for batch_index in range(nr_of_elements // 32): elements = generator.next() true_labels = [_get_labelname(class_indices, label) for label in elements[1]] predicted_labels = neural_network.network.predict(elements[0]) dimensions_reduced = neural_network.dimension_reducer.transform(predicted_labels) \ if should_reduce else predicted_labels data["x"].extend(dimensions_reduced[:, 0]) data["y"].extend(dimensions_reduced[:, 1]) if nr_of_dimensions == 3: data["z"].extend(dimensions_reduced[:, 2]) data["true_label"].extend(true_labels) filenames = [generator.filenames[generator.index_array[file_index + 32 * batch_index]] for file_index in range(32)] data["filename"].extend(filenames) return data
def main(): dataset = Uji() OPTIMIZERS = [ tf.train.RMSPropOptimizer, tf.train.AdamOptimizer, tf.train.GradientDescentOptimizer, tf.train.AdagradOptimizer, tf.train.AdadeltaOptimizer ] LEARNING_RATES = [ 0.01, 0.02, 0.05, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7 ] TRAINING_EPOCHS = [10, 20, 50, 100, 200, 300, 400, 500, 700, 1000] RESULTS = [] for optimizer in OPTIMIZERS: print(optimizer) for learning_rate in LEARNING_RATES: print("Learning rate: {}".format(learning_rate)) with NeuralNetwork(input_nodes=dataset.feature_number, hidden_nodes=200, output_nodes=dataset.label_number, learning_rate=learning_rate, batch_size=100, training_epochs=TRAINING_EPOCHS[-1], dropout=0.6, optimizer=optimizer, debug=False) as nn: accuracy_results = nn.fit(dataset, TRAINING_EPOCHS) for accuracy, training_epoch in zip(accuracy_results, TRAINING_EPOCHS): RESULTS.append( tuple([ accuracy, optimizer, learning_rate, training_epoch ])) RESULTS.sort(key=lambda row: row[0][0], reverse=True) with open(os.path.join(BASE_DIR, 'parametr_stats_by_accuracy.txt'), 'w') as fh: for snapshot_results, optimizer, learning_rate, training_epochs in RESULTS: fh.write( "Accuracy {0} Cost {1} Optimizer {2} Learning Rate {3}, Training Epoches: {4}\n" .format(snapshot_results[0], snapshot_results[1], optimizer, learning_rate, training_epochs)) RESULTS.sort(key=lambda row: row[0][1]) with open(os.path.join(BASE_DIR, 'parametr_stats_by_cost.txt'), 'w') as fh: for snapshot_results, optimizer, learning_rate, training_epochs in RESULTS: fh.write( "Accuracy {0} Cost {1} Optimizer {2} Learning Rate {3}, Training Epoches: {4}\n" .format(snapshot_results[0], snapshot_results[1], optimizer, learning_rate, training_epochs))
def main(): # dataset = Mnist() dataset = Uji() # joblib.dump(dataset, os.path.join(OBJECT_DIR, "Mnist")) joblib.dump(dataset, os.path.join(OBJECT_DIR, "Uji")) # Konfiguracja ~ 64 % # input_nodes=dataset.feature_number, hidden_nodes=200, output_nodes=dataset.label_number, # learning_rate=0.01, batch_size=100, training_epochs=350, dropout=0.6, # optimizer=tf.train.AdamOptimizer, debug=False with NeuralNetwork(input_nodes=dataset.feature_number, hidden_nodes=200, output_nodes=dataset.label_number, learning_rate=0.3, batch_size=100, training_epochs=500, dropout=0.6, optimizer=tf.train.AdagradOptimizer, debug=False) as nn: nn.fit(dataset) nn.save_model()
def parse_input(args): """ Parse commandline arguments and create plots :param args: Commandline arguments """ filename = args.network.replace('.h5', '') directory = args.directory nr_of_clusters_to_plot = args.nr_of_clusters_to_plot cluster_plot_dimensions = args.cluster_plot_dimension network = NeuralNetwork.load(f'{filename}.h5', directory) history = None if (os.path.exists(f'{filename}.json')): with open(f'{filename}.json') as f: history = json.loads(f.read().strip('"').replace("'", '"')) filename = filename.split(os.sep)[-1] if history is not None: _plot_accuracy(network, history, filename) _plot_clusters(network, nr_of_clusters_to_plot, cluster_plot_dimensions, filename)
def evaluate(args): results_dt_path = results_path.format(args.evaluate) x, y = load_data(args.evaluate) n_classes = y.shape[1] rs = np.random.RandomState(25) # defaults = get_defaults(x, n_classes, rs) with open(path.join(results_dt_path, 'best.csv'), 'r') as best: content = best.read() bests = {} for line in content.split('\n'): b = line.split(',') if len(b) > 1: bests[b[0]] = float(b[1]) bests['n_neurons'] = int(bests['n_neurons']) bests['n_layers'] = int(bests['n_layers']) r = bests['r'] a = bests['alpha'] b = bests['beta'] bs = int(bests['batchsize'] * x.shape[0]) w = [weights(bests['n_neurons'], x.shape[1] + 1, rs)] w += [weights(bests['n_neurons'], bests['n_neurons'] + 1, rs) for _ in range(bests['n_layers'] - 1)] w += [weights(n_classes, bests['n_neurons'] + 1, rs)] # # Number of folds to get 20% test set k = int(np.ceil(x.shape[0] / (x.shape[0] * 0.2))) model = NeuralNetwork(w, r, a, b) scores = cross_validate(model, x, y, k, bs, rs) mean = np.mean(scores) std = np.std(scores) print("Mean F1-Score = {} +- {}".format(round(mean, 3), round(std, 3)))
from src.loss_function import SquareLoss, CrossEntropy from src.neural_network import NeuralNetwork from src.layers import Conv2D, Dense, Activation import numpy as np # loss = SquareLoss() # print(loss.loss(np.random.normal(1,0.2, size=(256,10)), np.ones((256,10)))) # print(type(SquareLoss),type(CrossEntropy) X = model = NeuralNetwork(SquareLoss()) model.add(Conv2D)
if args.randomseed: seed = Game.generate_seed() if args.multimap: seed = [] for i in range(50): seed.append(Game.generate_seed()) generations = 100 if args.generations: generations = int(args.generations) nn = NeuralNetwork(False, seed, generations, True) winner = nn.run() if not args.noviz: nn.visualize_results() if args.save: with open(args.save, 'wb') as f: pickle.dump(winner, f) if args.load: genome = pickle.load(open(args.load, 'rb')) try: seed = pickle.load(open(args.load + "_seed", 'rb'))
def run(x, y, w, r, a, b, bs, rs): model = NeuralNetwork(w, r, a, b) return cross_validate(model, x, y, k_fold, bs, rs)
import numpy as np import time from src.neural_network import NeuralNetwork layers = [2, 10, 10, 5] network = NeuralNetwork(layers) print(network.feed_forward([10, 10]))
str(time.time()), type=str, help='Desired Filename') parser.add_argument('-r', '--learningrate', default=0.007, type=float, help='Learning Rate') args = parser.parse_args() filename = args.filename learningRate = args.learningrate layers = [64, 20, 10] network = NeuralNetwork(layers, learningRate) print(network.activationFunctions) print(network.trainMethod) print(network.inputSize) print(network.outputSize) train_inputs = [] train_outputs = [] with open('test_files/handwritten_digits/optdigits_train.txt', 'r') as f: for line in f: array = list(map(int, line.split(","))) value = array[-1] output = np.zeros(10) assert value < 10 output[value] = 1.0
def run_network(properties, train_data, test_data, iteration): """ Creates and runs a neural network using the data in properties, creates a confusion matrix and returns a dictionary with the metrics of the neural network and the las pair of predicted and expected classes """ neural_network = NeuralNetwork(properties, Sigmoid()) precision = [] recall = [] f1 = [] cost = [] epoch = [] expected, predictions = predict(neural_network, test_data) precision.append(precision_score( expected, predictions, average='weighted')) recall.append(recall_score(expected, predictions, average='weighted')) f1.append(f1_score(expected, predictions, average='weighted')) cost.append(calculate_cost(expected, predictions)) epoch.append(0) for i in range(1, properties["epoch"]+1): if i % properties["sampling_rate"] == 0: print("Epoch {:4}:".format(i)) expected, predictions = predict(neural_network, test_data) epoch_precision = precision_score( expected, predictions, average='weighted') epoch_recall = recall_score( expected, predictions, average='weighted') epoch_f1 = f1_score(expected, predictions, average='weighted') epoch_cost = calculate_cost(expected, predictions) print("{:>14} {:.4}".format('Precision:', epoch_precision)) print("{:>14} {:.4}".format('Recall:', epoch_recall)) print("{:>14} {:.4}".format('F1:', epoch_cost)) print("{:>14} {:.4}".format('Cost:', epoch_cost)) precision.append(epoch_precision) recall.append(epoch_recall) f1.append(epoch_f1) cost.append(epoch_cost) epoch.append(i) train(neural_network, train_data) print("Finished") print("Calculating confusion matrix") classes = np.array(["setosa", "versicolor", "virginica"]) title = "Iteration {}".format(iteration) plot_iteraton( title, (precision, recall, f1, cost, epoch), (expected, predictions, classes)) metrics = { "iteration:": iteration, "precision": precision, "recall": recall, "f1": f1, "cost": cost, "last_expected": expected, "last_predicted": predictions } return metrics
def main(args): np.set_printoptions(precision=precision) network_path = args.files[0] initial_weights_path = args.files[1] dataset_path = args.files[2] r, n_inputs, n_neurons, n_outputs = load_network(network_path) initial_weights = load_weights(initial_weights_path) x, y = load_benchmark(dataset_path) epsilon = 0.0000010000 n = x.shape[0] model = NeuralNetwork(deepcopy(initial_weights), r, 0.99, 0) print("Parâmetro de regularização lambda={}\n".format(round(r, 3))) print("Inicializando rede com a seguinte estrutura de neurônios por camadas: {}\n".format([n_inputs] + n_neurons + [n_outputs])) for i in range(len(initial_weights)): print("Theta{} inicial (pesos de cada neurônio, incluindo bias, armazenados nas linhas):\n{}".format(i + 1, str_matrix(initial_weights[i], '\t'))) print("Conjunto de treinamento") for i in range(x.shape[0]): print("\tExemplo {}".format(i + 1)) print("\t\tx: {}".format(x[i, :])) print("\t\ty: {}".format(y[i, :])) print("\n--------------------------------------------") print("Calculando erro/custo J da rede") for i in range(x.shape[0]): print("\tProcessando exemplo de treinamento {}".format(i + 1)) print("\tPropagando entrada {}".format(x[i, :])) f = model.forward_propagation(x[i, :]) cost = model.cost_x(y[i, :], f) print("\t\ta1: {}\n".format(model.a[0])) for l in range(1, model.n_layers + 1): print("\t\tz{}: {}".format(l + 1, model.z[l])) print("\t\ta{}: {}\n".format(l + 1, model.a[l])) print("\t\tf(x[{}]): {}".format(i + 1, f)) print("\tSaida predita para o exemplo {}: {}".format(i + 1, f)) print("\tSaida esperada para o exemplo {}: {}".format(i + 1, y[i, :])) print("\tJ do exemplo {}: {}\n".format(i + 1, cost)) print("J total do dataset (com regularizacao): {}\n".format(model.cost(x, y))) print("\n--------------------------------------------") print("Rodando backpropagation") for i in range(n): print("\tCalculando gradientes com base no exemplo {}".format(i + 1)) model.g = [np.zeros(model.w[i].shape) for i in range(model.n_layers)] model.m = [np.zeros(model.w[i].shape) for i in range(model.n_layers)] pred = model.forward_propagation(x[i, :]) model.d[model.last_layer] = pred - y[i, :] model.update_deltas(x[i, :]) for d in range(model.last_layer, -1, -1): print("\t\tdelta{}: {}".format(d + 2, model.d[d])) model.accumulate_gradients() for t in range(model.last_layer, -1, -1): print("\t\tGradientes de Theta{} com base no exemplo {}:\n{}".format(t + 1, i + 1, str_matrix(model.g[t], '\t\t\t'))) print("\tDataset completo processado. Calculando gradientes regularizados") model.final_gradients(n) for t in range(model.n_layers): print("\t\tGradientes finais para Theta{} (com regularizacao):\n{}".format(t + 1, str_matrix(model.g[t], '\t\t\t'))) print("\n--------------------------------------------") print("Rodando verificacao numerica de gradientes (epsilon={})".format(epsilon)) backprop_gradients = deepcopy(model.g) model.g = [np.zeros(model.w[i].shape) for i in range(model.n_layers)] for t in range(model.n_layers): for i in range(model.g[t].shape[0]): for j in range(model.g[t].shape[1]): w = model.w[t][i, j] model.w[t][i, j] = w + epsilon c1 = model.cost(x, y) model.w[t][i, j] = w - epsilon c2 = model.cost(x, y) model.g[t][i, j] += (c1 - c2) / (2 * epsilon) model.w[t][i, j] = w print("\tGradiente numerico de Theta{}:\n{}".format(t + 1, str_matrix(model.g[t], '\t\t'))) print("\n--------------------------------------------") print("Verificando corretude dos gradientes com base nos gradientes numericos:") for t in range(model.n_layers): errors = np.sum(np.abs(model.g[t] - backprop_gradients[t])) print("\tErro entre gradiente via backprop e gradiente numerico para Theta{}: {}".format(t + 1, errors))