def train_prediction( net: Neural_network.NeuralNet, inputs_train: Tensor, targets_train: Tensor, inputs_test: Tensor, targets_test: Tensor, loss: Loss.Loss = Loss.MeanSquareError(), optimizer: OptimizerClass.Optimizer = OptimizerClass.SGD(), num_epochs: int = 5000, batch_size: int = 32): Data = pd.DataFrame(columns=('MSE_train', 'MSE_test', 'error_round_train', 'error_round_test')) size_training = inputs_train.shape[0] for epoch in range(num_epochs): Chi2_train = 0.0 error_round_train = 0.0 nbr_batch = 0 for i in range(0, size_training, batch_size): nbr_batch += 1 # 1) feed forward y_actual = net.forward(inputs_train[i:i + batch_size]) # 2) compute the loss and the gradients Chi2_train += loss.loss(targets_train[i:i + batch_size], y_actual) grad_ini = loss.grad(targets_train[i:i + batch_size], y_actual) # 3)feed backwards grad_fini = net.backward(grad_ini) # 4) update the net optimizer.step(net, n_epoch=epoch) error_round_train += Error_round.error_round( targets_train[i:i + batch_size], y_actual) Chi2_train = Chi2_train / nbr_batch error_round_train = error_round_train / nbr_batch y_actual_test = net.forward(inputs_test) Chi2_test = loss.loss(targets_test, y_actual_test) error_round_test = Error_round.error_round(targets_test, y_actual_test) if epoch % 100 == 0: print('epoch : ' + str(epoch) + "/" + str(num_epochs) + "\r", end="") datanew = pd.DataFrame({ 'MSE_train': [Chi2_train], 'MSE_test': [Chi2_test], 'error_round_train': [error_round_train], 'error_round_test': [error_round_test] }) Data = Data.append(datanew) os.chdir(path_ini) Data.to_csv('Opt_num_epoch_backup.csv', index=False) return Data
def initialize_new(): """Initializes a new example neural net with one hidden layer.""" result = NeuralNet(2) result.add_relu(100) result.add_relu(3) result.add_softmax() result.add_cross_entropy_loss() return result
def train(net: Neural_network.NeuralNet, inputs: Tensor, targets: Tensor, loss: Loss = Loss.MeanSquareError(), optimizer: OptimizerClass.Optimizer = OptimizerClass.SGD(), num_epochs: int = 5000, batch_size: int = 32) -> tuple: chi2_list = [] round_error_list = [] size_training = inputs.shape[0] for epoch in range(num_epochs): chi2_loss = 0.0 round_error_loss = 0.0 nbr_batch = 0 for i in range(0, size_training, batch_size): nbr_batch += 1 # 1) Feed forward y_actual = net.forward(inputs[i:i + batch_size]) # 2) Compute the loss and the gradient chi2_loss += loss.loss(targets[i:i + batch_size], y_actual) round_error_loss += Error_round.error_round( targets[i:i + batch_size], y_actual) grad_ini = loss.grad(targets[i:i + batch_size], y_actual) # 3) Feed backwards grad_fini = net.backward(grad_ini) # 4) Update the net optimizer.step(net, n_epoch=epoch) chi2_loss = chi2_loss / nbr_batch round_error_loss = round_error_loss / nbr_batch chi2_list.append(chi2_loss) round_error_list.append(round_error_loss) # Print status every 50 iterations if epoch % 50 == 0: print('\r epoch : ' + str(epoch) + "/" + str(num_epochs) + ", training mean squared error : " + str(chi2_loss) + "\r", end="") print('epoch : ' + str(epoch) + "/" + str(num_epochs) + ", training final mean squared error : " + str(chi2_loss) + '\n') return chi2_list, round_error_list
def eliminate(self, bot, replace=False): self.time_since_last_death = 0.0 self.bots.remove(bot) color = bot.RGB neural_net_example = NeuralNet((2, 3, 3), ("sigmoid", "softmax")) if replace: example_bot = Bot(neural_net_example, color, self) self.bots.append(example_bot)
def __init__(self, size, mutation_rate, no_food): assert (size >= 5) assert (0 < mutation_rate < 1) self.SIZE = size self.mutation_rate = mutation_rate self.bots = [] self.food = [] self.time_since_last_death = 0.0 # The neural network will have 1 neuron in the input layer, 1 hidden # layer with 2 neurons, and 4 neurons in the output layer. The sigmoid # activation function will be used on the hidden layer, and a softmax # activation function will be used on the output layer. Input consists # of the bot's direction and if there is or isn't food in the bots field # of vision. Output consists of whether or not to move foward, turn # left, turn right, or do nothing. neural_net_example = NeuralNet((2, 3, 3), ("sigmoid", "softmax")) colors = [(255, 0, 0), (0, 255, 0)] # spawning equal no of carni_bots and herbi_bots for i in range(size): random_rgb = colors[0] example_bot = Bot(neural_net_example, random_rgb, self) self.bots.append(example_bot) for i in range(size): random_rgb = colors[1] example_bot = Bot(neural_net_example, random_rgb, self) self.bots.append(example_bot) for i in range(no_food): self.food.append(Food(self)) # append strongest parents from previous run if os.path.isfile("modelc.h5"): self.bots.append( Bot(NeuralNet(model_file="modelc.h5"), (255, 0, 0), self)) self.bots.append( Bot(NeuralNet(model_file="modelh.h5"), (0, 255, 0), self))
def prediction_calc(tic, data=None): if data == None: data = pd.read_csv(RAW_DATA_FOLDER + tic + '.csv').tail(30) data.index = range(len(data)) data = normalize(data) date = data.index[-1] print(date) inputs = getInputs(date, data) weights = np.load(WEIGHTS_FILE_PATH, allow_pickle=True) network = nn.NeuralNet([0, 0, 7], [41, 100, 50, 1]) network.weights = weights prediction = network.calculateOutput(inputs, single_input=True) if prediction > .5: confidence = ((prediction - .5) / .4) * 100 return (1, confidence) else: confidence = np.sqrt((.5 - prediction) / .4) * 100 return (0, confidence)
def optim(): """Determine best lambda parameter""" #list of lambda values to try lambdas = [ 0, 0.01, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64, 1.28, 2.56, 5.12, 10.24 ] #lambdas = [2.56, 5.12, 10, 20, 40, 80, 160, 320, 640, 1280, 2560] #list to record training parameters histories = [] J_train = [] J_test = [] accuracies = [] for l in lambdas: #define and train model model = NeuralNet(alpha=0.003, n_iter=40000, lamb=l, hidden_size=28, n_labels=2) Theta1, Theta2, cost_history = model.fit(X_train, Y_train) #append parameters to lists histories.append(cost_history) J_train.append(cost_history[-1]) X_test_1 = model.add_intercept(X_test) J_test.append(model.forward_prop(X_test_1, Y_test, Theta1, Theta2)[4]) accuracies.append(model.predict(Theta1, Theta2, X_test, Y_test)[1]) #plot costs VS lambda fig, ax = plt.subplots() ax.plot(lambdas, J_train, color="C0", label="training") ax.plot(lambdas, J_test, color="C2", label="testing") ax.set_xlabel("Lambda") ax.set_ylabel("Cost") ax.legend() plt.show() #plot accuracies VS lambdas fig, ax = plt.subplots() ax.plot(lambdas, accuracies) ax.set_xlabel("Lambda") ax.set_ylabel("Accuracy on test sample") plt.show() return accuracies
def train(): """Train the neural network""" model = NeuralNet(alpha=0.003, n_iter=300000, lamb=5, hidden_size=28, n_labels=2) start = time.time() Theta1, Theta2, cost_history = model.fit(X, Y) stop = time.time() elapsed = stop - start pred, accu = model.predict(Theta1, Theta2, X_test, Y_test) print("Time to train: {:.1f} seconds".format(elapsed)) print("Model accuracy: {:.3f}".format(accu)) model.plot_history(cost_history) return Theta1, Theta2, pred, accu, cost_history
def feed(self, bot, food, is_bot=False): bot.score = 1.0 if is_bot == False: # # print "Bot-Food collision" self.food.remove(food) self.food.append(Food(self)) else: # bot-bot collision # # print "Bot-Bot collision" idx = self.bots.index(bot) self.bots.pop(idx) if len(self.bots) <= 7: # # print "Creating new Bots!" neural_net_example = NeuralNet((2, 3, 3), ("sigmoid", "softmax")) colors = [(255, 0, 0), (0, 255, 0)] for i in range(self.SIZE): random_rgb = colors[np.random.randint(0, 2)] example_bot = Bot(neural_net_example, random_rgb, self) self.bots.append(example_bot) num_to_replace = int(self.SIZE / 7 - 1) if num_to_replace < 2: num_to_replace = 2 for i in range(num_to_replace): weakest = self.bots[0] for other in self.bots: if other.score < weakest.score: weakest = other self.eliminate(weakest) colors = [(255, 0, 0), (0, 255, 0)] # asexual reproduction - mutation for i in range(num_to_replace): if np.random.uniform(0, 1) <= self.mutation_rate: new_rgb = colors[np.random.randint(0, 2)] new_bot = Bot(bot.nnet, new_rgb, self) # new_bot.x = bot.x + Bot.HITBOX_RADIUS * 4 * np.random.uniform(0, 1) * np.random.choice((-1, 1)) # new_bot.y = bot.y + Bot.HITBOX_RADIUS * 4 * np.random.uniform(0, 1) * np.random.choice((-1, 1)) new_bot.x = np.random.randint( settings. WINDOW_WIDTH) + Bot.HITBOX_RADIUS * 4 * np.random.uniform( 0, 1) * np.random.choice((-1, 1)) new_bot.y = np.random.randint( settings. WINDOW_HEIGHT) + Bot.HITBOX_RADIUS * 4 * np.random.uniform( 0, 1) * np.random.choice((-1, 1)) # nn nb_c = new_bot.nnet.get_all_weights() mutated = False while not mutated: # for k in range(len(nb_c)-1): # # # print "len: ",len(nb_c) # # # print "from: ",len(nb_c[k]) # # # print "to: ",len(nb_c[k][0]-1) # for i in range(len(nb_c[k])-1): # for j in range(len(nb_c[k][0])): # if np.random.uniform(0, 1) <= self.mutation_rate: # nb_c[k][i][j] = nb_c[k][i][j] * np.random.normal(1, 0.5) + np.random.standard_normal() # mutated = True for k in range(len(nb_c)): if np.random.uniform(0, 1) <= self.mutation_rate: i = np.random.randint(0, len(nb_c[k])) j = np.random.randint(0, len(nb_c[k][0])) nb_c[k][i][j] = nb_c[k][i][j] * np.random.normal( 1, 0.5) + np.random.standard_normal() mutated = True new_bot.nnet.set_all_weights(nb_c) self.bots.append(new_bot) self.bots[self.bots.index(new_bot)].RGB = (255, 255, 255) # sexual - crossover else: # # print "Sexual! :P" sorted_bots_by_score = sorted(self.bots, key=lambda x: x.score, reverse=True) # get first 2 strongest bots bot1, bot2 = sorted_bots_by_score[0], sorted_bots_by_score[1] self.bots[self.bots.index(bot1)].change_color((255, 255, 255)) self.bots[self.bots.index(bot2)].change_color((255, 255, 255)) conn1 = bot1.nnet.get_all_weights() conn2 = bot2.nnet.get_all_weights() # get random weight to crossover idx1 = np.random.randint(len(conn1)) idx2 = np.random.randint(len(conn1[idx1])) idx3 = np.random.randint(len(conn1[idx1][idx2])) conn3 = conn1 # print "booyah: ",conn3[idx1][idx2][idx3] , conn2[idx1][idx2][idx3] # raw_input() conn3[idx1][idx2][idx3] = conn2[idx1][idx2][idx3] new_bot = Bot(bot.nnet, bot.RGB, self) new_bot.nnet.set_all_weights(conn3) new_bot.x = bot.x + Bot.HITBOX_RADIUS * 4 * np.random.uniform( 0, 1) * np.random.choice((-1, 1)) new_bot.y = bot.y + Bot.HITBOX_RADIUS * 4 * np.random.uniform( 0, 1) * np.random.choice((-1, 1)) self.bots.append(new_bot)
def run_classifier(folder_path, optimizer_string, workers, device, calculate_precision=False): data_loader = load_data(250, workers, "train.csv", folder_path, 32) data_load_time = 0 neural_net = NeuralNet().to(device) criterion = nn.BCELoss() optimizer = get_optimizer(optimizer_string, neural_net) loss_outer_array = [] precision1_outer_array = [] precision3_outer_array = [] epoch_start_time = time.monotonic() for epoch in range(5): if calculate_precision: loss_array = [] precision1_array = [] precision3_array = [] enumerator = enumerate(data_loader) data_load_start_time = time.monotonic() compute_start_time = data_load_start_time data_load_time = 0 compute_time = 0 for i, data in enumerator: data_load_end_time = time.monotonic() data_load_time = data_load_time + (data_load_end_time - data_load_start_time) input_tensor, input_label_tensor = data['image_tensor'].to( device), data['image_label_tensor'].to(device) optimizer.zero_grad() output_label_tensor = neural_net(input_tensor).float() loss = criterion(output_label_tensor, input_label_tensor) loss.backward() optimizer.step() compute_end_time = time.monotonic() compute_time = compute_time + (compute_end_time - compute_start_time) if calculate_precision: precision = accuracy(output_label_tensor, input_label_tensor) precision1_array.append(round(precision[0], 2)) precision3_array.append(round(precision[1], 2)) loss_array.append(round(loss.item(), 2)) compute_start_time = time.monotonic() data_load_start_time = time.monotonic() if calculate_precision: loss_outer_array.append(loss_array) precision1_outer_array.append(precision1_array) precision3_outer_array.append(precision3_array) epoch_end_time = time.monotonic() epoch_time = epoch_end_time - epoch_start_time print("-----------------Training done------------------------") if calculate_precision: print( "-----------------Printing precision stats------------------------" ) for epoch in range(5): for i, data in enumerate(data_loader, 0): print( "Epoch = {}, batch = {}, Loss = {}, Precision@1 {}, Precision@3 {}. \n" .format(epoch, i, loss_outer_array[epoch][i], precision1_outer_array[epoch][i], precision3_outer_array[epoch][i])) average_loss = sum(sum(x) for x in loss_outer_array) / 600.0 average_precision_1 = sum(sum(x) for x in precision1_outer_array) / 600.0 average_precision_3 = sum(sum(x) for x in precision3_outer_array) / 600.0 print("Average loss over 5 epoch = {}".format(average_loss)) print("Average precision@1 over 5 epoch = {}".format( average_precision_1)) print("Average precision@3 over 5 epoch = {}".format( average_precision_3)) print("-----------------Printing aggregate stats------------------------") print("Aggregated time for data loading = {}".format(data_load_time)) print( "Aggregated time for mini-batch computation = {}".format(compute_time)) print("Aggregated time for each epoch = {}".format(epoch_time)) print("-----------------Printing average stats------------------------") print("Average time per mini batch for data loading = {}".format( round(data_load_time / 600.0, 2))) print("Average time per minibatch for computation = {}".format( round(compute_time / 600.0, 2))) print("Average time for each epoch = {}".format(round(epoch_time / 5.0, 2)))
if __name__ == "__main__": learning_rate = 0.05 hidden_units = 7 num_epochs = 50 train = "datasets/heart_train.json" test = "datasets/heart_test.json" # parse the json file for data X_train, y_train, meta_train = parse_json(train) X_test, y_test, meta_test = parse_json(test) print("model,epoch,F1_train,F1_test") # Run logistic regression for epoch in range(1,num_epochs+1): logistic = NeuralNet(learning_rate=learning_rate, num_epoch=epoch,hidden_dim=()) logistic.fit(X_train, y_train, meta_train) F1_train = logistic.predict(X_train, y_true=y_train, F1=True) F1_test = logistic.predict(X_test, y_true=y_test, F1=True) print("logistic-0.05-heart,{},{},{}".format(epoch,F1_train,F1_test)) # Run neural network for epoch in range(1,num_epochs+1): nnet = NeuralNet(learning_rate=learning_rate, num_epoch=epoch,hidden_dim=(hidden_units,)) nnet.fit(X_train, y_train, meta_train) F1_train = nnet.predict(X_train, y_true=y_train, F1=True) F1_test = nnet.predict(X_test, y_true=y_test, F1=True) print("nnet-0.05-7-heart,{},{},{}".format(epoch,F1_train,F1_test))
from parse_json import parse_json import sys import json import numpy as np import pandas as pd if __name__ == "__main__": # <learning-rate> <#hidden-units> <#epochs> <train-set-file> <test-set-file> if len(sys.argv) == 1: learning_rate = 0.05 hidden_units = 7 num_epochs = 20 train = "datasets/heart_train.json" test = "datasets/heart_test.json" else: tmp, learning_rate, hidden_units, num_epochs, train, test = sys.argv learning_rate = float(learning_rate) hidden_units = int(hidden_units) num_epochs = int(num_epochs) # parse the json file for data X_train, y_train, meta_train = parse_json(train) X_test, y_test, meta_test = parse_json(test) nn = NeuralNet(learning_rate=learning_rate, num_epoch=num_epochs, hidden_dim=(hidden_units, )) nn.fit(X_train, y_train, meta_train, verbose=True) nn.predict(X_test, y_true=y_test, verbose=True)
def prediction(net: Neural_network.NeuralNet, inputs: Tensor) -> Tensor: return net.forward(inputs)
return [j for j in results if type(0) != type(j)] if __name__ == "__main__": save_string = 'prediction threshold: ' + str(PREDICTION_THRESHOLD) + '\n' save_string = save_string + 'gain threshold: ' + str(GAIN_THRESHOLD) + '\n' save_string = save_string + 'minimum price: ' + str(MINIMUM_PRICE) + '\n' save_string = save_string + 'days ahead: ' + str(DAYS_AHEAD) + '\n' save_string = save_string + 'attributes: ' + str(ATTRIBUTES) + '\n' save_string = save_string + '\nSTATS:\n\n' #gets network and tickers weights = np.load('test_results/Great+03/end_weights.npy', allow_pickle=True) network = nn.NeuralNet([0, 0, 7], [41, 100, 50, 1]) network.weights = weights stock_tickers = pd.read_csv('data/stock_names.csv')['Ticker'] #removes tickers wihtout data for tic in stock_tickers: if not os.path.exists('data/normalized_data/' + tic + '.csv'): stock_tickers = stock_tickers[stock_tickers != tic] stock_tickers = stock_tickers.sample(frac=PERCENT_STOCKS) #collects data print('collecting data...') pool = Pool() results = list( tqdm(pool.imap(testing_network, stock_tickers), total=len(stock_tickers))) pool.close()
countin += 1 flotar = "" print 'complete.' print scorecard return scorecard if __name__ == '__main__': print 'Starting neural network to recognize audio... maybe....' input_nodes = 80000 # 200x200 pixel image == 40,000 pixels, each px has red + green values to save so we need 80k hidden_nodes = 185 output_nodes = 10 # 0 == hello, 1 == yes, 2 == no learning_rate = 0.1 nn = NeuralNet(input_nodes, hidden_nodes, output_nodes, learning_rate) # Train train_the_neural_net(nn, epochs=1) # Test test_results = numpy.asarray(test_the_neural_net(nn)) # Print results print('Neural network is {}% accurate at predicting audio.... maybe....'. format(test_results.sum() / float(test_results.size) * 100.0)) thyme = time.time() thyme -= ti print str(thyme) + " seconds"
def testing_network(tic): #gets data weights = np.load('test_results/Great+03/end_weights.npy', allow_pickle=True) network = nn.NeuralNet([0, 0, 7], [41, 100, 50, 1]) network.weights = weights stock_data_normalized = pd.read_csv('data/normalized_data/' + tic + '.csv', index_col='date').tail(1520) stock_data = pd.read_csv( 'data/historical_stock_data/' + tic + '.csv', index_col='date', ).tail(1520) stock_data = stock_data[stock_data_normalized.index[0]:] results = [0] * len(stock_data_normalized.index[20:-DAYS_AHEAD]) i = 0 for date in stock_data_normalized.index[20:-DAYS_AHEAD]: input_data = dp.getInputs(date, stock_data_normalized) prediction = network.calculateOutput(input_data, single_input=True) #disregards low priced stocks if stock_data.loc[date, 'close'] > MINIMUM_PRICE: percent_change = ( (stock_data.loc[stock_data.index[20 + i + DAYS_AHEAD], 'close'] - stock_data.loc[date, 'close']) / stock_data.loc[date, 'close']) * 100 crossed_threshold, high = day_change_check( stock_data, 20 + i, ATTRIBUTES, GAIN_THRESHOLD, days_ahead=DAYS_AHEAD, ) if crossed_threshold: results[i] = (prediction, 1, percent_change, high) else: """ if (percent_change > .03) & (prediction >.8): print('high') print('5 day close price:',stock_data.loc[stock_data.index[i+5],'close']) print('compare price:', stock_data.loc[date,'close']) elif (percent_change < -.5)& (prediction >.8): print('low') print('5 day close price:',stock_data.loc[stock_data.index[i+5],'close']) print('compare price:', stock_data.loc[date,'close']) """ if percent_change > 3: print('date:', date) print('five day date:', stock_data.index[20 + i + DAYS_AHEAD]) print('current days price: ', stock_data.loc[date, 'close']) print( 'five day close: ', stock_data.loc[stock_data.index[20 + i + DAYS_AHEAD], 'close']) print('day change high:', high) """ if percent_change < -3: print('date:', date) print('five day date:', stock_data.index[20+i+DAYS_AHEAD]) print('current days price: ', stock_data.loc[date,'close']) print('five day close: ', stock_data.loc[stock_data.index[20+i+DAYS_AHEAD],'close']) print('day change high:',high) print('percent change: ', percent_change) """ results[i] = (prediction, 0, percent_change, high) i = i + 1 return [j for j in results if type(0) != type(j)]
print 'complete.' return scorecard if __name__ == '__main__': print 'Starting neural network to recognize handwritten digits.' input_nodes = 102 hidden_nodes = 30 output_nodes = 1 learning_rate = 0.1 weight_file = open('sd.txt', 'r') weight_list = weight_file.readlines() weight_list = numpy.asfarray(weight_list) weight_file.close() nn = NeuralNet(input_nodes, hidden_nodes, output_nodes, learning_rate, weight_list) # Train train_the_neural_net(nn, epochs=100) # Test test_results = numpy.asarray(test_the_neural_net(nn)) # Print results print('Neural network is {}% accurate at predicting handwritten digits.'. format(test_results.sum() / float(test_results.size) * 100.0))
9 : Step """ if __name__ == "__main__": #counters numPredictions = 0 training_errors = [0] * NUM_ITERATIONS testing_errors = None start_weights = None end_weights = None time_per_stock = 0 time_per_iteration = 0 overview_string = '' #create neural network network = nn.NeuralNet(ACTIVATION_FUNCTIONS, NODES_PER_LAYER) #get tickers stock_tickers = pd.read_csv('data/stock_names.csv')['Ticker'] #removes tickers wihtout data for tic in stock_tickers: if not os.path.exists('data/training/' + tic + '.csv'): stock_tickers = stock_tickers[stock_tickers != tic] if not os.path.exists('data/normalized_data/' + tic + '.csv'): stock_tickers = stock_tickers[stock_tickers != tic] print("loading training data") #get input training dictionaries loading_time = -time.time() #spawns a process for each stock data that needs to be loaded pool = Pool() results = list( tqdm.tqdm(pool.imap(inputsForBackProp, stock_tickers),
print("Sanity check") fig, ax = plt.subplots(figsize=(5, 5)) circ = plt.Circle((0.5, 0.5), math.sqrt(1 / 2 / math.pi), fill=False) plt.scatter(data_input[:, 0].cpu(), data_input[:, 1].cpu()) plt.xlabel('x0') plt.ylabel('x1') plt.title('Scatter of input') ax.add_patch(circ) plt.show() print('Average of train_target==1 is', torch.mean(data_target * 1.0).item()) print(" ") net1 = NeuralNet(layer_type='Sequential', learning_rate=2e-3, regularization=0.05, iteration=1000, batch_size=64, dtype=dtype, device=device) net1.append( Module(layer_type='Linear', if_batchnorm=True, activation_type='relu', params_shape=(2, 100), dtype=dtype, device=device)) net1.append( Module(layer_type='Linear', if_batchnorm=True, activation_type='relu', params_shape=(100, 50),