def my_solution2(X_train, X_test, y_train, y_test, hyperparams=None): if hyperparams == None: hyperparams = { 'hidden_layer_sizes': (3, ), 'learning_rate': 0.1, 'epoch': 3852, 'momentum': 0.04, 'tol': 1e-10, 'reg_coef': 0 } if not hasattr(hyperparams, 'batch_size'): hyperparams['batch_size'] = X_train.shape[0] # train with self nn = NeuralNetwork(**hyperparams) nn.fit(X_train, y_train) y_pred = nn.predict(X_test) y_pred = y_pred.argmax(axis=1) y_test = y_test.argmax(axis=1) # evaluate print('My implements2:') print(classification_report(y_test, y_pred)) return nn, y_pred, y_test
def main(filename='data/iris-virginica.txt'): # Load data data = read_data('%s/%s' % (filepath, filename)) X, y = data[:,:-1].astype(float), data[:,-1] class_vec = list(set(y)) K = len(class_vec) Y = pd.get_dummies(y).astype(int).as_matrix() # Define parameters n = X.shape[0] d = X.shape[1] # # # Define layer sizes print(n,d,K) layers = [d, 5, K] model = NeuralNetwork(layers=layers, num_epochs=1000, learning_rate=0.10, alpha=0.9, activation_func='sigmoid', epsilon=0.001, print_details=True) model.fit(X, Y) Y_hat = model.predict(X) accuracy = compute_acc(Y_hat, Y) print('Model training accuracy:\t%.2f' % (accuracy))
def train(iterations, rate): global training_inputs, temp_training_outputs, test_inputs, test_outputs NeuralNetwork.train(NumberNet, training_inputs=training_inputs, training_outputs=temp_training_outputs, training_iterations=iterations, learning_rate=rate)
def task_cv_single(t, modeLearn: ModeLearn, f:ActivFunct, theta:dict, errorFunct:ActivFunct, miniBatchDim = None): (trSet, vlSet) = t nn = NeuralNetwork(trSet, f, theta) nn.learn(modeLearn, errorFunct, miniBatchDim) vecErr = np.array([nn.getError(vlSet, i, 1/len(vlSet), errorFunct) for i in range(nn.hyp['OutputUnits'])]) return norm(vecErr,2)
def main(filename='data/iris-virginica.txt'): # Load data data = read_data('%s/%s' % (filepath, filename)) X, y = data[:, :-1].astype(float), data[:, -1] class_vec = list(set(y)) K = len(class_vec) Y = pd.get_dummies(y).astype(int).as_matrix() # Define parameters n = X.shape[0] d = X.shape[1] # # # Define layer sizes print(n, d, K) layers = [d, 5, K] model = NeuralNetwork(layers=layers, num_epochs=1000, learning_rate=0.10, alpha=0.9, activation_func='sigmoid', epsilon=0.001, print_details=True) model.fit(X, Y) Y_hat = model.predict(X) accuracy = compute_acc(Y_hat, Y) print('Model training accuracy:\t%.2f' % (accuracy))
def __init__(self, depth): self.depth = depth # self.heuristic = heuristic n = NeuralNetwork(64, 10, 0.003) n.readfile() self.heuristic = n.heuristic
def exp_500(): net = NeuralNetwork([28*28, 500, 10]) trainNetwork(net, cross_entropy=True) testNetwork(net) # Dump the network into a file, load it and test it again with open("network_files/mnist_500_network_file", "wb") as f: net.dump(f)
def __init__(self, state_size, action_size): self.state_size = state_size self.action_size = action_size self.brain = NeuralNetwork(self.state_size, self.action_size) self.brain_target = NeuralNetwork(self.state_size, self.action_size) self.memory = Memory(BUFFER_SIZE) self.epsilon = EPSILON
def recognize(src): nn = NeuralNetwork([784, 250, 10], 'logistic') img_list = GetCutZip(src) final_result = '' for img_array in img_list: img_array = img_array.flatten() result_list = nn.predict(img_array) result = np.argmax(result_list) final_result = final_result + str(result) return final_result
class NoisyLabelNeuralNetwork: def __init__(self, train_x=None, train_y=None, test_x=None, test_y=None): self.NN = NeuralNetwork(train_x=train_x, train_y=train_y, test_x=test_x, test_y=test_y) def set_model_path(self, model_path): self.NN.model_path = model_path def pretrain(self, labels=None, noise_level=None, save_state=None, batch_size=100, epoch_size=10): # direct to NN function self.NN.train_NN(self, labels=labels, noise_level=noise_level, save_state=save_state, batch_size=batch_size, epoch_size=epoch_size) def run_NLNN(self, labels=None, noise_level=None, it_nr=15, batch_size=100, epoch_size=10): # set number of iterations of NLNN, 'before' model must already exist # retrieve predictions and initializes EM module, prints accuracy for information prob_y = self.NN.restored_prob_y(noise_level, state='before') self.NN.get_acc(noise_level, 'before') self.EM = EMModule(initializer=prob_y, labels=labels) # iterate, gets improved theta, updates NN and checks for convergence for it in range(it_nr): prev_theta = self.EM.theta c, new_theta = self.EM.iteration(it_nr=it, new_prob_y=prob_y) acc, prob_y = self.NN.train_NN(save_state='after', labels=c, noise_level=noise_level, batch_size=batch_size, epoch_size=epoch_size) if ut.dist(prev_theta, new_theta) < 10**-3: print('Converged after %s iterations\n'%it_nr) break # print accuracy for information self.NN.get_acc(noise_level, state='after')
def main(): Info.printTitle() # get options from arguments. dicOptions = Opts.getOptions() if dicOptions == None: Info.printHelp() return False workClass = dicOptions[PAR_MODE] # get variables for NN. dicNNVariables = NNet.getNNVariables() # set up parameters. workClass.setOptions(dicOptions) workClass.setNNVariables(dicNNVariables) # run work-function. if workClass.run() == False: print("Fail.") return False print("Complete.") return True
def test(): user = {} if request.method == "GET": user['clean'] = request.args.get("img") else: if 'file' not in request.files: flash('No file part') return redirect("/") file = request.files['file'] # if user does not select file, browser also # submit a empty part without filename if file.filename == '': flash('No selected file') return redirect("/") if file and allowed(file.filename): filename = secure_filename(file.filename) file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename)) user['clean'] = filename else: flash('Format file tidak didukung(png,jpeg,jpg)') return redirect("/") user['guess'] = NN.test(user['clean']) #return user return render_template('test.html', title='Test', info=user)
def test(num_tests): global training_inputs, temp_training_outputs, test_inputs, test_outputs correct = 0 incorrect = 0 # Test NN amount of times user specifies with num_tests for i in range(num_tests): # Get random test data r = randint(0, 59999) # Get test data in correct format new_test_input = np.array([[0.0 for x in range(784)]]) temp = test_inputs[:, r] for j in range(784): new_test_input[0][j] = temp[j] new_test_input = new_test_input.T # Get NN's guess for input nnGuess = NeuralNetwork.think(NumberNet, test_input=new_test_input) # Log test results if test_outputs[r] == nnGuess: correct += 1 else: incorrect += 1 # Print test results print("Correct: " + str(correct)) print("Incorrect: " + str(incorrect))
def train(): user = { 'l1': NN.nodes_in_input_layer, 'l2': NN.nodes_in_hidden_layer, 'l3': NN.nodes_in_output_layer, 'akurasi': NN.accuration() } user['exist'] = os.path.isfile("./NN/mnist_train.csv") return render_template('training.html', title='Training', info=user)
class NoisyLabelNeuralNetwork: def __init__(self, train_x=None, train_y=None, test_x=None, test_y=None): self.NN = NeuralNetwork(train_x=train_x, train_y=train_y, test_x=test_x, test_y=test_y) def set_model_path(self, model_path): self.NN.model_path = model_path def pretrain(self, labels=None, noise_level=None, save_state=None, batch_size=100, epoch_size=10): # direct to NN function self.NN.train_NN(self, labels=labels, noise_level=noise_level, save_state=save_state, batch_size=batch_size, epoch_size=epoch_size) def run_NLNN(self, labels=None, noise_level=None, it_nr=15, batch_size=100, epoch_size=10): # set number of iterations of NLNN, 'before' model must already exist # retrieve predictions and initializes EM module, prints accuracy for information prob_y = self.NN.restored_prob_y(noise_level, state='before') self.NN.get_acc(noise_level, 'before') self.EM = EMModule(initializer=prob_y, labels=labels) # iterate, gets improved theta, updates NN and checks for convergence for it in range(it_nr): prev_theta = self.EM.theta c, new_theta = self.EM.iteration(it_nr=it, new_prob_y=prob_y) acc, prob_y = self.NN.train_NN(save_state='after', labels=c, noise_level=noise_level, batch_size=batch_size, epoch_size=epoch_size) if ut.dist(prev_theta, new_theta) < 10**-3: print('Converged after %s iterations\n' % it_nr) break # print accuracy for information self.NN.get_acc(noise_level, state='after')
def hello(): user = { 'l1': NN.nodes_in_input_layer, 'l2': NN.nodes_in_hidden_layer, 'l3': NN.nodes_in_output_layer, 'akurasi': NN.accuration() } user['file'] = filter(None, [ v if v.split('.').pop() in allowed_file else "" for v in os.listdir("static/clean") ]) return render_template('home.html', title='Home', info=user)
def NN_fit(train_set, val_set, nn=5, epochs=10, width=10, layers=2): from NN import NeuralNetwork last_error = 100 last_predicated = None fnn = None for x in range(nn): nn = NeuralNetwork() nn.train( train_set, val_set, epochs=epochs, width=width, layers=layers, batch_size=20, learning_rate=0.001, ) predicted_y, _ = nn.predict(train_set.X) logger.info( f"NN train MSE {mean_squared_error(train_set.Y, predicted_y)}") predicted_y, _ = nn.predict(val_set.X) error = mean_squared_error(val_set.Y, predicted_y) logger.info(f"NN dev MSE {error}") if error < last_error: last_error = error fnn = nn return fnn
def do_a_graph(params, a_list): fig = plt.figure() ax = fig.add_subplot(111, projection='3d') x_min = params["Grid"]["x_min"] x_step_size = params["Grid"]["x_step_size"] x_steps = params["Grid"]["x_steps"] x_max = x_min + (x_steps - 1) * x_step_size n_time_steps = len(a_list) T_mat = params["PayoffChars"]["T"] t_min = 0. t_max = T_mat t_steps = params["PayoffChars"]["NTime"] t_step_size = T_mat / n_time_steps # t = np.arange(t_min, t_max, t_step_size) # x = np.arange(x_min, x_max, x_step_size) t = np.linspace(t_min, t_max, num=t_steps) x = np.linspace(x_min, x_max, num=x_steps) X, T = np.meshgrid(x, t) a_np = np.array(a_list) zs = a_np Z = zs.reshape(T.shape) ax.plot_surface(X, T, Z) ax.set_xlabel(r'$x$') ax.set_ylabel(r'$t$') # ax.set_zlabel(r'$\tilde a^{\hat \theta^*}$') title = r"$\tilde a^{\hat \theta^*}$ Surface" # plt.title(title, pad=20) if params["General"]["Run"]["ShowEvalAOnGrid"]: plt.show() conf_type, diffusion_type = NeuralNetwork.get_conf_and_diffusion_types( params["Conf"], params["Diffusion"]["Type"]) a_type = "a_" + params["a"] for angle_step in range(12): for azim_step in range(3): azim = 30 * (azim_step - 1) angle = 30 * angle_step file_path = "./figures/" + conf_type + "/" + diffusion_type + "/" + a_type + "/a_surface/" + str( azim_step - 1 ) + "/graph_a_surface_" + conf_type + "_" + diffusion_type + "_" + str( azim) + "_" + str(angle) + ".png" directory = os.path.dirname(file_path) if not os.path.exists(directory): os.makedirs(directory) ax.view_init(azim, angle) fig.savefig(file_path) plt.close()
def dbnunfoldtonn(dbn, outputsize): #DBNUNFOLDTONN Unfolds a DBN to a NN # dbnunfoldtonn(dbn, outputsize ) returns the unfolded dbn with a final # layer of size outputsize added. if outputsize is not None: size = np.append(dbn.dbnSizes, outputsize) else: size = np.array(dbn.sizes) nn = NeuralNetwork.NN(size) for i in range(0, len(dbn.RBM)): nn.W[i][:, 0] = dbn.RBM[i].C.squeeze() nn.W[i][:, 1:] = dbn.RBM[i].W return nn
def trainOR(): Network0 = NeuralNetwork(sturcture=[2, 3, 1], learningRate=0.5) for i in range(1000): Network0.train([0, 0], [0]) Network0.train([0, 1], [1]) Network0.train([1, 0], [1]) Network0.train([1, 1], [1]) print(i, "", round((1 - Network.costFunction) * 100, 0)) pass print(round((1 - Network0.costFunction) * 100, 2)) Network0.answer([0, 0]) print(Network0.neurons[2][0].getOutput()) pass
def trainXOR(): print("\ntrain XOR") sturcture = [2, 3, 1] Network0 = NeuralNetwork(sturcture=sturcture, learningRate=1) for i in range(10000): Network0.train([0, 0], [0]) Network0.train([0, 1], [1]) Network0.train([1, 0], [1]) Network0.train([1, 1], [0]) #print(i,"",round((1-Network0.costFunction)*100, 0)) print(i, "", round(abs(Network0.neurons[len(sturcture) - 1][0].a - 0), 3)) print(Network0.answer([0, 0])) print(Network0.answer([1, 0])) print(Network0.answer([0, 1])) print(Network0.answer([1, 1])) pass
def do_single_run(params): nn = NeuralNetwork(params) final_loss = nn.train( number_of_batches=params["Run"]["NumberOfBatchesForTraining"], learning_rate=params["Run"]["LearningRate"]) if params["General"]["Run"]["DoEvalAOnGrid"]: a_list = nn.eval_a_on_grid(params["Grid"]["x_min"], params["Grid"]["x_steps"], params["Grid"]["x_step_size"]) do_a_graph(params, a_list) monte_carlo_price, monte_carlo_std = nn.monte_carlo_price( number_of_batches=params["Run"]["NumberOfBatchesForEval"]) print("real price: ", nn.bachelier_call_price()) print("monte_carlo_price: ", monte_carlo_price) print("monte_carlo_std: ", monte_carlo_std) _, _, = nn.eval(number_of_batches=params["Run"]["NumberOfBatchesForEval"]) if params["General"]["Run"]["DoRobustGraphs"]: params["General"]["Run"]["RatioRobustGraphs"] = params["General"][ "Run"]["RatioNumberOfBatchesForEval"] * params["General"]["Run"][ "RatioNumberOfBatchesForTraining"] do_robust_graphs(params, nn) nn.sess.graph.finalize() tf.reset_default_graph() nn.sess.close() tf.reset_default_graph() conf_type, diffusion_type = NeuralNetwork.get_conf_and_diffusion_types( params["Conf"], params["Diffusion"]["Type"]) a_type = "a_" + params["a"] file_path_final_loss = "./figures/" + conf_type + "/" + diffusion_type + "/" + a_type + "/graph_" + conf_type + "_" + diffusion_type + "_final_loss.txt" f = open(file_path_final_loss, "w") file_string = "Var(Z) for single run is: " + str(final_loss) f.write(file_string) f.close()
def __init__(self, train_x=None, train_y=None, test_x=None, test_y=None): self.NN = NeuralNetwork(train_x=train_x, train_y=train_y, test_x=test_x, test_y=test_y)
sys.path.append("../../..") import csv import numpy as np from NN import NeuralNetwork # Need to be decompressed from kaggle_test_dataset.tar file imgs_test = "test.csv" if __name__ == "__main__": # Create network from serialization file net = NeuralNetwork() with open("../network_files/mnist_500_network_file", "rb") as f: net.load(f) # Open images files imgs = open(imgs_test, "r") imgs_reader = csv.reader(imgs, delimiter=',') # Make the output print("ImageId,Label") first = True index = 0 for i in imgs_reader: if first:
import numpy as np from NN import NeuralNetwork import pygame from pygame.locals import * side = 400 screen = pygame.display.set_mode((side, side)) running = 1 brain = NeuralNetwork(2, 4, 1, 0.3) #brain.readState('XOR.state.json') training_data = [{ 'input': [0, 0], 'target': [0] }, { 'input': [1, 0], 'target': [1] }, { 'input': [0, 1], 'target': [1] }, { 'input': [1, 1], 'target': [0] }] def drawPrediction(): resolution = 10 cols = rows = int(side / resolution)
def do_graph(params): number_of_steps = params["Graph"]["NumberOfSteps"] if params["Payoff"] == "Call" or params["Payoff"] == "Calls&Puts": real_prices_array = np.full(number_of_steps, 0.) mc_prices_array = np.full(number_of_steps, 0.) mc_stds_array = np.full(number_of_steps, 0.) ad_prices_array = np.full(number_of_steps, 0.) ad_stds_array = np.full(number_of_steps, 0.) x_array = np.full(number_of_steps, 0.) x_min = params["Graph"]["XMin"] step_size = params["Graph"]["StepSize"] number_of_batches_for_training = params["Run"][ "NumberOfBatchesForTraining"] number_of_batches_for_eval = params["Run"]["NumberOfBatchesForEval"] nn_list = [None for i in range(number_of_steps)] x_original = copy.deepcopy(params["Diffusion"]["X"]) all_final_losses = np.full(number_of_steps, 0.) for i in range(number_of_steps): x = x_min + i * step_size params["Diffusion"]["X"] = x nn_list[i] = NeuralNetwork(params) mc_prices_array[i], mc_stds_array[i] = nn_list[i].monte_carlo_price( number_of_batches=number_of_batches_for_eval) if params["Run"]["DoAutomaticLearningRate"]: variance = np.square( mc_stds_array[i] ) * number_of_batches_for_eval * params["NN"]["NBatchSize"] log10_variance = np.floor(np.log10(variance)) learning_rate = params["Run"][ "BaseForAutomaticLearningRate"] / np.power(10, log10_variance) elif params["Run"]["DoListLearningRates"]: learning_rate = params["Run"]["ListLearningRates"][i] else: learning_rate = params["Run"]["LearningRate"] if params["NN"]["DoAutomaticLambdaConstraint"]: variance = np.square( mc_stds_array[i] ) * number_of_batches_for_eval * params["NN"]["NBatchSize"] log10_variance = np.floor(np.log10(variance)) lambda_constraint = params["NN"][ "BaseForAutomaticLambdaConstraint"] * np.power( 10, log10_variance) nn_list[i].params["NN"]["LambdaConstraint"] = lambda_constraint nn_list[i].lambda_constraint = lambda_constraint tf.Session().graph.finalize() all_final_losses[i] = nn_list[i].train( number_of_batches=number_of_batches_for_training, learning_rate=learning_rate) ad_prices_array[i], ad_stds_array[i] = nn_list[i].eval( number_of_batches=number_of_batches_for_eval) print("step_number =", i) print("x = ", x) print("learning_rate = ", learning_rate) print("lambda_constraint = ", lambda_constraint) print("mc_price = ", mc_prices_array[i]) print("ad_price = ", ad_prices_array[i]) print("mc_std = ", mc_stds_array[i]) print("ad_std = ", ad_stds_array[i]) if params["Diffusion"]["Type"] != "LV": if params["Payoff"] == "Call" or params["Payoff"] == "Calls&Puts": real_prices_array[i] = nn_list[i].bachelier_call_price() x_array[i] = x nn_list[i].sess.graph.finalize() tf.reset_default_graph() nn_list[i].sess.close() tf.reset_default_graph() params["Diffusion"]["X"] = x_original fig, ax1 = plt.subplots() # ax1.set_xlabel(r'$x_0$') if params["Payoff"] == "Call" or params["Payoff"] == "Calls&Puts": if params["Diffusion"]["Type"] != "LV": ax1.plot(x_array, real_prices_array, label=r'$\textnormal{Bachelier Price}$', color='black') ax1.plot(x_array, ad_prices_array, linestyle=':', label=r'$\textnormal{Adaptative Price}$', color='tab:blue') """ ax1.plot(x_array, ad_prices_array + 300 * ad_stds_array, linestyle=':', label=r'$\textnormal{Adaptative + 300 std}$', color='tab:cyan') ax1.plot(x_array, ad_prices_array - 300 * ad_stds_array, linestyle=':', label=r'$\textnormal{Adaptative - 300 std}$', color='tab:cyan') """ ax1.plot(x_array, mc_prices_array, linestyle='-', label=r'$\textnormal{MC Price}$', color='tab:red') """ ax1.plot(x_array, mc_prices_array + 300 * mc_stds_array, linestyle=':', label=r'$\textnormal{MC + 300 std}$', color='tab:pink') ax1.plot(x_array, mc_prices_array - 300 * mc_stds_array, linestyle=':', label=r'$\textnormal{MC - 300 std}$', color='tab:pink') """ ax1.tick_params(axis='y') if params["Diffusion"]["Type"] == "Bachelier": my_title = r'Bachelier Prices vs Adaptative Prices vs MC Prices' elif params["Diffusion"]["Type"] == "LV": my_title = r'Adaptative Prices vs MC Prices for Local Volatility Diffusion' # plt.title(my_title) # plt.tight_layout() # plt.subplots_adjust(bottom=0.3) # ax1.legend(loc='upper center', bbox_to_anchor=(0.5, - 0.2)) if params["Graph"]["ShowGraph"]: plt.show() if params["Graph"]["Save"]: conf_type, diffusion_type = NeuralNetwork.get_conf_and_diffusion_types( params["Conf"], params["Diffusion"]["Type"]) a_type = "a_" + params["a"] file_path = "./figures/" + conf_type + "/" + diffusion_type + "/" + a_type + "/graph_" + conf_type + "_" + diffusion_type + ".png" directory = os.path.dirname(file_path) if not os.path.exists(directory): os.makedirs(directory) fig.savefig(file_path) file_path_json = "./figures/" + conf_type + "/" + diffusion_type + "/" + a_type + "/json_" + conf_type + "_" + diffusion_type + ".json" params_json = json.dumps(params, indent=4) f = open(file_path_json, "w") f.write(params_json) f.close() plt.close() fig, ax1 = plt.subplots() # ax1.set_xlabel(r'$x_0$') """ if params["Payoff"] == "Call" or params["Payoff"] == "Calls&Puts": error_adaptative = ad_prices_array - real_prices_array error_mc = mc_prices_array - real_prices_array """ handle_list = [None for i in range(3)] label_list = [None for i in range(3)] # ax1.plot(x_array, error_adaptative, label=r'$\textnormal{Adaptative Error}$', color='tab:blue') # ax1.plot(x_array, error_mc, label=r'$\textnormal{MC Error}$', color='tab:red') handle_list[0], = ax1.plot( x_array, ad_stds_array, label=r'$\textnormal{Adaptative Standard Deviation}$', color='tab:blue', linestyle=":") label_list[0] = r'$\textnormal{Adaptative Standard Deviation}$' handle_list[1], = ax1.plot(x_array, mc_stds_array, label=r'$\textnormal{MC Standard Deviataion}$', color='tab:red', linestyle="-") label_list[1] = r'$\textnormal{MC Standard Deviataion}$' # ax1.set_ylabel(r"Standard Deviation") ax1.tick_params(axis='y', labelcolor='tab:blue') my_title = r'Adaptative vs MC Errors and Standard Deviations' # plt.title(my_title) # plt.tight_layout() # ax1.legend(loc=2) ax2 = ax1.twinx() handle_list[2], = ax2.plot( x_array, ad_stds_array / mc_stds_array, label=r'Ratio of Adaptive and MC Standard Deviations', color="tab:orange", linestyle="--") label_list[2] = r'Ratio of Adaptive and MC Standard Deviations' # ax2.legend(loc=1) # ax2.set_ylabel(r"Ratio") ax2.set_ylim(0.0, 1.5) ax2.tick_params(axis="y", labelcolor="tab:orange") # plt.subplots_adjust(bottom=0.3) # plt.legend(handles=handle_list, labels=label_list, loc='upper center', bbox_to_anchor=(0.5, - 0.2)) if params["Graph"]["ShowGraph"]: plt.show() if params["Graph"]["Save"]: conf_type, diffusion_type = NeuralNetwork.get_conf_and_diffusion_types( params["Conf"], params["Diffusion"]["Type"]) a_type = "a_" + params["a"] file_path = "./figures/" + conf_type + "/" + diffusion_type + "/" + a_type + "/graph_" + conf_type + "_" + diffusion_type + "_errors_and_stds.png" directory = os.path.dirname(file_path) if not os.path.exists(directory): os.makedirs(directory) fig.savefig(file_path) plt.close() file_path_all_final_losses = "./figures/" + conf_type + "/" + diffusion_type + "/" + a_type + "/graph_" + conf_type + "_" + diffusion_type + "_all_final_losses.txt" f = open(file_path_all_final_losses, "w") file_string = "" for i in range(len(all_final_losses)): file_string += "var(Z) for step i = " + str(i) + " is: " + str( all_final_losses[i]) + "\n" f.write(file_string) f.close()
def main(): parser = argparse.ArgumentParser( description='Script to train a language model') parser.add_argument("--training_size", default=10000, type=int, help="define training data set size") parser.add_argument("--test_size", default=1000, type=int, help="define test data set size") parser.add_argument( "--data_biased", default="../data/target_biased.txt", type=str, help="text file containing the source biased data (all)") parser.add_argument("--data_neutral", default="../data/target_neutral.txt", type=str, help="text file containing the source neutral data") parser.add_argument( "--model", default="NB", type=str, help= "choose model ||| NB --> Naïve Bayes ||| SVM --> Support Vector Machine ||| NN --> Neural Network" ) parser.add_argument( "--user_input", default=False, type=bool, help="Command line input for custom sentence classification") parser.add_argument("--load_model", default=False, type=bool, help="Load the best performing SVM model") args = parser.parse_args() #Load data from files biased_data = load_data(args.data_biased, "biased") neutral_data = load_data(args.data_neutral, "neutral") #Split data into sets training_data, dev_data, test_data = prepare_data(args.training_size, args.test_size, biased_data, neutral_data) #Get POS tags training_data_pos = pos_tag(training_data) dev_data_pos = pos_tag(dev_data) test_data_pos = pos_tag(test_data) print("Length of TRAINING data: ", len(training_data[0])) print("Length of DEVELOPMENT data: ", len(dev_data[0])) print("Length of TEST data: ", len(test_data[0])) if (args.model == "NB"): model = NaiveBayes(training_data_pos, test_data_pos) elif (args.model == "SVM"): if args.load_model: model = SVM(training_data_pos, test_data_pos, False, "SVM10000POS.joblib") else: model = SVM(training_data_pos, test_data_pos, False) elif (args.model == "NN"): model = NeuralNetwork(training_data_pos, dev_data_pos, test_data_pos, False) if args.user_input: user_input = input("Enter the sentence that you want to classify: ") user_label = input( "Do you think its biased or not? Enter 1 (for biased) | 0 (for neutral): " ) user_data = [[user_input], [user_label]] model.predict(pos_tag(user_data)) else: model.predict()
def generate(): Digit_NN = NeuralNetwork(no_of_in_nodes=image_pixels, no_of_out_nodes=10, no_of_hidden_nodes=100, learning_rate=0.1) ''' Letter_NN = NeuralNetwork(no_of_in_nodes = image_pixels, no_of_out_nodes = 26, no_of_hidden_nodes = 100, learning_rate = 0.1) ''' Meta_NN = NeuralNetwork(Digit_NN.no_of_out_nodes, 2, 100, 0.05) #Train Digit NN for i in range(len(digits_train_imgs)): Digit_NN.train(digits_train_imgs[i], digits_train_labels_one_hot[i]) #Display Statistics for Digits corrects, wrongs = Digit_NN.evaluate(digits_train_imgs, digits_train_labels) print("accuracy train: ", corrects / (corrects + wrongs)) corrects, wrongs = Digit_NN.evaluate(digits_test_imgs, digits_test_labels) print("accuracy: test", corrects / (corrects + wrongs)) #Train MetaNN, save NN output vectors to be evaluated later for i in range(len(mixed_train_imgs)): Meta_NN.train(np.sort(Digit_NN.run(mixed_train_imgs[i]).T), mixed_train_labels[i]) #Display Statistics for Meta #TODO: Investigate whether this has redundant code corrects, wrongs = Meta_NN.metaEval(Digit_NN, mixed_train_imgs, mixed_train_labels) train_accuracy = corrects / (corrects + wrongs) print("Train Accuracy: ", train_accuracy) print("Train Confusion Matrix: ") print( Meta_NN.meta_confusion_matrix(Digit_NN, mixed_train_imgs, mixed_train_labels, mixed_train_values)) corrects, wrongs = Meta_NN.metaEval(Digit_NN, mixed_test_imgs, mixed_test_labels) test_accuracy = corrects / (corrects + wrongs) print("Test Accuracy: ", test_accuracy) print("Test Confusion Matrix: ") print( Meta_NN.meta_confusion_matrix(Digit_NN, mixed_test_imgs, mixed_test_labels, mixed_test_values)) return train_accuracy, test_accuracy, Digit_NN, Meta_NN
def double_cross_validation(workers: int, testFolder: int, nFolder: int, dataSet, f: ActivFunct, learnRate: list, momRate: list, regRate: list, ValMax: list, HiddenUnits: list, OutputUnits: list, MaxEpochs: list, Tolerance: list, startTime, errorFunct=None, modeLearn: ModeLearn = ModeLearn.BATCH, miniBatchDim=None, errorVlFunct=None, hiddenF: ActivFunct = None): if testFolder <= 1: raise ValueError("Wrong value of num. folders inserted") cp = dataSet.copy() #Rimescolo il data set. rnd.shuffle(cp) #Costruisco la sottolista dei dati divisibile esattamente per testFolder. h = len(cp) - len(cp) % testFolder dataSetExact = cp[0:h] #Creo la lista dei folders. folderDim = int(len(dataSetExact) / testFolder) folders = [ cp[i * folderDim:(i + 1) * folderDim] for i in range(testFolder) ] #Inserisco gli elementi di avanzo. for i in range(len(dataSet) - h): folders[i].append(cp[i + h]) errList = list() for i in range(len(folders)): foldersCopy = folders.copy() testSet = foldersCopy[i] del (foldersCopy[i]) vlSet = list() for j in range(len(foldersCopy)): vlSet += foldersCopy[j] e = cross_validation(workers, nFolder, modeLearn, vlSet, f, errorFunct, learnRate, momRate, regRate, ValMax, HiddenUnits, OutputUnits, MaxEpochs, Tolerance, startTime, miniBatchDim=miniBatchDim, errorVlFunct=errorVlFunct, hiddenF=hiddenF) theta = getBestResult(e)[0] nn = NeuralNetwork(vlSet, f, new_hyp=theta, Hiddenf=hiddenF) (_, testErr, _, _) = nn.learn(modeLearn, errorFunct, miniBatchDim, testSet) errList.append(testErr[-1]) return 1 / testFolder * sum(errList)
def k_fold_CV_single(k: int, dataSet, f: ActivFunct, theta, errorFunct=None, modeLearn: ModeLearn = ModeLearn.BATCH, miniBatchDim=None, errorVlFunct=None, hiddenF: ActivFunct = None): if k <= 0: raise ValueError("Wrong value of num. folders inserted") cp = dataSet.copy() #Rimescolo il data set. rnd.shuffle(cp) #Costruisco la sottolista dei dati divisibile esattamente per k. h = len(cp) - len(cp) % k dataSetExact = cp[0:h] #Creo la lista dei folders. folderDim = int(len(dataSetExact) / k) folders = [cp[i * folderDim:(i + 1) * folderDim] for i in range(k)] #Inserisco gli elementi di avanzo. for i in range(len(cp) - h): folders[i].append(cp[i + h]) errore = list() #per stampare l'errore sul traininig set e sul validation set trErrorPlot = list() vlErrorPlot = list() for i in range(len(folders)): lcopy = folders.copy() del (lcopy[i]) vlSet = folders[i] trSet = list() for j in range(len(lcopy)): trSet += lcopy[j] nn = NeuralNetwork(trSet, f, theta, Hiddenf=hiddenF) (trErr, vlErr, trAcc, vlAcc) = nn.learn(modeLearn, errorFunct, miniBatchDim, vlSet, errorVlFunct=errorVlFunct) trErrorPlot.append(trErr) vlErrorPlot.append(vlErr) errore.append(nn.getError(vlSet, 1 / len(vlSet), errorVlFunct)) err = sum(errore) / k #controllo che tutti gli errorPlot abbiano la stessa lunghezza maxLen = len(trErrorPlot[0]) for i in range(1, len(trErrorPlot)): if len(trErrorPlot[i]) > maxLen: maxLen = len(trErrorPlot[i]) for i in range(len(trErrorPlot)): if len(trErrorPlot[i]) < maxLen: for j in range(maxLen - len(trErrorPlot[i])): trErrorPlot[i].append(trErrorPlot[i][-1]) vlErrorPlot[i].append(vlErrorPlot[i][-1]) trErrorArray = np.array(trErrorPlot[0]) vlErrorArray = np.array(vlErrorPlot[0]) for i in range(1, len(trErrorPlot)): trErrorArray = trErrorArray + np.array(trErrorPlot[i]) trErrorArray = trErrorArray / k for i in range(1, len(vlErrorPlot)): vlErrorArray = vlErrorArray + np.array(vlErrorPlot[i]) vlErrorArray = vlErrorArray / k return (err, trErrorArray, vlErrorArray)
def genetic_algorithm(training_inputs, training_groundtruth, test_inputs, test_groundtruth, num_population,times, invasion, hidden_nodes, lr, lr_decay, mf, batch_size, epoch): ''' In this function, I will perform the genetic_algorithm to find out the best combination of hyperparameters for the training. [training_inputs, training_groundtruth, test_inputs, test_groundtruth] is a training, testing set which are obtained for evalute the training performance. num_population: the number of random candidates with the random number of feathers. times: total times of "cross" for the parents to exchange the features which uis aiming at getting 'progeny" with the better performance. invasion: number of new "invasion" individuals, besides the progeny got from the 'cross', for each time I also introduce some new individuals with different features, which is aimming at making the whole population get more information for optimization. [hidden_nodes, lr, lr_decay, mf, batch_size, epoch] are the six features i planned to use as the hyperparameter features that needs to be optimized They are all put in as the two-element list, which represents the range. In the function, I will use rand funvtion to generate a random number between the range. ''' # generate the parents population print('generating '+ str(num_population)+' individuals') # makeing sure the input are correct assert hidden_nodes[0] < hidden_nodes[1], 'something went wrong!' assert lr[0] < lr[1], 'something went wrong!' assert lr_decay[0] < lr_decay[1], 'something went wrong!' assert mf[0] < mf[1], 'something went wrong!' assert batch_size[0] < batch_size[1], 'something went wrong!' assert epoch[0] < epoch[1], 'something went wrong!' # generating a bunch of individuals based on the range that provided individuals_genom = [] individuals_phyno = [] for i in range(num_population): # randonly generate the feature _hidden_nodes = int(rand(hidden_nodes[0],hidden_nodes[1])) _lr = rand(lr[0], lr[1]) _lr_decay = rand(lr_decay[0], lr_decay[1]) _mf = rand(mf[0],mf[1]) _batch_size = int(rand(batch_size[0],batch_size[1] )) _epoch = int(rand(epoch[0], epoch[1])) # build an individual and put it into the whole set individuals_genom.append( [_hidden_nodes, _lr, _lr_decay, _mf, _batch_size, _epoch]) NN = NeuralNetwork(input_layer=68, hidden_layer= _hidden_nodes, output_layer=1, lr = _lr, lr_decay= _lr_decay, iteration= _epoch, batch_size= _batch_size, mf= _mf) NN.make_weights() NN.train(training_inputs, training_groundtruth) # also store the individual's performance in a list vector individuals_phyno.append(AUROC_cruve(NN, test_inputs, test_groundtruth, Fig=False)) # take the best performance people and keep it for the next generation my_phyno = max(individuals_phyno) idx = individuals_phyno.index(my_phyno) my_genome = individuals_genom[idx] n = invasion # begin the cross, do N times of cross for t in range(times): print('For the time '+str(t)+' the best candidates and the best result is') print(my_genome) print(my_phyno) if t >=1: if len(individuals_genom) % 2 == 0: # add the new invasion people, # make sure that the number is even add = n else: add = n+1 for i in range(add): _hidden_nodes = int(rand(hidden_nodes[0],hidden_nodes[1])) _lr = rand(lr[0] , lr[1]) _lr_decay = rand(lr_decay[0], lr_decay[1]) _mf = rand(mf[0],mf[1]) _batch_size = int(rand(batch_size[0],batch_size[1] )) _epoch = int(rand(epoch[0], epoch[1])) individuals_genom.append( [_hidden_nodes, _lr, _lr_decay, _mf, _batch_size, _epoch]) NN = NeuralNetwork(input_layer=68, hidden_layer= _hidden_nodes, output_layer=1, lr = _lr, lr_decay= _lr_decay, iteration= _epoch, batch_size= _batch_size, mf= _mf) NN.make_weights() NN.train(training_inputs, training_groundtruth) individuals_phyno.append(AUROC_cruve(NN, test_inputs, test_groundtruth, Fig=False)) # next_generation_genom = [] next_generation_phyno = [] next_generation_genom.append(my_genome) next_generation_phyno.append(my_phyno) assert len(individuals_genom) %2 == 0, 'something wrong!' for i in range(int(len(individuals_genom)/2)): child_genom, child_phyno = cross(individuals_genom[i*2],individuals_genom[ i*2 +1], training_inputs, training_groundtruth, test_inputs, test_groundtruth) # get the child, put the child into the next generation next_generation_genom.append(child_genom) next_generation_phyno.append(child_phyno) # next is now the parents individuals_genom = next_generation_genom individuals_phyno = next_generation_phyno # shift it randomly index = list(range(len(individuals_genom))) random.shuffle(index) individuals_phyno = [individuals_phyno[x] for x in index] individuals_genom = [individuals_genom[x] for x in index] # still get the best performance my_phyno = max(individuals_phyno) idx = individuals_phyno.index(my_phyno) my_genome = individuals_genom[idx] print('The whole crossing process is done') my_phyno = max(individuals_phyno) idx = individuals_phyno.index(my_phyno) my_genome = individuals_genom[idx] print('the best candidates and the best result is') print(my_genome) print(my_phyno)
def cross(Father, Mother, training_inputs, training_groundtruth, test_inputs, test_groundtruth ): ''' This cross function is used for the genetic algorithm optimization. the inputs are two individuals, and will return a "child" with a better training performance. This function will first generate two children. And for each child's one single feature, it will either be from the 'mother' or the 'father' And then a quick training and testing process will be used to evalue the AUROC score. And the 'child' with a better performance will be returned. ''' Child_1 = [] Child_2 = [] for i in range(6): coin = rand(-1,1) if coin>= 0: Child_1.append(Father[i]) Child_2.append(Mother[i]) else: Child_1.append(Mother[i]) Child_2.append(Father[i]) Child_1 = mutate(Child_1) Child_2 = mutate(Child_2) # build the network, make weights, and train it NN_1 = NeuralNetwork(input_layer=68, hidden_layer= Child_1[0], output_layer=1, lr = Child_1[1], lr_decay= Child_1[2], iteration= Child_1[5], batch_size= Child_1[4], mf= Child_1[3]) NN_2 = NeuralNetwork(input_layer=68, hidden_layer= Child_2[0], output_layer=1, lr = Child_2[1], lr_decay= Child_2[2], iteration= Child_2[5], batch_size= Child_2[4], mf= Child_2[3]) NN_1.make_weights() NN_2.make_weights() NN_1.train(training_inputs, training_groundtruth) NN_2.train(training_inputs, training_groundtruth) Score_1 = AUROC_cruve(NN_1, test_inputs, test_groundtruth, Fig=False) Score_2 = AUROC_cruve(NN_2, test_inputs, test_groundtruth, Fig=False) if Score_1 > Score_2: return Child_1, Score_1 else: return Child_2, Score_2
from NN import NeuralNetwork from Normalizer import Normalizer import numpy as np from sklearn.datasets import load_breast_cancer from sklearn.model_selection import train_test_split cancer = load_breast_cancer() X = cancer['data'] y = cancer['target'] length = len(cancer['feature_names']) nn = NeuralNetwork([length, 30, 20, 10, 5, 1]) X_train, X_test, y_train, y_test = train_test_split(X, y) normalize = Normalizer() normalize.fit(X_train) X_train = normalize.transform(X_train) X_test = normalize.transform(X_test) nn.fit(X_train, y_train, epochs=1000, verbose=False) predictions = nn.predict(X_test) print(nn.cost(predictions, y_test))
import pandas import pygame import numpy as np from pygame.locals import * from NN import NeuralNetwork side = 28 * 8 step = 8 screen = pygame.display.set_mode((side, side)) pygame.font.init() myfont = pygame.font.SysFont('Comic Sans MS', 30) running = 1 numberIndex = 0 brain = NeuralNetwork(784, 100, 10, 0.3) brain.readState('digit-recognizer.state.json') numbers = pandas.read_csv('digit-recognizer/test.csv') numberIndex = 0 def drawNumber(index): number = numbers.iloc[index] pixelX = 0 pixelY = 0 for (columnName, color) in number.iteritems(): if (pixelX == 28): pixelX = 0 pixelY += 1
def do_robust_graphs(params, nn): if params["Diffusion"]["Type"] == "LV": parameters = ["X", "A", "B", "Rho", "M", "Sigma"] elif params["Diffusion"]["Type"] == "Bachelier": parameters = ["X", "Sigma"] for parameter in parameters: base_value = params["Diffusion"][parameter] number_of_steps = 20 step_size = base_value * 0.8 / number_of_steps sigma_array_adaptive = np.full(number_of_steps, 0.) sigma_array_mc = np.full(number_of_steps, 0.) base_values_array = np.full(number_of_steps, 0.) ratio = params["General"]["Run"]["RatioRobustGraphs"] for step in range(number_of_steps): cur_parameter = base_value * 0.6 + step * step_size base_values_array[step] = cur_parameter if parameter == "X": nn.params["Diffusion"]["X"] = cur_parameter av_price, glob_std = nn.eval( params["Run"]["NumberOfBatchesForEval"] // ratio, doing_robust_graph=True) mc_price, mc_std = nn.monte_carlo_price( params["Run"]["NumberOfBatchesForEval"] // ratio) sigma_array_adaptive[step] = glob_std sigma_array_mc[step] = mc_std nn.params["Diffusion"]["X"] = base_value print("Finished step=", step, "of do_robust_graphs with parameter", parameter) if parameter == "A": nn.params["Diffusion"]["A"] = cur_parameter av_price, glob_std = nn.eval( params["Run"]["NumberOfBatchesForEval"] // ratio, doing_robust_graph=True) mc_price, mc_std = nn.monte_carlo_price( params["Run"]["NumberOfBatchesForEval"] // ratio) sigma_array_adaptive[step] = glob_std sigma_array_mc[step] = mc_std nn.params["Diffusion"]["A"] = base_value print("Finished step=", step, "of do_robust_graphs with parameter", parameter) if parameter == "B": nn.params["Diffusion"]["B"] = cur_parameter av_price, glob_std = nn.eval( params["Run"]["NumberOfBatchesForEval"] // ratio, doing_robust_graph=True) mc_price, mc_std = nn.monte_carlo_price( params["Run"]["NumberOfBatchesForEval"] // ratio) sigma_array_adaptive[step] = glob_std sigma_array_mc[step] = mc_std nn.params["Diffusion"]["B"] = base_value print("Finished step=", step, "of do_robust_graphs with parameter", parameter) if parameter == "M": nn.params["Diffusion"]["M"] = cur_parameter av_price, glob_std = nn.eval( params["Run"]["NumberOfBatchesForEval"] // ratio, doing_robust_graph=True) mc_price, mc_std = nn.monte_carlo_price( params["Run"]["NumberOfBatchesForEval"] // ratio) sigma_array_adaptive[step] = glob_std sigma_array_mc[step] = mc_std nn.params["Diffusion"]["M"] = base_value print("Finished step=", step, "of do_robust_graphs with parameter", parameter) if parameter == "Rho": nn.params["Diffusion"]["Rho"] = cur_parameter av_price, glob_std = nn.eval( params["Run"]["NumberOfBatchesForEval"] // ratio, doing_robust_graph=True) mc_price, mc_std = nn.monte_carlo_price( params["Run"]["NumberOfBatchesForEval"] // ratio) sigma_array_adaptive[step] = glob_std sigma_array_mc[step] = mc_std nn.params["Diffusion"]["Rho"] = base_value print("Finished step=", step, "of do_robust_graphs with parameter", parameter) if parameter == "Sigma": nn.params["Diffusion"]["Sigma"] = cur_parameter av_price, glob_std = nn.eval( params["Run"]["NumberOfBatchesForEval"] // ratio, doing_robust_graph=True) mc_price, mc_std = nn.monte_carlo_price( params["Run"]["NumberOfBatchesForEval"] // ratio) sigma_array_adaptive[step] = glob_std sigma_array_mc[step] = mc_std nn.params["Diffusion"]["Sigma"] = base_value print("Finished step=", step, "of do_robust_graphs with parameter", parameter) fig, ax1 = plt.subplots() ax1.plot(base_values_array, sigma_array_mc, color='tab:blue', linestyle=":") ax1.tick_params(axis="y", labelcolor="tab:blue") ax1.plot(base_values_array, sigma_array_adaptive, color="tab:red", linestyle="-") ax1.set_ylim(bottom=0.0) ax2 = ax1.twinx() ax2.plot(base_values_array, sigma_array_adaptive / sigma_array_mc, color="tab:orange", linestyle="--") ax2.set_ylim(0.0, 1.5) ax2.tick_params(axis="y", labelcolor="tab:orange") if params["Graph"]["ShowGraph"]: plt.show() if params["Graph"]["Save"]: conf_type, diffusion_type = NeuralNetwork.get_conf_and_diffusion_types( params["Conf"], params["Diffusion"]["Type"]) a_type = "a_" + params["a"] file_path = "./figures/" + conf_type + "/" + diffusion_type + "/" + a_type + "/graph_" + conf_type + "_" + diffusion_type + "_robust_" + parameter + ".png" directory = os.path.dirname(file_path) if not os.path.exists(directory): os.makedirs(directory) fig.savefig(file_path) plt.close() return