def train_prediction( net: Neural_network.NeuralNet, inputs_train: Tensor, targets_train: Tensor, inputs_test: Tensor, targets_test: Tensor, loss: Loss.Loss = Loss.MeanSquareError(), optimizer: OptimizerClass.Optimizer = OptimizerClass.SGD(), num_epochs: int = 5000, batch_size: int = 32): Data = pd.DataFrame(columns=('MSE_train', 'MSE_test', 'error_round_train', 'error_round_test')) size_training = inputs_train.shape[0] for epoch in range(num_epochs): Chi2_train = 0.0 error_round_train = 0.0 nbr_batch = 0 for i in range(0, size_training, batch_size): nbr_batch += 1 # 1) feed forward y_actual = net.forward(inputs_train[i:i + batch_size]) # 2) compute the loss and the gradients Chi2_train += loss.loss(targets_train[i:i + batch_size], y_actual) grad_ini = loss.grad(targets_train[i:i + batch_size], y_actual) # 3)feed backwards grad_fini = net.backward(grad_ini) # 4) update the net optimizer.step(net, n_epoch=epoch) error_round_train += Error_round.error_round( targets_train[i:i + batch_size], y_actual) Chi2_train = Chi2_train / nbr_batch error_round_train = error_round_train / nbr_batch y_actual_test = net.forward(inputs_test) Chi2_test = loss.loss(targets_test, y_actual_test) error_round_test = Error_round.error_round(targets_test, y_actual_test) if epoch % 100 == 0: print('epoch : ' + str(epoch) + "/" + str(num_epochs) + "\r", end="") datanew = pd.DataFrame({ 'MSE_train': [Chi2_train], 'MSE_test': [Chi2_test], 'error_round_train': [error_round_train], 'error_round_test': [error_round_test] }) Data = Data.append(datanew) os.chdir(path_ini) Data.to_csv('Opt_num_epoch_backup.csv', index=False) return Data
def train(config, exp_name, data_path, resume=False, tune=False): results_dir = os.path.join(data_path, exp_name) if os.path.exists(results_dir) and not (resume or tune): print("{} already exists, no need to train.\n".format(results_dir)) return if not os.path.exists(results_dir): os.makedirs(results_dir) json.dump(config, open(os.path.join(results_dir, 'config.json'), 'w'), sort_keys=True, separators=(',\n', ': ')) is_dev = config['is_dev'] print("\n***{} MODE***\n".format('DEV' if is_dev else 'TEST')) if not is_dev: print("\n***Changing TEST to DEV***\n") config['is_dev'] = True data_set = data_loader.read_dataset(data_path, results_dir, dev_mode=True, max_examples=float('inf')) cuda = torch.cuda.is_available() print("Number of training data points {}".format(len(data_set['train']))) print("Number of dev data points {}".format(len(data_set['test']))) # Provide train and dev data to negative sampler for filtering positives data = copy.copy(data_set['train']) data.extend(data_set['test']) model, neg_sampler, evaluator = build_model(data, config, results_dir, data_set['num_ents'], data_set['num_rels']) model = is_gpu(model, cuda) state = None if resume or tune: params_path = os.path.join(results_dir, '{}_params.pt'.format(config['model'])) model.load_state_dict(torch.load(params_path)) if resume: state_path = os.path.join(results_dir, '{}_optim_state.pt'.format(config['model'])) state = torch.load(state_path) if config['neg_sampler'] == 'rl': sgd = optimizer.Reinforce(data_set['train'], data_set['dev'], model, neg_sampler, evaluator, results_dir, config, state) else: sgd = optimizer.SGD(data_set['train'], data_set['dev'], model, neg_sampler, evaluator, results_dir, config, state) start = time.time() sgd.minimize() end = time.time() hours = int((end - start) / 3600) minutes = ((end - start) % 3600) / 60. profile_string = "Finished Training! Took {} hours and {} minutes\n".format( hours, minutes) with open(os.path.join(results_dir, 'train_time'), 'w') as f: f.write(profile_string + "Raw seconds {}\n".format(end - start)) print(profile_string)
def build_optimizer_from_layer_parameters(layer_parameters: Dict): if _OPTIMIZER not in layer_parameters: _optimizer = optimiser.SGD() # Use SGD() as default else: optimizer_spec = layer_parameters[_OPTIMIZER] assert \ _SCHEME in optimizer_spec, \ "Invalid optimizer specification %s" % optimizer_spec optimizer_scheme = optimizer_spec[_SCHEME].lower() assert optimizer_scheme in optimiser.SCHEMES, \ "Invalid optimizer scheme %s. Scheme must be one of %s." \ % (optimizer_scheme, optimiser.SCHEMES) _optimizer = \ optimiser.SCHEMES[optimizer_scheme].build(optimizer_spec[_PARAMETERS]) \ if _PARAMETERS in optimizer_spec else optimiser.SGD() return _optimizer
def Opt_learning_rate(list_learning_rate): for my_lr in list_learning_rate: Data = User.train(my_NN, data_train_input, data_train_target, num_epochs=Nmax, optimizer=OptimizerClass.SGD(lr=my_lr), batch_size=my_batch_size)[1] plt.plot(range(Nmax), Data, label=str(my_lr)) plt.xlabel('Epoch') plt.ylabel(' Training mean squared error') plt.legend(title='Learning rate impact') plt.show()
def __init__(self, name: str, num_nodes: int, W: np.ndarray, posteriors: Optional[List[Layer]] = None, optimizer: optimiser.Optimizer = optimiser.SGD(), log_level: int = logging.ERROR): """Initialize a matmul layer that has 'num_nodes' nodes Input X:(N,D) is a batch. D is number of features NOT including bias Weight W:(M, D+1) is the layer weight including bias weight. Args: name: Layer identity name num_nodes: Number of nodes in the layer W: Weight of shape(M=num_nodes, D+1). A row is a weight vector of a node. posteriors: Post layers to which forward the matmul layer output optimizer: Gradient descent implementation e.g SGD, Adam. log_level: logging level """ super().__init__(name=name, num_nodes=num_nodes, log_level=log_level) # -------------------------------------------------------------------------------- # W: weight matrix of shape(M,D) where M=num_nodes # Gradient dL/dW has the same shape shape(M, D) with W because L is scalar. # # Not use WT because W keeps updated every cycle, hence need to update WT as well. # Hence not much performance gain and risk of introducing bugs. # self._WT: np.ndarray = W.T # transpose of W # -------------------------------------------------------------------------------- assert W.shape[0] == num_nodes, \ f"W shape needs to be ({num_nodes}, D) but {W.shape}." self._D = W.shape[1] # number of features in x including bias self._W: np.ndarray = copy.deepcopy(W) # node weight vectors self._dW: np.ndarray = np.empty(0, dtype=TYPE_FLOAT) # -------------------------------------------------------------------------------- # State of the layer # -------------------------------------------------------------------------------- self._S = {} self.logger.debug( "Matmul[%s] W.shape is [%s], number of nodes is [%s]", name, W.shape, num_nodes) # -------------------------------------------------------------------------------- # Optimizer for gradient descent # Z(n+1) = optimiser.update((Z(n), dL/dZ(n)+regularization) # -------------------------------------------------------------------------------- assert isinstance(optimizer, optimiser.Optimizer) self._optimizer: optimiser.Optimizer = optimizer
def __init__(self, X_train, Y_train, Net='LeNet5', opti='SGDMomentum'): # Prepare Data: Load, Shuffle, Normalization, Batching, Preprocessing self.X_train = X_train self.Y_train = Y_train self.batch_size = 64 # D_in: input depth of network, 784, 28*28 input grayscale image self.D_in = 784 # D_out: output depth of network = 10, the 10 digits self.D_out = 10 print(' Net: ' + str(Net)) print(' batch_size: ' + str(self.batch_size)) print(' D_in: ' + str(self.D_in)) print(' D_out: ' + str(self.D_out)) print(' Optimizer: ' + opti) # ======================= if Net == 'TwoLayerNet': # H is the size of the one hidden layer. H = 400 self.model = ANN.TwoLayerNet(self.D_in, H, self.D_out) elif Net == 'ThreeLayerNet': ####################################### ############ TODO ################## ####################################### # H1, H2 are the size of the two hidden layers. #self.model = ANN.ThreeLayerNet (self.D_in, H1, H2, self.D_out) print('Not Implemented.') exit(0) elif Net == 'LeNet5': self.model = CNN.LeNet5() # store training loss over iterations, for later visualization self.losses = [] if opti == 'SGD': self.opti = optimizer.SGD(self.model.get_params(), lr=0.0001, reg=0) else: self.opti = optimizer.SGDMomentum(self.model.get_params(), lr=0.0001, momentum=0.80, reg=0.00003) self.criterion = loss.CrossEntropyLoss()
def train(net: Neural_network.NeuralNet, inputs: Tensor, targets: Tensor, loss: Loss = Loss.MeanSquareError(), optimizer: OptimizerClass.Optimizer = OptimizerClass.SGD(), num_epochs: int = 5000, batch_size: int = 32) -> tuple: chi2_list = [] round_error_list = [] size_training = inputs.shape[0] for epoch in range(num_epochs): chi2_loss = 0.0 round_error_loss = 0.0 nbr_batch = 0 for i in range(0, size_training, batch_size): nbr_batch += 1 # 1) Feed forward y_actual = net.forward(inputs[i:i + batch_size]) # 2) Compute the loss and the gradient chi2_loss += loss.loss(targets[i:i + batch_size], y_actual) round_error_loss += Error_round.error_round( targets[i:i + batch_size], y_actual) grad_ini = loss.grad(targets[i:i + batch_size], y_actual) # 3) Feed backwards grad_fini = net.backward(grad_ini) # 4) Update the net optimizer.step(net, n_epoch=epoch) chi2_loss = chi2_loss / nbr_batch round_error_loss = round_error_loss / nbr_batch chi2_list.append(chi2_loss) round_error_list.append(round_error_loss) # Print status every 50 iterations if epoch % 50 == 0: print('\r epoch : ' + str(epoch) + "/" + str(num_epochs) + ", training mean squared error : " + str(chi2_loss) + "\r", end="") print('epoch : ' + str(epoch) + "/" + str(num_epochs) + ", training final mean squared error : " + str(chi2_loss) + '\n') return chi2_list, round_error_list
def backward(self, actual): """ Backward propagation Args: actual(np.ndarray): actual y's, right lables """ # for every layer reversed we backpropagate error = None # Radnoming i for extracting ith sample from tenosr # for stochastic gradient descent i = random.randint(0, actual.shape[0] - 1) for layer in reversed(self.layers): error = layer.backward(i, error, actual, self.loss) # Updating weights optim = optimizer.SGD(layer, self.learning_rate) optim.step()
def train_model(model, train_input, train_target, loss_fnc, nb_epochs, lr): ''' Supervised machine learning model training. ''' print("-------------------- Training --------------------") optimizer = optim.SGD(model.param(), lr) mini_batch_size = 100 base = int(nb_epochs / 10) model.initParameters() l = [] for e in range(nb_epochs + 1): for b in range(0, train_input.size(0), mini_batch_size): output = model(train_input.narrow(0, b, mini_batch_size)) loss = loss_fnc(output, train_target.narrow(0, b, mini_batch_size)) model.zero_grad() loss.backward() optimizer.step() if e % base == 0: print('Epochs', e, ': Loss ', loss.loss.item()) l.append(loss.loss.item()) print("--------------------------------------------------\n") return l
def Opt_nbr_epoch(): ''' Evolution of the chi2 et our special round error of the training and testing set according to the number of epoch. ''' Data = train_prediction(my_NN, data_train_input, data_train_target, data_test_input, data_test_target, num_epochs=Nmax, optimizer=OptimizerClass.SGD(lr=my_lr), batch_size=my_batch_size) print(Data) plt.plot(range(Nmax), Data['MSE_train'], label='training') plt.plot(range(Nmax), Data['MSE_test'], label='testing') plt.xlabel('Epoch') plt.ylabel(r'Mean Squared Error') plt.legend() plt.title('Learning Curve') plt.show()
def __init__(self, const): self.const = const lastLayer = layer.InputLayer() for l in range(len(const)): layerInfo = const[l] if layerInfo['type'] == 'Aff': if layerInfo['opt'] == 'Adam': opt = op.Adam(layerInfo['size']) elif layerInfo['opt'] == 'SGD': opt = op.SGD() lastLayer = layer.AffineLayer(layerInfo['size'], lastLayer, opt) elif layerInfo['type'] == 'Act': if layerInfo['func'] == 'ReLU': act = ac.ReLU() elif layerInfo['func'] == 'Sigmoid': act = ac.Sigmoid() elif layerInfo['func'] == 'Softmax': act = ac.Softmax() lastLayer = layer.ActLayer(lastLayer.outNode, lastLayer, act) self.outputLayer = lastLayer
if epoch % 100 == 0: print('epoch : ' + str(epoch) + "/" + str(num_epochs) + "\r", end="") for k in range(9): if k not in destroyed_NN: my_NN = list_net[k] return my_NN, Result_chi2 ## User's function Results = train_simultaneousNN(data_train_input, data_train_target, num_epochs=Nmax, optimizer=OptimizerClass.SGD(lr=my_lr), batch_size=my_batch_size) for k in range(9): Y = Results[1][k] X = range(len(Y)) plt.plot(X, Y) plt.ylabel('Mean Squared Error') plt.xlabel('Epoch') plt.title('Simultaneous NN : training curve') '''testing ''' data_test_prediction = User.prediction(Results[0], data_test_input) error = Error_round.error_round(data_test_prediction, data_test_target) print('% of false error for testing set : ', error) '''histogram of predictions'''
(x_train, y_train), (x_test, y_test) = load_mnist( normalize=True, flatten=True, one_hot_label=False) if flag == "train": return x_train, y_train elif flag == "test": return x_test, y_test else: print("data type error!!!") batch_size = 100 # 批数量 lr = 0.1 EPOCH = 100 net = simpleFC(784, 50, 10) optim = optimizer.SGD(lr=lr) x_train, y_train = get_data(flag="train") train_size = x_train.shape[0] x_test, y_test = get_data(flag="test") train_loss = [] train_acc = [] test_loss = [] test_acc = [] for epoch in tqdm(range(EPOCH)): # train for i in range(0, len(x_train), batch_size): batch_mask = np.random.choice(train_size, batch_size) x = x_train[batch_mask] y_ = y_train[batch_mask]
X_train -= np.mean(X_train) X_test -= np.mean(X_test) X_train = X_train.reshape(X_train.shape[0], 1, 28, 28) X_test = X_test.reshape(X_test.shape[0], 1, 28, 28) #X_train = X_train[:6000] #Y_train = Y_train[:6000] #X_test = X_test[:1000] #Y_test = Y_test[:1000] batch_size = 16 D_out = 10 model = nn.LeNet5() losses = [] optim = optimizer.SGD(model.get_params(), lr=0.00003) #optim = optimizer.SGDMomentum(model.get_params(), lr=0.00003, momentum=0.80, reg=0.0003) criterion = loss.SoftmaxLoss() # Train ITER = 30000 for i in range(ITER): # get batch, make onehot X_batch, Y_batch = util.get_batch(X_train, Y_train, batch_size) Y_batch = util.MakeOneHot(Y_batch, D_out) # forward, loss, backward, step Y_pred = model.forward(X_batch) loss, dout = criterion.get(Y_pred, Y_batch) model.backward(dout) optim.step()
print('\tOptim parameters:\t{}'.format(optim_params)) print('\tModel parameters:\t{}'.format(fit_params)) ### Init model, loss and optimzer model = model.Model(input_size=X.shape[1], init_weights=fit_params['init_weights']) my_loss = loss.LogisticLoss(reg_coeff=loss_params['weight_decay']) # Choose optimizer: if optim_params['type'] == 'gd': optim = optimizer.GD(params=model.weights, loss=my_loss, learn_rate=optim_params['lr'], tollerance=optim_params['tollerance']) elif optim_params['type'] == 'sgd': optim = optimizer.SGD(params=model.weights, loss=my_loss, learn_rate=optim_params['lr'], tollerance=optim_params['tollerance']) elif optim_params['type'] == 'svrg': optim = optimizer.SVRG(params=model.weights, loss=my_loss, learn_rate=optim_params['lr'], tollerance=optim_params['tollerance'], iter_epoch=optim_params['iter_epoch']) else: raise NotImplementedError ### Shape info print('-Displaying shape of data and variable') print('\tInput: {}'.format(X.shape)) print('\tTarget: {}'.format(y.shape)) print('\tWeights (with bias): {}'.format(model.weights.shape))
activation.ReLU(), layer.Linear(25, 50), activation.ReLU(), layer.Linear(50, 50), activation.ReLU(), layer.Linear(50, 25), activation.ReLU(), layer.Linear(25, 1), activation.Sigmoid() ) train_target[((train_input-0.5)**2).sum(1) < 1/(2*math.pi)] = 0 train_target[((train_input-0.5)**2).sum(1) >= 1/(2*math.pi)] = 1 test_target[((test_input-0.5)**2).sum(1) < 1/(2*math.pi)] = 0 test_target[((test_input-0.5)**2).sum(1) >= 1/(2*math.pi)] = 1 ps = model.parameters() optim = optimizer.SGD(model.parameters(), lr=0.05, decay=500) #for plotting later levels = [0, 0.25, 0.5, 0.75, 1] #for accuracy computation sigmoid = True else: epochs = int(input("Please input how many epochs you want to train over [DEFAULT = 4000] : ") or "4000") print("") criterion = loss.LossMSE() model = sequential.Sequential( layer.Linear(2, 25), activation.ReLU(), layer.Linear(25, 50), activation.ReLU(), layer.Linear(50, 50),
def run(args=None): usage = "usage : %prog [options]" parser = optparse.OptionParser(usage = usage) parser.add_option("--test", action = "store_true", dest = "test", default = False) # Paramsfile includes hyperparameters for training parser.add_option('--params_file', dest = "params_file", default = './params/exp_params.json', help = "Path to the file containing the training settings") parser.add_option('--data_dir', dest = "data_dir", default = './trees', help = "Directory containing the trees") # Directory containing the model to test parser.add_option("--model_directory", dest = "test_dir", type = "string") parser.add_option("--data", dest = "data", type = "string", default = "train") (opts, args) = parser.parse_args(args) results_dir = "./results" if opts.test: pass else: results_dir_current_job = os.path.join(results_dir, utils.now_as_str_f()) while os.path.isdir(results_dir_current_job): # generate a new timestamp if the current one already exists results_dir_current_job = os.path.join(results_dir, utils.now_as_str_f()) os.makedirs(results_dir_current_job) # Load training settings (e.g. hyperparameters) params = utils.Params(opts.params_file) if opts.test: pass else: # Copy the settings file into the results directory copyfile(opts.params_file, os.path.join(results_dir_current_job, os.path.basename(opts.params_file))) # Get the logger if opts.test: log_path = os.path.join(opts.test_dir, 'testing.log') else: log_path = os.path.join(results_dir_current_job, 'training.log') log_level = params.log_level if hasattr(params, 'log_level') else logging.DEBUG log = utils.get_logger(log_path, log_level) if opts.test: log.info("Testing directory: " + opts.test_dir) log.info("Dataset used for testing: " + opts.data) else: log.info("Results directory: " + results_dir_current_job) log.info("Minibatch: " + str(params.optimizer_settings['minibatch'])) log.info("Optimizer: " + params.optimizer) log.info("Epsilon: " + str(params.optimizer_settings['epsilon'])) log.info("Alpha: " + str(params.optimizer_settings['alpha'])) log.info("Number of samples used: " + str(params.sample_size)) # Testing if opts.test: test(opts.test_dir, opts.data) return log.info("Loading data...") # load training data trees = tr.loadTrees(sample_size = params.sample_size) params.numWords = len(tr.loadWordMap()) overall_performance = pd.DataFrame() rnn = nnet.RNN(params.wvecDim, params.outputDim, params.numWords, params.optimizer_settings['minibatch']) rnn.initParams() sgd = optimizer.SGD(rnn, alpha = params.optimizer_settings['alpha'], minibatch = params.optimizer_settings['minibatch'], optimizer = params.optimizer, epsilon = params.optimizer_settings['epsilon']) best_val_cost = float('inf') best_epoch = 0 for e in range(params.num_epochs): start = time.time() log.info("Running epoch %d" % e) df, updated_model, train_cost, train_acc = sgd.run(trees) end = time.time() log.info("Time per epoch : %f" % (end - start)) log.info("Training accuracy : %f" % train_acc) # VALIDATION val_df, val_cost, val_acc = validate(updated_model, results_dir_current_job) if val_cost < best_val_cost: # best validation cost we have seen so far log.info("Validation score improved, saving model") best_val_cost = val_cost best_epoch = e best_epoch_row = {"epoch": e, "train_cost": train_cost, "val_cost": val_cost, "train_acc": train_acc, "val_acc": val_acc} with open(results_dir_current_job + "/checkpoint.bin", 'w') as fid: pickle.dump(params, fid) pickle.dump(sgd.costt, fid) rnn.toFile(fid) val_df.to_csv(results_dir_current_job + "/validation_preds_epoch_ " + str(e) + ".csv", header = True, index = False) df.to_csv(results_dir_current_job + "/training_preds_epoch_" + str(e) + ".csv", header = True, index = False) row = {"epoch": e, "train_cost": train_cost, "val_cost": val_cost, "train_acc": train_acc, "val_acc": val_acc} overall_performance = overall_performance.append(row, ignore_index = True) # break if no val loss improvement in the last epochs if (e - best_epoch) >= params.num_epochs_early_stop: log.tinfo("No improvement in the last {num_epochs_early_stop} epochs, stop training.".format(num_epochs_early_stop=params.num_epochs_early_stop)) break overall_performance = overall_performance.append(best_epoch_row, ignore_index = True) overall_performance.to_csv(results_dir_current_job + "/train_val_costs.csv", header = True, index = False) log.info("Experiment end")
toll = 0.001 epochs = 40 starting_point = sp = -5 loss_fn = loss.LogisticLoss(reg_coeff=reg_coeff) W = np.arange(lims[0], lims[1], lims[2]) losses = [ np.round(loss_fn.compute_loss(X, y, np.array([[w]])), 3) for w in W ] gd = optimizer.GD(params=np.array([[sp]]), loss=loss_fn, learn_rate=lr, tollerance=toll) sgd = optimizer.SGD(params=np.array([[sp]]), loss=loss_fn, learn_rate=lr, tollerance=toll) #sag = optimizer.SAG(params=np.array([[sp]]), loss=loss_fn, learn_rate=lr, tollerance=toll) svrg = optimizer.SVRG(params=np.array([[sp]]), loss=loss_fn, learn_rate=lr, tollerance=toll, iter_epoch=10) results1 = gd.run(X, y, epochs) results2 = sgd.run(X, y, epochs // 3) results4 = svrg.run(X, y, epochs // 2) # print('\nComputed losses:\n\t{}\n'.format(losses)) # print('Results:') # print('\tLosses: {}'.format(np.round(results['loss_list'], 3))) # print('\tParams: {}'.format([np.round(float(i[0]), 3) for i in results['params_list']])) # print('\tOutput: {}'.format(np.dot(X, results['params_list'][-1]).transpose()))
train_target[((train_input-0.5)**2).sum(1) < 1/(2*math.pi)] = -1 train_target[((train_input-0.5)**2).sum(1) >= 1/(2*math.pi)] = 1 test_input = torch.rand((1000,2)) test_target = torch.rand((1000,1)) test_target[((test_input-0.5)**2).sum(1) < 1/(2*math.pi)] = -1 test_target[((test_input-0.5)**2).sum(1) >= 1/(2*math.pi)] = 1 #Normalization mu, std = train_input.mean(0), train_input.std(0) train_input.sub_(mu).div_(std) test_input.sub_(mu).div_(std) epochs = 10000 ps = model.parameters() optim = optimizer.SGD(model.parameters()) for i in range(epochs): output = model(train_input) optim.zero_grad() gradwrrtxL = criterion.backward(output, train_target) model.backward(gradwrrtxL) optim.step() if i % 10 == 0: test_accuracyV = test_accuracy(model, test_input, test_target) print(criterion.forward(output, train_target), test_accuracyV, test_accuracy(model, train_input, train_target))
if i % 100 == 0: print(i) self.forw = inparr[i] for i in range(len(self.layers) - 1): self.forw = self.layers[i].forward(self.forw) self.out = self.layers[len(self.layers) - 1].forward(self.forw) self.arr.append(self.out) self.arr = np.array(self.arr) return self.arr if __name__ == "__main__": layer1 = layerslib.Sigmoid(2, 2, True) layer2 = layerslib.Sigmoid(2, 1, True) layerarr = [layer1, layer2] opt = optimizer.SGD(0.5, 0.1, layerarr) model = Seqential(layerslib.MSE(), opt) inp = [[0, 0], [0, 1], [1, 0], [1, 1]] out = [0, 1, 1, 0] curloss = layerslib.MSE() for i in range(5000): for j in range(4): opt.zero_grad() ourout = model.forward(inp[j]) c = curloss.loss(ourout, out[j]) curloss.backward() opt.step()
def test_nbr_neuron(list_test): color_list=['r','g','b','k','m','c','y'] color_list *= 3 k=0 for i in list_test : my_layer1 = Layer.Linear(6,i) my_layer2 = ActivationFunctions.Tanh() my_layer5 = Layer.Linear(i,i) my_layer6 = ActivationFunctions.Tanh() my_layer3 = Layer.Linear(i,1) my_layer4 = ActivationFunctions.Sigmoid() my_NN = Neural_network.NeuralNet([my_layer1, my_layer2, my_layer5, my_layer6, my_layer3, my_layer4]) chi2_list, error_list = User.train(my_NN, data_train_input, data_train_target, num_epochs = num_epoch_max, optimizer = Optimizer.SGD(lr = my_lr), batch_size=my_batch_size) data_test_prediction = User.prediction(my_NN,data_test_input) error_final = Error_round.error_round(data_test_prediction, data_test_target) plt.plot(range(num_epoch_max), error_list, label= str(i), c=color_list[k]) plt.plot([num_epoch_max],[error_final], marker='o', c=color_list[k]) plt.xlabel('Epoch') plt.ylabel('Training round error') k+=1 plt.legend(title='Neurons') plt.title('Optimisation of the number of neurons') plt.show()
def test_nbr_layer(list_test, n_neuron): color_list=['r','g','b','k','m','c','y'] color_list *= 3 k=0 my_layerini1 = Layer.Linear(6,n_neuron) my_layerini2 = ActivationFunctions.Tanh() my_layerfini1 = Layer.Linear(n_neuron,1) my_layerfini2 = ActivationFunctions.Sigmoid() for i in list_test : layers_new = [my_layerini1, my_layerini2] for j in range(i) : layers_new += [Layer.Linear(n_neuron,n_neuron),ActivationFunctions.Tanh()] layers_new += [my_layerfini1, my_layerfini2] my_NN = Neural_network.NeuralNet(layers_new) chi2_list, error_list = User.train(my_NN, data_train_input, data_train_target, num_epochs = num_epoch_max,optimizer = Optimizer.SGD(lr = my_lr), batch_size=my_batch_size) data_test_prediction = User.prediction(my_NN,data_test_input) error_final = Error_round.error_round(data_test_prediction, data_test_target) plt.plot(range(num_epoch_max), error_list, label= str(i),c=color_list[k]) plt.plot([num_epoch_max],[error_final], marker='o', c=color_list[k]) plt.xlabel('Epoch') plt.ylabel('Training round error') k+=1 plt.legend(title='Hidden layers') plt.title('Optimisation of the number of hidden layers') plt.show()
def train_simultaneousNN( inputs_train: Tensor, targets_train: Tensor, loss: Loss.Loss = Loss.MeanSquareError(), optimizer: OptimizerClass.Optimizer = OptimizerClass.SGD(), num_epochs: int = 5000, batch_size: int = 32) -> tuple: size_training = inputs_train.shape[0] Result_chi2 = [[], [], [], [], [], [], [], [], []] list_epoch = np.array(range(10, 50, 5)) / 100 * num_epochs '''initialisation des 9 NN''' #verifier question seed() list_net = [] for i in range(9): layers = [] layers.append(Layer.Linear(6, 4)) layers.append(ActivationFunctions.Tanh()) layers.append(Layer.Linear(4, 2)) layers.append(ActivationFunctions.Tanh()) layers.append(Layer.Linear(2, 1)) layers.append(ActivationFunctions.Sigmoid()) list_net.append(Neural_network.NeuralNet(layers)) destroyed_NN = [] nbr_batch = size_training // batch_size ''' training des 9 NN''' for epoch in range(num_epochs): for k in range(9): if k not in destroyed_NN: Chi2_train = 0 for i in range(0, size_training, batch_size): # 1) feed forward y_actual = list_net[k].forward(inputs_train[i:i + batch_size]) # 2) compute the loss and the gradients Chi2_train += loss.loss(targets_train[i:i + batch_size], y_actual) grad_ini = loss.grad(targets_train[i:i + batch_size], y_actual) # 3)feed backwards grad_fini = list_net[k].backward(grad_ini) # 4) update the net optimizer.step(list_net[k], n_epoch=epoch) Chi2_train = Chi2_train / nbr_batch Result_chi2[k].append(Chi2_train) '''Supression du NN le moins efficace ''' if epoch in list_epoch: Comparaison = [[], []] for k in range(9): if k not in destroyed_NN: ErrorSlope = np.polyfit(np.array(range(epoch - 49, epoch)), Result_chi2[k][-50:-1], 1)[0] MixedError = Result_chi2[k][-1] * (1 - np.arctan(ErrorSlope) / (np.pi / 2)) Comparaison[0].append(k) Comparaison[1].append(MixedError) k = Comparaison[0][Comparaison[1].index(max(Comparaison[1]))] destroyed_NN.append(k) if epoch % 100 == 0: print('epoch : ' + str(epoch) + "/" + str(num_epochs) + "\r", end="") for k in range(9): if k not in destroyed_NN: my_NN = list_net[k] return my_NN, Result_chi2
'''training set''' data_train_input = np.array(Data_train[param][:train_size]) data_train_target = np.array(Data_train[['isSignal']][:train_size]) '''testing set''' data_test_input = np.array(Data_test[param][:test_size]) data_test_target = np.array(Data_test[['isSignal']][:test_size]) ## Basic use : training and testing print('basic example of utilisation :') '''training''' chi2_list, error_list = User.train(my_NN, data_train_input, data_train_target, num_epochs=my_num_epochs, optimizer=Optimizer.SGD(lr=my_lr), batch_size=my_batch_size) plt.plot(range(my_num_epochs), chi2_list) plt.xlabel('Epoch') plt.ylabel('Mean squared error') plt.title('Evolution of the training error : basic example') '''testing ''' data_test_prediction = User.prediction(my_NN, data_test_input) error = Error_round.error_round(data_test_prediction, data_test_target) print('% of false error for testing set : ', error) '''histogram of predictions''' plt.figure() S = data_test_prediction[data_test_target == 1] B = data_test_prediction[data_test_target == 0]