def test_nbr_neuron(list_test): color_list=['r','g','b','k','m','c','y'] color_list *= 3 k=0 for i in list_test : my_layer1 = Layer.Linear(6,i) my_layer2 = ActivationFunctions.Tanh() my_layer5 = Layer.Linear(i,i) my_layer6 = ActivationFunctions.Tanh() my_layer3 = Layer.Linear(i,1) my_layer4 = ActivationFunctions.Sigmoid() my_NN = Neural_network.NeuralNet([my_layer1, my_layer2, my_layer5, my_layer6, my_layer3, my_layer4]) chi2_list, error_list = User.train(my_NN, data_train_input, data_train_target, num_epochs = num_epoch_max, optimizer = Optimizer.SGD(lr = my_lr), batch_size=my_batch_size) data_test_prediction = User.prediction(my_NN,data_test_input) error_final = Error_round.error_round(data_test_prediction, data_test_target) plt.plot(range(num_epoch_max), error_list, label= str(i), c=color_list[k]) plt.plot([num_epoch_max],[error_final], marker='o', c=color_list[k]) plt.xlabel('Epoch') plt.ylabel('Training round error') k+=1 plt.legend(title='Neurons') plt.title('Optimisation of the number of neurons') plt.show()
def test_nbr_layer(list_test, n_neuron): color_list=['r','g','b','k','m','c','y'] color_list *= 3 k=0 my_layerini1 = Layer.Linear(6,n_neuron) my_layerini2 = ActivationFunctions.Tanh() my_layerfini1 = Layer.Linear(n_neuron,1) my_layerfini2 = ActivationFunctions.Sigmoid() for i in list_test : layers_new = [my_layerini1, my_layerini2] for j in range(i) : layers_new += [Layer.Linear(n_neuron,n_neuron),ActivationFunctions.Tanh()] layers_new += [my_layerfini1, my_layerfini2] my_NN = Neural_network.NeuralNet(layers_new) chi2_list, error_list = User.train(my_NN, data_train_input, data_train_target, num_epochs = num_epoch_max,optimizer = Optimizer.SGD(lr = my_lr), batch_size=my_batch_size) data_test_prediction = User.prediction(my_NN,data_test_input) error_final = Error_round.error_round(data_test_prediction, data_test_target) plt.plot(range(num_epoch_max), error_list, label= str(i),c=color_list[k]) plt.plot([num_epoch_max],[error_final], marker='o', c=color_list[k]) plt.xlabel('Epoch') plt.ylabel('Training round error') k+=1 plt.legend(title='Hidden layers') plt.title('Optimisation of the number of hidden layers') plt.show()
def __init__(self, args, vocab): self.model = nn.Sequential() self.model.add_layer(nn.WordEmbedding(args['vocab_size'], args['embedding_dim'], init_wt=args['init_wt'])) self.model.add_layer(nn.Linear(args['embedding_dim'] * args['context_len'], args['num_hid'], init_wt=args['init_wt'])) self.model.add_layer(nn.Sigmoid()) self.model.add_layer(nn.Linear(args['num_hid'], args['vocab_size'], init_wt=args['init_wt'])) self.model.add_layer(nn.SoftMax()) self.criterion = nn.CrossEntropy(args['vocab_size']) self.vocab = vocab
def test_XOR() : my_layer1 = Layer.Linear(2,3) my_layer2 = ActivationFunctions.Tanh() my_layer3 = Layer.Linear(3,1) my_layer4 = ActivationFunctions.Sigmoid() #my_layer3 = lib2.Arondi() my_NN = Neural_network.NeuralNet([my_layer1,my_layer2,my_layer3,my_layer4]) input =np.array([[0, 0], [1, 0], [0, 1], [1, 1]]) target = np.array([[0], [1], [1], [0]]) User.train(my_NN, input, target, batch_size = 1,,num_epochs= 1000) # By careful, we must have size_training = number of rows in our data input_predict = np.array([[0, 0], [1, 0], [0, 1], [1, 1]]) print(User.prediction(my_NN,input_predict)) ''' OK '''
def test_LinearLayer() : input = np.array([[1, 1, 1, 1, 1, 1]]) grad = np.array([[0.5,0.2,0.3,0.1]]) lin = Layer.Linear(6,4) #taille input et output print('Y', lin.forward(input)) print( 'grad',lin.backward(grad)) print('grad_w',lin.grad_w) print('grad_b',lin.grad_b) ''' OK '''
def __init__(self, depth=8, lrmul=0.01): super(G_mapping, self).__init__() layers = [layer.PixelNorm()] for i in range(depth): layers += [ layer.Linear(512, 512, lrmul=lrmul), nn.LeakyReLU(negative_slope=0.2) ] self.layers = nn.Sequential(*layers)
def test_NeuralNet() : my_layer1 = Layer.Linear(3,2) my_layer2 = ActivationFunctions.Tanh() my_NN = Neural_network.NeuralNet([my_layer1,my_layer2]) input = np.array([[1,2,3],[4,5,6]]) grad = np.array([[0.5,0.2],[0.1,0.3]]) print('forward', my_NN.forward(input)) print('backward', my_NN.backward(grad)) '''OK'''
def test_train_prediction() : my_layer1 = Layer.Linear(3,2) my_layer2 = ActivationFunctions.Tanh() my_NN = Neural_network.NeuralNet([my_layer1,my_layer2]) input = np.array([[1,2,3],[4,5,6]]) target = np.array([[0.5,0.2],[0.1,0.3]]) User.train(my_NN, input, target, batch_size = 1) #By careful, we must have size_training = number of rows in our data input_predict = np.array([[1,1,4],[0.5,2,4]]) print(User.prediction(my_NN,input_predict)) ''' OK '''
def train_simultaneousNN( inputs_train: Tensor, targets_train: Tensor, loss: Loss.Loss = Loss.MeanSquareError(), optimizer: OptimizerClass.Optimizer = OptimizerClass.SGD(), num_epochs: int = 5000, batch_size: int = 32) -> tuple: size_training = inputs_train.shape[0] Result_chi2 = [[], [], [], [], [], [], [], [], []] list_epoch = np.array(range(10, 50, 5)) / 100 * num_epochs '''initialisation des 9 NN''' #verifier question seed() list_net = [] for i in range(9): layers = [] layers.append(Layer.Linear(6, 4)) layers.append(ActivationFunctions.Tanh()) layers.append(Layer.Linear(4, 2)) layers.append(ActivationFunctions.Tanh()) layers.append(Layer.Linear(2, 1)) layers.append(ActivationFunctions.Sigmoid()) list_net.append(Neural_network.NeuralNet(layers)) destroyed_NN = [] nbr_batch = size_training // batch_size ''' training des 9 NN''' for epoch in range(num_epochs): for k in range(9): if k not in destroyed_NN: Chi2_train = 0 for i in range(0, size_training, batch_size): # 1) feed forward y_actual = list_net[k].forward(inputs_train[i:i + batch_size]) # 2) compute the loss and the gradients Chi2_train += loss.loss(targets_train[i:i + batch_size], y_actual) grad_ini = loss.grad(targets_train[i:i + batch_size], y_actual) # 3)feed backwards grad_fini = list_net[k].backward(grad_ini) # 4) update the net optimizer.step(list_net[k], n_epoch=epoch) Chi2_train = Chi2_train / nbr_batch Result_chi2[k].append(Chi2_train) '''Supression du NN le moins efficace ''' if epoch in list_epoch: Comparaison = [[], []] for k in range(9): if k not in destroyed_NN: ErrorSlope = np.polyfit(np.array(range(epoch - 49, epoch)), Result_chi2[k][-50:-1], 1)[0] MixedError = Result_chi2[k][-1] * (1 - np.arctan(ErrorSlope) / (np.pi / 2)) Comparaison[0].append(k) Comparaison[1].append(MixedError) k = Comparaison[0][Comparaison[1].index(max(Comparaison[1]))] destroyed_NN.append(k) if epoch % 100 == 0: print('epoch : ' + str(epoch) + "/" + str(num_epochs) + "\r", end="") for k in range(9): if k not in destroyed_NN: my_NN = list_net[k] return my_NN, Result_chi2
print("\t Sigmoïd Output Activation with Binary Cross Entropy Loss (0)") print("\t Tanh Output Activation with Mean Square Error Loss (1)") choice = int(input("[DEFAULT = 1] : ") or "1") print("") train_input = torch.rand((1000,2)) train_target = torch.rand((1000,1)) test_input = torch.rand((1000,2)) test_target = torch.rand((1000,1)) if choice == 0: epochs = int(input("Please input how many epochs you want to train over [DEFAULT = 6000] : ") or "6000") print("") criterion = loss.LossBCE() model = sequential.Sequential( layer.Linear(2, 25), activation.ReLU(), layer.Linear(25, 50), activation.ReLU(), layer.Linear(50, 50), activation.ReLU(), layer.Linear(50, 25), activation.ReLU(), layer.Linear(25, 1), activation.Sigmoid() ) train_target[((train_input-0.5)**2).sum(1) < 1/(2*math.pi)] = 0 train_target[((train_input-0.5)**2).sum(1) >= 1/(2*math.pi)] = 1 test_target[((test_input-0.5)**2).sum(1) < 1/(2*math.pi)] = 0 test_target[((test_input-0.5)**2).sum(1) >= 1/(2*math.pi)] = 1 ps = model.parameters()
'''Seed''' np.random.seed(1) '''size of the training set and the testing set ''' train_size = 3000 test_size = 1500 '''Maximal number of epochs ''' my_num_epochs = 500 '''size of the batch''' my_batch_size = 100 ''' learning rate''' my_lr = 0.001 my_initial_lr = 0.1 my_decay_coeff = 1 / 200 '''Construction of the neural network ''' my_layer1 = Layer.Linear(6, 5) my_layer2 = ActivationFunctions.Tanh() my_layer3 = Layer.Linear(5, 4) my_layer4 = ActivationFunctions.Tanh() my_layer5 = Layer.Linear(4, 3) my_layer6 = ActivationFunctions.Tanh() my_layer7 = Layer.Linear(3, 2) my_layer8 = ActivationFunctions.Tanh() my_layer9 = Layer.Linear(2, 1) my_layer10 = ActivationFunctions.Sigmoid() my_NN = Neural_network.NeuralNet([ my_layer1, my_layer2, my_layer3, my_layer4, my_layer5, my_layer6, my_layer7, my_layer8, my_layer9, my_layer10 ]) ## Importation of the training and testing data
import Neural_Network_Library.loss as Loss import Neural_Network_Library.layer as Layer import Neural_Network_Library.error_round as Error_round import Neural_Network_Library.activation_functions as ActivationFunctions import Neural_Network_Library.neural_network as Neural_network import Neural_Network_Library.optimizer as OptimizerClass import Neural_Network_Library.user as User ''' plt.close() # Parameters' choice '''seed ''' np.random.seed(1) '''Construction of the neural network ''' my_layer1 = Layer.Linear(6, 4) my_layer2 = ActivationFunctions.Tanh() my_layer5 = Layer.Linear(4, 2) my_layer6 = ActivationFunctions.Tanh() my_layer3 = Layer.Linear(2, 1) my_layer4 = ActivationFunctions.Sigmoid() my_NN = Neural_network.NeuralNet( [my_layer1, my_layer2, my_layer5, my_layer6, my_layer3, my_layer4]) '''Maximal number of epochs ''' Nmax = 50000 '''size of the training set and the testing set ''' train_size = 3000 test_size = 1500 '''size of the batch''' my_batch_size = 100 ''' learning rate'''
height=a_max_sent_size + 2 * (filter_width - 1), width=overlap_ndim + word_dim) non_line = layer.Activation(tf.tanh, nkernals=kernals) max_pool = layer.Maxpool( ksize=[1, a_max_sent_size + filter_width - 1, 1, 1]) flatten = layer.Flatten(batch_size=batch_size) net_a = layer.FeedForwardNet( [lookup_words, conv, non_line, max_pool, flatten]) net_a.set_input([x_a, x_a_overlap]) with tf.name_scope('pair_combine'): pair_combine = layer.PairCombine(shape1=kernals, shape2=kernals) pair_combine.set_input([net_q.output, net_a.output]) with tf.name_scope('hiddent_layer'): hidden_layer = layer.Linear(n_in=2 * kernals + 1, n_out=2 * kernals + 1) hidden_layer.set_input(pair_combine.output) with tf.name_scope('LR'): lr_layer = layer.LR(n_in=2 * kernals + 1, n_out=n_class) lr_layer.set_input(hidden_layer.output) #Save Graph tf.add_to_collection('x_q', x_q) tf.add_to_collection('x_q_overlap', x_q_overlap) tf.add_to_collection('x_a', x_a) tf.add_to_collection('x_a_overlap', x_a_overlap) tf.add_to_collection('pred', lr_layer.output) init = tf.global_variables_initializer()