def train_nn( net: NeuralNet, inputs: Tensor, targets: Tensor, epochs: int = 1, loss: Loss = MSE(), batch_iter: DataIterator = BatchIterator(), optimizer: Optimizer = SGD(), ) -> None: for epoch in range(epochs): epoch_loss = 0. for batch in batch_iter(inputs, targets): pred = net.forward(batch.input) batch_loss = loss.loss(pred, batch.target) epoch_loss += batch_loss loss_grad = loss.grad(pred, batch.target) net_grad = net.backward(loss_grad) optimizer.step(net) print(f'epoch:{epoch}, loss:{epoch_loss}')
def compile(self, loss): if loss == "binary_crossentropy": self.loss = BinaryCrossentropy(self) elif loss == "mse": self.loss = MSE(self) else: logger.warning("Invalid loss function.") self.loss = None
def __init__(self, encoder=None, sampler=None, decoder=None): super().__init__() if encoder != None and sampler != None and decoder != None: self.layers = encoder.layers + [sampler.mean, sampler.logVar ] + decoder.layers self.encoder = encoder self.sampler = sampler self.decoder = decoder self.decoder.loss = MSE()
def main(): train_data = np.array([[0, 0], [0, 1], [0, 2], [0, 3], [0, 4], [0, 5], [0, 6], [0, 7], [1, 0], [1, 1], [1, 2], [1, 3], [1, 4], [1, 5], [1, 6], [1, 7], [2, 0], [2, 1], [2, 2], [2, 3], [2, 4], [2, 5], [2, 6], [2, 7], [3, 0], [3, 1], [3, 2], [3, 3], [3, 4], [3, 5], [3, 6], [3, 7], [4, 0], [4, 1], [4, 2], [4, 3], [4, 4], [4, 5], [4, 6], [4, 7], [5, 0], [5, 1], [5, 2], [5, 3], [5, 4], [5, 5], [5, 6], [5, 7], [6, 0], [6, 1], [6, 2], [6, 3], [6, 4], [6, 5], [6, 6], [6, 7], [7, 0], [7, 1], [7, 2], [7, 3], [7, 4], [7, 5], [7, 6], [7, 7]]) label_data = np.array( [[0], [1], [2], [3], [4], [5], [6], [7], [1], [0], [3], [2], [5], [4], [7], [6], [2], [3], [0], [1], [6], [7], [4], [5], [3], [2], [1], [0], [7], [6], [5], [4], [4], [5], [6], [7], [0], [1], [2], [3], [5], [4], [7], [6], [1], [0], [3], [2], [6], [7], [4], [5], [2], [3], [0], [1], [7], [6], [5], [4], [3], [2], [1], [0]]) learn_rate = 0.15 epoch = 600000 loss_list = [] w1, b1, w2, b2 = init(input_size=2, hidden_size=16, output_size=1) #print(w1, w2, b1, b2) for i in range(epoch): affine1 = Affine(w1, b1) affine2 = Affine(w2, b2) sigmoid = Sigmoid() loss = MSE() x1 = affine1.forward(train_data) y1 = sigmoid.forward(x1) x2 = affine2.forward(y1) ls = loss.mean_square_error(x2, label_data) print(ls) loss_list.append(ls) dout = loss.backward(x2, label_data) dx = affine2.backward(dout) w2 = w2 - learn_rate * affine2.dw b2 = b2 - learn_rate * affine2.db dy1 = sigmoid.backward(dx) dx = affine1.backward(dy1) b1 = b1 - learn_rate * affine1.db w1 = w1 - learn_rate * affine1.dw #print(w1,w2,b1,b2) plt.plot(loss_list) plt.show() acc(w1, b1, w2, b2, train_data, label_data)
def train(self, dataset, loss=MSE(), epochs=1, metrics=["train_loss", "test_loss"], tensorboard=False, callbacks={}): super().train(dataset, loss=loss, epochs=epochs, metrics=metrics, tensorboard=tensorboard, callbacks=callbacks, autoencoder=True, noise=None)
def train(net: NeuralNet, inputs: Tensor, targets: Tensor, num_epochs: int = 5000, iterator: DataIterator = BatchIterator(), loss: Loss = MSE(), optimizer: Optimizer = SGD() ) -> None: for epoch in range(num_epochs): epoch_loss = 0.0 for batch in iterator(inputs, targets): predicted = net.forward(batch.inputs) epoch_loss += loss.loss(predicted, batch.targets) grad = loss.grad(predicted, batch.targets) net.backward(grad) optimizer.step(net) print(epoch, epoch_loss)
def train(net: NetWork, inputs: Tensor, targets: Tensor, epochs: int = 500, loss: Loss = MSE(), optimizer: Optimizer = SGD(), iterator: DataIterator = BatchIterator(), show_info: bool = False): for epoch in range(epochs): epoch_loss = .0 for batch_inputs, batch_targets in iterator(inputs, targets): predictions = net.forward(batch_inputs) epoch_loss += loss.loss(predictions, batch_targets) grad = loss.grad(predictions, batch_targets) net.backward(grad) optimizer.step(net) if show_info: print('epoch:{}, loss:{}'.format(epoch, epoch_loss))
def train(self, X, label, loss=MSE()): if self.configured == False: prevLayer = None for l in self.layers: l.configure(X.shape, "TRAIN", prevLayer) prevLayer = l self.configured = True self.Y = X for layer in self.layers: self.Y = layer.forward(self.Y) err = loss.calc(self.Y, label, self.batch) err_delta = loss.partial_derivative(self.Y, label) for layer in self.layers[::-1]: err_delta = layer.backward(err_delta) return err
def train(self, dataset, loss=MSE(), epochs=1, metrics=["train_loss", "test_loss"], tensorboard=False, callbacks={}, autoencoder=False, noise=None): metricsWriter = Writer(metrics, callbacks, tensorboard) self.loss = loss ind = 0 # number of samples processed for i in range(epochs): logging.debug(f" *** EPOCH {i+1}/{epochs} ***") for (train, test, batchSize) in dataset.batches(onehot_encoded=True, autoencoder=autoencoder, noise=noise): # set batch size before training for layer in self.layers: layer.setBatchSize(batchSize) self.feedforward(train[0]) self.backpropagate(train[1]) if ind % 1000 < batchSize: if "train_loss" in metrics: metricsWriter.add(metric="train_loss", index=ind, value=self.getLoss(train[1])) if "train_accuracy" in metrics: metricsWriter.add(metric="train_accuracy", index=ind, value=self.getAccuracy(train[1])) self.validate(test, ind, callbacks, writer=metricsWriter, metrics=metrics) ind += batchSize metricsWriter.close()
def load(self, name): modelDir = f"./models/{name}" self.encoder = MLP() self.decoder = MLP() self.decoder.loss = MSE() # load encoder and decoder for name, model in [("encoder", self.encoder), ("decoder", self.decoder)]: layerDir = [ dir for dir in os.listdir(modelDir) if os.path.isdir(os.path.join(modelDir, dir)) and name in dir ] layerDir.sort(key=lambda x: int(x.strip(f"{name}_layer"))) for dir in layerDir: layerFolder = os.path.join(modelDir, dir) if "dense.json" in os.listdir(layerFolder): # this is a dense layer newLayer = Dense() newLayer.load(layerFolder) model.layers.append(newLayer) # load aditional information about sampler with open(f"{modelDir}/sampler.json", "r") as file: data = json.load(file) inputDim = data["inputDim"] outputDim = data["outputDim"] self.sampler = Sampler(inputDim, outputDim) # load mean and logvar layer self.sampler.mean = Dense() self.sampler.mean.load(os.path.join(modelDir, f"sampler_mean")) self.sampler.logVar = Dense() self.sampler.logVar.load(os.path.join(modelDir, f"sampler_logvar")) self.layers = self.encoder.layers + [ self.sampler.mean, self.sampler.logVar ] + self.decoder.layers
], #FullyConnect(units=last_units)], # FullyConnect(units=last_units, input_shape=144)], learning_rate = learning_rate, optimizer=Momentum(0.9), batch=batchSize, dtype=np.float32 ) err_prev = 0 for e in range(epoch): err = 0 for batchIdx in range(0, dataNum, batchSize): batchData = dataset[batchIdx:batchIdx + batchSize,:,:] batchLabel = label[batchIdx:batchIdx + batchSize,:] err += net.train(batchData, batchLabel, loss=MSE()) print "epoch", e print "\t", err if abs(err_prev - err) < 10e-15: print "early_stop" break err_prev = err print net.predict(dataset[0, :, :].reshape(1,1,input_shape)), label[0,:] print net.predict(dataset[dataNum-1, :].reshape(1,1,input_shape)), label[dataNum-1,:] """ # 100 epoch 10 batch 8*8 image epoch = 100 batchSize = 10
def train(self, dataset, loss=MSE(), epochs=1, metrics=["generator_loss", "discriminator_loss"], tensorboard=False, callbacks={}, autoencoder=False, noise=None): metricsWriter = Writer(metrics, callbacks, tensorboard) imgNoise = np.random.standard_normal( size=(self.generator.layers[0].inputDim, 10)) ind = 0 # number of samples processed for i in range(epochs): logging.debug(f" *** EPOCH {i+1}/{epochs} ***") for (train, test, batchSize) in dataset.batches(onehot_encoded=True, autoencoder=autoencoder, noise=noise): # set batch size before training for layer in self.layers: layer.setBatchSize(batchSize) # 1. Train discriminator fake_img = self.generator.feedforward(self.sample(batchSize)) real_img = np.asarray(train[0]) input = np.concatenate((fake_img, real_img), axis=1) label = np.concatenate((np.zeros( (1, batchSize)), 0.9 * np.ones((1, batchSize))), axis=1) self.discriminator.feedforward(np.asarray(input)) self.discriminator.backpropagate(np.asarray(label)) discriminatorLoss = self.discriminator.getLoss( np.asarray(label)) # 2. Train generator #fake_img = self.generator.feedforward(self.sample(batchSize)) self.discriminator.feedforward(fake_img) self.discriminator.backpropagate(np.ones((1, batchSize)), updateParameters=False) discriminatorGradient = self.discriminator.layers[0].gradient self.generator.backpropagate(discriminatorGradient, useLoss=False) generatorLoss = self.discriminator.getLoss( np.ones((1, batchSize))) if ind % 1000 < batchSize: if "generator_loss" in metrics: metricsWriter.add(metric="generator_loss", index=ind, value=generatorLoss) if "discriminator_loss" in metrics: metricsWriter.add(metric="discriminator_loss", index=ind, value=discriminatorLoss) #self.validate(test, ind, callbacks, writer = metricsWriter, metrics = metrics) ind += batchSize self.generateImages(imgNoise, i) metricsWriter.close()
outputDim=1024, activation=LeakyReLU(0.2), optimizer=optimizer)) discriminator.addLayer( Dense(inputDim=1024, outputDim=512, activation=LeakyReLU(0.2), optimizer=optimizer)) discriminator.addLayer( Dense(inputDim=512, outputDim=256, activation=LeakyReLU(0.2), optimizer=optimizer)) discriminator.addLayer( Dense(inputDim=256, outputDim=1, activation=Sigmoid(), optimizer=optimizer)) gan = GAN(generator, discriminator) print(gan) gan.train(dataset, loss=MSE(), epochs=50, metrics=["generator_loss", "discriminator_loss"], tensorboard=True, callbacks=[]) gan.save("tryout_gan")
def main(args): total_size = 2000 train_size = 1000 test_size = 1000 data, target = generate_disc_set(total_size, random_state=1) train_data, train_target = data[:train_size], target[:train_size] test_data, test_target = data[test_size:], target[test_size:] colours = ['blue', 'green', 'red'] def colour_labels(labels): return list(map(lambda x: colours[x], labels)) plt.figure(figsize=(12, 5)) plt.subplot(1, 2, 1) plt.scatter(train_data[:, 0], train_data[:, 1], c=colour_labels(train_target.argmax(1)), edgecolors='none') plt.title('Train Data') plt.xlabel(r'$x_{1}$') plt.ylabel(r'$x_{2}$') plt.subplot(1, 2, 2) plt.scatter(test_data[:, 0], test_data[:, 1], c=colour_labels(test_target.argmax(1)), edgecolors='none') plt.title('Test Data') plt.xlabel(r'$x_{1}$') plt.ylabel(r'$x_{2}$') plt.pause(1) plt.show(block=False) if args.loss == 'mse': net_loss = MSE() net = Sequential(DenseLayer(2, 25), ReLU(), DenseLayer(25, 25), ReLU(), DenseLayer(25, 25), ReLU(), DenseLayer(25, 2)) elif args.loss == 'softmax_loss': net_loss = CrossEntropy() net = Sequential(DenseLayer(2, 25), ReLU(), DenseLayer(25, 25), ReLU(), DenseLayer(25, 25), ReLU(), DenseLayer(25, 2), SoftMax()) else: raise ValueError( args.loss + ' is invalid loss. Please use either \'mse\' or \'softmax_loss\'.') def sgd(x, dx, config): for cur_layer_x, cur_layer_dx in zip(x, dx): for cur_x, cur_dx in zip(cur_layer_x, cur_layer_dx): cur_old_grad = config['learning_rate'] * cur_dx if cur_old_grad.shape[0] == 1: cur_x = cur_x.reshape(cur_old_grad.shape) cur_x.add_(-cur_old_grad) def train_model(model, model_loss, train_data, train_target, lr=0.005, batch_size=1, n_epoch=50): optimizer_config = {'learning_rate': lr} train_loss_history = [] test_loss_history = [] for i in range(n_epoch): loss = 0 k = 0 for x_batch, y_batch in get_batches(train_data, train_target, batch_size): model.zero_grad_params() # Forward pred = model.forward(x_batch) loss += model_loss.forward(pred, y_batch) # Backward lg = model_loss.backward(pred, y_batch) model.backward(lg) # Update weights sgd(net.get_params(), net.get_grad_params(), optimizer_config) k += 1 train_loss_history.append(loss / k) test_pred = model.forward(test_data) test_loss = model_loss.forward(test_pred, test_target) test_loss_history.append(test_loss) print('#Epoch {}: current train loss = {:.4f}'.format( i + 1, loss.item() / k)) return train_loss_history, test_loss_history print('Training started...') train_loss_history, test_loss_history = train_model(net, net_loss, train_data, train_target, n_epoch=50) print('Final train loss: {:.4f}'.format(train_loss_history[-1])) print('Final test loss: {:.4f}'.format(test_loss_history[-1])) plt.figure(2, figsize=(8, 6)) plt.title("Train and Test Loss") plt.xlabel("#Epochs") plt.ylabel("loss") plt.plot(train_loss_history, 'b') plt.plot(test_loss_history, 'r') plt.legend(['train loss', 'test loss']) plt.pause(1) plt.show(block=False) train_res = net.forward(train_data) errors_train = compute_nb_errors(train_res, train_target) print("Number of errors on the train set: " + str(errors_train)) train_res = train_res.argmax(1) train_res[train_res != train_target.argmax(1)] = 2 test_res = net.forward(test_data) errors_test = compute_nb_errors(test_res, test_target) print("Number of errors on the test set: " + str(errors_test)) test_res = test_res.argmax(1) test_res[test_res != test_target.argmax(1)] = 2 plt.figure(figsize=(12, 5)) plt.subplot(1, 2, 1) plt.scatter(train_data[:, 0], train_data[:, 1], c=colour_labels(train_res), edgecolors='none') plt.xlabel(r'$x_{1}$') plt.ylabel(r'$x_{2}$') plt.title(f'Train Data, {errors_train} errors') plt.subplot(1, 2, 2) plt.scatter(test_data[:, 0], test_data[:, 1], c=colour_labels(test_res), edgecolors='none') plt.xlabel(r'$x_{1}$') plt.ylabel(r'$x_{2}$') plt.title(f'Test Data, {errors_test} errors') plt.show()
inputs = np.array([ [0, 0], [1, 0], [0, 1], [1, 1] ]) targets = np.array([ [1, 0], [0, 1], [0, 1], [1, 0] ]) net = NeuralNetwork([ LinearLayer(inputSize=2, outputSize=2), LeakyRelu(), LinearLayer(inputSize=2, outputSize=2), Tanh() ]) train(net, inputs, targets, loss= MSE(), num_epochs=5000, optimizer=MBGD(learningRate=0.01), showGraph=True) # net.loadParamsFromFile("/home/ayush/scratch/Net/aknet/serialized.json") for x, y in zip(inputs, targets): predicted = net.forward(x) print(x, predicted, y) net.serialize("serialized.json")
prefix = np.random.rand()*2 dataX[i,:] = np.linspace(prefix, prefix+np.pi*4, dataPoints) labelY = np.sin(dataX) epoch = 60 batchSize = 20 input_shape = 2048 units = 256 last_units = 64 lstm = Network([LSTM(units, input_shape), FullyConnect(units=last_units, input_shape=units) ], learning_rate=learning_rate, optimizer=Momentum(0.9), batch=batchSize, ) err_prev = 0 for e in range(epoch): err = 0 for batchIdx in range(0, dataNum, batchSize): batchData = labelY[batchIdx:batchIdx + batchSize, :] for timeIdx in range(0, dataPoints-input_shape, input_shape): err += lstm.train(batchData[:, timeIdx:timeIdx+input_shape], batchData[:, timeIdx+input_shape:timeIdx+input_shape+last_units], loss=MSE()) print "epoch", e print "\t", err/batchSize err_prev = err print lstm.predict(labelY[:batchSize, :input_shape]), labelY[:batchSize, input_shape:input_shape+units] nxt = lstm.predict(labelY[:batchSize,:input_shape]) plt.plot(dataX[0,input_shape:input_shape+last_units], nxt[0, :], dataX[0, input_shape:input_shape+last_units], labelY[0, input_shape:input_shape+last_units]) plt.show()