def __init__(self, batch_size): self.batch_size = batch_size self.train_x = mnist.train_images().reshape(60000, 784, 1)[0:50000] self.train_y = np.eye(10)[mnist.train_labels()].reshape(60000, 10, 1)[0:50000] self.test_x = mnist.test_images().reshape(10000, 784, 1) self.test_y = mnist.test_labels().reshape(10000, 1)
def makeMNISTFitness(): #load images = mnist.train_images() #reshape images = images.reshape( (images.shape[0], images.shape[1] * images.shape[2])) #normalize images = np.divide(images, 255) #labels labels = mnist.train_labels() def fitness(genome): error = 0 for i, image in enumerate(images): fx = cgp.evaluateGenome(genome, image) #find largest output x = 0.0 xi = 0 for j, output in enumerate(fx): if output > x: x = output xi = j if xi != labels[i]: error += 1 return error return fitness
def main(): #importing data train_images = mnist.train_images() train_y = mnist.train_labels() test_images = mnist.test_images() test_y = mnist.test_labels() #normalizing data train_x = (train_images / 255) - 0.5 test_x = (test_images / 255) - 0.5 train_x = train_x.reshape((-1, 784)) # 28*28 = 784 test_x = test_x.reshape((-1, 784)) #initialising the ANN model = tf.keras.Sequential() #adding the input layer and the first hidden layer model.add(tf.keras.layers.Dense(397, activation='relu', input_dim=784)) #adding the second hidden layer model.add(tf.keras.layers.Dense(397, activation='relu')) #adding the output layer model.add(tf.keras.layers.Dense(10, activation='softmax')) #compiling the ANN model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) #Fitting the ANN to the training set model.fit(train_x, to_categorical(train_y), epochs=5, batch_size=16)
def data_mnist(datadir='/tmp/', train_start=0, train_end=60000, test_start=0, test_end=10000): """ Load and preprocess MNIST dataset :param datadir: path to folder where data should be stored :param train_start: index of first training set example :param train_end: index of last training set example :param test_start: index of first test set example :param test_end: index of last test set example :return: tuple of four arrays containing training data, training labels, testing data and testing labels. """ assert isinstance(train_start, int) assert isinstance(train_end, int) assert isinstance(test_start, int) assert isinstance(test_end, int) import mnist X_train = mnist.train_images() / 255. Y_train = mnist.train_labels() X_test = mnist.test_images() / 255. Y_test = mnist.test_labels() X_train = np.expand_dims(X_train, -1) X_test = np.expand_dims(X_test, -1) X_train = X_train[train_start:train_end] Y_train = Y_train[train_start:train_end] X_test = X_test[test_start:test_end] Y_test = Y_test[test_start:test_end] Y_train = utils.to_categorical(Y_train, num_classes=10) Y_test = utils.to_categorical(Y_test, num_classes=10) return X_train, Y_train, X_test, Y_test
def makeCompiledMNistFitness(): #load images = mnist.train_images() #reshape images = images.reshape( (images.shape[0], images.shape[1] * images.shape[2])) #normalize images = np.divide(images, 255) #labels labels = mnist.train_labels() def compiledFitness(genome): genomeFunction = compiledGenome.compileGenome(genome) error = 0 for i, image in enumerate(images): genomeOutput = genomeFunction(image) #find largest output which is the digit called by the genome calledDigit = np.argmax(genomeOutput) # print(labels[i]) # print(genomeOutput) if calledDigit != labels[i]: error += 1 return error return compiledFitness
def main(): # Numpy Stuff # np.random.seed(1) # np.set_printoptions(threshold=np.inf) # Toy Data for testing # X_Data = np.array([[0, 0, 0], # [0, 0, 1], # [0, 1, 0], # [0, 1, 1], # [1, 0, 0], # [1, 0, 1], # [1, 1, 0], # [1, 1, 1]]) # Y_Data = np.array([[1, 0, 0, 0, 0, 0, 0, 0], # [0, 1, 0, 0, 0, 0, 0, 0], # [0, 0, 1, 0, 0, 0, 0, 0], # [0, 0, 0, 1, 0, 0, 0, 0], # [0, 0, 0, 0, 1, 0, 0, 0], # [0, 0, 0, 0, 0, 1, 0, 0], # [0, 0, 0, 0, 0, 0, 1, 0], # [0, 0, 0, 0, 0, 0, 0, 1]]) structure = [(784, '*'), (38, 'sigmoid'), (10, 'sigmoid')] nn = NeuralNet(structure, random_init_bound=0.05) # nn.load('models/Neon.json') # Load a model train_images = mnist.train_images() train_labels = mnist.train_labels() test_images = mnist.test_images() test_labels = mnist.test_labels() X_Test_Data = (test_images.reshape(test_images.shape[0], 784)) / 255.0 Y_Test_Data = np.zeros((10, 10000)) X_Train_Data = (train_images.reshape(train_images.shape[0], 784)) / 255.0 Y_Train_Data = np.zeros((10, 60000)) for i in range(Y_Train_Data.shape[1]): Y_Train_Data[train_labels[i]][i] = 1.0 for i in range(Y_Test_Data.shape[1]): Y_Test_Data[test_labels[i]][i] = 1.0 nn.fit(X_Train_Data, Y_Train_Data.T, 'MSE', 0.01, 0.05, 50, 50, print_mode=1) nn.test(X_Test_Data, Y_Test_Data.T, 'MSE') nn.save('models/Sodium.json') # Save model
def prepare_mnist(): print('[MNIST] Preparing dataset ...') # setting dir_image_output = './dataset/mnist/' # clean and create dir if any create_dir(dir_image_output + 'all') create_dir(dir_image_output + 'classes') for i in range(10): create_dir(dir_image_output + 'classes/{}'.format(i)) train_images = mnist.train_images() train_labels = mnist.train_labels() for i in range(train_images.shape[0]): img = train_images[i, :, :] label = train_labels[i] img = Image.fromarray(img).convert('RGB').resize((32, 32)) img.save(dir_image_output + 'all/img_{}.jpg'.format(i)) img.save(dir_image_output + 'classes/{}/img_{}.jpg'.format(label, i)) print('[MNIST] Preparing dataset was completed.') pass
def main(): train_images = mnist.train_images() train_labels = mnist.train_labels() train_images_flat = train_images.reshape( (train_images.shape[0], train_images.shape[1] * train_images.shape[2])) #print('train_images[0][0] is ' + repr(train_images[0][0])) #test_images = mnist.test_images() #test_labels = mnist.test_labels() my_net = Net((784, 32, 32, 10)) for n in range(len(train_images)): input_data = train_images_flat[n] desired_outputs = [ 0 if i != train_labels[n] else 100 for i in range(10) ] #print(input_data) print('{:5d} {:1d} '.format(n, train_labels[n]), end='') #print(desired_outputs) t = None for _ in range(1): t = my_net.tweak(input_data, desired_outputs, t) print(' {:6d} '.format(t), end='') print(my_net.layers[2].neurons) print() print(my_net.layers[2].neurons) print(my_net.error_score(desired_outputs))
def get_dataloaders(batch_size, seed, flatten=False): import mnist from sklearn.model_selection import train_test_split def preprocess(x, y, flatten): x = x.astype("float32") / 255. y = y.astype("int64") x = x.reshape(-1, 784) if flatten else x.reshape(-1, 1, 28, 28) return x, y images, labels = mnist.train_images(), mnist.train_labels() x_train, x_valid, y_train, y_valid = train_test_split(images, labels, test_size=0.2, random_state=seed) x_test, y_test = mnist.test_images(), mnist.test_labels() x_train, y_train = preprocess(x_train, y_train, flatten) x_valid, y_valid = preprocess(x_valid, y_valid, flatten) x_test, y_test = preprocess(x_test, y_test, flatten) train_loader = torch.utils.data.DataLoader(ImageDataset(x_train, y_train), batch_size=batch_size, shuffle=True) valid_loader = torch.utils.data.DataLoader(ImageDataset(x_valid, y_valid), batch_size=batch_size, shuffle=False) test_loader = torch.utils.data.DataLoader(ImageDataset(x_test, y_test), batch_size=batch_size, shuffle=False) return train_loader, valid_loader, test_loader
def train(self, epochs=10, batch_size=10): print("Training...") start = time.time() training_images = mnist.train_images() training_labels = mnist.train_labels() test_images = mnist.test_images() test_labels = mnist.test_labels() training_images = (training_images / 255.0) - 0.5 test_images = (test_images / 255.0) - 0.5 training_images = np.expand_dims(training_images, axis=3) test_images = np.expand_dims(test_images, axis=3) hist = self.model.fit(training_images, to_categorical(training_labels), batch_size=batch_size, epochs=epochs, validation_data=(test_images, to_categorical(test_labels))) end = time.time() elapsed = end - start acc = hist.history['acc'][-1] return acc, elapsed
def runCNN(): trainImgs = mnist.train_images() trainLabels = mnist.train_labels() print(trainImgs.shape) print(trainLabels.shape) testImgs = mnist.test_images() testLabels = mnist.test_labels() print(testImgs.shape) print(testLabels.shape) trainImgs = (trainImgs / 255) - 0.5 testImgs = (testImgs / 255) - 0.5 trainImgs = np.expand_dims(trainImgs, axis=3) testImgs = np.expand_dims(testImgs, axis=3) print(trainImgs.shape) print(testImgs.shape) model = Sequential() model.add(Conv2D(8, 3, input_shape=(28, 28, 1))) model.add(MaxPooling2D(pool_size=2)) model.add(Flatten()) model.add(Dense(10, activation='softmax')) model.load_weights("mnist_cnn.h5") predict = model.predict(testImgs[0:10]) print(predict) max = np.argmax(predict, axis=1) print(max)
def load_preprocessed_mnist(use_torch=False, flatten_images=False): """ Return mnist dataset with images normalized to [0, 1], scaled to (16, 16) and deskewed. Args: use_torch (bool, optional): If True, return torch tensors, otherwise numpy arrays (default: False). flatten_images (bool, optional): Flatten train and test images (default: False). Returns: Train images, train labels, test images, test labels as numpy arrays (or torch tensors, see parameter use_torch). """ train_images = mnist.train_images() / 255 train_images = preprocess(train_images, (16, 16), unskew=True) train_images = train_images.astype('float32') test_images = mnist.test_images() / 255 test_images = preprocess(test_images, (16, 16), unskew=True) test_images = test_images.astype('float32') train_labels = mnist.train_labels().astype( int) # original arrays are uint8 test_labels = mnist.test_labels().astype(int) if flatten_images: train_images = train_images.reshape(len(train_images), -1) test_images = test_images.reshape(len(test_images), -1) if use_torch: return torch.from_numpy(train_images).float(), torch.from_numpy(train_labels), \ torch.from_numpy(test_images).float(), torch.from_numpy(test_labels) else: return train_images, train_labels, test_images, test_labels
def main(): samples = 20 train_images = mnist.train_images()[0:samples] train_labels = mnist.train_labels()[0:samples] labels = [[1 if l > 4 else 0] for l in train_labels] features = vectorize(train_images) #print(train_images[5]) #print(features) #for feature in features: print(feature) #return #print(train_labels[0],labels[0]) #print(len(features[0])) #print(labels) #return def updates(epoch): print(epoch, nn.loss_avg[-1], nn.loss[-1]) nn = QuantumMNIST() nn.load(features=features, labels=labels) nn.train(progress=updates, updates=100) results = nn.predict(features) #print(np.array(nn.loss)) print(np.column_stack(( results , np.round(results) , np.array(labels) ))) x = [x for x in range(len(nn.loss))] plt.errorbar(x, nn.loss, yerr=nn.loss, errorevery=100) plt.yscale('log') plt.show()
def makeFitness(): #load images = mnist.train_images() #reshape images = images.reshape( (images.shape[0], images.shape[1] * images.shape[2])) #normalize images = np.divide(images, 255) #labels labels = mnist.train_labels() assert (len(images) == len(labels)) dataset = list(zip(images, labels)) def fitness(individual): errors = 0 for image, label in dataset: try: x = individual.output(image) except RecursionError as re: print(str(individual)) calledDigit = np.argmax(x) if calledDigit != label: errors += 1 return errors return fitness
def load_data(): """ Returns the tuple "(train_set, test_set)". "train_set" contains 60000 training examples (x, y), and "test_set" contains 10000 training examples (x, y). "x" has shape (n^{[0]}, m) and contains n^{[0]} input features for m training examples. "y" has shape (n^{[L]}, m), and contains n^{[L]} outputs for m training examples. """ train_images = mnist.train_images() train_labels = np.array([label_vector(i) for i in mnist.train_labels()]) train_labels = np.transpose(train_labels, (1, 2, 0)) train_labels = train_labels.reshape( train_labels.shape[0] * train_labels.shape[1], train_labels.shape[2]) test_images = mnist.test_images() test_labels = np.array([label_vector(i) for i in mnist.test_labels()]) test_labels = np.transpose(test_labels, (1, 2, 0)) test_labels = test_labels.reshape( test_labels.shape[0] * test_labels.shape[1], test_labels.shape[2]) x_train = train_images.reshape( train_images.shape[0], train_images.shape[1] * train_images.shape[2]).T y_train = train_labels x_test = test_images.reshape(test_images.shape[0], test_images.shape[1] * test_images.shape[2]).T y_test = test_labels train_set = (x_train, y_train) test_set = (x_test, y_test) return (train_set, test_set)
def loadDataMNIST(mlModel): """ Load the MNIST data set. No training file names need to be specified but mnist must be installed. """ try: import mnist except: raise ("Please run pip install mnist") train_images = mnist.train_images() train_labels = mnist.train_labels() test_images = mnist.test_images() test_labels = mnist.test_labels() mlModel.features = train_images.reshape( (train_images.shape[0], train_images.shape[1] * train_images.shape[2])) # Convert labels to one hot onehot_train_labels = np.zeros((train_labels.size, 10)) onehot_train_labels[np.arange(train_labels.size), train_labels] = 1 mlModel.labels = onehot_train_labels.T mlModel.testFeatures = test_images.reshape( (test_images.shape[0], test_images.shape[1] * test_images.shape[2])) # Convert labels to one hot onehot_test_labels = np.zeros((test_labels.size, 10)) onehot_test_labels[np.arange(test_labels.size), test_labels] = 1 mlModel.testLabels = onehot_test_labels.T return mlModel
def main(): #importing data train_images = mnist.train_images() train_y = mnist.train_labels() test_images = mnist.test_images() test_y = mnist.test_labels() #normalizing data train_x = (train_images / 255) - 0.5 test_x = (test_images / 255) - 0.5 train_x = train_x.reshape((-1, 784)) # 28*28 = 784 test_x = test_x.reshape((-1, 784)) model = KerasClassifier(build_fn=build_classifier) parameters = { 'batch_size': [32, 64], 'nb_epoch': [6, 10], 'nodes': [1, 2, 3], 'layer': [64, 397], 'optimizer': ['adam', 'rmsprop'] } grid_search = GridSearchCV(estimator=model, param_grid=parameters, scoring='accuracy', cv=10) grid_search = grid_search.fit(train_x, train_y) best_parameters = grid_search.best_params_ best_acc = grid_search.best_score_ print(best_acc, best_parameters)
def main(): epochs = 201 train_images = mnist.train_images() train_labels = mnist.train_labels() test_images = mnist.test_images() test_labels = mnist.test_labels() n_train, w, h = train_images.shape X_train = train_images.reshape((n_train, w * h)) # 维度为60000 * 784 Y_train = train_labels # 60000 * 1个label n_test, w, h = test_images.shape X_test = test_images.reshape((n_test, w * h)) Y_test = test_labels model = initialize(784, 128, 64, 32, 16, 10) # 初始化模型参数 batch_size = 600 train_error = [] test_acc = [] for epoch in range(epochs): loss_ep = 0 index = np.arange(60000) np.random.shuffle(index) X_train = X_train[index, :] Y_train = Y_train[index] for i in range(100): begin_num = i * batch_size end_num = i * batch_size + batch_size loss, model = forward(model, X_train[begin_num:end_num, :], Y_train[begin_num:end_num]) loss_ep += loss loss_ep = loss_ep / batch_size train_error.append(loss_ep) y_predict = predict(model, X_test) counter = 0 for i in range(len(y_predict)): if y_predict[i] == Y_test[i]: counter += 1 test_acc.append(counter / len(y_predict)) if epoch % 10 == 0: print('epoch:', epoch, 'loss:', loss_ep, 'test_acc:', counter / len(y_predict), '\n') plt.title(" training error curves") plt.xlabel("epochs") plt.ylabel("loss_average") x = np.arange(0, epochs) plt.plot(x, train_error) plt.show() plt.title(" test acc curves") plt.xlabel("epochs") plt.ylabel("loss_average") x = np.arange(0, epochs) plt.plot(x, test_acc) plt.show()
def runCNN(): trainImgs = mnist.train_images() trainLabels = mnist.train_labels() print(trainImgs.shape) print(trainLabels.shape) testImgs = mnist.test_images() testLabels = mnist.test_labels() print(testImgs.shape) print(testLabels.shape) trainImgs = (trainImgs / 255) - 0.5 testImgs = (testImgs / 255) - 0.5 trainImgs = np.expand_dims(trainImgs, axis=3) testImgs = np.expand_dims(testImgs, axis=3) print(trainImgs.shape) print(testImgs.shape) model = Sequential() model.add(Conv2D(8, 3, input_shape=(28, 28, 1))) model.add(MaxPooling2D(pool_size=2)) model.add(Flatten()) model.add(Dense(10, activation='softmax')) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) model.fit(x=trainImgs, y=to_categorical(trainLabels), epochs=5, validation_data=(testImgs, to_categorical(testLabels))) model.save("mnist_cnn.h5")
def main(): route = 2 if route == 1: gan = GAN(generator=[5, 6, 5, 7], discriminator=[7, 3, 2]) gan.fake_input_run() cost = gan.cost print("Initial cost: ", cost) diff = gan.generator_weight_dash[1][5, 4] print("weight dash: ", diff) wei = gan.generatorANN.weights[1][5, 3] print("the actual weight", wei) gan.generatorANN.weights[1][5, 3] *= 1.1 newwei = gan.generatorANN.weights[1][5, 3] print("the new weight", newwei) gan.fake_input_run() costnew = gan.get_cost() print("new cost = ", costnew) print("change = ", (costnew - cost) / (newwei - wei)) #b = ANN(layers = di, # cost_function = 'self.cross_entropy', # batch_length = 10, # learning_rate= 5, # randomise = False) cost = gan.cost print("Initial cost: ", cost) print(gan.discriminator_weight_dash[0][5, 2]) diff = gan.discriminator_weight_dash[0][5, 2] print("weight dash: ", diff) wei = gan.discriminatorANN.weights[0][5, 2] print("the actual weight", wei) gan.discriminatorANN.weights[0][5, 2] += 0.2 newwei = gan.discriminatorANN.weights[0][5, 2] print("the new weight", newwei) gan.fake_input_run() costnew = gan.get_cost() print("new cost = ", costnew) print("change = ", (costnew - cost) / (newwei - wei)) elif route == 2: X = np.array([x.flatten() / 256 for x in mnist.train_images()]) pos = np.where(mnist.train_labels() == 7) gan = GAN( generator=[100, 300, 500, 784], discriminator=[784, 30, 1], discriminator_learning_rate=1, generator_learning_rate=1.25, #X=X, X=X[pos], save=False) #gan.visualise_mnist(gan.train_images[64]) a = gan.run(no=300, disc_steps=1) #filehandler = open('testsave.obj', 'wb') #pickle.dump(gan, filehandler) gan.graphs() for ii in range(3): gan.visualise_mnist( gan.generatorANN.predict(gan.get_random_input())) #gan.visualise_mnist(gan.generated_digit_list[-1]) plt.show()
def get_dataloaders(batch_size, seed, flatten=False): import mnist from sklearn.model_selection import train_test_split def preprocess(x, y, flatten): x = x.astype("float32") / 255. y = y.astype("int64") x = x.reshape(-1, 28 * 28) if flatten else x.reshape(-1, 28, 28, 1) return x, y images, labels = mnist.train_images(), mnist.train_labels() x_train, x_valid, y_train, y_valid = train_test_split(images, labels, test_size=0.2, random_state=seed) x_test, y_test = mnist.test_images(), mnist.test_labels() x_train, y_train = preprocess(x_train, y_train, flatten) x_valid, y_valid = preprocess(x_valid, y_valid, flatten) x_test, y_test = preprocess(x_test, y_test, flatten) train_loader = tf.data.Dataset.from_tensor_slices( (x_train, y_train)).shuffle(buffer_size=1024).batch(batch_size) valid_loader = tf.data.Dataset.from_tensor_slices( (x_valid, y_valid)).batch(batch_size) test_loader = tf.data.Dataset.from_tensor_slices( (x_test, y_test)).batch(batch_size) return train_loader, valid_loader, test_loader
def load_data(): """ Ref: https://victorzhou.com/blog/keras-neural-network-tutorial/ """ import numpy as np import mnist from tensorflow.keras.utils import to_categorical train_images = mnist.train_images() train_labels = mnist.train_labels() test_images = mnist.test_images() test_labels = mnist.test_labels() # Normalize the images. train_images = (train_images / 255) - 0.5 test_images = (test_images / 255) - 0.5 # Flatten the images. train_images = train_images.reshape((-1, 784)) test_images = test_images.reshape((-1, 784)) # One-hot encoding train_labels, test_labels = to_categorical(train_labels), to_categorical( test_labels) return (train_images, train_labels, test_images, test_labels)
def __init__(self, dims=(120, 120, 64), batch_size=16, shuffle=True, validation=False, split=0.2, extend_dims=True, augment_data=True): self.dims = dims self.batch_size = batch_size self.extend_dims = extend_dims # Get MNIST files, Split based on validation if validation: files = mnist.test_images() labels = mnist.test_labels() else: files = mnist.train_images() labels = mnist.train_labels() # Take into account shuffling if shuffle: tmp = list(zip(files, labels)) random.shuffle(tmp) files, labels = zip(*tmp) labels = np.array(labels) self.files = files self.labels = labels
def mnist_autoencoder(): ''' Use features extracted by the encoder of an autoencoder ''' import mnist x_train = np.load('../mnist_train_autoencoder.npy') return x_train, mnist.train_labels()
def main(): #importing data train_images = mnist.train_images() train_y = mnist.train_labels() test_images = mnist.test_images() test_y = mnist.test_labels() #normalizing data train_x = (train_images / 255) - 0.5 test_x = (test_images / 255) - 0.5 train_x = train_x.reshape((-1, 784)) # 28*28 = 784 test_x = test_x.reshape((-1, 784)) #model model = tf.keras.Sequential() model.add(tf.keras.layers.Dense(397, activation='relu', input_dim=784)) model.add(tf.keras.layers.Dense(397, activation='relu')) model.add(tf.keras.layers.Dense(10, activation='softmax')) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) model.fit(train_x, to_categorical(train_y), epochs=10, batch_size=32) y_test = tf.keras.utils.to_categorical( tf.keras.datasets.mnist.load_data()[1][1], num_classes=10, dtype='float32') y_score = model.predict_proba(test_x) plot_pr(y_test, y_score) roc(y_test, y_score)
def __init__(self, mode='train'): if mode is 'train': self.data, self.labels = train_images(), train_labels() elif mode is 'test': self.data, self.labels = test_images(), test_labels() self.indexes = np.arange(self.data.shape[0]) np.random.shuffle(self.indexes)
def main(): route = 1 X = np.array([x.flatten() / 256 for x in mnist.train_images()]) y = mnist.train_labels() X_test = np.array([x.flatten() / 256 for x in mnist.test_images()]) y_test = np.array(mnist.test_labels()) if route == 1: b = ANN(layers = [784, 30, 10], X=X, y=y, cost_function='cross_entropy', activation_function='ReLU', #activation_function='sigmoid', #optimiser='ADAM', batch_length = 15, learning_rate = 1, randomise = True, X_test=X_test, y_test=y_test, ) b.train(epochs = 1) b.predict_test(X_test = X_test, y_test = y_test )
def test_keras_mnist(): train_images = mnist.train_images() train_labels = mnist.train_labels() n = train_images.shape[0] lb = LabelBinarizer() lb.fit(range(10)) selection = random.sample(range(n), 10000) # Use only a subsample y_train = lb.transform(train_labels[selection]) # One-hot encodings x_train = train_images.reshape(n, 28, 28, 1)[selection] x_train = x_train / 255 # Normalize train data pop = Population(GeneticCnnIndividual, x_train, y_train, size=20, crossover_rate=0.3, mutation_rate=0.1, additional_parameters={ 'kfold': 5, 'epochs': (20, 4, 1), 'learning_rate': (1e-3, 1e-4, 1e-5), 'batch_size': 32 }, maximize=True) ga = RussianRouletteGA(pop, crossover_probability=0.2, mutation_probability=0.8) ga.run(50)
def test_keras_model(): train_images = mnist.train_images() train_labels = mnist.train_labels() n = train_images.shape[0] lb = LabelBinarizer() lb.fit(range(10)) selection = random.sample(range(n), 10000) y_train = lb.transform(train_labels[selection]) x_train = train_images.reshape(n, 28, 28, 1)[selection] x_train = x_train / 255 # Normalize train data model = GeneticCnnModel( x_train, y_train, { 'S_1': '000', 'S_2': '0000000000' }, # Genes to test (3, 5), # Number of nodes per DAG (corresponds to gene bytes) (28, 28, 1), # Shape of input data (20, 50), # Number of kernels per layer ((5, 5), (5, 5)), # Sizes of kernels per layer 500, # Number of units in Dense layer 0.5, # Dropout probability 10, # Number of classes to predict kfold=5, epochs=(20, 4, 1), learning_rate=(1e-3, 1e-4, 1e-5), batch_size=128) print(model.cross_validate())
def get_train_set(self): X_train = mnist.train_images() y_train = mnist.train_labels() mixed_indexes = np.random.permutation(60000) X_train, y_train = X_train[mixed_indexes], y_train[mixed_indexes] X_train_prepared = self._prepare_data(X_train) return X_train_prepared, y_train
def main(): samples = 20 train_images = mnist.train_images()[0:samples] train_labels = mnist.train_labels()[0:samples] resolution = len(train_images[0]) * len(train_images[0]) labels = [[1 if l > 4 else 0] for l in train_labels] features = [ np.where(np.reshape(symbol, resolution) > 1, 1, 0) for symbol in train_images ] nn = ClassicalMNIST() nn.load(features=features, labels=labels) nn.train(progress=lambda epoch : print(epoch, np.average(nn.loss))) results = nn.predict(features) print(features[0]) print(len(features[0])) print(labels) print(np.array(nn.loss)) print(np.column_stack(( results , np.round(results) , np.array(labels) )))
def main(): # XOR revisited # training data xs = [[0., 0], [0., 1], [1., 0], [1., 1]] ys = [[0.], [1.], [1.], [0.]] random.seed(0) net = Sequential([ Linear(input_dim=2, output_dim=2), Sigmoid(), Linear(input_dim=2, output_dim=1) ]) import tqdm optimizer = GradientDescent(learning_rate=0.1) loss = SSE() with tqdm.trange(3000) as t: for epoch in t: epoch_loss = 0.0 for x, y in zip(xs, ys): predicted = net.forward(x) epoch_loss += loss.loss(predicted, y) gradient = loss.gradient(predicted, y) net.backward(gradient) optimizer.step(net) t.set_description(f"xor loss {epoch_loss:.3f}") for param in net.params(): print(param) # FizzBuzz Revisited from scratch.neural_networks import binary_encode, fizz_buzz_encode, argmax xs = [binary_encode(n) for n in range(101, 1024)] ys = [fizz_buzz_encode(n) for n in range(101, 1024)] NUM_HIDDEN = 25 random.seed(0) net = Sequential([ Linear(input_dim=10, output_dim=NUM_HIDDEN, init='uniform'), Tanh(), Linear(input_dim=NUM_HIDDEN, output_dim=4, init='uniform'), Sigmoid() ]) def fizzbuzz_accuracy(low: int, hi: int, net: Layer) -> float: num_correct = 0 for n in range(low, hi): x = binary_encode(n) predicted = argmax(net.forward(x)) actual = argmax(fizz_buzz_encode(n)) if predicted == actual: num_correct += 1 return num_correct / (hi - low) optimizer = Momentum(learning_rate=0.1, momentum=0.9) loss = SSE() with tqdm.trange(1000) as t: for epoch in t: epoch_loss = 0.0 for x, y in zip(xs, ys): predicted = net.forward(x) epoch_loss += loss.loss(predicted, y) gradient = loss.gradient(predicted, y) net.backward(gradient) optimizer.step(net) accuracy = fizzbuzz_accuracy(101, 1024, net) t.set_description(f"fb loss: {epoch_loss:.2f} acc: {accuracy:.2f}") # Now check results on the test set print("test results", fizzbuzz_accuracy(1, 101, net)) random.seed(0) net = Sequential([ Linear(input_dim=10, output_dim=NUM_HIDDEN, init='uniform'), Tanh(), Linear(input_dim=NUM_HIDDEN, output_dim=4, init='uniform') # No final sigmoid layer now ]) optimizer = Momentum(learning_rate=0.1, momentum=0.9) loss = SoftmaxCrossEntropy() with tqdm.trange(100) as t: for epoch in t: epoch_loss = 0.0 for x, y in zip(xs, ys): predicted = net.forward(x) epoch_loss += loss.loss(predicted, y) gradient = loss.gradient(predicted, y) net.backward(gradient) optimizer.step(net) accuracy = fizzbuzz_accuracy(101, 1024, net) t.set_description(f"fb loss: {epoch_loss:.3f} acc: {accuracy:.2f}") # Again check results on the test set print("test results", fizzbuzz_accuracy(1, 101, net)) # Load the MNIST data import mnist # This will download the data, change this to where you want it. # (Yes, it's a 0-argument function, that's what the library expects.) # (Yes, I'm assigning a lambda to a variable, like I said never to do.) mnist.temporary_dir = lambda: '/tmp' # Each of these functions first downloads the data and returns a numpy array. # We call .tolist() because our "tensors" are just lists. train_images = mnist.train_images().tolist() train_labels = mnist.train_labels().tolist() assert shape(train_images) == [60000, 28, 28] assert shape(train_labels) == [60000] import matplotlib.pyplot as plt fig, ax = plt.subplots(10, 10) for i in range(10): for j in range(10): # Plot each image in black and white and hide the axes. ax[i][j].imshow(train_images[10 * i + j], cmap='Greys') ax[i][j].xaxis.set_visible(False) ax[i][j].yaxis.set_visible(False) # plt.show() # Load the MNIST test data test_images = mnist.test_images().tolist() test_labels = mnist.test_labels().tolist() assert shape(test_images) == [10000, 28, 28] assert shape(test_labels) == [10000] # Recenter the images # Compute the average pixel value avg = tensor_sum(train_images) / 60000 / 28 / 28 # Recenter, rescale, and flatten train_images = [[(pixel - avg) / 256 for row in image for pixel in row] for image in train_images] test_images = [[(pixel - avg) / 256 for row in image for pixel in row] for image in test_images] assert shape(train_images) == [60000, 784], "images should be flattened" assert shape(test_images) == [10000, 784], "images should be flattened" # After centering, average pixel should be very close to 0 assert -0.0001 < tensor_sum(train_images) < 0.0001 # One-hot encode the test data train_labels = [one_hot_encode(label) for label in train_labels] test_labels = [one_hot_encode(label) for label in test_labels] assert shape(train_labels) == [60000, 10] assert shape(test_labels) == [10000, 10] # Training loop import tqdm def loop(model: Layer, images: List[Tensor], labels: List[Tensor], loss: Loss, optimizer: Optimizer = None) -> None: correct = 0 # Track number of correct predictions. total_loss = 0.0 # Track total loss. with tqdm.trange(len(images)) as t: for i in t: predicted = model.forward(images[i]) # Predict. if argmax(predicted) == argmax(labels[i]): # Check for correct += 1 # correctness. total_loss += loss.loss(predicted, labels[i]) # Compute loss. # If we're training, backpropagate gradient and update weights. if optimizer is not None: gradient = loss.gradient(predicted, labels[i]) model.backward(gradient) optimizer.step(model) # And update our metrics in the progress bar. avg_loss = total_loss / (i + 1) acc = correct / (i + 1) t.set_description(f"mnist loss: {avg_loss:.3f} acc: {acc:.3f}") # The logistic regression model for MNIST random.seed(0) # Logistic regression is just a linear layer followed by softmax model = Linear(784, 10) loss = SoftmaxCrossEntropy() # This optimizer seems to work optimizer = Momentum(learning_rate=0.01, momentum=0.99) # Train on the training data loop(model, train_images, train_labels, loss, optimizer) # Test on the test data (no optimizer means just evaluate) loop(model, test_images, test_labels, loss) # A deep neural network for MNIST random.seed(0) # Name them so we can turn train on and off dropout1 = Dropout(0.1) dropout2 = Dropout(0.1) model = Sequential([ Linear(784, 30), # Hidden layer 1: size 30 dropout1, Tanh(), Linear(30, 10), # Hidden layer 2: size 10 dropout2, Tanh(), Linear(10, 10) # Output layer: size 10 ]) # Training the deep model for MNIST optimizer = Momentum(learning_rate=0.01, momentum=0.99) loss = SoftmaxCrossEntropy() # Enable dropout and train (takes > 20 minutes on my laptop!) dropout1.train = dropout2.train = True loop(model, train_images, train_labels, loss, optimizer) # Disable dropout and evaluate dropout1.train = dropout2.train = False loop(model, test_images, test_labels, loss)