def __init__(self, batch_size):
self.batch_size = batch_size
self.train_x = mnist.train_images().reshape(60000, 784, 1)[0:50000]
self.train_y = np.eye(10)[mnist.train_labels()].reshape(60000, 10,
1)[0:50000]
self.test_x = mnist.test_images().reshape(10000, 784, 1)
self.test_y = mnist.test_labels().reshape(10000, 1)
def makeMNISTFitness():
#load
images = mnist.train_images()
#reshape
images = images.reshape(
(images.shape[0], images.shape[1] * images.shape[2]))
#normalize
images = np.divide(images, 255)
#labels
labels = mnist.train_labels()
def fitness(genome):
error = 0
for i, image in enumerate(images):
fx = cgp.evaluateGenome(genome, image)
#find largest output
x = 0.0
xi = 0
for j, output in enumerate(fx):
if output > x:
x = output
xi = j
if xi != labels[i]:
error += 1
return error
return fitness
def main():
#importing data
train_images = mnist.train_images()
train_y = mnist.train_labels()
test_images = mnist.test_images()
test_y = mnist.test_labels()
#normalizing data
train_x = (train_images / 255) - 0.5
test_x = (test_images / 255) - 0.5
train_x = train_x.reshape((-1, 784)) # 28*28 = 784
test_x = test_x.reshape((-1, 784))
#initialising the ANN
model = tf.keras.Sequential()
#adding the input layer and the first hidden layer
model.add(tf.keras.layers.Dense(397, activation='relu', input_dim=784))
#adding the second hidden layer
model.add(tf.keras.layers.Dense(397, activation='relu'))
#adding the output layer
model.add(tf.keras.layers.Dense(10, activation='softmax'))
#compiling the ANN
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
#Fitting the ANN to the training set
model.fit(train_x, to_categorical(train_y), epochs=5, batch_size=16)
def data_mnist(datadir='/tmp/', train_start=0, train_end=60000, test_start=0,
test_end=10000):
"""
Load and preprocess MNIST dataset
:param datadir: path to folder where data should be stored
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:return: tuple of four arrays containing training data, training labels,
testing data and testing labels.
"""
assert isinstance(train_start, int)
assert isinstance(train_end, int)
assert isinstance(test_start, int)
assert isinstance(test_end, int)
import mnist
X_train = mnist.train_images() / 255.
Y_train = mnist.train_labels()
X_test = mnist.test_images() / 255.
Y_test = mnist.test_labels()
X_train = np.expand_dims(X_train, -1)
X_test = np.expand_dims(X_test, -1)
X_train = X_train[train_start:train_end]
Y_train = Y_train[train_start:train_end]
X_test = X_test[test_start:test_end]
Y_test = Y_test[test_start:test_end]
Y_train = utils.to_categorical(Y_train, num_classes=10)
Y_test = utils.to_categorical(Y_test, num_classes=10)
return X_train, Y_train, X_test, Y_test
def makeCompiledMNistFitness():
#load
images = mnist.train_images()
#reshape
images = images.reshape(
(images.shape[0], images.shape[1] * images.shape[2]))
#normalize
images = np.divide(images, 255)
#labels
labels = mnist.train_labels()
def compiledFitness(genome):
genomeFunction = compiledGenome.compileGenome(genome)
error = 0
for i, image in enumerate(images):
genomeOutput = genomeFunction(image)
#find largest output which is the digit called by the genome
calledDigit = np.argmax(genomeOutput)
# print(labels[i])
# print(genomeOutput)
if calledDigit != labels[i]:
error += 1
return error
return compiledFitness
def main():
# Numpy Stuff
# np.random.seed(1)
# np.set_printoptions(threshold=np.inf)
# Toy Data for testing
# X_Data = np.array([[0, 0, 0],
# [0, 0, 1],
# [0, 1, 0],
# [0, 1, 1],
# [1, 0, 0],
# [1, 0, 1],
# [1, 1, 0],
# [1, 1, 1]])
# Y_Data = np.array([[1, 0, 0, 0, 0, 0, 0, 0],
# [0, 1, 0, 0, 0, 0, 0, 0],
# [0, 0, 1, 0, 0, 0, 0, 0],
# [0, 0, 0, 1, 0, 0, 0, 0],
# [0, 0, 0, 0, 1, 0, 0, 0],
# [0, 0, 0, 0, 0, 1, 0, 0],
# [0, 0, 0, 0, 0, 0, 1, 0],
# [0, 0, 0, 0, 0, 0, 0, 1]])
structure = [(784, '*'), (38, 'sigmoid'), (10, 'sigmoid')]
nn = NeuralNet(structure, random_init_bound=0.05)
# nn.load('models/Neon.json') # Load a model
train_images = mnist.train_images()
train_labels = mnist.train_labels()
test_images = mnist.test_images()
test_labels = mnist.test_labels()
X_Test_Data = (test_images.reshape(test_images.shape[0], 784)) / 255.0
Y_Test_Data = np.zeros((10, 10000))
X_Train_Data = (train_images.reshape(train_images.shape[0], 784)) / 255.0
Y_Train_Data = np.zeros((10, 60000))
for i in range(Y_Train_Data.shape[1]):
Y_Train_Data[train_labels[i]][i] = 1.0
for i in range(Y_Test_Data.shape[1]):
Y_Test_Data[test_labels[i]][i] = 1.0
nn.fit(X_Train_Data,
Y_Train_Data.T,
'MSE',
0.01,
0.05,
50,
50,
print_mode=1)
nn.test(X_Test_Data, Y_Test_Data.T, 'MSE')
nn.save('models/Sodium.json') # Save model
def prepare_mnist():
print('[MNIST] Preparing dataset ...')
# setting
dir_image_output = './dataset/mnist/'
# clean and create dir if any
create_dir(dir_image_output + 'all')
create_dir(dir_image_output + 'classes')
for i in range(10):
create_dir(dir_image_output + 'classes/{}'.format(i))
train_images = mnist.train_images()
train_labels = mnist.train_labels()
for i in range(train_images.shape[0]):
img = train_images[i, :, :]
label = train_labels[i]
img = Image.fromarray(img).convert('RGB').resize((32, 32))
img.save(dir_image_output + 'all/img_{}.jpg'.format(i))
img.save(dir_image_output + 'classes/{}/img_{}.jpg'.format(label, i))
print('[MNIST] Preparing dataset was completed.')
pass
def main():
train_images = mnist.train_images()
train_labels = mnist.train_labels()
train_images_flat = train_images.reshape(
(train_images.shape[0], train_images.shape[1] * train_images.shape[2]))
#print('train_images[0][0] is ' + repr(train_images[0][0]))
#test_images = mnist.test_images()
#test_labels = mnist.test_labels()
my_net = Net((784, 32, 32, 10))
for n in range(len(train_images)):
input_data = train_images_flat[n]
desired_outputs = [
0 if i != train_labels[n] else 100 for i in range(10)
]
#print(input_data)
print('{:5d} {:1d} '.format(n, train_labels[n]), end='')
#print(desired_outputs)
t = None
for _ in range(1):
t = my_net.tweak(input_data, desired_outputs, t)
print(' {:6d} '.format(t), end='')
print(my_net.layers[2].neurons)
print()
print(my_net.layers[2].neurons)
print(my_net.error_score(desired_outputs))
def get_dataloaders(batch_size, seed, flatten=False):
import mnist
from sklearn.model_selection import train_test_split
def preprocess(x, y, flatten):
x = x.astype("float32") / 255.
y = y.astype("int64")
x = x.reshape(-1, 784) if flatten else x.reshape(-1, 1, 28, 28)
return x, y
images, labels = mnist.train_images(), mnist.train_labels()
x_train, x_valid, y_train, y_valid = train_test_split(images,
labels,
test_size=0.2,
random_state=seed)
x_test, y_test = mnist.test_images(), mnist.test_labels()
x_train, y_train = preprocess(x_train, y_train, flatten)
x_valid, y_valid = preprocess(x_valid, y_valid, flatten)
x_test, y_test = preprocess(x_test, y_test, flatten)
train_loader = torch.utils.data.DataLoader(ImageDataset(x_train, y_train),
batch_size=batch_size,
shuffle=True)
valid_loader = torch.utils.data.DataLoader(ImageDataset(x_valid, y_valid),
batch_size=batch_size,
shuffle=False)
test_loader = torch.utils.data.DataLoader(ImageDataset(x_test, y_test),
batch_size=batch_size,
shuffle=False)
return train_loader, valid_loader, test_loader
def train(self, epochs=10, batch_size=10):
print("Training...")
start = time.time()
training_images = mnist.train_images()
training_labels = mnist.train_labels()
test_images = mnist.test_images()
test_labels = mnist.test_labels()
training_images = (training_images / 255.0) - 0.5
test_images = (test_images / 255.0) - 0.5
training_images = np.expand_dims(training_images, axis=3)
test_images = np.expand_dims(test_images, axis=3)
hist = self.model.fit(training_images,
to_categorical(training_labels),
batch_size=batch_size,
epochs=epochs,
validation_data=(test_images,
to_categorical(test_labels)))
end = time.time()
elapsed = end - start
acc = hist.history['acc'][-1]
return acc, elapsed
def runCNN():
trainImgs = mnist.train_images()
trainLabels = mnist.train_labels()
print(trainImgs.shape)
print(trainLabels.shape)
testImgs = mnist.test_images()
testLabels = mnist.test_labels()
print(testImgs.shape)
print(testLabels.shape)
trainImgs = (trainImgs / 255) - 0.5
testImgs = (testImgs / 255) - 0.5
trainImgs = np.expand_dims(trainImgs, axis=3)
testImgs = np.expand_dims(testImgs, axis=3)
print(trainImgs.shape)
print(testImgs.shape)
model = Sequential()
model.add(Conv2D(8, 3, input_shape=(28, 28, 1)))
model.add(MaxPooling2D(pool_size=2))
model.add(Flatten())
model.add(Dense(10, activation='softmax'))
model.load_weights("mnist_cnn.h5")
predict = model.predict(testImgs[0:10])
print(predict)
max = np.argmax(predict, axis=1)
print(max)
def load_preprocessed_mnist(use_torch=False, flatten_images=False):
"""
Return mnist dataset with images normalized to [0, 1], scaled to (16, 16) and deskewed.
Args:
use_torch (bool, optional): If True, return torch tensors, otherwise numpy arrays
(default: False).
flatten_images (bool, optional): Flatten train and test images (default: False).
Returns:
Train images, train labels, test images, test labels as numpy arrays
(or torch tensors, see parameter use_torch).
"""
train_images = mnist.train_images() / 255
train_images = preprocess(train_images, (16, 16), unskew=True)
train_images = train_images.astype('float32')
test_images = mnist.test_images() / 255
test_images = preprocess(test_images, (16, 16), unskew=True)
test_images = test_images.astype('float32')
train_labels = mnist.train_labels().astype(
int) # original arrays are uint8
test_labels = mnist.test_labels().astype(int)
if flatten_images:
train_images = train_images.reshape(len(train_images), -1)
test_images = test_images.reshape(len(test_images), -1)
if use_torch:
return torch.from_numpy(train_images).float(), torch.from_numpy(train_labels), \
torch.from_numpy(test_images).float(), torch.from_numpy(test_labels)
else:
return train_images, train_labels, test_images, test_labels
def main():
samples = 20
train_images = mnist.train_images()[0:samples]
train_labels = mnist.train_labels()[0:samples]
labels = [[1 if l > 4 else 0] for l in train_labels]
features = vectorize(train_images)
#print(train_images[5])
#print(features)
#for feature in features: print(feature)
#return
#print(train_labels[0],labels[0])
#print(len(features[0]))
#print(labels)
#return
def updates(epoch): print(epoch, nn.loss_avg[-1], nn.loss[-1])
nn = QuantumMNIST()
nn.load(features=features, labels=labels)
nn.train(progress=updates, updates=100)
results = nn.predict(features)
#print(np.array(nn.loss))
print(np.column_stack((
results
, np.round(results)
, np.array(labels)
)))
x = [x for x in range(len(nn.loss))]
plt.errorbar(x, nn.loss, yerr=nn.loss, errorevery=100)
plt.yscale('log')
plt.show()
def makeFitness():
#load
images = mnist.train_images()
#reshape
images = images.reshape(
(images.shape[0], images.shape[1] * images.shape[2]))
#normalize
images = np.divide(images, 255)
#labels
labels = mnist.train_labels()
assert (len(images) == len(labels))
dataset = list(zip(images, labels))
def fitness(individual):
errors = 0
for image, label in dataset:
try:
x = individual.output(image)
except RecursionError as re:
print(str(individual))
calledDigit = np.argmax(x)
if calledDigit != label:
errors += 1
return errors
return fitness
def load_data():
"""
Returns the tuple "(train_set, test_set)".
"train_set" contains 60000 training examples (x, y), and "test_set"
contains 10000 training examples (x, y).
"x" has shape (n^{[0]}, m) and contains n^{[0]} input features for
m training examples. "y" has shape (n^{[L]}, m), and contains n^{[L]}
outputs for m training examples.
"""
train_images = mnist.train_images()
train_labels = np.array([label_vector(i) for i in mnist.train_labels()])
train_labels = np.transpose(train_labels, (1, 2, 0))
train_labels = train_labels.reshape(
train_labels.shape[0] * train_labels.shape[1], train_labels.shape[2])
test_images = mnist.test_images()
test_labels = np.array([label_vector(i) for i in mnist.test_labels()])
test_labels = np.transpose(test_labels, (1, 2, 0))
test_labels = test_labels.reshape(
test_labels.shape[0] * test_labels.shape[1], test_labels.shape[2])
x_train = train_images.reshape(
train_images.shape[0], train_images.shape[1] * train_images.shape[2]).T
y_train = train_labels
x_test = test_images.reshape(test_images.shape[0],
test_images.shape[1] * test_images.shape[2]).T
y_test = test_labels
train_set = (x_train, y_train)
test_set = (x_test, y_test)
return (train_set, test_set)
def loadDataMNIST(mlModel):
"""
Load the MNIST data set. No training file names need to be specified but
mnist must be installed.
"""
try:
import mnist
except:
raise ("Please run pip install mnist")
train_images = mnist.train_images()
train_labels = mnist.train_labels()
test_images = mnist.test_images()
test_labels = mnist.test_labels()
mlModel.features = train_images.reshape(
(train_images.shape[0], train_images.shape[1] * train_images.shape[2]))
# Convert labels to one hot
onehot_train_labels = np.zeros((train_labels.size, 10))
onehot_train_labels[np.arange(train_labels.size), train_labels] = 1
mlModel.labels = onehot_train_labels.T
mlModel.testFeatures = test_images.reshape(
(test_images.shape[0], test_images.shape[1] * test_images.shape[2]))
# Convert labels to one hot
onehot_test_labels = np.zeros((test_labels.size, 10))
onehot_test_labels[np.arange(test_labels.size), test_labels] = 1
mlModel.testLabels = onehot_test_labels.T
return mlModel
def main():
#importing data
train_images = mnist.train_images()
train_y = mnist.train_labels()
test_images = mnist.test_images()
test_y = mnist.test_labels()
#normalizing data
train_x = (train_images / 255) - 0.5
test_x = (test_images / 255) - 0.5
train_x = train_x.reshape((-1, 784)) # 28*28 = 784
test_x = test_x.reshape((-1, 784))
model = KerasClassifier(build_fn=build_classifier)
parameters = {
'batch_size': [32, 64],
'nb_epoch': [6, 10],
'nodes': [1, 2, 3],
'layer': [64, 397],
'optimizer': ['adam', 'rmsprop']
}
grid_search = GridSearchCV(estimator=model,
param_grid=parameters,
scoring='accuracy',
cv=10)
grid_search = grid_search.fit(train_x, train_y)
best_parameters = grid_search.best_params_
best_acc = grid_search.best_score_
print(best_acc, best_parameters)
def main():
epochs = 201
train_images = mnist.train_images()
train_labels = mnist.train_labels()
test_images = mnist.test_images()
test_labels = mnist.test_labels()
n_train, w, h = train_images.shape
X_train = train_images.reshape((n_train, w * h)) # 维度为60000 * 784
Y_train = train_labels # 60000 * 1个label
n_test, w, h = test_images.shape
X_test = test_images.reshape((n_test, w * h))
Y_test = test_labels
model = initialize(784, 128, 64, 32, 16, 10) # 初始化模型参数
batch_size = 600
train_error = []
test_acc = []
for epoch in range(epochs):
loss_ep = 0
index = np.arange(60000)
np.random.shuffle(index)
X_train = X_train[index, :]
Y_train = Y_train[index]
for i in range(100):
begin_num = i * batch_size
end_num = i * batch_size + batch_size
loss, model = forward(model, X_train[begin_num:end_num, :],
Y_train[begin_num:end_num])
loss_ep += loss
loss_ep = loss_ep / batch_size
train_error.append(loss_ep)
y_predict = predict(model, X_test)
counter = 0
for i in range(len(y_predict)):
if y_predict[i] == Y_test[i]:
counter += 1
test_acc.append(counter / len(y_predict))
if epoch % 10 == 0:
print('epoch:', epoch, 'loss:', loss_ep, 'test_acc:',
counter / len(y_predict), '\n')
plt.title(" training error curves")
plt.xlabel("epochs")
plt.ylabel("loss_average")
x = np.arange(0, epochs)
plt.plot(x, train_error)
plt.show()
plt.title(" test acc curves")
plt.xlabel("epochs")
plt.ylabel("loss_average")
x = np.arange(0, epochs)
plt.plot(x, test_acc)
plt.show()
def runCNN():
trainImgs = mnist.train_images()
trainLabels = mnist.train_labels()
print(trainImgs.shape)
print(trainLabels.shape)
testImgs = mnist.test_images()
testLabels = mnist.test_labels()
print(testImgs.shape)
print(testLabels.shape)
trainImgs = (trainImgs / 255) - 0.5
testImgs = (testImgs / 255) - 0.5
trainImgs = np.expand_dims(trainImgs, axis=3)
testImgs = np.expand_dims(testImgs, axis=3)
print(trainImgs.shape)
print(testImgs.shape)
model = Sequential()
model.add(Conv2D(8, 3, input_shape=(28, 28, 1)))
model.add(MaxPooling2D(pool_size=2))
model.add(Flatten())
model.add(Dense(10, activation='softmax'))
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.fit(x=trainImgs,
y=to_categorical(trainLabels),
epochs=5,
validation_data=(testImgs, to_categorical(testLabels)))
model.save("mnist_cnn.h5")
def main():
route = 2
if route == 1:
gan = GAN(generator=[5, 6, 5, 7], discriminator=[7, 3, 2])
gan.fake_input_run()
cost = gan.cost
print("Initial cost: ", cost)
diff = gan.generator_weight_dash[1][5, 4]
print("weight dash: ", diff)
wei = gan.generatorANN.weights[1][5, 3]
print("the actual weight", wei)
gan.generatorANN.weights[1][5, 3] *= 1.1
newwei = gan.generatorANN.weights[1][5, 3]
print("the new weight", newwei)
gan.fake_input_run()
costnew = gan.get_cost()
print("new cost = ", costnew)
print("change = ", (costnew - cost) / (newwei - wei))
#b = ANN(layers = di,
# cost_function = 'self.cross_entropy',
# batch_length = 10,
# learning_rate= 5,
# randomise = False)
cost = gan.cost
print("Initial cost: ", cost)
print(gan.discriminator_weight_dash[0][5, 2])
diff = gan.discriminator_weight_dash[0][5, 2]
print("weight dash: ", diff)
wei = gan.discriminatorANN.weights[0][5, 2]
print("the actual weight", wei)
gan.discriminatorANN.weights[0][5, 2] += 0.2
newwei = gan.discriminatorANN.weights[0][5, 2]
print("the new weight", newwei)
gan.fake_input_run()
costnew = gan.get_cost()
print("new cost = ", costnew)
print("change = ", (costnew - cost) / (newwei - wei))
elif route == 2:
X = np.array([x.flatten() / 256 for x in mnist.train_images()])
pos = np.where(mnist.train_labels() == 7)
gan = GAN(
generator=[100, 300, 500, 784],
discriminator=[784, 30, 1],
discriminator_learning_rate=1,
generator_learning_rate=1.25,
#X=X,
X=X[pos],
save=False)
#gan.visualise_mnist(gan.train_images[64])
a = gan.run(no=300, disc_steps=1)
#filehandler = open('testsave.obj', 'wb')
#pickle.dump(gan, filehandler)
gan.graphs()
for ii in range(3):
gan.visualise_mnist(
gan.generatorANN.predict(gan.get_random_input()))
#gan.visualise_mnist(gan.generated_digit_list[-1])
plt.show()
def get_dataloaders(batch_size, seed, flatten=False):
import mnist
from sklearn.model_selection import train_test_split
def preprocess(x, y, flatten):
x = x.astype("float32") / 255.
y = y.astype("int64")
x = x.reshape(-1, 28 * 28) if flatten else x.reshape(-1, 28, 28, 1)
return x, y
images, labels = mnist.train_images(), mnist.train_labels()
x_train, x_valid, y_train, y_valid = train_test_split(images,
labels,
test_size=0.2,
random_state=seed)
x_test, y_test = mnist.test_images(), mnist.test_labels()
x_train, y_train = preprocess(x_train, y_train, flatten)
x_valid, y_valid = preprocess(x_valid, y_valid, flatten)
x_test, y_test = preprocess(x_test, y_test, flatten)
train_loader = tf.data.Dataset.from_tensor_slices(
(x_train, y_train)).shuffle(buffer_size=1024).batch(batch_size)
valid_loader = tf.data.Dataset.from_tensor_slices(
(x_valid, y_valid)).batch(batch_size)
test_loader = tf.data.Dataset.from_tensor_slices(
(x_test, y_test)).batch(batch_size)
return train_loader, valid_loader, test_loader
def load_data():
"""
Ref: https://victorzhou.com/blog/keras-neural-network-tutorial/
"""
import numpy as np
import mnist
from tensorflow.keras.utils import to_categorical
train_images = mnist.train_images()
train_labels = mnist.train_labels()
test_images = mnist.test_images()
test_labels = mnist.test_labels()
# Normalize the images.
train_images = (train_images / 255) - 0.5
test_images = (test_images / 255) - 0.5
# Flatten the images.
train_images = train_images.reshape((-1, 784))
test_images = test_images.reshape((-1, 784))
# One-hot encoding
train_labels, test_labels = to_categorical(train_labels), to_categorical(
test_labels)
return (train_images, train_labels, test_images, test_labels)
def __init__(self,
dims=(120, 120, 64),
batch_size=16,
shuffle=True,
validation=False,
split=0.2,
extend_dims=True,
augment_data=True):
self.dims = dims
self.batch_size = batch_size
self.extend_dims = extend_dims
# Get MNIST files, Split based on validation
if validation:
files = mnist.test_images()
labels = mnist.test_labels()
else:
files = mnist.train_images()
labels = mnist.train_labels()
# Take into account shuffling
if shuffle:
tmp = list(zip(files, labels))
random.shuffle(tmp)
files, labels = zip(*tmp)
labels = np.array(labels)
self.files = files
self.labels = labels
def mnist_autoencoder():
'''
Use features extracted by the encoder of an autoencoder
'''
import mnist
x_train = np.load('../mnist_train_autoencoder.npy')
return x_train, mnist.train_labels()
def main():
#importing data
train_images = mnist.train_images()
train_y = mnist.train_labels()
test_images = mnist.test_images()
test_y = mnist.test_labels()
#normalizing data
train_x = (train_images / 255) - 0.5
test_x = (test_images / 255) - 0.5
train_x = train_x.reshape((-1, 784)) # 28*28 = 784
test_x = test_x.reshape((-1, 784))
#model
model = tf.keras.Sequential()
model.add(tf.keras.layers.Dense(397, activation='relu', input_dim=784))
model.add(tf.keras.layers.Dense(397, activation='relu'))
model.add(tf.keras.layers.Dense(10, activation='softmax'))
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.fit(train_x, to_categorical(train_y), epochs=10, batch_size=32)
y_test = tf.keras.utils.to_categorical(
tf.keras.datasets.mnist.load_data()[1][1],
num_classes=10,
dtype='float32')
y_score = model.predict_proba(test_x)
plot_pr(y_test, y_score)
roc(y_test, y_score)
def __init__(self, mode='train'):
if mode is 'train':
self.data, self.labels = train_images(), train_labels()
elif mode is 'test':
self.data, self.labels = test_images(), test_labels()
self.indexes = np.arange(self.data.shape[0])
np.random.shuffle(self.indexes)
def main():
route = 1
X = np.array([x.flatten() / 256 for x in mnist.train_images()])
y = mnist.train_labels()
X_test = np.array([x.flatten() / 256 for x in mnist.test_images()])
y_test = np.array(mnist.test_labels())
if route == 1:
b = ANN(layers = [784, 30, 10],
X=X,
y=y,
cost_function='cross_entropy',
activation_function='ReLU',
#activation_function='sigmoid',
#optimiser='ADAM',
batch_length = 15,
learning_rate = 1,
randomise = True,
X_test=X_test,
y_test=y_test,
)
b.train(epochs = 1)
b.predict_test(X_test = X_test,
y_test = y_test
)
def test_keras_mnist():
train_images = mnist.train_images()
train_labels = mnist.train_labels()
n = train_images.shape[0]
lb = LabelBinarizer()
lb.fit(range(10))
selection = random.sample(range(n), 10000) # Use only a subsample
y_train = lb.transform(train_labels[selection]) # One-hot encodings
x_train = train_images.reshape(n, 28, 28, 1)[selection]
x_train = x_train / 255 # Normalize train data
pop = Population(GeneticCnnIndividual,
x_train,
y_train,
size=20,
crossover_rate=0.3,
mutation_rate=0.1,
additional_parameters={
'kfold': 5,
'epochs': (20, 4, 1),
'learning_rate': (1e-3, 1e-4, 1e-5),
'batch_size': 32
},
maximize=True)
ga = RussianRouletteGA(pop,
crossover_probability=0.2,
mutation_probability=0.8)
ga.run(50)
def test_keras_model():
train_images = mnist.train_images()
train_labels = mnist.train_labels()
n = train_images.shape[0]
lb = LabelBinarizer()
lb.fit(range(10))
selection = random.sample(range(n), 10000)
y_train = lb.transform(train_labels[selection])
x_train = train_images.reshape(n, 28, 28, 1)[selection]
x_train = x_train / 255 # Normalize train data
model = GeneticCnnModel(
x_train,
y_train,
{
'S_1': '000',
'S_2': '0000000000'
}, # Genes to test
(3, 5), # Number of nodes per DAG (corresponds to gene bytes)
(28, 28, 1), # Shape of input data
(20, 50), # Number of kernels per layer
((5, 5), (5, 5)), # Sizes of kernels per layer
500, # Number of units in Dense layer
0.5, # Dropout probability
10, # Number of classes to predict
kfold=5,
epochs=(20, 4, 1),
learning_rate=(1e-3, 1e-4, 1e-5),
batch_size=128)
print(model.cross_validate())
def get_train_set(self): X_train = mnist.train_images() y_train = mnist.train_labels() mixed_indexes = np.random.permutation(60000) X_train, y_train = X_train[mixed_indexes], y_train[mixed_indexes] X_train_prepared = self._prepare_data(X_train) return X_train_prepared, y_train
def main():
samples = 20
train_images = mnist.train_images()[0:samples]
train_labels = mnist.train_labels()[0:samples]
resolution = len(train_images[0]) * len(train_images[0])
labels = [[1 if l > 4 else 0] for l in train_labels]
features = [
np.where(np.reshape(symbol, resolution) > 1, 1, 0)
for symbol in train_images
]
nn = ClassicalMNIST()
nn.load(features=features, labels=labels)
nn.train(progress=lambda epoch : print(epoch, np.average(nn.loss)))
results = nn.predict(features)
print(features[0])
print(len(features[0]))
print(labels)
print(np.array(nn.loss))
print(np.column_stack((
results
, np.round(results)
, np.array(labels)
)))
def main():
# XOR revisited
# training data
xs = [[0., 0], [0., 1], [1., 0], [1., 1]]
ys = [[0.], [1.], [1.], [0.]]
random.seed(0)
net = Sequential([
Linear(input_dim=2, output_dim=2),
Sigmoid(),
Linear(input_dim=2, output_dim=1)
])
import tqdm
optimizer = GradientDescent(learning_rate=0.1)
loss = SSE()
with tqdm.trange(3000) as t:
for epoch in t:
epoch_loss = 0.0
for x, y in zip(xs, ys):
predicted = net.forward(x)
epoch_loss += loss.loss(predicted, y)
gradient = loss.gradient(predicted, y)
net.backward(gradient)
optimizer.step(net)
t.set_description(f"xor loss {epoch_loss:.3f}")
for param in net.params():
print(param)
# FizzBuzz Revisited
from scratch.neural_networks import binary_encode, fizz_buzz_encode, argmax
xs = [binary_encode(n) for n in range(101, 1024)]
ys = [fizz_buzz_encode(n) for n in range(101, 1024)]
NUM_HIDDEN = 25
random.seed(0)
net = Sequential([
Linear(input_dim=10, output_dim=NUM_HIDDEN, init='uniform'),
Tanh(),
Linear(input_dim=NUM_HIDDEN, output_dim=4, init='uniform'),
Sigmoid()
])
def fizzbuzz_accuracy(low: int, hi: int, net: Layer) -> float:
num_correct = 0
for n in range(low, hi):
x = binary_encode(n)
predicted = argmax(net.forward(x))
actual = argmax(fizz_buzz_encode(n))
if predicted == actual:
num_correct += 1
return num_correct / (hi - low)
optimizer = Momentum(learning_rate=0.1, momentum=0.9)
loss = SSE()
with tqdm.trange(1000) as t:
for epoch in t:
epoch_loss = 0.0
for x, y in zip(xs, ys):
predicted = net.forward(x)
epoch_loss += loss.loss(predicted, y)
gradient = loss.gradient(predicted, y)
net.backward(gradient)
optimizer.step(net)
accuracy = fizzbuzz_accuracy(101, 1024, net)
t.set_description(f"fb loss: {epoch_loss:.2f} acc: {accuracy:.2f}")
# Now check results on the test set
print("test results", fizzbuzz_accuracy(1, 101, net))
random.seed(0)
net = Sequential([
Linear(input_dim=10, output_dim=NUM_HIDDEN, init='uniform'),
Tanh(),
Linear(input_dim=NUM_HIDDEN, output_dim=4, init='uniform')
# No final sigmoid layer now
])
optimizer = Momentum(learning_rate=0.1, momentum=0.9)
loss = SoftmaxCrossEntropy()
with tqdm.trange(100) as t:
for epoch in t:
epoch_loss = 0.0
for x, y in zip(xs, ys):
predicted = net.forward(x)
epoch_loss += loss.loss(predicted, y)
gradient = loss.gradient(predicted, y)
net.backward(gradient)
optimizer.step(net)
accuracy = fizzbuzz_accuracy(101, 1024, net)
t.set_description(f"fb loss: {epoch_loss:.3f} acc: {accuracy:.2f}")
# Again check results on the test set
print("test results", fizzbuzz_accuracy(1, 101, net))
# Load the MNIST data
import mnist
# This will download the data, change this to where you want it.
# (Yes, it's a 0-argument function, that's what the library expects.)
# (Yes, I'm assigning a lambda to a variable, like I said never to do.)
mnist.temporary_dir = lambda: '/tmp'
# Each of these functions first downloads the data and returns a numpy array.
# We call .tolist() because our "tensors" are just lists.
train_images = mnist.train_images().tolist()
train_labels = mnist.train_labels().tolist()
assert shape(train_images) == [60000, 28, 28]
assert shape(train_labels) == [60000]
import matplotlib.pyplot as plt
fig, ax = plt.subplots(10, 10)
for i in range(10):
for j in range(10):
# Plot each image in black and white and hide the axes.
ax[i][j].imshow(train_images[10 * i + j], cmap='Greys')
ax[i][j].xaxis.set_visible(False)
ax[i][j].yaxis.set_visible(False)
# plt.show()
# Load the MNIST test data
test_images = mnist.test_images().tolist()
test_labels = mnist.test_labels().tolist()
assert shape(test_images) == [10000, 28, 28]
assert shape(test_labels) == [10000]
# Recenter the images
# Compute the average pixel value
avg = tensor_sum(train_images) / 60000 / 28 / 28
# Recenter, rescale, and flatten
train_images = [[(pixel - avg) / 256 for row in image for pixel in row]
for image in train_images]
test_images = [[(pixel - avg) / 256 for row in image for pixel in row]
for image in test_images]
assert shape(train_images) == [60000, 784], "images should be flattened"
assert shape(test_images) == [10000, 784], "images should be flattened"
# After centering, average pixel should be very close to 0
assert -0.0001 < tensor_sum(train_images) < 0.0001
# One-hot encode the test data
train_labels = [one_hot_encode(label) for label in train_labels]
test_labels = [one_hot_encode(label) for label in test_labels]
assert shape(train_labels) == [60000, 10]
assert shape(test_labels) == [10000, 10]
# Training loop
import tqdm
def loop(model: Layer,
images: List[Tensor],
labels: List[Tensor],
loss: Loss,
optimizer: Optimizer = None) -> None:
correct = 0 # Track number of correct predictions.
total_loss = 0.0 # Track total loss.
with tqdm.trange(len(images)) as t:
for i in t:
predicted = model.forward(images[i]) # Predict.
if argmax(predicted) == argmax(labels[i]): # Check for
correct += 1 # correctness.
total_loss += loss.loss(predicted, labels[i]) # Compute loss.
# If we're training, backpropagate gradient and update weights.
if optimizer is not None:
gradient = loss.gradient(predicted, labels[i])
model.backward(gradient)
optimizer.step(model)
# And update our metrics in the progress bar.
avg_loss = total_loss / (i + 1)
acc = correct / (i + 1)
t.set_description(f"mnist loss: {avg_loss:.3f} acc: {acc:.3f}")
# The logistic regression model for MNIST
random.seed(0)
# Logistic regression is just a linear layer followed by softmax
model = Linear(784, 10)
loss = SoftmaxCrossEntropy()
# This optimizer seems to work
optimizer = Momentum(learning_rate=0.01, momentum=0.99)
# Train on the training data
loop(model, train_images, train_labels, loss, optimizer)
# Test on the test data (no optimizer means just evaluate)
loop(model, test_images, test_labels, loss)
# A deep neural network for MNIST
random.seed(0)
# Name them so we can turn train on and off
dropout1 = Dropout(0.1)
dropout2 = Dropout(0.1)
model = Sequential([
Linear(784, 30), # Hidden layer 1: size 30
dropout1,
Tanh(),
Linear(30, 10), # Hidden layer 2: size 10
dropout2,
Tanh(),
Linear(10, 10) # Output layer: size 10
])
# Training the deep model for MNIST
optimizer = Momentum(learning_rate=0.01, momentum=0.99)
loss = SoftmaxCrossEntropy()
# Enable dropout and train (takes > 20 minutes on my laptop!)
dropout1.train = dropout2.train = True
loop(model, train_images, train_labels, loss, optimizer)
# Disable dropout and evaluate
dropout1.train = dropout2.train = False
loop(model, test_images, test_labels, loss)
