def load_data():
"""
Returns the tuple "(train_set, test_set)".
"train_set" contains 60000 training examples (x, y), and "test_set"
contains 10000 training examples (x, y).
"x" has shape (n^{[0]}, m) and contains n^{[0]} input features for
m training examples. "y" has shape (n^{[L]}, m), and contains n^{[L]}
outputs for m training examples.
"""
train_images = mnist.train_images()
train_labels = np.array([label_vector(i) for i in mnist.train_labels()])
train_labels = np.transpose(train_labels, (1, 2, 0))
train_labels = train_labels.reshape(
train_labels.shape[0] * train_labels.shape[1], train_labels.shape[2])
test_images = mnist.test_images()
test_labels = np.array([label_vector(i) for i in mnist.test_labels()])
test_labels = np.transpose(test_labels, (1, 2, 0))
test_labels = test_labels.reshape(
test_labels.shape[0] * test_labels.shape[1], test_labels.shape[2])
x_train = train_images.reshape(
train_images.shape[0], train_images.shape[1] * train_images.shape[2]).T
y_train = train_labels
x_test = test_images.reshape(test_images.shape[0],
test_images.shape[1] * test_images.shape[2]).T
y_test = test_labels
train_set = (x_train, y_train)
test_set = (x_test, y_test)
return (train_set, test_set)
def main():
# Numpy Stuff
# np.random.seed(1)
# np.set_printoptions(threshold=np.inf)
# Toy Data for testing
# X_Data = np.array([[0, 0, 0],
# [0, 0, 1],
# [0, 1, 0],
# [0, 1, 1],
# [1, 0, 0],
# [1, 0, 1],
# [1, 1, 0],
# [1, 1, 1]])
# Y_Data = np.array([[1, 0, 0, 0, 0, 0, 0, 0],
# [0, 1, 0, 0, 0, 0, 0, 0],
# [0, 0, 1, 0, 0, 0, 0, 0],
# [0, 0, 0, 1, 0, 0, 0, 0],
# [0, 0, 0, 0, 1, 0, 0, 0],
# [0, 0, 0, 0, 0, 1, 0, 0],
# [0, 0, 0, 0, 0, 0, 1, 0],
# [0, 0, 0, 0, 0, 0, 0, 1]])
structure = [(784, '*'), (38, 'sigmoid'), (10, 'sigmoid')]
nn = NeuralNet(structure, random_init_bound=0.05)
# nn.load('models/Neon.json') # Load a model
train_images = mnist.train_images()
train_labels = mnist.train_labels()
test_images = mnist.test_images()
test_labels = mnist.test_labels()
X_Test_Data = (test_images.reshape(test_images.shape[0], 784)) / 255.0
Y_Test_Data = np.zeros((10, 10000))
X_Train_Data = (train_images.reshape(train_images.shape[0], 784)) / 255.0
Y_Train_Data = np.zeros((10, 60000))
for i in range(Y_Train_Data.shape[1]):
Y_Train_Data[train_labels[i]][i] = 1.0
for i in range(Y_Test_Data.shape[1]):
Y_Test_Data[test_labels[i]][i] = 1.0
nn.fit(X_Train_Data,
Y_Train_Data.T,
'MSE',
0.01,
0.05,
50,
50,
print_mode=1)
nn.test(X_Test_Data, Y_Test_Data.T, 'MSE')
nn.save('models/Sodium.json') # Save model
def runCNN():
trainImgs = mnist.train_images()
trainLabels = mnist.train_labels()
print(trainImgs.shape)
print(trainLabels.shape)
testImgs = mnist.test_images()
testLabels = mnist.test_labels()
print(testImgs.shape)
print(testLabels.shape)
trainImgs = (trainImgs / 255) - 0.5
testImgs = (testImgs / 255) - 0.5
trainImgs = np.expand_dims(trainImgs, axis=3)
testImgs = np.expand_dims(testImgs, axis=3)
print(trainImgs.shape)
print(testImgs.shape)
model = Sequential()
model.add(Conv2D(8, 3, input_shape=(28, 28, 1)))
model.add(MaxPooling2D(pool_size=2))
model.add(Flatten())
model.add(Dense(10, activation='softmax'))
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.fit(x=trainImgs,
y=to_categorical(trainLabels),
epochs=5,
validation_data=(testImgs, to_categorical(testLabels)))
model.save("mnist_cnn.h5")
def main():
#importing data
train_images = mnist.train_images()
train_y = mnist.train_labels()
test_images = mnist.test_images()
test_y = mnist.test_labels()
#normalizing data
train_x = (train_images / 255) - 0.5
test_x = (test_images / 255) - 0.5
train_x = train_x.reshape((-1, 784)) # 28*28 = 784
test_x = test_x.reshape((-1, 784))
model = KerasClassifier(build_fn=build_classifier)
parameters = {
'batch_size': [32, 64],
'nb_epoch': [6, 10],
'nodes': [1, 2, 3],
'layer': [64, 397],
'optimizer': ['adam', 'rmsprop']
}
grid_search = GridSearchCV(estimator=model,
param_grid=parameters,
scoring='accuracy',
cv=10)
grid_search = grid_search.fit(train_x, train_y)
best_parameters = grid_search.best_params_
best_acc = grid_search.best_score_
print(best_acc, best_parameters)
def mnist_autoencoder_test():
'''
Use features extracted by the encoder of an autoencoder
'''
import mnist
x_test = np.load('../mnist_test_autoencoder.npy')
return x_test, mnist.test_labels()
def main():
route = 1
X = np.array([x.flatten() / 256 for x in mnist.train_images()])
y = mnist.train_labels()
X_test = np.array([x.flatten() / 256 for x in mnist.test_images()])
y_test = np.array(mnist.test_labels())
if route == 1:
b = ANN(layers = [784, 30, 10],
X=X,
y=y,
cost_function='cross_entropy',
activation_function='ReLU',
#activation_function='sigmoid',
#optimiser='ADAM',
batch_length = 15,
learning_rate = 1,
randomise = True,
X_test=X_test,
y_test=y_test,
)
b.train(epochs = 1)
b.predict_test(X_test = X_test,
y_test = y_test
)
def main():
#importing data
train_images = mnist.train_images()
train_y = mnist.train_labels()
test_images = mnist.test_images()
test_y = mnist.test_labels()
#normalizing data
train_x = (train_images / 255) - 0.5
test_x = (test_images / 255) - 0.5
train_x = train_x.reshape((-1, 784)) # 28*28 = 784
test_x = test_x.reshape((-1, 784))
#initialising the ANN
model = tf.keras.Sequential()
#adding the input layer and the first hidden layer
model.add(tf.keras.layers.Dense(397, activation='relu', input_dim=784))
#adding the second hidden layer
model.add(tf.keras.layers.Dense(397, activation='relu'))
#adding the output layer
model.add(tf.keras.layers.Dense(10, activation='softmax'))
#compiling the ANN
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
#Fitting the ANN to the training set
model.fit(train_x, to_categorical(train_y), epochs=5, batch_size=16)
def __init__(self, batch_size):
self.batch_size = batch_size
self.train_x = mnist.train_images().reshape(60000, 784, 1)[0:50000]
self.train_y = np.eye(10)[mnist.train_labels()].reshape(60000, 10,
1)[0:50000]
self.test_x = mnist.test_images().reshape(10000, 784, 1)
self.test_y = mnist.test_labels().reshape(10000, 1)
def loadDataMNIST(mlModel):
"""
Load the MNIST data set. No training file names need to be specified but
mnist must be installed.
"""
try:
import mnist
except:
raise ("Please run pip install mnist")
train_images = mnist.train_images()
train_labels = mnist.train_labels()
test_images = mnist.test_images()
test_labels = mnist.test_labels()
mlModel.features = train_images.reshape(
(train_images.shape[0], train_images.shape[1] * train_images.shape[2]))
# Convert labels to one hot
onehot_train_labels = np.zeros((train_labels.size, 10))
onehot_train_labels[np.arange(train_labels.size), train_labels] = 1
mlModel.labels = onehot_train_labels.T
mlModel.testFeatures = test_images.reshape(
(test_images.shape[0], test_images.shape[1] * test_images.shape[2]))
# Convert labels to one hot
onehot_test_labels = np.zeros((test_labels.size, 10))
onehot_test_labels[np.arange(test_labels.size), test_labels] = 1
mlModel.testLabels = onehot_test_labels.T
return mlModel
def data_mnist(datadir='/tmp/', train_start=0, train_end=60000, test_start=0,
test_end=10000):
"""
Load and preprocess MNIST dataset
:param datadir: path to folder where data should be stored
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:return: tuple of four arrays containing training data, training labels,
testing data and testing labels.
"""
assert isinstance(train_start, int)
assert isinstance(train_end, int)
assert isinstance(test_start, int)
assert isinstance(test_end, int)
import mnist
X_train = mnist.train_images() / 255.
Y_train = mnist.train_labels()
X_test = mnist.test_images() / 255.
Y_test = mnist.test_labels()
X_train = np.expand_dims(X_train, -1)
X_test = np.expand_dims(X_test, -1)
X_train = X_train[train_start:train_end]
Y_train = Y_train[train_start:train_end]
X_test = X_test[test_start:test_end]
Y_test = Y_test[test_start:test_end]
Y_train = utils.to_categorical(Y_train, num_classes=10)
Y_test = utils.to_categorical(Y_test, num_classes=10)
return X_train, Y_train, X_test, Y_test
def get_dataloaders(batch_size, seed, flatten=False):
import mnist
from sklearn.model_selection import train_test_split
def preprocess(x, y, flatten):
x = x.astype("float32") / 255.
y = y.astype("int64")
x = x.reshape(-1, 784) if flatten else x.reshape(-1, 1, 28, 28)
return x, y
images, labels = mnist.train_images(), mnist.train_labels()
x_train, x_valid, y_train, y_valid = train_test_split(images,
labels,
test_size=0.2,
random_state=seed)
x_test, y_test = mnist.test_images(), mnist.test_labels()
x_train, y_train = preprocess(x_train, y_train, flatten)
x_valid, y_valid = preprocess(x_valid, y_valid, flatten)
x_test, y_test = preprocess(x_test, y_test, flatten)
train_loader = torch.utils.data.DataLoader(ImageDataset(x_train, y_train),
batch_size=batch_size,
shuffle=True)
valid_loader = torch.utils.data.DataLoader(ImageDataset(x_valid, y_valid),
batch_size=batch_size,
shuffle=False)
test_loader = torch.utils.data.DataLoader(ImageDataset(x_test, y_test),
batch_size=batch_size,
shuffle=False)
return train_loader, valid_loader, test_loader
def runCNN():
trainImgs = mnist.train_images()
trainLabels = mnist.train_labels()
print(trainImgs.shape)
print(trainLabels.shape)
testImgs = mnist.test_images()
testLabels = mnist.test_labels()
print(testImgs.shape)
print(testLabels.shape)
trainImgs = (trainImgs / 255) - 0.5
testImgs = (testImgs / 255) - 0.5
trainImgs = np.expand_dims(trainImgs, axis=3)
testImgs = np.expand_dims(testImgs, axis=3)
print(trainImgs.shape)
print(testImgs.shape)
model = Sequential()
model.add(Conv2D(8, 3, input_shape=(28, 28, 1)))
model.add(MaxPooling2D(pool_size=2))
model.add(Flatten())
model.add(Dense(10, activation='softmax'))
model.load_weights("mnist_cnn.h5")
predict = model.predict(testImgs[0:10])
print(predict)
max = np.argmax(predict, axis=1)
print(max)
def train(self, epochs=10, batch_size=10):
print("Training...")
start = time.time()
training_images = mnist.train_images()
training_labels = mnist.train_labels()
test_images = mnist.test_images()
test_labels = mnist.test_labels()
training_images = (training_images / 255.0) - 0.5
test_images = (test_images / 255.0) - 0.5
training_images = np.expand_dims(training_images, axis=3)
test_images = np.expand_dims(test_images, axis=3)
hist = self.model.fit(training_images,
to_categorical(training_labels),
batch_size=batch_size,
epochs=epochs,
validation_data=(test_images,
to_categorical(test_labels)))
end = time.time()
elapsed = end - start
acc = hist.history['acc'][-1]
return acc, elapsed
def main():
#importing data
train_images = mnist.train_images()
train_y = mnist.train_labels()
test_images = mnist.test_images()
test_y = mnist.test_labels()
#normalizing data
train_x = (train_images / 255) - 0.5
test_x = (test_images / 255) - 0.5
train_x = train_x.reshape((-1, 784)) # 28*28 = 784
test_x = test_x.reshape((-1, 784))
#model
model = tf.keras.Sequential()
model.add(tf.keras.layers.Dense(397, activation='relu', input_dim=784))
model.add(tf.keras.layers.Dense(397, activation='relu'))
model.add(tf.keras.layers.Dense(10, activation='softmax'))
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.fit(train_x, to_categorical(train_y), epochs=10, batch_size=32)
y_test = tf.keras.utils.to_categorical(
tf.keras.datasets.mnist.load_data()[1][1],
num_classes=10,
dtype='float32')
y_score = model.predict_proba(test_x)
plot_pr(y_test, y_score)
roc(y_test, y_score)
def main():
epochs = 201
train_images = mnist.train_images()
train_labels = mnist.train_labels()
test_images = mnist.test_images()
test_labels = mnist.test_labels()
n_train, w, h = train_images.shape
X_train = train_images.reshape((n_train, w * h)) # 维度为60000 * 784
Y_train = train_labels # 60000 * 1个label
n_test, w, h = test_images.shape
X_test = test_images.reshape((n_test, w * h))
Y_test = test_labels
model = initialize(784, 128, 64, 32, 16, 10) # 初始化模型参数
batch_size = 600
train_error = []
test_acc = []
for epoch in range(epochs):
loss_ep = 0
index = np.arange(60000)
np.random.shuffle(index)
X_train = X_train[index, :]
Y_train = Y_train[index]
for i in range(100):
begin_num = i * batch_size
end_num = i * batch_size + batch_size
loss, model = forward(model, X_train[begin_num:end_num, :],
Y_train[begin_num:end_num])
loss_ep += loss
loss_ep = loss_ep / batch_size
train_error.append(loss_ep)
y_predict = predict(model, X_test)
counter = 0
for i in range(len(y_predict)):
if y_predict[i] == Y_test[i]:
counter += 1
test_acc.append(counter / len(y_predict))
if epoch % 10 == 0:
print('epoch:', epoch, 'loss:', loss_ep, 'test_acc:',
counter / len(y_predict), '\n')
plt.title(" training error curves")
plt.xlabel("epochs")
plt.ylabel("loss_average")
x = np.arange(0, epochs)
plt.plot(x, train_error)
plt.show()
plt.title(" test acc curves")
plt.xlabel("epochs")
plt.ylabel("loss_average")
x = np.arange(0, epochs)
plt.plot(x, test_acc)
plt.show()
def Test_Network(n_network):
print("Preparing Test Data Set.")
test_images = [
x.reshape(1, x.shape[0] * x.shape[1]).tolist()[0]
for x in mnist.test_images()
]
print("Preparing Test Data Set Completed.\n")
print("Testing Start.")
test_labels = mnist.test_labels()
successes = 0
for x in range(len(test_images)):
output = n_network.feedforward(test_images[x])
pos = 0
max_probability = 0
for z in range(10):
if output[z] > max_probability:
max_probability = output[z]
pos = z
if pos == test_labels[x]:
successes += 1
print("Testing End.\n")
print("Successes : {0:.2f}%".format(successes / len(test_labels) * 100))
def get_dataloaders(batch_size, seed, flatten=False):
import mnist
from sklearn.model_selection import train_test_split
def preprocess(x, y, flatten):
x = x.astype("float32") / 255.
y = y.astype("int64")
x = x.reshape(-1, 28 * 28) if flatten else x.reshape(-1, 28, 28, 1)
return x, y
images, labels = mnist.train_images(), mnist.train_labels()
x_train, x_valid, y_train, y_valid = train_test_split(images,
labels,
test_size=0.2,
random_state=seed)
x_test, y_test = mnist.test_images(), mnist.test_labels()
x_train, y_train = preprocess(x_train, y_train, flatten)
x_valid, y_valid = preprocess(x_valid, y_valid, flatten)
x_test, y_test = preprocess(x_test, y_test, flatten)
train_loader = tf.data.Dataset.from_tensor_slices(
(x_train, y_train)).shuffle(buffer_size=1024).batch(batch_size)
valid_loader = tf.data.Dataset.from_tensor_slices(
(x_valid, y_valid)).batch(batch_size)
test_loader = tf.data.Dataset.from_tensor_slices(
(x_test, y_test)).batch(batch_size)
return train_loader, valid_loader, test_loader
def reload(dir="D:/python-ml/deepLearning-cases/keras_handwrinting/",
fileName="hd_digits_3nn_pred.h5"):
# Build the model.
# the same nn framework should be used
# the layers of NN should be the same with the training model
model = Sequential([
Dense(64, activation='relu', input_shape=(784, )),
Dense(64, activation='relu'),
Dense(10, activation='softmax'),
])
# Load the model's saved weights.
model.load_weights(str(dir) + str(fileName))
# predict the test data
# load data
test_X_orig = mnist.test_images() #shape(10000,28,28)
test_y = mnist.test_labels()
#flatten the image data into 2D. (60000,784)
test_X = test_X_orig.reshape((test_X_orig.shape[0], -1))
# normalize data by 255 into [-0.5,0.5]
test_X = test_X / 255 - 0.5
predictions = model.predict(test_X[:5])
predictions = pd.DataFrame(predictions)
predictions_me = predictions.apply(np.argmax, axis=1)
predictions_test = pd.DataFrame(test_y[:5])
predictions = pd.concat([predictions_me, predictions_test], axis=1)
print(predictions.head())
def __init__(self, mode='train'):
if mode is 'train':
self.data, self.labels = train_images(), train_labels()
elif mode is 'test':
self.data, self.labels = test_images(), test_labels()
self.indexes = np.arange(self.data.shape[0])
np.random.shuffle(self.indexes)
def play(method_name, takes_time):
w = []
for i in range(10):
w.append(get_vecs('./out/' + method_name + '_w_' + str(i)))
# print(w[i])
test_set_matrix = mnist.test_images()
test_set = [
test_set_matrix[i].flatten() for i in range(len(test_set_matrix))
]
test_set = [
np.append(test_set_matrix[i].flatten(), 1)
for i in range(len(test_set_matrix))
]
# test_set = [test_set[i]/np.max(test_set[i]) for i in range(len(test_set))]
# for i in range(len(test_set)):
# test_set[i] = np.insert(test_set[i], 0, 1)
label_set = mnist.test_labels()
bien_respond = 0
path_ = './result/result_' + method_name
with open(path_, "w") as f:
f.write('L = %f, train = %i, N_algo = %i\n' % (L, N_train, N_algo))
f.write('Takes time %f\n' % takes_time)
for j in range(len(test_set)):
vals = [np.dot(test_set[j], w[i]) for i in range(10)]
argmax = np.argmax(vals)
# f.write(str(argmax) + ' ' + str(label_set[j]) + '\n')
bien_respond += int(argmax == label_set[j])
f.write(
str(bien_respond) + ' ' + str(len(test_set)) + ' ' +
str(bien_respond / len(test_set)))
def load_data():
"""
Ref: https://victorzhou.com/blog/keras-neural-network-tutorial/
"""
import numpy as np
import mnist
from tensorflow.keras.utils import to_categorical
train_images = mnist.train_images()
train_labels = mnist.train_labels()
test_images = mnist.test_images()
test_labels = mnist.test_labels()
# Normalize the images.
train_images = (train_images / 255) - 0.5
test_images = (test_images / 255) - 0.5
# Flatten the images.
train_images = train_images.reshape((-1, 784))
test_images = test_images.reshape((-1, 784))
# One-hot encoding
train_labels, test_labels = to_categorical(train_labels), to_categorical(
test_labels)
return (train_images, train_labels, test_images, test_labels)
def __init__(self,
dims=(120, 120, 64),
batch_size=16,
shuffle=True,
validation=False,
split=0.2,
extend_dims=True,
augment_data=True):
self.dims = dims
self.batch_size = batch_size
self.extend_dims = extend_dims
# Get MNIST files, Split based on validation
if validation:
files = mnist.test_images()
labels = mnist.test_labels()
else:
files = mnist.train_images()
labels = mnist.train_labels()
# Take into account shuffling
if shuffle:
tmp = list(zip(files, labels))
random.shuffle(tmp)
files, labels = zip(*tmp)
labels = np.array(labels)
self.files = files
self.labels = labels
def load_preprocessed_mnist(use_torch=False, flatten_images=False):
"""
Return mnist dataset with images normalized to [0, 1], scaled to (16, 16) and deskewed.
Args:
use_torch (bool, optional): If True, return torch tensors, otherwise numpy arrays
(default: False).
flatten_images (bool, optional): Flatten train and test images (default: False).
Returns:
Train images, train labels, test images, test labels as numpy arrays
(or torch tensors, see parameter use_torch).
"""
train_images = mnist.train_images() / 255
train_images = preprocess(train_images, (16, 16), unskew=True)
train_images = train_images.astype('float32')
test_images = mnist.test_images() / 255
test_images = preprocess(test_images, (16, 16), unskew=True)
test_images = test_images.astype('float32')
train_labels = mnist.train_labels().astype(
int) # original arrays are uint8
test_labels = mnist.test_labels().astype(int)
if flatten_images:
train_images = train_images.reshape(len(train_images), -1)
test_images = test_images.reshape(len(test_images), -1)
if use_torch:
return torch.from_numpy(train_images).float(), torch.from_numpy(train_labels), \
torch.from_numpy(test_images).float(), torch.from_numpy(test_labels)
else:
return train_images, train_labels, test_images, test_labels
def train():
global data_folder
global output_folder
#load data
train_images = mnist.train_images()
train_labels = mnist.train_labels()
test_images = mnist.test_images()
test_labels = mnist.test_labels()
# X, Y
X = train_images
Y = keras.utils.to_categorical(train_labels, num_classes=num_classes)
X_test = test_images
Y_test = keras.utils.to_categorical(test_labels, num_classes=num_classes)
print('Image data format', K.image_data_format())
if K.image_data_format() == 'channels_first':
X = X.reshape(X.shape[0], 1, img_rows, img_cols)
X_test = test_images.reshape(X_test.shape[0], 1, img_rows, img_cols)
input_shape = (1, img_rows, img_cols)
else:
X = X.reshape(X.shape[0], img_rows, img_cols, 1)
X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1)
input_shape = (img_rows, img_cols, 1)
trainer = ConvTrainer(input_shape)
# X, Y = trainer.prepare_data(X, Y)
print('X shape', X.shape)
print('Y shape', Y.shape)
model = trainer.create_model()
model.summary()
model.fit(X,
Y,
validation_data=(X_test, Y_test),
shuffle=True,
epochs=epochs,
batch_size=500)
print('* Done training')
# serialize model to JSON
print("* Saving model to disk..")
model_json = model.to_json()
with open(output_folder + "mnist_model.json", "w") as json_file:
json_file.write(model_json)
# serialize weights to HDF5
model.save_weights(output_folder + "mnist_model.h5")
print("* Saved model to disk")
#Compute accuracy scores
print("* Computing accuracy scores..")
score = model.evaluate(X, Y, verbose=0)
print("%s: %.2f%%" % (model.metrics_names[0], score[0] * 100))
print("%s: %.2f%%" % (model.metrics_names[1], score[1] * 100))
def __init__(self):
self.fullset = mnist.train_images() # 返回一个形状为[60000,28,28]的numpy数组,第一个索引代表图片，后两个指定28x28图片上的像素点
self.fullset = self.fullset - numpy.mean(numpy.mean(self.fullset[:, :, :])) # 为张量中的每一个元素都减去平均值
self.testset = mnist.test_images() # 返回一个形状为[10000,28,28]的numpy数组,第一个索引代表图片，后两个指定28x28图片上的像素点
self.fullset_label = mnist.train_labels() # 返回一个形状为[60000]的numpy数组,代表对应图片的label
self.testset_label = mnist.test_labels() # 返回一个形状为[10000]的numpy数组,代表对应图片的label
self.trainset = self.fullset[0:50000, :, :] # 从60000张图片中选取前50000张做训练集
self.trainset_label = self.fullset_label[0:50000] # 从60000张图片中选取前50000个标签做训练集的label
self.validationset = self.fullset[50000:60000, :, :] # 从60000张图片中选取后10000个图片做验证集
self.validation_label = self.fullset[50000:60000] # 从60000张图片中选取后10000个标签做验证集的label
# 转tensor
self.testset = torch.from_numpy(self.testset)
self.testset_label = torch.from_numpy(self.testset_label.astype(numpy.int64))
self.trainset = torch.from_numpy(self.trainset)
self.trainset_label = torch.from_numpy(self.trainset_label.astype(numpy.int64))
# 训练参数
self.learningRate = 1e-2
self.learningRateDecay = 1e-4
self.weightDecay = 1e-3
self.momentum = 1e-4
self.batch_size = 200
# 先转换成 torch 能识别的 TensorDataset
self.torch_trainset = torch.utils.data.TensorDataset(self.trainset, self.trainset_label)
# 把 TensorDataset 放入 DataLoader
self.train_loader = torch.utils.data.DataLoader(
dataset=self.torch_trainset,
batch_size=self.batch_size,
shuffle=True, # 打乱数据
num_workers=2, # 多线程来读数据
)
# 先转换成 torch 能识别的 TensorDataset
self.torch_testset = torch.utils.data.TensorDataset(self.testset, self.testset_label)
# 把 TensorDataset 放入 DataLoader
self.test_loader = torch.utils.data.DataLoader(
dataset=self.torch_testset,
batch_size=self.batch_size,
shuffle=True, # 打乱数据
num_workers=2, # 多线程来读数据
)
# 定义代价函数，使用交叉熵验证
self.criterion = nn.CrossEntropyLoss(reduction='sum')
# 下面开始搭建神经网络模型
self.model = nn.Sequential( # 创建一个时序容器
nn.Conv2d(1, 16, 5, 1, 2),
nn.ReLU(),
nn.MaxPool2d(kernel_size=2),
nn.Conv2d(16, 32, 5, 1, 2),
nn.ReLU(),
nn.MaxPool2d(2),
Reshape(-1, 32 * 7 * 7),
nn.Linear(32 * 7 * 7, 10)
# nn.Softmax(dim=1)
)
# 使用GPU计算
self.model = self.model.cuda()
# 直接定义优化器，而不是调用backward
self.optimizer = torch.optim.Adam(self.model.parameters(), lr=0.0001)
# 调用参数初始化方法初始化网络参数
self.model.apply(self.weight_init)
def load_data():
train_images = mnist.train_images()
train_labels = mnist.train_labels()
test_images = mnist.test_images()
test_labels = mnist.test_labels()
return train_images, test_images, train_labels, test_labels
def load_data():
x_train = (mnist.train_images() / 255.0) - 0.5
y_train = onehot_encode(mnist.train_labels())
x_test = (mnist.test_images()[:1000] / 255.0) - 0.5
y_test = onehot_encode(mnist.test_labels()[:1000])
return (x_train, y_train), (x_test, y_test)
def load_test(padding=((0, 0), (0, 0), (3,
0))) -> Tuple[np.ndarray, np.ndarray]:
"""
Returns testing data, X, y.
"""
test_images = mnist.test_images()
return np.pad(test_images, padding), mnist.test_labels()
def load_mnist():
train_images = mnist.train_images()
train_labels = mnist.train_labels()
test_images = mnist.test_images()
test_labels = mnist.test_labels()
return ((train_images, train_labels), (test_images, test_labels))
def download():
print("================= Downloading the data ===============")
# Download the train images and train label
train_images = mnist.train_images()
train_labels = np.asarray(mnist.train_labels(), np.int32)
test_imgaes = mnist.test_images()
test_labels = np.asarray(mnist.test_labels(), np.int32)
return train_images, train_labels, test_imgaes, test_labels
def mnist_784_test():
'''
Converts 28x28 mnist digits to [16x16]
[samples x pixels] ([N X 256])
'''
import mnist
z = (mnist.test_images() / 255)
z = z.reshape(-1, (784))
return z, mnist.test_labels()
def main():
# XOR revisited
# training data
xs = [[0., 0], [0., 1], [1., 0], [1., 1]]
ys = [[0.], [1.], [1.], [0.]]
random.seed(0)
net = Sequential([
Linear(input_dim=2, output_dim=2),
Sigmoid(),
Linear(input_dim=2, output_dim=1)
])
import tqdm
optimizer = GradientDescent(learning_rate=0.1)
loss = SSE()
with tqdm.trange(3000) as t:
for epoch in t:
epoch_loss = 0.0
for x, y in zip(xs, ys):
predicted = net.forward(x)
epoch_loss += loss.loss(predicted, y)
gradient = loss.gradient(predicted, y)
net.backward(gradient)
optimizer.step(net)
t.set_description(f"xor loss {epoch_loss:.3f}")
for param in net.params():
print(param)
# FizzBuzz Revisited
from scratch.neural_networks import binary_encode, fizz_buzz_encode, argmax
xs = [binary_encode(n) for n in range(101, 1024)]
ys = [fizz_buzz_encode(n) for n in range(101, 1024)]
NUM_HIDDEN = 25
random.seed(0)
net = Sequential([
Linear(input_dim=10, output_dim=NUM_HIDDEN, init='uniform'),
Tanh(),
Linear(input_dim=NUM_HIDDEN, output_dim=4, init='uniform'),
Sigmoid()
])
def fizzbuzz_accuracy(low: int, hi: int, net: Layer) -> float:
num_correct = 0
for n in range(low, hi):
x = binary_encode(n)
predicted = argmax(net.forward(x))
actual = argmax(fizz_buzz_encode(n))
if predicted == actual:
num_correct += 1
return num_correct / (hi - low)
optimizer = Momentum(learning_rate=0.1, momentum=0.9)
loss = SSE()
with tqdm.trange(1000) as t:
for epoch in t:
epoch_loss = 0.0
for x, y in zip(xs, ys):
predicted = net.forward(x)
epoch_loss += loss.loss(predicted, y)
gradient = loss.gradient(predicted, y)
net.backward(gradient)
optimizer.step(net)
accuracy = fizzbuzz_accuracy(101, 1024, net)
t.set_description(f"fb loss: {epoch_loss:.2f} acc: {accuracy:.2f}")
# Now check results on the test set
print("test results", fizzbuzz_accuracy(1, 101, net))
random.seed(0)
net = Sequential([
Linear(input_dim=10, output_dim=NUM_HIDDEN, init='uniform'),
Tanh(),
Linear(input_dim=NUM_HIDDEN, output_dim=4, init='uniform')
# No final sigmoid layer now
])
optimizer = Momentum(learning_rate=0.1, momentum=0.9)
loss = SoftmaxCrossEntropy()
with tqdm.trange(100) as t:
for epoch in t:
epoch_loss = 0.0
for x, y in zip(xs, ys):
predicted = net.forward(x)
epoch_loss += loss.loss(predicted, y)
gradient = loss.gradient(predicted, y)
net.backward(gradient)
optimizer.step(net)
accuracy = fizzbuzz_accuracy(101, 1024, net)
t.set_description(f"fb loss: {epoch_loss:.3f} acc: {accuracy:.2f}")
# Again check results on the test set
print("test results", fizzbuzz_accuracy(1, 101, net))
# Load the MNIST data
import mnist
# This will download the data, change this to where you want it.
# (Yes, it's a 0-argument function, that's what the library expects.)
# (Yes, I'm assigning a lambda to a variable, like I said never to do.)
mnist.temporary_dir = lambda: '/tmp'
# Each of these functions first downloads the data and returns a numpy array.
# We call .tolist() because our "tensors" are just lists.
train_images = mnist.train_images().tolist()
train_labels = mnist.train_labels().tolist()
assert shape(train_images) == [60000, 28, 28]
assert shape(train_labels) == [60000]
import matplotlib.pyplot as plt
fig, ax = plt.subplots(10, 10)
for i in range(10):
for j in range(10):
# Plot each image in black and white and hide the axes.
ax[i][j].imshow(train_images[10 * i + j], cmap='Greys')
ax[i][j].xaxis.set_visible(False)
ax[i][j].yaxis.set_visible(False)
# plt.show()
# Load the MNIST test data
test_images = mnist.test_images().tolist()
test_labels = mnist.test_labels().tolist()
assert shape(test_images) == [10000, 28, 28]
assert shape(test_labels) == [10000]
# Recenter the images
# Compute the average pixel value
avg = tensor_sum(train_images) / 60000 / 28 / 28
# Recenter, rescale, and flatten
train_images = [[(pixel - avg) / 256 for row in image for pixel in row]
for image in train_images]
test_images = [[(pixel - avg) / 256 for row in image for pixel in row]
for image in test_images]
assert shape(train_images) == [60000, 784], "images should be flattened"
assert shape(test_images) == [10000, 784], "images should be flattened"
# After centering, average pixel should be very close to 0
assert -0.0001 < tensor_sum(train_images) < 0.0001
# One-hot encode the test data
train_labels = [one_hot_encode(label) for label in train_labels]
test_labels = [one_hot_encode(label) for label in test_labels]
assert shape(train_labels) == [60000, 10]
assert shape(test_labels) == [10000, 10]
# Training loop
import tqdm
def loop(model: Layer,
images: List[Tensor],
labels: List[Tensor],
loss: Loss,
optimizer: Optimizer = None) -> None:
correct = 0 # Track number of correct predictions.
total_loss = 0.0 # Track total loss.
with tqdm.trange(len(images)) as t:
for i in t:
predicted = model.forward(images[i]) # Predict.
if argmax(predicted) == argmax(labels[i]): # Check for
correct += 1 # correctness.
total_loss += loss.loss(predicted, labels[i]) # Compute loss.
# If we're training, backpropagate gradient and update weights.
if optimizer is not None:
gradient = loss.gradient(predicted, labels[i])
model.backward(gradient)
optimizer.step(model)
# And update our metrics in the progress bar.
avg_loss = total_loss / (i + 1)
acc = correct / (i + 1)
t.set_description(f"mnist loss: {avg_loss:.3f} acc: {acc:.3f}")
# The logistic regression model for MNIST
random.seed(0)
# Logistic regression is just a linear layer followed by softmax
model = Linear(784, 10)
loss = SoftmaxCrossEntropy()
# This optimizer seems to work
optimizer = Momentum(learning_rate=0.01, momentum=0.99)
# Train on the training data
loop(model, train_images, train_labels, loss, optimizer)
# Test on the test data (no optimizer means just evaluate)
loop(model, test_images, test_labels, loss)
# A deep neural network for MNIST
random.seed(0)
# Name them so we can turn train on and off
dropout1 = Dropout(0.1)
dropout2 = Dropout(0.1)
model = Sequential([
Linear(784, 30), # Hidden layer 1: size 30
dropout1,
Tanh(),
Linear(30, 10), # Hidden layer 2: size 10
dropout2,
Tanh(),
Linear(10, 10) # Output layer: size 10
])
# Training the deep model for MNIST
optimizer = Momentum(learning_rate=0.01, momentum=0.99)
loss = SoftmaxCrossEntropy()
# Enable dropout and train (takes > 20 minutes on my laptop!)
dropout1.train = dropout2.train = True
loop(model, train_images, train_labels, loss, optimizer)
# Disable dropout and evaluate
dropout1.train = dropout2.train = False
loop(model, test_images, test_labels, loss)
