def __init__(self, random_state=None, dtype=np.float32): import mnist super().__init__(random_state) self.dtype = dtype mnist_dir = MNIST_DIR if os.path.isdir(MNIST_DIR) else None self.train_images = mnist.download_and_parse_mnist_file( 'train-images-idx3-ubyte.gz', target_dir=mnist_dir) self.test_images = mnist.download_and_parse_mnist_file( 't10k-images-idx3-ubyte.gz', target_dir=mnist_dir) self.train_images = self.train_images / 255.0 self.test_images = self.train_images / 255.0
def load_dataset() -> (np.array, np.array, np.array, np.array, np.ndarray): """ """ path = 'dataset/Mnist_Fashion' x_train_file, y_train_file = 'train-images-idx3-ubyte', 'train-labels-idx1-ubyte' x_test, y_test = 't10k-images-idx3-ubyte', 't10k-labels-idx1-ubyte' data_names = [x_train_file, y_train_file, x_test, y_test] Data = [ download_and_parse_mnist_file(fname, target_dir=path) for fname in data_names ] x_train, y_train, x_test, y_test = Data[0], Data[1], Data[2], Data[3] # Add a new dimension to images : # it hsould be 28,28,1 not 28 ,28 x_train = np.expand_dims(x_train, axis=-1) y_train = np.expand_dims(y_train, axis=-1) x_test = np.expand_dims(x_test, axis=-1) y_test = np.expand_dims(y_test, axis=-1) avilable_classes = [ 'T-shirt', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle', 'boot' ] return x_train, y_train, x_test, y_test, avilable_classes
def download(data): # Download data y = mnist.download_and_parse_mnist_file(data) return y
#!/usr/bin/env python # coding: utf-8 # In[1]: import numpy as np import mnist import matplotlib.pyplot as plt from pylab import cm X = mnist.download_and_parse_mnist_file( "train-images-idx3-ubyte.gz") #教師用画像データ 60000×28×28 Y = mnist.download_and_parse_mnist_file( "train-labels-idx1-ubyte.gz") #対応する数値データ 60000 # In[2]: #定数 SIZE = 28 d = SIZE * SIZE #画素数 M = 100 #中間層のノード数 C = 10 #クラス数 B = 100 #バッチサイズ RATE = 0.01 #学習率η # In[3]: #各画素の値(0~255)を0~1に正規化 def Normalize(data): N = np.where(data < 128, 0, 1) return N
import numpy as np from keras.models import Model, Sequential from keras.layers import Dense, Activation from sklearn.preprocessing import OneHotEncoder from sklearn.preprocessing import StandardScaler import mnist train_images = mnist.download_and_parse_mnist_file( fname="train-images-idx3-ubyte.gz") train_images = np.reshape(train_images, (len(train_images), 784)) standardScaler = StandardScaler() train_images = standardScaler.fit_transform(train_images) test_images = mnist.download_and_parse_mnist_file( fname="t10k-images-idx3-ubyte.gz") test_images = np.reshape(test_images, (len(test_images), 784)) test_images = standardScaler.transform(test_images) train_labels = mnist.download_and_parse_mnist_file( fname="train-labels-idx1-ubyte.gz") train_labels = np.reshape(train_labels, (-1, 1)) test_labels = mnist.download_and_parse_mnist_file( fname="t10k-labels-idx1-ubyte.gz") ohe = OneHotEncoder() train_labels = ohe.fit_transform(train_labels).toarray() model = Sequential() model.add(Dense(units=32, input_shape=(784, ), activation="relu")) model.add(Dense(units=10, activation="softmax"))
import numpy as np import mnist X_train = mnist.download_and_parse_mnist_file("train-images-idx3-ubyte.gz") Y_train = mnist.download_and_parse_mnist_file("train-labels-idx1-ubyte.gz") X_test = mnist.download_and_parse_mnist_file("t10k-images-idx3-ubyte.gz") Y_test = mnist.download_and_parse_mnist_file("t10k-labels-idx1-ubyte.gz") def shuffle_batches(X,Y,batch_size): data_num = X.shape[0] mini_batches = [] index = np.arange(data_num) np.random.shuffle(index) shuffled_X = X[index] shuffled_Y = Y[index] num_minibatches = int(data_num / batch_size) for k in range (0,num_minibatches): mini_batch_X = shuffled_X[(k * batch_size): ((k+1) * batch_size)] mini_batch_Y = shuffled_Y[(k * batch_size): ((k+1) * batch_size)] mini_batch = (mini_batch_X, mini_batch_Y) #((100,784),(100,10)) mini_batches.append(mini_batch) return mini_batches # b = shuffle_batches(X_test,Y_test,100) # print(b[0][0].reshape(100,1,28,28).shape)
import numpy as np import matplotlib.pyplot as plt import mnist train_X = mnist.download_and_parse_mnist_file("train-images-idx3-ubyte.gz") # print(type(np.array(train_X[0]))) # print(train_X.shape) train_Y = mnist.download_and_parse_mnist_file("train-labels-idx1-ubyte.gz") test_X = mnist.download_and_parse_mnist_file("t10k-images-idx3-ubyte.gz") test_Y = mnist.download_and_parse_mnist_file("t10k-labels-idx1-ubyte.gz") # print(len(test_X)) # from pylab import cm # idx = 105 # plt.imshow(train_X[idx], cmap=cm.gray) # plt.show() # print (train_Y[idx]) # print(np.array(train_X[0]).reshape(1,784)) class params: def __init__(self, M, d): np.random.seed(seed=32) self.W = np.random.normal(0, 1/d, (d, M)) self.b = np.random.normal(0, 1/d, (1, M)) self.eta = 0.01 def update(self, dW, db): self.W -= self.eta * dW self.b -= self.eta * db
import numpy as np import mnist import matplotlib.pylab as plt from network3 import Network3 from pylab import cm X = mnist.download_and_parse_mnist_file("t10k-images-idx3-ubyte.gz") Y = mnist.download_and_parse_mnist_file("t10k-labels-idx1-ubyte.gz") # 入力獲得 flag = True while flag: i = input("Enter a number between 0 and 9999: ") if i.isdigit(): i = (int)(i) if i >= 0 and i <= 9999: flag = False else: print("Invalid Input!") else: print("Invalid Input!") # ネットワークサイズ指定 l0 = X.shape[1] * X.shape[2] l1 = 50 #中間層ノード数 l2 = len(np.unique(Y)) sizes = np.array([l0, l1, l2]) # ネットワーク構築 net = Network3(sizes, batch_size=100,
import numpy as np import mnist from Lena import Activation, Dense, Dropout, Model, Convolution2d, MaxPooling, AveragePooling, Input, EarlyStopping, ModelCheckpointMgr x_test = mnist.download_and_parse_mnist_file('t10k-images-idx3-ubyte.gz', 'data/') y_test = mnist.download_and_parse_mnist_file('t10k-labels-idx1-ubyte.gz', 'data/') x_test = x_test[:, np.newaxis, :, :] / 255 model = Model() model.add(Input((1, 28, 28))) model.add(Convolution2d([10, 5, 5], padding='SAME')) model.add(Activation('sigmoid')) model.add(MaxPooling((2, 2))) model.add(Convolution2d([15, 5, 5], padding='SAME')) model.add(Activation('sigmoid')) model.add(MaxPooling((2, 2))) model.add(Convolution2d([20, 6, 6], padding='VALID')) model.add(Activation('relu')) model.add(MaxPooling((2, 2))) model.add(Dense(20)) model.add(Activation('sigmoid')) model.add(Dense(10)) model.add(Activation('softmax')) model.comp('gradient_descent', 'binary_crossentropy') chpoint_mgr = ModelCheckpointMgr(model) chpoint_mgr.LoadWeights('ConvolutionNN002006.txt')
formated_test_images = _test_images.reshape(len(_test_images), 1, 28, 28) y_hat, loss = first_net.predict(formated_test_images, one_hot_test_labels) y_hat = np.argmax(y_hat, axis=0) test_accuracy2 = accuracy_score(y_hat, _test_labels) test_accuracy_list.append(test_accuracy2) loss_list.append(loss) print(f' test精度 in test : {np.mean(np.asarray(test_accuracy_list))} , test平均loss : {np.mean(np.asarray(loss_list))}') if __name__ == '__main__': mnist_root = '../data/' # train_images = mnist.download_and_parse_mnist_file(fname='train-images-idx3-ubyte.gz', target_dir=mnist_root) # train_images = (train_images / 255) - 0.5 # train_labels = mnist.download_and_parse_mnist_file(fname='train-labels-idx1-ubyte.gz', target_dir=mnist_root) test_images = mnist.download_and_parse_mnist_file(fname='t10k-images-idx3-ubyte.gz', target_dir=mnist_root) # test_images = (test_images / 255) - 0.5 test_labels = mnist.download_and_parse_mnist_file(fname='t10k-labels-idx1-ubyte.gz', target_dir=mnist_root) # train_images = train_images[:3000] # train_labels = train_labels[:3000] test_images = test_images[3000:6000] test_labels = test_labels[3000:6000] # train_images1 = np.rot90(train_images, 1, (1, 2)) # # train_images2 = np.rot90(train_images, 3, (1, 2)) # train_images = np.concatenate((train_images, train_images1), axis=0) # # train_images = np.concatenate((train_images, train_images2), axis=0) # train_labels = np.concatenate((train_labels, train_labels), axis=0)