def reader(): if filename == 'train': dataset = mnist_data.fetch_traingset() else: dataset = mnist_data.fetch_testingset() for i in range(n): data = np.array(dataset['images'][i]) data = np.reshape(data, (28, 28)) yield data, dataset['labels'][i]
def process(settings, filename): if filename == 'train': dataset = mnist.fetch_traingset() else: dataset = mnist.fetch_testingset() train_images = dataset['images'] train_labels = dataset['labels'] num_images = len(train_images) for i in range(num_images): yield {'image': train_images[i], 'label': int(train_labels[i])}
def load_data(num_train=50000, num_test=10000, num_val=10000, num_dev=5000): """ - X (N, D) - y (N, ) """ X_train = fetch_traingset()['images'][0: num_train] X_train = np.array(X_train).reshape(num_train, -1) y_train = fetch_traingset()['labels'][0:num_train] y_train = np.array(y_train) X_val = fetch_traingset()['images'][num_train: num_train+num_val] X_val = np.array(X_val).reshape(num_val, -1) y_val = fetch_traingset()['labels'][num_train: num_train+num_val] y_val = np.array(y_val) X_test = fetch_testingset()['images'][0: num_test] X_test = np.array(X_test).reshape(num_test, -1) y_test = fetch_testingset()['labels'][0:num_test] y_test = np.array(y_test) X_dev = fetch_traingset()['images'][0:num_dev] X_dev = np.array(X_dev).reshape(num_dev, -1) y_dev = fetch_traingset()['labels'][0:num_dev] y_dev = np.array(y_dev) #print(X_train.shape, X_test.shape, y_train.shape, y_test.shape) # data normalization mean_img = np.mean(X_train, axis=0) #std_img = np.std(X_train, axis=0) X_train -= mean_img X_test -= mean_img X_val -= mean_img X_dev -= mean_img return X_train, y_train, X_test, y_test, X_val, y_val, X_dev, y_dev
import mnist_data #plt.rcParams['font.sas-serig']=['SimHei'] plt.rcParams['axes.unicode_minus'] = False xaaa = [] yaaa = [] #print(range(1,60000,1000)) for m in range(1, 6000, 100): print(m) xaaa.append(m) # import some data to play with dataset = mnist_data.fetch_traingset() # split train and test X_train, y_train = dataset['images'][:], dataset['labels'][:] # shuffle X_train = np.array(X_train) y_train = np.array(y_train) X_train = X_train.reshape(60000, 784) y_train = y_train.reshape(60000, 1) idx = np.arange(X_train.shape[0]) np.random.seed(0) np.random.shuffle(idx) X_train = X_train[idx] y_train = y_train[idx]
syn0 += train_list.T.dot(l1_delta) * 0.000045 #update loss_1, train_acc_1, test_acc_1 = mnist_calculate.calLoss( image_train[:, :-1], image_train[:, -1:], image_test, label_test, syn0, syn1, syn2, syn3, 60000) loss.append(loss_1) train_acc.append(train_acc_1) test_acc.append(test_acc_1) return syn0, syn1, syn2, syn3, loss, train_acc, test_acc #read minist data data_train = {} data_test = {} data_test = mnist_data.fetch_testingset() data_train = mnist_data.fetch_traingset() image_train = np.array(data_train['images']) label_train = np.array(data_train['labels']).reshape(60000, 1) image_test = data_test['images'] label_test = np.array(data_test['labels']).reshape(10000, 1) loss = [] train_acc = [] image_train = np.append(image_train, label_train, axis=1) # start to train syn0, syn1, syn2, syn3, loss, train_acc, test_acc = train_onsigmoid( image_train, image_test, label_test) index = range(1, 1000) plt.plot(index, train_acc, label='train_accuracy')