def reader():
if filename == 'train':
dataset = mnist_data.fetch_traingset()
else:
dataset = mnist_data.fetch_testingset()
for i in range(n):
data = np.array(dataset['images'][i])
data = np.reshape(data, (28, 28))
yield data, dataset['labels'][i]
def process(settings, filename):
if filename == 'train':
dataset = mnist.fetch_traingset()
else:
dataset = mnist.fetch_testingset()
train_images = dataset['images']
train_labels = dataset['labels']
num_images = len(train_images)
for i in range(num_images):
yield {'image': train_images[i], 'label': int(train_labels[i])}
def load_data(num_train=50000, num_test=10000, num_val=10000, num_dev=5000):
"""
- X (N, D)
- y (N, )
"""
X_train = fetch_traingset()['images'][0: num_train]
X_train = np.array(X_train).reshape(num_train, -1)
y_train = fetch_traingset()['labels'][0:num_train]
y_train = np.array(y_train)
X_val = fetch_traingset()['images'][num_train: num_train+num_val]
X_val = np.array(X_val).reshape(num_val, -1)
y_val = fetch_traingset()['labels'][num_train: num_train+num_val]
y_val = np.array(y_val)
X_test = fetch_testingset()['images'][0: num_test]
X_test = np.array(X_test).reshape(num_test, -1)
y_test = fetch_testingset()['labels'][0:num_test]
y_test = np.array(y_test)
X_dev = fetch_traingset()['images'][0:num_dev]
X_dev = np.array(X_dev).reshape(num_dev, -1)
y_dev = fetch_traingset()['labels'][0:num_dev]
y_dev = np.array(y_dev)
#print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)
# data normalization
mean_img = np.mean(X_train, axis=0)
#std_img = np.std(X_train, axis=0)
X_train -= mean_img
X_test -= mean_img
X_val -= mean_img
X_dev -= mean_img
return X_train, y_train, X_test, y_test, X_val, y_val, X_dev, y_dev
import mnist_data
#plt.rcParams['font.sas-serig']=['SimHei']
plt.rcParams['axes.unicode_minus'] = False
xaaa = []
yaaa = []
#print(range(1,60000,1000))
for m in range(1, 6000, 100):
print(m)
xaaa.append(m)
# import some data to play with
dataset = mnist_data.fetch_traingset()
# split train and test
X_train, y_train = dataset['images'][:], dataset['labels'][:]
# shuffle
X_train = np.array(X_train)
y_train = np.array(y_train)
X_train = X_train.reshape(60000, 784)
y_train = y_train.reshape(60000, 1)
idx = np.arange(X_train.shape[0])
np.random.seed(0)
np.random.shuffle(idx)
X_train = X_train[idx]
y_train = y_train[idx]
syn0 += train_list.T.dot(l1_delta) * 0.000045 #update
loss_1, train_acc_1, test_acc_1 = mnist_calculate.calLoss(
image_train[:, :-1], image_train[:, -1:], image_test, label_test,
syn0, syn1, syn2, syn3, 60000)
loss.append(loss_1)
train_acc.append(train_acc_1)
test_acc.append(test_acc_1)
return syn0, syn1, syn2, syn3, loss, train_acc, test_acc
#read minist data
data_train = {}
data_test = {}
data_test = mnist_data.fetch_testingset()
data_train = mnist_data.fetch_traingset()
image_train = np.array(data_train['images'])
label_train = np.array(data_train['labels']).reshape(60000, 1)
image_test = data_test['images']
label_test = np.array(data_test['labels']).reshape(10000, 1)
loss = []
train_acc = []
image_train = np.append(image_train, label_train, axis=1)
# start to train
syn0, syn1, syn2, syn3, loss, train_acc, test_acc = train_onsigmoid(
image_train, image_test, label_test)
index = range(1, 1000)
plt.plot(index, train_acc, label='train_accuracy')