def main():
#test_data_format = np.zeros((100, 10))
#print(test_data_format.shape)
#training_data, training_labels = expecatation_propagation.getRandomTrainingData(test_data_format)
training_data, training_labels = mnist.load_mnist("training")
print("training_data")
print(training_data.shape)
print("training_labels")
print(training_labels.shape)
#test_data = expecatation_propagation.getRandomTrainingData(test_data_format) # wrong, please provide meaningful test data
test_data, test_labels = mnist.load_mnist("training")
print("test_data_X")
print(test_data[0].shape)
print("test_data_y")
print(test_data[1].shape)
# learn parameters
K = kernel.compute(training_data, training_data, 1) # Calling this with the MNIST Data throws: ValueError: XA must be a 2-dimensional array.
print("kernel")
print(K.shape)
v, t = expecatation_propagation.EP_binary_classification(K, training_labels)
print("v, t")
print(v)
print(t)
probability = prediction.classify(v, t, training_data, K, training_labels, kernel.compute, test_data)
print(probability)
def getData(fraction=1, deskew=False, subtractMean=False, whiten=False):
assert 0 < fraction <= 1 , '0 < fraction <= 1 !'
maxIndex = lambda x, y: int(x*y)
trainSet, trainLabels = mnist.load_mnist('training', path=MNIST_PATH, selection=slice(0, maxIndex(fraction, 60000)))
testSet, testLabels = mnist.load_mnist('testing', path=MNIST_PATH, selection=slice(0, maxIndex(fraction, 10000)))
if deskew:
trainSet = deskewSet(trainSet)
testSet = deskewSet(testSet)
trainSet = reshapeDataset(trainSet)
testSet = reshapeDataset(testSet)
if subtractMean:
trainSet, mean = doSubtractMean(trainSet)
testSet = doSubtractMean(testSet, mean)
if whiten:
trainSet, filter = doWhiten(trainSet)
testSet = doWhiten(testSet, filter)
trainData = (trainSet, trainLabels)
testData = (testSet, testLabels)
return trainData, testData
def train_dbn(path, epochs, learning_rate, dims):
#train dbn on mnist
data = mnist.load_mnist('../../../mnist_test.csv')
data = rbm.insert_biases(data)
dims = [784, 100, 50, 10]
dbn_weights = generate_dbn_weights(dims)
dbn_weights = train_dbm(dbm_weights, data, learning_rate = learning_rate, max_epochs = 300)
def look_through_sevens(k):
images, labels = load_mnist(digits=[7], path='.')
# Displaying the mean image for digit 9.
indices = random.sample(range(len(images)), k)
for i in indices:
plt.imshow(images[i], cmap='gray')
plt.show()
def ParseTestSet(): X, Y = mnist.load_mnist(dataset = 'testing') X = X.reshape(10000, 28 * 28) Y = (Y.repeat(10, axis = 1) == np.array([[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]] * 10000)) Y = Y + np.zeros(Y.shape) X = X.astype(float) / 255 X = X.tolist() return X, Y
def loadData(dataset):
img, numbers = load_mnist(dataset=dataset, path='../OpenCl_DNN/demo/mnist-data/')
images = img.astype(np.float32).reshape(
(img.shape[0], img.shape[1] * img.shape[2])).transpose()
output = np.array([numbers == i for i in range(10)], dtype=np.float32)
return images, output, numbers
def loadData(dataset, N=100):
img, numbers = load_mnist(
dataset=dataset, path='mnist-data/', selection=slice(0, N))
images = img.astype(np.float32).reshape(
(img.shape[0], img.shape[1] * img.shape[2])).transpose()
output = np.array([numbers == i for i in range(10)], dtype=np.float32)
return images, output, numbers
def main():
#test_data_format = np.zeros((100, 10))
#print(test_data_format.shape)
#training_data, training_labels = expecatation_propagation.getRandomTrainingData(test_data_format)
training_data, training_labels = mnist.load_mnist("training", digits=[1,7])
#test_data = expecatation_propagation.getRandomTrainingData(test_data_format) # wrong, please provide meaningful test data
test_data, test_labels = mnist.load_mnist("testing", digits=[1,7])
# only 1 or -1 are allowed in the binary classifier
training_labels[training_labels == 7] = -1
test_labels[test_labels == 7] = -1
# reshape the data, to have an 2D array
training_data = training_data.reshape(training_data.shape[0], -1)
test_data = test_data.reshape(test_data.shape[0], -1)
# the data array is too big. We have to decrease the size
size = 10
training_data = training_data[:size,]
training_labels = training_labels[:size]
test_data = test_data[:size,]
test_labels = test_labels[:size]
print("training_data")
print(training_data.shape)
print("training_labels")
print(training_labels.shape)
print("test_data_X")
print(test_data[0].shape)
print("test_data_y")
print(test_data[1].shape)
# learn parameters
K = kernel.compute(training_data, training_data, 1)
print("kernel")
print(K.shape)
np.set_printoptions(threshold=np.nan)
print(training_labels)
v, t = expecatation_propagation.EP_binary_classification(K, training_labels)
print("v, t")
print(v, t)
probability = prediction.classify(v, t, training_data, K, training_labels, kernel.compute, test_data)
print(probability)
def compare_precisions_by_nb_of_components():
kmeans = kmeans_.load_kmeans('kmeans-20.dat')
train_data, train_labels = load_mnist(dataset='training', path=args.path)
train_data = np.reshape(train_data, (train_data.shape[0], 784))
test_data, test_labels = load_mnist(dataset='testing', path=args.path)
test_data = np.reshape(test_data, (test_data.shape[0], 784))
d = 40
reducer = sklearn.decomposition.PCA(n_components=d)
reducer.fit(train_data)
train_data_reduced = reducer.transform(train_data)
test_data_reduced = reducer.transform(test_data)
kmeans_reduced = reducer.transform(kmeans)
label_set = set(train_labels)
precisions = []
ks = list(range(1, 11)) + [15, 20, 30]
for k in ks:
print('learning {} components'.format(k))
model = classifier.classifier(k, covariance_type='full',
model_type='gmm',
means_init_heuristic='kmeans',
means=kmeans_reduced,
verbose=False)
model.fit(train_data_reduced, train_labels)
predicted_labels = model.predict(test_data_reduced, label_set)
expected_labels = test_labels
precision = np.mean(predicted_labels == expected_labels)
precisions.append((k, precision))
print('precision: {}'.format(precision))
print(precisions)
def dt_baseline():
training_data, training_labels = mnist.load_mnist('training', path=DIR_PATH, selection=slice(0, 60000))
test_data, test_labels = mnist.load_mnist('testing', path=DIR_PATH, selection=slice(0, 10000))
#mnist.show(training_data[0])
listTrainImg = []
for img in training_data:
trainImg = img.flatten()
listTrainImg.append(list(trainImg))
listTestImg = []
for img in test_data:
testImg = img.flatten()
listTestImg.append(list(testImg))
# Preprocessing
mean = np.mean(listTrainImg, axis=0)
print mean
listTrainImg = listTrainImg - np.tile(mean, [np.asarray(listTrainImg).shape[0], 1])
listTestImg = listTestImg - np.tile(mean, [np.asarray(listTestImg).shape[0], 1])
"""
param_grid = {
'criterion' : ['entropy', 'gini'],
'max_depth' : [10, 50, 70, 100],
'max_features' : [50, 100, 250, 500]
}
classifier = GridSearchCV(tree.DecisionTreeClassifier(), param_grid, cv=5, n_jobs=-1)
print classifier.fit(listTrainImg,training_labels)
print classifier.best_params_
"""
#classifier = tree.DecisionTreeClassifier()
classifier = tree.DecisionTreeClassifier(criterion='entropy', max_depth=50, max_features=500)
classifier.fit(listTrainImg,training_labels)
predictions = classifier.predict(listTestImg)
print score(predictions, test_labels)
# pydot Visualization
#pydotVisualization(classifier, filename="treePlot")
#pixel importance
pixelPlot(classifier, filename="treePixels")
def ParseTrainSet(n, m): X, Y = mnist.load_mnist() X = X[n:m] Y = Y[n:m] Y = (Y.repeat(10, axis = 1) == np.array([[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]] * (m - n))) Y = Y + np.zeros(Y.shape) X = X.reshape(m - n, 28 * 28) X = X.astype(float) / 255 X = X.tolist() return X, Y
def svm_baseline():
training_data, training_labels = mnist.load_mnist('training', path=DIR_PATH, selection=slice(0, 60000))
test_data, test_labels = mnist.load_mnist('testing', path=DIR_PATH, selection=slice(0, 10000))
listTrainImg = []
for img in training_data:
trainImg = img.flatten()
listTrainImg.append(list(trainImg))
listTestImg = []
for img in test_data:
testImg = img.flatten()
listTestImg.append(list(testImg))
# Preprocessing
print "preprocessing"
mean = np.mean(listTrainImg, axis=0)
listTrainImg = listTrainImg - np.tile(mean, [np.asarray(listTrainImg).shape[0], 1])
listTestImg = listTestImg - np.tile(mean, [np.asarray(listTestImg).shape[0], 1])
# train
# Default SVM
# classifier = svm.SVC()
# Parameter tuning using GridSearchCV
param_grid = [
{'C': [1, 10, 100, 1000], 'kernel': ['linear']},
{'C': [1, 10, 100, 1000], 'gamma': [0.001, 0.0001], 'kernel': ['rbf']},
{'C': [1, 10, 100, 1000], 'degree': [2, 3, 5], 'kernel': ['poly']},
]
#classifier = GridSearchCV(svm.SVC(cache_size=2000), param_grid, cv=5, n_jobs=-1)
# Polynomial Kernel
classifier = svm.SVC(kernel='poly', degree=2, C=1000)
classifier.fit(listTrainImg, training_labels)
#print classifier.best_params_
predictions = classifier.predict(listTestImg)
print predictions
print score(predictions, test_labels)
def testNetwork(net):
for i in range(0, 10):
num = i
test_ds = SupervisedDataSet(28 * 28, 1)
test_images, test_labels = mnist.load_mnist(dataset='testing', digits=[num])
for image, label in zip(test_images, test_labels):
test_ds.addSample(ravel(image), label)
p = net.activateOnDataset(test_ds)
res = np.round( p )
total = len(res)
count = sum(res == num)
print num,
print " ", count/float(total)*100
def generate_training_and_testing_sets():
training_set_1 = []
training_set_2 = []
training_set_3 = []
testing_set = []
for digit in xrange(10):
images, labels = load_mnist(digits=[digit], path='.')
training_indices = random.sample(range(len(images)), 1100)
testing_indices = random.sample(training_indices, 100)
training_indices = [x for x in training_indices if x not in testing_indices]
if digit in [0, 1, 2]:
training_set_1.extend([(images[i], labels[i]) for i in training_indices])
if digit in [3, 4, 5]:
training_set_2.extend([(images[i], labels[i]) for i in training_indices])
if digit in [6, 7, 8, 9]:
training_set_3.extend([(images[i], labels[i]) for i in training_indices])
testing_set.extend([(images[i], labels[i]) for i in testing_indices])
library.save_pickle(training_set_1, 'training_set_1.p')
library.save_pickle(training_set_2, 'training_set_2.p')
library.save_pickle(training_set_3, 'training_set_3.p')
library.save_pickle(testing_set, 'testing_set.p')
def select_data(training_set_size, testing_set_size, save_test_set=False):
"""
Choose examples for training and testing data, randomly, as described in 1C.
:param training_set_size: Total number of training examples
:param testing_set_size: Total number of testing examples
:param save_test_set: If True, pickle testing set
:return: training_set, training_labels, testing_set, testing_labels
"""
training_set = []
training_labels = []
testing_set = []
testing_labels = []
for digit in xrange(10):
# load digit:
images, labels = load_mnist(digits=[digit], path='.')
# choose random digits to add to training and testing sets:
if (training_set_size+testing_set_size)/10 <= len(images):
combined_sample_size = (training_set_size+testing_set_size)/10
testing_sample_size = testing_set_size/10
else:
combined_sample_size = len(images)
testing_sample_size = testing_set_size/10 * combined_sample_size / ((training_set_size+testing_set_size)/10)
training_indices = random.sample(range(len(images)), combined_sample_size)
testing_indices = random.sample(training_indices, testing_sample_size)
training_indices = [x for x in training_indices if x not in testing_indices]
# add to training set:
training_set.extend(images[i] for i in training_indices)
training_labels.extend(labels[i] for i in training_indices)
# add to testing set:
testing_set.extend([images[i] for i in testing_indices])
testing_labels.extend([labels[i] for i in testing_indices])
if save_test_set:
pickle.dump(testing_set, open('testing_set_2.p', 'w'))
pickle.dump(testing_labels, open('testing_labels_2.p', 'w'))
return training_set, training_labels, testing_set, testing_labels
import numpy as np
import mnist
import math
from PIL import Image
from numpy import array
import matplotlib.pyplot as plt
x1=784
y1=30
z1=10
theta1=pickle.load(open("theta1_new.npy","rb"))
theta2=pickle.load(open("theta2_new.npy","rb"))
images,labels = mnist.load_mnist('training')
a2=np.empty([y1+1,1],dtype='float64') #31x1
output=np.empty([z1,1],dtype='float64') #10x1
def g(z):
return (1.0/(1.0+math.exp(-z)))
def feedforward2(input1):
z2=np.dot(theta1,input1)
z2=z2/100.0
#print theta1
for i in range (1,(y1+1)):
a2[i] = g(z2[i-1])
a2[0]=1.0
z3=np.dot(theta2,a2)
for i in range (0,z1):
import autograd.numpy as np
from autograd import grad
from params import Params
import nn_base
import nn_types
import nn_helpers
import sys
import os
from tqdm import tqdm
import mnist
# Load mnist.
training_data, training_labels = mnist.load_mnist('training', path=os.getcwd())
validation_data, validation_labels = mnist.load_mnist('testing', path=os.getcwd())
DATA_SIZE = 784
NUM_CLASSES = 10
DROPOUT_RATE = 0.1
# Network construction.
network = nn_base.NNetwork()
network.set_input_shape((-1, DATA_SIZE))
# network.add_node(nn_types.ReshapeNode((-1, 1, 28, 28)))
# network.add_node(nn_types.DropoutNode(DROPOUT_RATE))
# network.add_node(nn_types.ConvolutionNode(8, 3))
# # TODO Normalize
# network.add_node(nn_types.ReluNode())
# network.add_node(nn_types.MaxPoolNode())
grads['b2'] = self.layers['A2'].db
grads['W3'] = self.layers['A3'].dW
grads['b3'] = self.layers['A3'].db
return grads
def train(self, batch, epoch, learning_rate):
for i in range(epoch):
batch_mask = np.random.choice(self.data.shape[0], batch)
x = self.data[batch_mask]
t = self.label[batch_mask]
grads = self.gradient(x, t)
for grad in grads:
self.params[grad] -= learning_rate * grads[grad]
train_acc = accuracy(self.predict(x), t)
print('epoch {} accuracy : {}'.format(i, train_acc))
def test(self, test_set, test_label):
predict = self.predict(test_set)
test_acc = accuracy(predict, test_label)
return test_acc
(x_train, y_train), (x_test, y_test) = load_mnist(flatten=True, normalize=True, one_hot_label=True)
model = MLP(data=x_train, label=y_train, activation='relu', weight_init='he')
model.train(batch=100, epoch=10000, learning_rate=0.0005)
test_acc = model.test(x_test, y_test)
print(test_acc)
def get_mnist_data():
# load_mnnist 를 한번 해두면 mnist.pkl 파일로 덤프해두어 이후 로딩시 빠르게 로딩된다.
(train_img, train_label), (test_img,
test_label) = load_mnist(flatten=True,
normalize=False)
return train_img, train_label, test_img, test_label
# coding: utf-8
import numpy as np
from mnist import load_mnist
from PIL import Image
def img_show(img):
pil_img = Image.fromarray(np.uint8(img))
pil_img.show()
(x_train, t_train), (x_test, t_test) = load_mnist(flatten=True,
normalize=False)
img = x_train[0]
label = t_train[0]
print(label) # 5
print(img.shape) # (784,)
img = img.reshape(28, 28) # 형상을 원래 이미지의 크기로 변형
print(img.shape) # (28, 28)
img_show(img)
def get_data():
(x_train, t_train), (x_test, t_test) = load_mnist(normalize=True,
flatten=True,
one_hot_label=False)
return x_test, t_test
from mnist import load_mnist
from train import train
from test import test
from model import net, lenet2
if __name__ == "__main__":
print("THIS IS JUST AN EXAMPLE.")
print("Please don't take these numbers as a benchmark.")
print("While the optimizer's parameters have been grid-searched,")
print("a fair comparison would run all experiments multiple times AND RUN MORE THAN FIVE EPOCHS.")
batch_size = 64
(Xtrain, ytrain), (Xval, yval), (Xtest, ytest) = load_mnist()
criterion = df.ClassNLLCriterion()
def run(optim):
progress = make_progressbar('Training with ' + str(optim), 5)
progress.start()
model = net()
model.training()
for epoch in range(5):
train(Xtrain, ytrain, model, optim, criterion, batch_size, 'train')
train(Xtrain, ytrain, model, optim, criterion, batch_size, 'stats')
progress.update(epoch+1)
progress.finish()
def _get_test_data(self):
(x_train, t_train), (x_test, t_test) = load_mnist(flatten=True,
normalize=False)
return x_test, t_test
def get_data(normalize=True, flatten=True, one_hot_label=True):
(x_train, t_train), (x_test,
t_test) = load_mnist(normalize=normalize,
flatten=flatten,
one_hot_label=one_hot_label)
return x_train, t_train, x_test, t_test
# -*- coding:utf-8 -*-
# Author: Raina
# Date: 2020/06/05
import numpy as np
import matplotlib.pyplot as plt
from mnist import load_mnist
from optimizer import SGD, Momentum, AdaGrad, Adam
from multi_layer_net import MultiLayerNet
from utils import smooth_curve
if __name__ == "__main__":
# 读取 mnist 数据
(train_img, train_label), (test_img,
test_label) = load_mnist(normalize=True)
train_size = train_img.shape[0]
batch_size = 128
iters_times = 2000
# 初始化
optimizers = {}
optimizers['SGD'] = SGD()
optimizers['Momentum'] = Momentum()
optimizers['AdaGrad'] = AdaGrad()
optimizers['Adam'] = Adam()
networks = {}
train_loss = {}
for key in optimizers.keys():
networks[key] = MultiLayerNet(input_size=784,
def main(data_name, method, dimZ, dimH, n_channel, batch_size, K_mc,
checkpoint, lbd):
# set up dataset specific stuff
from config import config
labels, n_iter, dimX, shape_high, ll = config(data_name, n_channel)
if data_name == 'mnist':
from mnist import load_mnist
if data_name == 'notmnist':
from notmnist import load_notmnist
# import functionalities
if method == 'onlinevi':
from bayesian_generator import generator_head, generator_shared, \
generator, construct_gen
from onlinevi import construct_optimizer, init_shared_prior, \
update_shared_prior, update_q_sigma
if method in ['ewc', 'noreg', 'laplace', 'si']:
from generator import generator_head, generator_shared, generator, construct_gen
if method in ['ewc', 'noreg']:
from vae_ewc import construct_optimizer, lowerbound
if method == 'ewc': from vae_ewc import update_ewc_loss, compute_fisher
if method == 'laplace':
from vae_laplace import construct_optimizer, lowerbound
from vae_laplace import update_laplace_loss, compute_fisher, init_fisher_accum
if method == 'si':
from vae_si import construct_optimizer, lowerbound, update_si_reg
# then define model
n_layers_shared = 2
batch_size_ph = tf.placeholder(tf.int32, shape=(), name='batch_size')
dec_shared = generator_shared(dimX, dimH, n_layers_shared, 'sigmoid',
'gen')
# initialise sessions
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
string = method
if method in ['ewc', 'laplace', 'si']:
string = string + '_lbd%.1f' % lbd
if method == 'onlinevi' and K_mc > 1:
string = string + '_K%d' % K_mc
path_name = data_name + '_%s/' % string
if not os.path.isdir('save/'):
os.mkdir('save/')
if not os.path.isdir('save/' + path_name):
os.mkdir('save/' + path_name)
print('create path save/' + path_name)
filename = 'save/' + path_name + 'checkpoint'
if checkpoint < 0:
print('training from scratch')
old_var_list = init_variables(sess)
else:
load_params(sess, filename, checkpoint)
checkpoint += 1
# visualise the samples
N_gen = 10**2
path = 'figs/' + path_name
if not os.path.isdir('figs/'):
os.mkdir('figs/')
if not os.path.isdir(path):
os.mkdir(path)
print('create path ' + path)
X_ph = tf.placeholder(tf.float32, shape=(batch_size, dimX), name='x_ph')
# now start fitting
N_task = len(labels)
gen_ops = []
X_valid_list = []
X_test_list = []
eval_func_list = []
result_list = []
if method == 'onlinevi':
shared_prior_params = init_shared_prior()
if method in ['ewc', 'noreg']:
ewc_loss = 0.0
if method == 'laplace':
F_accum = init_fisher_accum()
laplace_loss = 0.0
if method == 'si':
old_params_shared = None
si_reg = None
n_layers_head = 2
n_layers_enc = n_layers_shared + n_layers_head - 1
for task in range(1, N_task + 1):
# first load data
if data_name == 'mnist':
X_train, X_test, _, _ = load_mnist(digits=labels[task - 1],
conv=False)
if data_name == 'notmnist':
X_train, X_test, _, _ = load_notmnist(data_path,
digits=labels[task - 1],
conv=False)
N_train = int(X_train.shape[0] * 0.9)
X_valid_list.append(X_train[N_train:])
X_train = X_train[:N_train]
X_test_list.append(X_test)
# define the head net and the generator ops
dec = generator(
generator_head(dimZ, dimH, n_layers_head, 'gen_%d' % task),
dec_shared)
enc = encoder(dimX, dimH, dimZ, n_layers_enc, 'enc_%d' % task)
gen_ops.append(construct_gen(dec, dimZ, sampling=False)(N_gen))
print('construct eval function...')
eval_func_list.append(construct_eval_func(X_ph, enc, dec, ll, \
batch_size_ph, K = 100, sample_W = False))
# then construct loss func and fit func
print('construct fit function...')
if method == 'onlinevi':
fit = construct_optimizer(X_ph, enc, dec, ll, X_train.shape[0], batch_size_ph, \
shared_prior_params, task, K_mc)
if method in ['ewc', 'noreg']:
bound = lowerbound(X_ph, enc, dec, ll)
fit = construct_optimizer(X_ph, batch_size_ph, bound,
X_train.shape[0], ewc_loss)
if method == 'ewc':
fisher, var_list = compute_fisher(X_ph, batch_size_ph, bound,
X_train.shape[0])
if method == 'laplace':
bound = lowerbound(X_ph, enc, dec, ll)
fit = construct_optimizer(X_ph, batch_size_ph, bound,
X_train.shape[0], laplace_loss)
fisher, var_list = compute_fisher(X_ph, batch_size_ph, bound,
X_train.shape[0])
if method == 'si':
bound = lowerbound(X_ph, enc, dec, ll)
fit, shared_var_list = construct_optimizer(X_ph, batch_size_ph,
bound, X_train.shape[0],
si_reg,
old_params_shared, lbd)
if old_params_shared is None:
old_params_shared = sess.run(shared_var_list)
# initialise all the uninitialised stuff
old_var_list = init_variables(sess, old_var_list)
# start training for each task
if method == 'si':
new_params_shared, w_params_shared = fit(sess, X_train, n_iter, lr)
else:
fit(sess, X_train, n_iter, lr)
# plot samples
x_gen_list = sess.run(gen_ops, feed_dict={batch_size_ph: N_gen})
for i in range(len(x_gen_list)):
plot_images(x_gen_list[i], shape_high, path, \
data_name+'_gen_task%d_%d' % (task, i+1))
x_list = [x_gen_list[i][:1] for i in range(len(x_gen_list))]
x_list = np.concatenate(x_list, 0)
tmp = np.zeros([10, dimX])
tmp[:task] = x_list
if task == 1:
x_gen_all = tmp
else:
x_gen_all = np.concatenate([x_gen_all, tmp], 0)
# print test-ll on all tasks
tmp_list = []
for i in range(len(eval_func_list)):
print('task %d' % (i + 1))
test_ll = eval_func_list[i](sess, X_valid_list[i])
tmp_list.append(test_ll)
result_list.append(tmp_list)
# save param values
save_params(sess, filename, checkpoint)
checkpoint += 1
# update regularisers/priors
if method == 'ewc':
# update EWC loss
print('update ewc loss...')
X_batch = X_train[np.random.permutation(range(
X_train.shape[0]))[:batch_size]]
ewc_loss = update_ewc_loss(sess, ewc_loss, var_list, fisher, lbd,
X_batch)
if method == 'laplace':
# update EWC loss
print('update laplace loss...')
X_batch = X_train[np.random.permutation(range(
X_train.shape[0]))[:batch_size]]
laplace_loss, F_accum = update_laplace_loss(
sess, F_accum, var_list, fisher, lbd, X_batch)
if method == 'onlinevi':
# update prior
print('update prior...')
shared_prior_params = update_shared_prior(sess,
shared_prior_params)
# reset the variance of q
update_q_sigma(sess)
if method == 'si':
# update regularisers/priors
print('update SI big omega matrices...')
si_reg, _ = update_si_reg(sess, si_reg, new_params_shared, \
old_params_shared, w_params_shared)
old_params_shared = new_params_shared
plot_images(x_gen_all, shape_high, path, data_name + '_gen_all')
for i in range(len(result_list)):
print(result_list[i])
# save results
fname = 'results/' + data_name + '_%s.pkl' % string
import pickle
pickle.dump(result_list, open(fname, 'wb'))
print('test-ll results saved in', fname)
from pylab import *
from numpy import *
from mnist import load_mnist
sum = 0
for i in range(10):
images, labels = load_mnist('training', digits=[i])
num = 0
sum = sum + len(images)
print '%d - %d' % (i, len(images))
print '-------'
for img in images:
imsave('./all_images/%d_%d.png' % (i, num), img)
num = num + 1
print 'total=%d' % sum
# print labels
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import sys, os
import pickle
import numpy as np
import matplotlib.pyplot as plt
from active_func import *
from common_func import *
from es_net import *
from es_nn_layer import *
from mnist import load_mnist
if __name__ == '__main__':
(train_data, train_label), (test_data, test_label) = load_mnist()
scale = 0.01
image_size = 28 * 28
hidden_nodes = 100
output_nodes = 10
update_class = Momentum
# random init the dnn param
dnn = es_net()
scale = weight_init_scale(input_size=image_size, active_func='relu')
dnn_weight_arr = scale * np.random.randn(image_size, hidden_nodes)
dnn_bias_arr = np.zeros(hidden_nodes)
updater_obj = update_class(learning_rate=0.1)
layer_tmp = affine_layer(weight=dnn_weight_arr, bias=dnn_bias_arr, \
updater=updater_obj)
dnn.add_layer(layer_obj=layer_tmp)
pickle.dump(classifier, open('classifier_1.p', 'w'))
pickle.dump(training_set, open('training_set_1.p', 'w'))
pickle.dump(training_labels, open('training_labels_1.p', 'w'))
def classify(images, classifier):
#runs the classifier on a set of images.
return classifier.predict(images)
def error_measure(predicted, actual):
return np.count_nonzero(abs(predicted - actual))/float(len(predicted))
if __name__ == "__main__":
# Code for loading data
imagesData, labelsData = load_mnist(digits=range(0, 10), path='.')
print "1 Finish loading data"
# preprocessing
images = preprocess(imagesData)
labels = labelsData
print "2 Finish preprocessing"
# pick training and testing set
# YOU HAVE TO CHANGE THIS TO PICK DIFFERENT SET OF DATA
training_set = images[0:54000]
training_labels = labels[0:54000]
testing_set = images[-6000:]
testing_labels = labels[-6000:]
print "3 Finish training testing split"
#build_classifier is a function that takes in training data and outputs an sklearn classifier.
classifier = build_classifier(training_set, training_labels)
import sys,os
sys.path.append(os.pardir)
import numpy as np
from mnist import load_mnist
from TwoLayerNet import TwoLayerNet
#最後にbackpropagationを用いたニューラルネットワークの学習を行う
#データの読み込みとネットワークの構築
(X_train,t_train),(X_test,t_test) = load_mnist(normalize=True,one_hot_label=True)
network = TwoLayerNet(input_size=784,hidden_size=50,output_size=10)
n_iter = 10000
train_size = X_train.shape[0]
batch_size = 100
eta = 0.1
train_loss_list = []
train_acc_list = []
test_acc_list = []
iter_per_epoch = max(train_size,batch_size,1) #100
for i in range(n_iter):
batch_mask = np.random.choice(train_size,batch_size)
X_batch = X_train[batch_mask]
t_batch = t_train[batch_mask]
#backpropagationを用いた勾配の学習
grads = network.gradient(X_batch,t_batch)
import matplotlib.pyplot as plt
import numpy as np
from mnist import load_mnist
"""
###PROBLEM 1A
images, labels = load_mnist(digits=range(0,10), path = '.')
print "total number of images: " + str (len(images))
for i in range(0,10):
images, labels = load_mnist(digits=[i], path='.')
print str(i) + " number of images: " + str(len(images))
"""
###PROBLEM 1B
images, labels = load_mnist(digits=[0], path = '.')
for i in range(0,50):
plt.imshow(images[i], cmap = 'gray')
plt.show()
import numpy as np
# Define Sigmoid Function
def sig(x):
return 1 / (1 + np.exp(-x))
# Read Test Data
import pickle
from mnist import load_mnist
(x_train, label_train), (x_test, label_test) = load_mnist(normalize=False,flatten=True)
# Display Test Data
number = 0
for i in range(0, 28):
for j in range(0, 28):
print("{0:3d}".format(x_test[number][28*i+j]), end=' ')
print('')
print(label_test[number])
#print(x_test.shape)
'''
# Read Trained Wx and bx Data
with open("sample_weight.pkl", 'rb') as f:
network = pickle.load(f)
W1, W2, W3 = network['W1'].T, network['W2'].T, network['W3'].T
b1, b2, b3 = network['b1'], network['b2'], network['b3']
#print(W1)
#print(W1.shape)
#print(W2.shape)
#print(W3.shape)
This function takes as input
images as load_mnist returns (without the labels)
and returns the whitened data (an array
of the arrays that will DIRECTLY be passed
into the neural net)
"""
flattened_images = images.reshape(DATA_SIZE, images.shape[1] * images.shape[2])
zero_mean_images = flattened_images - mean(flattened_images, axis=0)
normalized_images = zero_mean_images / (std(zero_mean_images, axis=0) + 0.01)
cov = dot(normalized_images.T, normalized_images) / normalized_images.shape[0]
U,S,V = linalg.svd(cov)
rot_reduced = dot(normalized_images, U[:,:COMPONENTS])
white = rot_reduced / sqrt(S[:COMPONENTS] + 1e-5)
return white
images, labels = load_mnist('training', selection=slice(0,DATA_SIZE))
test_images, test_labels = load_mnist('testing', selection=slice(0,DATA_SIZE))
net = buildNetwork(COMPONENTS, 1000, 500, 100, 10, bias=True)
data = SupervisedDataSet(COMPONENTS, 10)
white = preprocess(images)
test_white = preprocess(test_images)
for whitened, label in zip(white, labels):
data.addSample(whitened, one_to_many(label))
vals = []
testing_vals = []
trainer = BackpropTrainer(net, data, learningrate=0.0001)
for epoch in range(200):
import sys
sys.path.append('../common/')
sys.path.append('../dataset/')
import numpy as np
import matplotlib.pyplot as plt
from mnist import load_mnist
from simple_convnet import SimpleConvNet
from optimizer import *
from trainer import Trainer
# データの読み込み
(x_train, t_train), (x_test, t_test) = load_mnist(flatten=False)
# 処理に時間のかかる場合はデータを削減
x_train, t_train = x_train[:5000], t_train[:5000]
x_test, t_test = x_test[:1000], t_test[:1000]
# ハイパーパラメータの設定
max_epochs = 20
batch_size = 50
model = SimpleConvNet(input_dim=(1, 28, 28),
conv_param={
'filter_num': 30,
'filter_size': 5,
'pad': 0,
'stride': 1
},
hidden_size=100,
output_size=10,
# coding: utf-8
import numpy as np
import matplotlib.pyplot as plt
from mnist import load_mnist
from util import smooth_curve
from multi_layer_net import MultiLayerNet
from optimizer import SGD
# 0. MNIST 데이터 읽기==========
(x_train, t_train), (x_test, t_test) = load_mnist(normalize=True)
train_size = x_train.shape[0]
batch_size = 128
max_iterations = 2000
# 1. 실험용 설정==========
weight_init_types = {'std=0.01': 0.01, 'Xavier': 'sigmoid', 'He': 'relu'}
optimizer = SGD(lr=0.01)
networks = {}
train_loss = {}
for key, weight_type in weight_init_types.items():
networks[key] = MultiLayerNet(input_size=784, hidden_size_list=[100, 100, 100, 100],
output_size=10, weight_init_std=weight_type)
train_loss[key] = []
# 2. 훈련 시작==========
for i in range(max_iterations):
# -*- coding: utf-8 -*-
import tensorflow as tf
import sys
# data mnist
train_size, test_size = 6000, 1000
from mnist import download_mnist, load_mnist, key_file
download_mnist()
X_train = load_mnist(key_file["train_img"])[8:train_size + 8, :]
X_test = load_mnist(key_file["test_img"], )[8:test_size + 8, :]
y_train = load_mnist(key_file["train_label"], 1)[:train_size, 0]
y_test = load_mnist(key_file["test_label"], 1)[:test_size, 0]
labels = tf.placeholder(tf.int64, [None])
y_ = tf.one_hot(labels, depth=10, dtype=tf.float32)
print(y_train[:10])
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
print(sess.run(y_, feed_dict={labels: y_train[:10]}))
def gradient(f, x):
h = 0.0001
grad = np.zeros_like(x)
for idx in range(x.size):
tmp = x[idx]
fxph = f(tmp + h)
fxmh = f(tmp - h)
grad[idx] = (fxph - fxmh) / (2 * h)
return grad
# mnist 이미지 가져오기
(x_train, t_train), (x_test, t_test) = \
load_mnist(normalize=True, one_hot_label=True)
# 데이터 차원 살펴보기
# print(x_train.shape)
# print(x_test.shape)
# print(t_train.shape)
# print(t_test.shape)
# 데이터 하나 확인해보기
# I = x_train[0]
# I = I.reshape(28, 28)
# imshow(I)
train_size = x_train.shape[0]
import sys, os
sys.path.append(os.pardir)
import numpy as np
from mnist import load_mnist
from PIL import Image
def img_show(img):
pil_img = Image.fromarray(np.uint8(img))
pil_img.show()
(x_train, t_train),(x_test, t_test)=\
load_mnist(flatten=True, normalize=False)
img = x_train[0]
label = t_train[0]
print(label)
print('____________')
print(img.shape)
img = img.reshape(28,28)
print(img.shape)
print('____________')
img_show(img)
def get_data():
(x_train, t_train), (x_test, t_test) = load_mnist(flatten = True, normalize = True)
return x_test, t_test
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import sys, os
import pickle
import numpy as np
import matplotlib.pyplot as plt
from active_func import *
from common_func import *
from es_net import *
from es_nn_layer import *
from mnist import load_mnist
if __name__=='__main__':
(train_data, train_label), (test_data, test_label) = load_mnist(flatten=False)
weight_init_std=0.01
image_size = 28 * 28
hidden_nodes = 100
output_nodes = 10
input_dim = (1, 28, 28)
# conv param
filter_num = 30
filter_size = 5
filter_pad = 0
filter_stride = 1
input_size = input_dim[1]
conv_output_size = (input_size - filter_size + 2*filter_pad) / filter_stride + 1
pool_output_size = int(filter_num * (conv_output_size/2) * (conv_output_size/2))
def one_hot(n):
a = numpy.zeros(10)
a[n] = 1
return a
def test(X, Y):
predictions = numpy.array([forward(layers, x) for x in X])
Y_predict = numpy.argmax(predictions, axis=1).reshape(-1, 1)
return (Y_predict == Y).sum() / len(Y)
if __name__ == "__main__":
from mnist import load_mnist
from pickle import dump, load
import sys
X, Y = load_mnist('training', path="./mnist")
X = X.reshape(-1, 28*28)/256
Y_hot = numpy.array([one_hot(y) for y in Y])
X_test, Y_test = load_mnist('testing', path="./mnist")
X_test = X_test.reshape(-1, 28*28)/256
print("Loaded")
t = sympy.Symbol("t")
y = sympy.Symbol("y")
sigmoid = 1/(1+sympy.exp(-y))
cross_entropy = y * sympy.ln(t)+(1 - y) * sympy.ln(1 - t)
cost = (y - t) ** 2 / 2
def add_salt_and_pepper(data, proportion):
num_samples = np.shape(data)[0]
original_shape = np.shape(data[0])
new_data = np.reshape(data, (num_samples,-1))
num_features = np.shape(new_data)[1]
salt_and_pepper = np.random.binomial(1, proportion,
size=(num_samples,num_features))
new_data = (1-new_data)*salt_and_pepper + new_data*(1-salt_and_pepper)
return np.reshape(new_data, (num_samples, original_shape[0],
original_shape[1]))
# load data
ret = load_mnist(path='../datasets/mnist/downloads/', digits=[0,1,2])
X = ret[0]
Y = ret[1]
# show one example
fig = plt.figure('mnist_example')
for i in range(3):
plt.subplot(1,3,i)
sample = X[i]
plt.imshow(sample)
plt.show()
sap_X = add_salt_and_pepper(X, 0.25)
fig = plt.figure('mnist_salt_and_pepper')
for i in range(3):
net = caffe.Net(
caffe_root + "examples/mnist/lenet.prototxt",
caffe_root + "examples/mnist_lmdb/lenet_iter_10000.caffemodel",
caffe.TEST,
)
# set net to batch size of n1
n1 = 100
net.blobs["data"].reshape(n1, 1, 28, 28)
# load mnist databse
mnist_path = caffe_root + "data/mnist"
print("loading test dataset ...")
images_test, labels_test = mnist.load_mnist("testing", np.arange(10), mnist_path)
# feed data to the network
images_test = images_test.reshape(len(images_test), 1, 28, 28)
net.blobs["data"].data[...] = images_test[0:n1, :, :]
# perfrom forward operation
out = net.forward()
print("Predictions are #{}.".format(out["prob"].argmax(axis=1)))
print("The labels are:#{}.".format(labels_test[0:n1, 0]))
def compare_listcomp(x, y):
z = [i for i, j in zip(x, y) if i == j]
accuracy = float(len(z)) / len(x)
import pickle
import numpy as np
import mnist
import math
from PIL import Image
from numpy import array
x1 = 784
y1 = 30
z1 = 10
theta1 = pickle.load(open("theta1_new.npy", "rb"))
theta2 = pickle.load(open("theta2_new.npy", "rb"))
images, labels = mnist.load_mnist("training")
a2 = np.empty([y1 + 1, 1], dtype="float64") # 31x1
output = np.empty([z1, 1], dtype="float64") # 10x1
def g(z):
return 1.0 / (1.0 + math.exp(-z))
def feedforward2(input1):
z2 = np.dot(theta1, input1)
z2 = z2 / 100.0
# print theta1
for i in range(1, (y1 + 1)):
a2[i] = g(z2[i - 1])
a2[0] = 1.0
from mnist import load_mnist
import numpy
from preprocessing import deskew
from sklearn.metrics import accuracy_score
[feature,lable]=load_mnist("training")
[tfeature,tlable]=load_mnist("testing")
for i in range(0,60000):
temp=deskew(feature[i,:])
temp =feature.reshape(60000,28*28)
templ = numpy.squeeze(numpy.asarray(lable))
ttemp=tfeature.reshape(10000,28*28)
ttempl=numpy.squeeze(numpy.asarray(tlable))
smalldata=temp[0:20000,:]
smalllable=numpy.squeeze(numpy.asarray(lable[0:20000,:]))
print 'preprocessing finished'
#print smalldata.shape
#print smalllable.shape
#print
#print templ
#print templ.shape
from sklearn import svm
from sklearn.externals import joblib
print "Training"
clf=svm.SVC(degree=3,kernel='poly',decision_function_shape='ovr')
clf.fit(temp,templ)
print "Train finished"
joblib.dump(clf, 'poly.pkl')
res=clf.predict(ttemp)
acc=accuracy_score(ttempl,res)
def main(data_name, method, dimZ, dimH, n_channel, batch_size, K_mc, checkpoint, lbd):
# set up dataset specific stuff
from config import config
labels, n_iter, dimX, shape_high, ll = config(data_name, n_channel)
if data_name == 'mnist':
from mnist import load_mnist
if data_name == 'notmnist':
from notmnist import load_notmnist
# import functionalities
if method == 'onlinevi':
from bayesian_generator import generator_head, generator_shared, \
generator, construct_gen
if method in ['ewc', 'noreg', 'si', 'laplace']:
from generator import generator_head, generator_shared, generator, construct_gen
# then define model
n_layers_shared = 2
batch_size_ph = tf.placeholder(tf.int32, shape=(), name='batch_size')
dec_shared = generator_shared(dimX, dimH, n_layers_shared, 'sigmoid', 'gen')
# initialise sessions
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
string = method
if method in ['ewc', 'laplace', 'si']:
string = string + '_lbd%.1f' % lbd
if method == 'onlinevi' and K_mc > 1:
string = string + '_K%d' % K_mc
path_name = data_name + '_%s/' % string
assert os.path.isdir('save/'+path_name)
filename = 'save/' + path_name + 'checkpoint'
# visualise the samples
N_gen = 10**2
X_ph = tf.placeholder(tf.float32, shape=(batch_size, dimX), name = 'x_ph')
# now start fitting
N_task = len(labels)
gen_ops = []
X_valid_list = []
X_test_list = []
eval_func_list = []
result_list = []
n_layers_head = 2
n_layers_enc = n_layers_shared + n_layers_head - 1
for task in xrange(1, N_task+1):
# first load data
# first load data
if data_name == 'mnist':
X_train, X_test, _, _ = load_mnist(digits = labels[task-1], conv = False)
if data_name == 'notmnist':
X_train, X_test, _, _ = load_notmnist(data_path, digits = labels[task-1], conv = False)
N_train = int(X_train.shape[0] * 0.9)
X_valid_list.append(X_train[N_train:])
X_train = X_train[:N_train]
X_test_list.append(X_test)
# define the head net and the generator ops
dec = generator(generator_head(dimZ, dimH, n_layers_head, 'gen_%d' % task), dec_shared)
enc = encoder(dimX, dimH, dimZ, n_layers_enc, 'enc_%d' % task)
gen_ops.append(construct_gen(dec, dimZ, sampling=False)(N_gen))
eval_func_list.append(construct_eval_func(X_ph, enc, dec, ll, batch_size_ph,
K = 5000, sample_W = False))
# then load the trained model
load_params(sess, filename, checkpoint=task-1, init_all = False)
# plot samples
x_gen_list = sess.run(gen_ops, feed_dict={batch_size_ph: N_gen})
x_list = []
for i in xrange(len(x_gen_list)):
ind = np.random.randint(len(x_gen_list[i]))
x_list.append(x_gen_list[i][ind:ind+1])
x_list = np.concatenate(x_list, 0)
tmp = np.zeros([10, dimX])
tmp[:task] = x_list
if task == 1:
x_gen_all = tmp
else:
x_gen_all = np.concatenate([x_gen_all, tmp], 0)
# print test-ll on all tasks
tmp_list = []
for i in xrange(len(eval_func_list)):
print 'task %d' % (i+1),
test_ll = eval_func_list[i](sess, X_test_list[i])
tmp_list.append(test_ll)
result_list.append(tmp_list)
#x_gen_all = 1.0 - x_gen_all
if not os.path.isdir('figs/visualisation/'):
os.mkdir('figs/visualisation/')
print 'create path figs/visualisation/'
plot_images(x_gen_all, shape_high, 'figs/visualisation/', data_name+'_gen_all_'+method)
for i in xrange(len(result_list)):
print result_list[i]
# save results
fname = 'results/' + data_name + '_%s.pkl' % string
import pickle
pickle.dump(result_list, open(fname, 'wb'))
print 'test-ll results saved in', fname
from pylab import *
import numpy as np
from numpy import *
import mnist
images, labels = mnist.load_mnist('training', digits=np.arange([10]))
print np.shape(images)
print np.shape(labels)
#imshow(images[0], cmap=cm.gray)
#show()
# Network: TwoLayerNet2
# Test: Backpropagation Gradient vs Numerical Gradient
import os
import sys
import numpy as np
from pathlib import Path
try:
sys.path.append(os.path.join(Path(os.getcwd()).parent, 'lib'))
from mnist import load_mnist
import twolayernet2 as network
except ImportError:
print('Library Module Can Not Found')
# 1.load training/test data
(train_x, train_t), (test_x, test_t) = load_mnist(normalize=True,
flatten=True,
one_hot_label=True)
# 2.initialize network
network.initialize(input_size=train_x.shape[1],
hidden_size=50,
output_size=train_t.shape[1])
# 3. batch by 3
train_x_batch = train_x[:3]
train_t_batch = train_t[:3]
# 4. gradient
gradient_numerical = network.numerical_gradient_net(train_x_batch,
train_t_batch)
gradient_backpropagation = network.backpropagation_gradient_net(
pickle.dump(classifier, open('classifier_2.p', 'w'))
pickle.dump(training_set, open('training_set_2.p', 'w'))
pickle.dump(training_labels, open('training_labels_2.p', 'w'))
def classify(images, classifier):
#runs the classifier on a set of images.
return classifier.predict(images)
def error_measure(predicted, actual):
return np.count_nonzero(abs(predicted - actual))/float(len(predicted))
if __name__ == "__main__":
# Code for loading data
images, labels = load_mnist(digits=range(0,10),path='.')
num_images = len(labels)
# preprocessing
images = preprocess(images)
# pick training and testing set
# YOU HAVE TO CHANGE THIS TO PICK SET DIFFERENT OF DATA
X = 10000
training_set = images[:X]
training_labels = labels[:X]
testing_set = images[X:2*X]
testing_labels = labels[X:2*X]
import numpy as np
import time
np.random.seed(11785)
#initialize neural parameters
learning_rate = 0.004
momentum = 0.996 #0.956
num_bn_layers = 1
mini_batch_size = 10
epochs = 40
# initialize training, validation and testing data
train, val, test = mnist.load_mnist()
net = mlp.MLP(
784, 10, [64, 32],
[ac.Sigmoid(), ac.Sigmoid(), ac.Sigmoid()], ac.SoftmaxCrossEntropy(),
learning_rate, momentum, num_bn_layers)
start = time.time()
#training neural network
net.fit(train, val, epochs, mini_batch_size)
end = time.time()
print("Training time(sec.) =", end - start)
#testing neural network
sg3 = sg2 + Ng2
initv = 16
inith = 4
spk_rec_mon = np.arange(N_NEURONS, dtype='int')
print("##### Loading Data ")
exp_name = 'eta' + str(eta) + '_sig' + str(sig) + '_inputfact' + \
str(inp_fact) + '_inith' + str(inith) + '_wgg' + str(wgg)
exp_name_test = 'test_eta' + str(eta) + '_sig' + str(sig) + '_inputfact' + str(
inp_fact) + '_inith' + str(inith) + '_wgg' + str(wgg)
data_train, targets_train = mnist.load_mnist(
'/share/data/mnist/train-images-idx3-ubyte',
'/share/data/mnist/train-labels-idx1-ubyte',
50000,
with_labels=True)
data_classify, targets_classify = mnist.load_mnist(
'/share/data/mnist/t10k-images-idx3-ubyte',
'/share/data/mnist/t10k-labels-idx1-ubyte',
10000,
with_labels=False)
np.random.seed(100)
###########################################################################
print(
"###################### Train Stimulus Creation ##################################"
)
sim_ticks = N_train * t_sample_train
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import sys, os
import pickle
import numpy as np
import matplotlib.pyplot as plt
from active_func import *
from common_func import *
from es_net import *
from es_nn_layer import *
from mnist import load_mnist
if __name__ == '__main__':
(train_data, train_label), (test_data,
test_label) = load_mnist(flatten=False)
scale = 0.1
image_size = 28 * 28
hidden_nodes = 100
output_nodes = 10
input_dim = (1, 28, 28)
update_class = SGD
# random init the dnn param
dnn = es_net()
filter_num = 30
filter_size = 5
filter_pad = 0
filter_stride = 1
# Conv layer
meanErr = sum(tempErr) / float(foldNum)
print tempErr, 'mean error rate:', meanErr
classifier = pickle.load(open('classifier_2.p'))
tempErr = []
for i in range(foldNum):
predicted = classify(testSetList[i], classifier)
tempErr.append(error_measure(predicted, testLabelList[i]))
meanErr = sum(tempErr) / float(foldNum)
print tempErr, 'mean error rate:', meanErr
if __name__ == "__main__":
# Code for loading data
images, labels = load_mnist(digits=range(10), path='.')
# preprocessing
images = preprocess(images)
# compare two classifier
# expForComp(images, labels)
# number of train = numDataToUse - numTest
crossData, crossLabel, trainData, trainLabel, testData, testLabel = dataSplit(
images, labels, numDataToUse=50000, numCross=10000, numTest=20000)
# to submit, set the para to best as default
classifier = build_classifier(trainData, trainLabel)
# save_classifier(classifier, trainData, trainLabel)
# classifier = pickle.load(open('classifier_1.p'))
# print 'finish build'
@author: Funato """ # -*- coding: utf-8 -*- """ Created on Thu Aug 15 13:48:14 2019 @author: Funato Kazuyuki """ #このプログラムは自由に使用、改変していただけますが、それにより生じた、またそれを利用したことにより生じたいかなる損害についても責任を負いません。 import numpy as np import mnist import matplotlib.pyplot as plt train_img, train_label, test_img, test_label = mnist.load_mnist() # 学習データの準備 train_img = np.asarray(train_img) train_label = np.asarray(train_label) test_img = np.asarray(test_img) test_label = np.asarray(test_label) learning_rate = 0.1 #学習係数の設定 number = 10000 #学習回数の設定 batch_size = 100 # data_size = train_img.shape[0] t_data_size = test_img.shape[0] test_count = 1000 #テスト回数 # ネットワークの構造の設定 全体は3層構造
tanh = False
regTrainedNetworks = ["1notanh.xml", "2notanh.xml", "3notanh.xml", "3notanhx1.xml", "3notanhx10.xml"]
tanhTrainedNetworks = ["1tanh.xml", "2tanh.xml", "3tanh.xml"]
if readFile:
for f in regTrainedNetworks + tanhTrainedNetworks:
if os.path.isfile(f):
print "reading network",
print f
net = NetworkReader.readFrom(f)
testNetwork(net)
else:
print "file",file,"does not exist"
else:
file = "newNetwork.xml"
train_ds = SupervisedDataSet(28 * 28, 1)
train_images, train_labels = mnist.load_mnist(dataset='training')
for image, label in zip(train_images, train_labels):
train_ds.addSample(ravel(image), label)
if tanh:
net = buildNetwork(train_ds.indim, 98, 98, 49, train_ds.outdim, bias=True, hiddenclass=TanhLayer)
else:
net = buildNetwork(train_ds.indim, 98, train_ds.outdim, bias=True)
trainer = BackpropTrainer(net, train_ds)
print "start training"
for i in range(5):
trainer.trainEpochs(1)
print "epoch: %4d" % trainer.totalepochs
testNetwork(net)
import pickle
import numpy as np
import mnist
import math
from PIL import Image
from numpy import array
x1 = 784
y1 = 30
z1 = 10
theta1 = pickle.load(open("theta1_new.npy", "rb"))
theta2 = pickle.load(open("theta2_new.npy", "rb"))
images, labels = mnist.load_mnist('training')
a2 = np.empty([y1 + 1, 1], dtype='float64') #31x1
output = np.empty([z1, 1], dtype='float64') #10x1
def g(z):
return (1.0 / (1.0 + math.exp(-z)))
def feedforward2(input1):
z2 = np.dot(theta1, input1)
z2 = z2 / 100.0
#print theta1
for i in range(1, (y1 + 1)):
a2[i] = g(z2[i - 1])
a2[0] = 1.0
z3 = np.dot(theta2, a2)
return vis_states
def reconstruct(weights, hid_states):
for i in range(len(weights) + 1)[1:]:
hid_states = rbm.reconstruct(weights[-i], hid_states)
return hid_states
def pickle_weights(weights, path):
data = {"weights": weights}
output = open(path, "wb")
pickle.dump(data, output)
output.close()
def unpickle(path):
pkl_file = open(path, "rb")
data = pickle.load(pkl_file)
pkl_file.close()
return data["weights"]
if __name__ == "__main__":
data = mnist.load_mnist("../../../mnist_test.csv")
data = rbm.insert_biases(data)
dims = [784, 200, 50, 10]
dbn_weights = generate_dbn_weights(dims)
dbn_weights = unpickle("../weights/300_dbn.pkl")
# dbn_weights = train_dbn(dbn_weights, data, max_epochs = 2000)
def main():
# # Arg parse
# parser = argparse.ArgumentParser()
# parser.add_argument('--data_dir', type=str, default='./input_data/',
# help='Type mnist data path')
# parser.add_argument('--verbose', type=int, default=1)
#
# args = parser.parse_args()
#
# data_dir = args.data_dir
# verbose = args.verbose
# Import data
train_csv = "./data/train.csv"
valid_csv = "./data/valid.csv"
test_csv = "./data/test.csv"
mnist = load_mnist(train_csv, valid_csv, test_csv)
# Create the model
with tf.variable_scope('input'):
inputs = tf.placeholder(tf.float32, [None, 28, 28, 1])
with tf.variable_scope('conv1'):
W_conv1 = tf.Variable(tf.random_normal([5, 5, 1, 32]))
b_conv1 = tf.Variable(tf.random_normal([32]))
h_conv1 = tf.nn.relu(conv2d(inputs, W_conv1) + b_conv1)
h_pool1 = max_pool_2x2(h_conv1)
with tf.variable_scope('conv2'):
W_conv2 = tf.Variable(tf.random_normal([5, 5, 32, 64]))
b_conv2 = tf.Variable(tf.random_normal([64]))
h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
h_pool2 = max_pool_2x2(h_conv2)
with tf.variable_scope('fc'):
flat = tf.reshape(h_pool2, [-1, 7 * 7 * 64])
with tf.variable_scope('fc1'):
W_fc1 = tf.Variable(tf.random_normal([7 * 7 * 64, 1024]))
b_fc1 = tf.Variable(tf.random_normal([1024]))
h_fc1 = tf.nn.relu(tf.matmul(flat, W_fc1) + b_fc1)
with tf.variable_scope('fc2'):
W_fc2 = tf.Variable(tf.random_normal([1024, 10]))
b_fc2 = tf.Variable(tf.random_normal([10]))
outputs = tf.matmul(h_fc1, W_fc2) + b_fc2
# Label placeholder
labels = tf.placeholder(tf.float32, [None, 10])
# Correct
correct_prediction = tf.equal(tf.argmax(outputs, 1), tf.argmax(labels, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# Loss and Optimizer
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits=outputs))
train_step = tf.train.AdamOptimizer().minimize(cross_entropy)
# Session
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Train
for epoch in range(nb_epochs):
sys.stdout.write("Epoch %d/%d\n" % (epoch + 1, nb_epochs))
for iter, (batch_xs,
batch_ys) in enumerate(mnist.train.next_batch(batch_size)):
feed_dict = {inputs: batch_xs, labels: batch_ys}
_, train_loss, train_accuracy = sess.run(
[train_step, cross_entropy, accuracy], feed_dict=feed_dict)
if verbose == 1:
length = 30
percentage = float(iter * batch_size / mnist.train.num_data)
bar = "[" + "=" * int(length * percentage) + "-" * (
length - int(length * percentage)) + "]"
display = "\r{} / {} {} " \
"loss: {:.4f} - acc: {:.4f}" \
.format(iter * batch_size, mnist.train.num_data, bar, train_loss, train_accuracy)
sys.stdout.write(display)
sys.stdout.flush()
# validation
valid_loss = 0
valid_accuracy = 0
for iter, (batch_xs,
batch_ys) in enumerate(mnist.valid.next_batch(batch_size)):
feed_dict = {inputs: batch_xs, labels: batch_ys}
v_l, v_a = sess.run([cross_entropy, accuracy], feed_dict=feed_dict)
valid_loss += v_l * len(batch_xs)
valid_accuracy += v_a * len(batch_ys)
valid_loss /= valid_num
valid_accuracy /= valid_num
if verbose == 1:
display = " - val_loss : {:.4f} - val_acc : {:.4f}\n" \
.format(valid_loss, valid_accuracy)
sys.stdout.write(display)
sys.stdout.write("\nComplete training !!\n")
# Test trained model
feed_dict = {inputs: mnist.train.images, labels: mnist.train.labels}
test_loss, test_accuracy = sess.run([cross_entropy, accuracy],
feed_dict=feed_dict)
display = "test_loss : {:.4f} - test_acc : {:.4f}\n" \
.format(test_loss, test_accuracy)
sys.stdout.write(display)
"3notanhx10.xml"
]
tanhTrainedNetworks = ["1tanh.xml", "2tanh.xml", "3tanh.xml"]
if readFile:
for f in regTrainedNetworks + tanhTrainedNetworks:
if os.path.isfile(f):
print "reading network",
print f
net = NetworkReader.readFrom(f)
testNetwork(net)
else:
print "file", file, "does not exist"
else:
file = "newNetwork.xml"
train_ds = SupervisedDataSet(28 * 28, 1)
train_images, train_labels = mnist.load_mnist(dataset='training')
for image, label in zip(train_images, train_labels):
train_ds.addSample(ravel(image), label)
if tanh:
net = buildNetwork(train_ds.indim,
98,
98,
49,
train_ds.outdim,
bias=True,
hiddenclass=TanhLayer)
else:
net = buildNetwork(train_ds.indim, 98, train_ds.outdim, bias=True)
trainer = BackpropTrainer(net, train_ds)
new_data = np.reshape(data, (num_samples,-1))
num_features = np.shape(new_data)[1]
salt_and_pepper = np.random.binomial(1, proportion,
size=(num_samples,num_features))
new_data = (1-new_data)*salt_and_pepper + new_data*(1-salt_and_pepper)
return np.reshape(new_data, original_shape)
def add_gaussian_noise(data,mean,std):
new_data = data + np.random.normal(mean,std,size=np.shape(data))
new_data = new_data - np.min(new_data)
new_data /= np.max(new_data)
return new_data
# load data
if __name__ == "__main__":
for dataset in ['training', 'testing']:
ret = load_mnist(path=PATH_MNIST, dataset=dataset)
X = ret[0]
Y = ret[1]
bin_X = np.array(binaryze_dataset(X,bin_threshold),dtype=int)
sap_bin_X = np.array(add_salt_and_pepper(bin_X,
salt_pepper_proportion),dtype=bool)
np.save(path.join(PATH_SAVE, "{}_X".format(dataset)), sap_bin_X)
np.save(path.join(PATH_SAVE, "{}_Y".format(dataset)), Y)
