class TestMNIST(unittest.TestCase):
def setUp(self):
skip_if_no_data()
self.train = MNIST(which_set='train')
self.test = MNIST(which_set='test')
def test_range(self):
"""Tests that the data spans [0,1]"""
for X in [self.train.X, self.test.X]:
assert X.min() == 0.0
assert X.max() == 1.0
def test_topo(self):
"""Tests that a topological batch has 4 dimensions"""
topo = self.train.get_batch_topo(1)
assert topo.ndim == 4
def test_topo_c01b(self):
"""
Tests that a topological batch with axes ('c',0,1,'b')
can be dimshuffled back to match the standard ('b',0,1,'c')
format.
"""
batch_size = 100
c01b_test = MNIST(which_set='test', axes=('c', 0, 1, 'b'))
c01b_X = c01b_test.X[0:batch_size, :]
c01b = c01b_test.get_topological_view(c01b_X)
assert c01b.shape == (1, 28, 28, batch_size)
b01c = c01b.transpose(3, 1, 2, 0)
b01c_X = self.test.X[0:batch_size, :]
assert c01b_X.shape == b01c_X.shape
assert np.all(c01b_X == b01c_X)
b01c_direct = self.test.get_topological_view(b01c_X)
assert b01c_direct.shape == b01c.shape
assert np.all(b01c_direct == b01c)
def test_y_index_space(self):
"""
Tests that requesting the targets to be in IndexSpace and iterating
over them works
"""
data_specs = (IndexSpace(max_labels=10, dim=1), 'targets')
it = self.test.iterator(mode='sequential',
data_specs=data_specs,
batch_size=100)
for y in it:
pass
def test_y_vector_space(self):
"""
Tests that requesting the targets to be in VectorSpace and iterating
over them works
"""
data_specs = (VectorSpace(dim=10), 'targets')
it = self.test.iterator(mode='sequential',
data_specs=data_specs,
batch_size=100)
for y in it:
pass
class TestMNIST(unittest.TestCase):
def setUp(self):
skip_if_no_data()
self.train = MNIST(which_set = 'train')
self.test = MNIST(which_set = 'test')
def test_range(self):
"""Tests that the data spans [0,1]"""
for X in [self.train.X, self.test.X ]:
assert X.min() == 0.0
assert X.max() == 1.0
def test_topo(self):
"""Tests that a topological batch has 4 dimensions"""
topo = self.train.get_batch_topo(1)
assert topo.ndim == 4
def test_topo_c01b(self):
"""
Tests that a topological batch with axes ('c',0,1,'b')
can be dimshuffled back to match the standard ('b',0,1,'c')
format.
"""
batch_size = 100
c01b_test = MNIST(which_set='test', axes=('c', 0, 1, 'b'))
c01b_X = c01b_test.X[0:batch_size,:]
c01b = c01b_test.get_topological_view(c01b_X)
assert c01b.shape == (1, 28, 28, batch_size)
b01c = c01b.transpose(3,1,2,0)
b01c_X = self.test.X[0:batch_size,:]
assert c01b_X.shape == b01c_X.shape
assert np.all(c01b_X == b01c_X)
b01c_direct = self.test.get_topological_view(b01c_X)
assert b01c_direct.shape == b01c.shape
assert np.all(b01c_direct == b01c)
def test_sample_ae():
"""
Visualize some samples from the trained unsupervised GSN.
"""
with open("gsn_ae_example.pkl") as f:
gsn = pickle.load(f)
# random point to start at
mb_data = MNIST(which_set='test').X[105:106, :]
history = gsn.get_samples([(0, mb_data)],
walkback=1000,
symbolic=False,
include_first=True)
history = list(itertools.chain(*history))
history = np.vstack(history)
tiled = image.tile_raster_images(history,
img_shape=[28, 28],
tile_shape=[50, 50],
tile_spacing=(2, 2))
image.save("gsn_ae_example.png", tiled)
# code to get log likelihood from kernel density estimator
# this crashed on GPU (out of memory), but works on CPU
pw = ParzenWindows(MNIST(which_set='test').X, .20)
print(pw.get_ll(history))
def model1():
#pdb.set_trace()
# train set X has dim (60,000, 784), y has dim (60,000, 10)
train_set = MNIST(which_set='train', one_hot=True)
# test set X has dim (10,000, 784), y has dim (10,000, 10)
valid_set = MNIST(which_set='test', one_hot=True)
test_set = MNIST(which_set='test', one_hot=True)
#import pdb
#pdb.set_trace()
#print train_set.X.shape[1]
# =====<Create the MLP Model>=====
h2_layer = NoisyRELU(layer_name='h1',
sparse_init=15,
noise_factor=5,
dim=1000,
desired_active_rate=0.2,
bias_factor=20,
max_col_norm=1)
#h2_layer = RectifiedLinear(layer_name='h2', dim=100, sparse_init=15, max_col_norm=1)
#print h1_layer.get_params()
#h2 = RectifiedLinear(layer_name='h2', dim=500, sparse_init=15, max_col_norm=1)
y_layer = Softmax(layer_name='y', n_classes=10, irange=0., max_col_norm=1)
mlp = MLP(batch_size=200,
input_space=VectorSpace(dim=train_set.X.shape[1]),
layers=[h2_layer, y_layer])
# =====<Create the SGD algorithm>=====
sgd = SGD(init_momentum=0.1,
learning_rate=0.01,
monitoring_dataset={'valid': valid_set},
cost=MethodCost('cost_from_X'),
termination_criterion=MonitorBased(
channel_name='valid_y_misclass', prop_decrease=0.001, N=50))
#sgd.setup(model=mlp, dataset=train_set)
# =====<Extensions>=====
ext = [MomentumAdjustor(start=1, saturate=10, final_momentum=0.9)]
# =====<Create Training Object>=====
save_path = './mlp_model1.pkl'
train_obj = Train(dataset=train_set,
model=mlp,
algorithm=sgd,
extensions=ext,
save_path=save_path,
save_freq=0)
#train_obj.setup_extensions()
#import pdb
#pdb.set_trace()
train_obj.main_loop()
# =====<Run the training>=====
'''
def main():
#import pdb
#pdb.set_trace()
###################
#BUILD THE DATASET#
###################
print 'build the dataset'
train_set = MNIST(which_set='train', one_hot=False)
test_set = MNIST(which_set='test', one_hot=False)
train_setX, valid_setX = split(train_set.X, [50000], axis=0)
train_sety, valid_sety = split(train_set.y, [50000], axis=0)
#import pdb
#pdb.set_trace()
##################
#BUILD THE LAYERS#
##################
print 'build the layers'
input_size = len(train_setX[0])
h1 = Tanh(prev_layer_size=input_size, this_layer_size=2000)
output_layer = Softmax(prev_layer_size=h1.this_layer_size, this_layer_size=10,
W_range=[0,0], b_range=[0,0])
#y_layer = Sigmoid(prev_layer_size=h2.this_layer_size, this_layer_size=[10,1])
mlp = MLP(input_size=input_size,
layers=[h1, output_layer],
train_set=[train_setX, train_sety],
valid_set=[valid_setX, valid_sety],
test_set=[test_set.X, test_set.y],
error_function=loglikehood,
batch_size=20,
learning_rate=0.1)
print 'start training'
mlp.train()
#p = plt.plot(mlp.epoch, mlp.valid_error)
#plots.append(p)
with open('batches_clean.pkl', 'wb') as bat:
cPickle.dump(mlp.epoch, bat)
with open('errors_clean.pkl', 'wb') as err:
cPickle.dump(mlp.valid_error, err)
with open('legends_clean.pkl', 'wb') as leg:
cPickle.dump(['2000-tanh'], leg)
def model2():
#pdb.set_trace()
# train set X has dim (60,000, 784), y has dim (60,000, 10)
train_set = MNIST(which_set='train', one_hot=True)
# test set X has dim (10,000, 784), y has dim (10,000, 10)
test_set = MNIST(which_set='test', one_hot=True)
# =====<Create the MLP Model>=====
h1_layer = RectifiedLinear(layer_name='h1', dim=1000, irange=0.5)
#print h1_layer.get_params()
h2_layer = RectifiedLinear(layer_name='h2',
dim=1000,
sparse_init=15,
max_col_norm=1)
y_layer = Softmax(layer_name='y',
n_classes=train_set.y.shape[1],
irange=0.5)
mlp = MLP(batch_size=100,
input_space=VectorSpace(dim=train_set.X.shape[1]),
layers=[h1_layer, h2_layer, y_layer])
# =====<Create the SGD algorithm>=====
sgd = SGD(batch_size=100,
init_momentum=0.1,
learning_rate=0.01,
monitoring_dataset={
'valid': train_set,
'test': test_set
},
cost=SumOfCosts(costs=[
MethodCost('cost_from_X'),
WeightDecay(coeffs=[0.00005, 0.00005, 0.00005])
]),
termination_criterion=MonitorBased(
channel_name='valid_y_misclass', prop_decrease=0.0001, N=5))
#sgd.setup(model=mlp, dataset=train_set)
# =====<Extensions>=====
ext = [MomentumAdjustor(start=1, saturate=10, final_momentum=0.99)]
# =====<Create Training Object>=====
save_path = './mlp_model2.pkl'
train_obj = Train(dataset=train_set,
model=mlp,
algorithm=sgd,
extensions=ext,
save_path=save_path,
save_freq=0)
#train_obj.setup_extensions()
train_obj.main_loop()
def mnist():
#import pdb
#pdb.set_trace()
###################
#BUILD THE DATASET#
###################
print 'build the dataset'
train_set = MNIST(which_set='train')
test_set = MNIST(which_set='test')
train_setX, valid_setX = split(train_set.X, [50000], axis=0)
train_sety, valid_sety = split(train_set.y, [50000], axis=0)
#import pdb
#pdb.set_trace()
##################
#BUILD THE LAYERS#
##################
print 'build the layers'
input_size = len(train_setX[0])
h1 = NoisyRELU(prev_layer_size=input_size, this_layer_size=1000, threshold=5, noise_factor=1)
h2 = NoisyRELU(prev_layer_size=h1.get_size(), this_layer_size=1000, threshold=5, noise_factor=1)
#h3 = NoisyRELU(prev_layer_size=h2.get_size(), this_layer_size=1000, threshold=5, noise_factor=1)
output_layer = Softmax(prev_layer_size=h1.this_layer_size, this_layer_size=10,
W_range=[0,0], b_range=[0,0])
#y_layer = Sigmoid(prev_layer_size=h2.this_layer_size, this_layer_size=[10,1])
mlp = MLP(input_size=input_size,
layers=[h1, h2, output_layer],
train_set=[train_setX, train_sety],
valid_set=[valid_setX, valid_sety],
test_set=[test_set.X, test_set.y],
error_function=loglikehood,
batch_size=20,
learning_rate=0.1)
print 'start training'
mlp.train(save_freq=1)
def setup():
"""
Create pickle file with a simple model.
"""
# tearDown is guaranteed to run pop_load_data.
control.push_load_data(False)
with open('dbm.pkl', 'wb') as f:
dataset = MNIST(which_set='train', start=0, stop=100, binarize=True)
vis_layer = BinaryVector(nvis=784, bias_from_marginals=dataset)
hid_layer1 = BinaryVectorMaxPool(layer_name='h1',
pool_size=1,
irange=.05,
init_bias=-2.,
detector_layer_dim=50)
hid_layer2 = BinaryVectorMaxPool(layer_name='h2',
pool_size=1,
irange=.05,
init_bias=-2.,
detector_layer_dim=10)
model = DBM(batch_size=20,
niter=2,
visible_layer=vis_layer,
hidden_layers=[hid_layer1, hid_layer2])
model.dataset_yaml_src = """
!obj:pylearn2.datasets.binarizer.Binarizer {
raw: !obj:pylearn2.datasets.mnist.MNIST {
which_set: "train",
start: 0,
stop: 100
}
}
"""
model.layer_to_chains = model.make_layer_to_state(1)
cPickle.dump(model, f, protocol=cPickle.HIGHEST_PROTOCOL)
def test_sample_supervised(idxs=None, noisy=True):
"""
Visualize samples and labels produced by GSN.
"""
with open("gsn_sup_example.pkl") as f:
gsn = pickle.load(f)
gsn._corrupt_switch = noisy
ds = MNIST(which_set='test')
if idxs is None:
data = ds.X[100:150]
else:
data = ds.X[idxs]
# change the walkback parameter to make the data fill up rows in image
samples = gsn.get_samples([(0, data)],
indices=[0, 2],
walkback=21,
symbolic=False,
include_first=True)
stacked = vis_samples(samples)
tiled = image.tile_raster_images(stacked,
img_shape=[28, 28],
tile_shape=[50, 50],
tile_spacing=(2, 2))
image.save("gsn_sup_example.png", tiled)
def test_decision_function(self):
"""
Test DenseMulticlassSVM.decision_function.
"""
dataset = MNIST(which_set='train')
X = dataset.X[0:20, :]
y = dataset.y[0:20]
for i in xrange(10):
assert (y == i).sum() > 0
model = DenseMulticlassSVM(kernel='poly', C=1.0).fit(X, y)
f = model.decision_function(X)
print(f)
yhat_f = np.argmax(f, axis=1)
yhat = np.cast[yhat_f.dtype](model.predict(X))
print(yhat_f)
print(yhat)
assert (yhat_f != yhat).sum() == 0
def test_mnist():
"""
Test the mnist.yaml file from the maxout
paper on random input
"""
skip_if_no_gpu()
train = load_train_file(
os.path.join(pylearn2.__path__[0], "scripts/papers/maxout/mnist.yaml"))
# Load fake MNIST data
init_value = control.load_data
control.load_data = [False]
train.dataset = MNIST(which_set='train',
one_hot=1,
axes=['c', 0, 1, 'b'],
start=0,
stop=100)
train.algorithm._set_monitoring_dataset(train.dataset)
control.load_data = init_value
# Train shortly and prevent saving
train.algorithm.termination_criterion = EpochCounter(max_epochs=1)
train.extensions.pop(0)
train.save_freq = 0
train.main_loop()
def try_pylearn2_generator():
dataset = MNIST('train')
dataset_generator = Pylearn2OldGenerator(dataset,
batch_size=1000,
num_batches=-1)
for b in dataset_generator:
print b.shape
assert b.shape == (1000, 784)
def load_dataset(self):
# TODO: we might need other variables for identifying what kind of
# extra preprocessing was done such as features product and number
# of features kept based on MI.
#base_path = get_data_path(self.state)
#self.base_path = base_path
#import pdb
#pdb.set_trace()
if self.state.dataset == 'mnist':
self.test_ddm = MNIST(which_set='test', one_hot=True)
dataset = MNIST(which_set='train', shuffle=True, one_hot=True)
train_X, valid_X = np.split(dataset.X, [50000])
train_y, valid_y = np.split(dataset.y, [50000])
self.train_ddm = DenseDesignMatrix(X=train_X, y=train_y)
self.valid_ddm = DenseDesignMatrix(X=valid_X, y=valid_y)
elif self.state.dataset == 'svhn':
self.train_ddm = SVHN(which_set='splitted_train')
self.test_ddm = SVHN(which_set='test')
self.valid_ddm = SVHN(which_set='valid')
elif self.state.dataset == 'cifar10':
self.train_ddm = My_CIFAR10(which_set='train', one_hot=True)
self.test_ddm = None
self.valid_ddm = My_CIFAR10(which_set='test', one_hot=True)
if self.train_ddm is not None:
self.nvis = self.train_ddm.X.shape[1]
self.nout = self.train_ddm.y.shape[1]
print "nvis, nout :", self.nvis, self.nout
self.ntrain = self.train_ddm.X.shape[0]
print "ntrain :", self.ntrain
if self.valid_ddm is not None:
self.nvalid = self.valid_ddm.X.shape[0]
print "nvalid :", self.nvalid
if self.test_ddm is not None:
self.ntest = self.test_ddm.X.shape[0]
print "ntest :", self.ntest
def get_valid(ds, limit_size=-1, fold=0):
if ds == 'mnist':
data = MNIST('train', start=50000, stop=60000)
return data.X[:limit_size]
elif ds == 'tfd':
data = TFD('valid', fold=fold, scale=True)
return data.X
else:
raise ValueError("Unknow dataset: {}".format(args.dataet))
def test_topo_c01b(self):
"""
Tests that a topological batch with axes ('c',0,1,'b')
can be dimshuffled back to match the standard ('b',0,1,'c')
format.
"""
batch_size = 100
c01b_test = MNIST(which_set='test', axes=('c', 0, 1, 'b'))
c01b_X = c01b_test.X[0:batch_size,:]
c01b = c01b_test.get_topological_view(c01b_X)
assert c01b.shape == (1, 28, 28, batch_size)
b01c = c01b.transpose(3,1,2,0)
b01c_X = self.test.X[0:batch_size,:]
assert c01b_X.shape == b01c_X.shape
assert np.all(c01b_X == b01c_X)
b01c_direct = self.test.get_topological_view(b01c_X)
assert b01c_direct.shape == b01c.shape
assert np.all(b01c_direct == b01c)
def MNISTmain():
# TODO: max_col_norm
h0 = maxout.Maxout(layer_name='h0', num_units=1200, num_pieces=2, W_lr_scale=1.0, irange=0.005, b_lr_scale=1.0)
h1 = maxout.Maxout(layer_name='h1', num_units=1200, num_pieces=2, W_lr_scale=1.0, irange=0.005, b_lr_scale=1.0)
# h2 = maxout.Maxout(layer_name='h2, num_units=1200, num_pieces=2, W_lr_scale=1.0, irange=0.005, b_lr_scale=1.0)
outlayer = mlp.Softmax(layer_name='y', n_classes=10, irange=0)
# layers = [h0, h1, h2, outlayer]
layers = [h0, h1, outlayer]
model = mlp.MLP(layers, nvis=784)
train = MNIST('train', one_hot=1, start=0, stop=50000)
valid = MNIST('train', one_hot=1, start=50000, stop=60000)
test = MNIST('test', one_hot=1, start=0, stop=10000)
monitoring = dict(valid=valid)
termination = MonitorBased(channel_name="valid_y_misclass", N=100)
extensions = [best_params.MonitorBasedSaveBest(channel_name="valid_y_misclass",
save_path="/data/mcr10/train_best.pkl")]
algorithm = sgd.SGD(0.1, batch_size=100, cost=Dropout(),
monitoring_dataset = monitoring, termination_criterion = termination)
save_path = "/data/mcr10/train_best.pkl"
if not args.train and os.path.exists(save_path):
model = serial.load(save_path)
else:
print 'Running training'
train_job = Train(train, model, algorithm, extensions=extensions, save_path="/data/mcr10/train.pkl", save_freq=1)
train_job.main_loop()
X = model.get_input_space().make_batch_theano()
Y = model.fprop(X)
y = T.argmax(Y, axis=1)
f = function(inputs=[X], outputs=y)
yhat = f(test.X)
y = np.squeeze(test.get_targets())
print 'accuracy', (y==yhat).sum() / y.size
def main():
###################
#BUILD THE DATASET#
###################
print 'build the dataset'
train_set = MNIST(which_set='train', one_hot=False)
test_set = MNIST(which_set='test', one_hot=False)
train_setX, valid_setX = split(train_set.X, [50000], axis=0)
train_sety, valid_sety = split(train_set.y, [50000], axis=0)
#import pdb
#pdb.set_trace()
##################
#BUILD THE LAYERS#
##################
print 'build the layers'
input_size = len(train_setX[0])
h1 = NoisyRELU(prev_layer_size=input_size,
this_layer_size=200,
noise_factor=0,
activation_threshold=0)
output_layer = Softmax(prev_layer_size=h1.this_layer_size,
this_layer_size=10,
W_range=[0, 0],
b_range=[0, 0])
#y_layer = Sigmoid(prev_layer_size=h2.this_layer_size, this_layer_size=[10,1])
# import pdb
# pdb.set_trace()
print 'build the model'
mlp = MLP(input_size=input_size,
layers=[h1, output_layer],
train_set=[train_setX, train_sety],
valid_set=[valid_setX, valid_sety],
test_set=[test_set.X, test_set.y],
error_function=loglikehood,
batch_size=20)
print 'start training'
mlp.train_batch(100000)
def test_topo_c01b(self):
"""
Tests that a topological batch with axes ('c',0,1,'b')
can be dimshuffled back to match the standard ('b',0,1,'c')
format.
"""
batch_size = 100
# TODO: the one_hot=True is only necessary because one_hot=False is
# broken, remove it after one_hot=False is fixed.
c01b_test = MNIST(which_set='test', axes=('c', 0, 1, 'b'),
one_hot=True)
c01b_X = c01b_test.X[0:batch_size,:]
c01b = c01b_test.get_topological_view(c01b_X)
assert c01b.shape == (1, 28, 28, batch_size)
b01c = c01b.transpose(3,1,2,0)
b01c_X = self.test.X[0:batch_size,:]
assert c01b_X.shape == b01c_X.shape
assert np.all(c01b_X == b01c_X)
b01c_direct = self.test.get_topological_view(b01c_X)
assert b01c_direct.shape == b01c.shape
assert np.all(b01c_direct == b01c)
class TestMNIST(unittest.TestCase):
def setUp(self):
skip_if_no_data()
self.train = MNIST(which_set = 'train')
self.test = MNIST(which_set = 'test')
def test_range(self):
"""Tests that the data spans [0,1]"""
for X in [self.train.X, self.test.X ]:
assert X.min() == 0.0
assert X.max() == 1.0
def test_topo(self):
"""Tests that a topological batch has 4 dimensions"""
topo = self.train.get_batch_topo(1)
assert topo.ndim == 4
class TestMNIST(unittest.TestCase):
def setUp(self):
skip_if_no_data()
self.train = MNIST(which_set='train')
self.test = MNIST(which_set='test')
def test_range(self):
"""Tests that the data spans [0,1]"""
for X in [self.train.X, self.test.X]:
assert X.min() == 0.0
assert X.max() == 1.0
def test_topo(self):
"""Tests that a topological batch has 4 dimensions"""
topo = self.train.get_batch_topo(1)
assert topo.ndim == 4
def setUp(self):
skip_if_no_h5py()
import h5py
skip_if_no_data()
from pylearn2.datasets.mnist import MNIST
# save MNIST data to HDF5
train = MNIST(which_set='train', one_hot=1, start=0, stop=100)
for name, dataset in [('train', train)]:
with h5py.File("{}.h5".format(name), "w") as f:
f.create_dataset('X', data=dataset.get_design_matrix())
f.create_dataset('topo_view',
data=dataset.get_topological_view())
f.create_dataset('y', data=dataset.get_targets())
# instantiate Train object
self.train = yaml_parse.load(trainer_yaml)
def get_dim_input(state):
if state.dataset == 'mnist':
dataset = MNIST(which_set='test')
dim = dataset.X.shape[1]
elif state.dataset == 'svhn':
dataset = SVHN(which_set='test')
dim = dataset.X.shape[1]
elif state.dataset == 'cifar10':
dataset = My_CIFAR10(which_set='test')
dim = dataset.X.shape[1]
else:
raise ValueError(
'only mnist, cifar10 and svhn are supported for now in get_dim_input'
)
del dataset
return dim
def get_valid(ds, limit_size=-1, fold=0):
if ds == 'mnist':
data = MNIST('train', start=50000, stop=60000)
return data.X[:limit_size]
elif ds == 'tfd':
data = TFD('valid', fold=fold, scale=True)
return data.X
elif ds == 'lfwcrop':
# HACK
return LFW(
axes=('c', 0, 1, 'b'),
gcn=55,
lfw_path=
'/afs/cs.stanford.edu/u/jgauthie/scr/lfwcrop_color/faces32',
filelist_path=
'/afs/cs.stanford.edu/u/jgauthie/scr/lfwcrop_color/filelist.dev.ids.txt',
embedding_file=
'/afs/cs.stanford.edu/u/jgauthie/scr/lfw-lsa/LFW_attributes_30d.npz',
img_shape=(3, 32, 32)).X
else:
raise ValueError("Unknow dataset: {}".format(args.dataet))
def ais_data(fname, do_exact=True):
rbm_params = load_rbm_params(fname)
# load data to set visible biases to ML solution
from pylearn2.datasets.mnist import MNIST
dataset = MNIST(which_set='train', one_hot=True)
data = numpy.asarray(dataset.X, dtype=config.floatX)
# run ais using B=0 model with ML visible biases
t1 = time.time()
(logz, log_var_dz), aisobj = \
rbm_tools.rbm_ais(rbm_params, n_runs=100, seed=123, data=data)
print 'AIS logZ : %f' % logz
print ' log_variance : %f' % log_var_dz
print 'Elapsed time: ', time.time() - t1
if do_exact:
exact_logz = compute_logz(rbm_params)
print 'Exact logZ = %f' % exact_logz
numpy.testing.assert_almost_equal(exact_logz, logz, decimal=0)
def get_ddm_mnist(self, ddm_id):
row = self.db.executeSQL(
"""
SELECT which_set, center, shuffle, one_hot, binarize, start,
stop, axes
FROM hps3.ddm_mnist
WHERE ddm_id = %s
""", (ddm_id, ), self.db.FETCH_ONE)
if not row or row is None:
raise HPSData("No cifar100 ddm for ddm_id="\
+str(ddm_id))
(which_set, center, shuffle, one_hot, binarize, start, stop,
axes_char) = row
axes = self.get_axes(axes_char)
return MNIST(which_set=which_set,
center=center,
shuffle=shuffle,
binarize=binarize,
axes=axes,
start=start,
stop=stop,
one_hot=one_hot)
def mnist_test(model_paths=["dae_l1.pkl", "dae_l2.pkl", "dae_l3.pkl"],
sparse_coef=0,
sparse_p=0.1):
cost = yaml_parse.load(
'cost : !obj:pylearn2.costs.cost.SumOfCosts {costs: [!obj:pylearn2.costs.autoencoder.MeanSquaredReconstructionError {}, !obj:pylearn2.costs.autoencoder.SparseActivation {coeff: '
+ str(sparse_coef) + ',p: ' + str(sparse_p) + '}]}')['cost']
model = load_model(model_paths, cost)
DUconfig.dataset = MNIST(which_set='train', start=0, stop=50000)
#reconstructed_dataset = represent(DUconfig.dataset,model)
#print reconstructed_dataset[0]
print recon_errors(DUconfig.dataset.X, model)
#reconstruct(model,img_path="mnist7.png",out_path="out7.png")
ipaths = ["mnist1.png", "mnist4.png", "mnist5.png", "mnist7.png"]
imgs = []
for img in ipaths:
i = Image.open(img)
imgs.append(np.reshape(i, (1, i.size[0] * i.size[1])))
flatimgs = np.concatenate(imgs)
out = decode(encode(flatimgs, model), model)
scipy.misc.imsave("out_prime.png",
out.reshape([flatimgs.shape[0] * 28, 28]))
save_as_patches(show_weights(model), [28, 28], out_path="outpatches.png")
def _get_dataset(dataset, no_imgs):
""" Dump machine learning dataset into C source file and create a header file with
array declaration and few macro definition
"""
dataset = dataset.lower()
if (dataset not in dataset_factory['name']):
print('Dataset is not in the factory')
sys.exit()
if (dataset == 'mnist'):
print('Reading MNIST dataset from pylearn2 database')
data = MNIST(which_set='test')
if (no_imgs > data.X.shape[0]):
cprint(
'Only {:d} images are available in this dataset. Dumping only those many'
.format(data.X.shape[0]), 'red')
data_x = data.X[0:min(no_imgs, data.X.shape[0])]
data_y = data.y[0:min(no_imgs, data.y.shape[0])]
data_x = np.uint8(data_x * 255) # conversion from 0 -> 0.99 to 0->255
elif (dataset == 'cifar10'):
print('Reading CIFAR-10 dataset from pylearn2 database')
data = CIFAR10(which_set='test')
if (no_imgs > data.X.shape[0]):
cprint(
'Only {:d} images are available in this dataset. Dumping only those many'
.format(data.X.shape[0]), 'red')
data_x = data.X[0:min(no_imgs, data.X.shape[0])]
data_y = data.y[0:min(no_imgs, data.y.shape[0])]
data_x = np.uint8(data_x) # already in the range 0 -> 255
elif (dataset == 'svhn'):
cprint('Not supported', 'red')
return
return data_x, data_y
def test_classify():
"""
See how well a (supervised) GSN performs at classification.
"""
with open("gsn_sup_example.pkl") as f:
gsn = pickle.load(f)
gsn = JointGSN.convert(gsn)
# turn off corruption
gsn._corrupt_switch = False
ds = MNIST(which_set='test')
mb_data = ds.X
y = ds.y
for i in xrange(1, 10):
y_hat = gsn.classify(mb_data, trials=i)
errors = np.abs(y_hat - y).sum() / 2.0
# error indices
#np.sum(np.abs(y_hat - y), axis=1) != 0
print(i, errors, errors / mb_data.shape[0])
def load_data(dataset, train_percent=0.8, val_percent=0.2):
""" Load MNIST, CIFAR-10 dataset
dataset: string: MNIST or CIFAR-10
train_percent: float: percentage of the dataset to be used for training
val_per: string: percentage of the dataset to be used for the validation purpose
Output:
(train_x, val_x, test_x, train_y, val_y, test_y)
"""
zero_mean = False
if(dataset.lower() == 'mnist'):
print('Loading MNIST dataset from pylearn2')
train_set_size = int(_DATASET_SIZE['mnist'] * train_percent)
train_data = MNIST(which_set='train', start=0, stop=train_set_size, center=zero_mean)
val_data = MNIST(which_set='train', start=train_set_size, stop=_DATASET_SIZE[dataset], center=zero_mean)
test_data = MNIST(which_set='test', center=zero_mean)
# convert labels into 1D array
train_data.y = np.hstack(train_data.y)
val_data.y = np.hstack(val_data.y)
test_data.y = np.hstack(test_data.y)
# create 10 dimensional vector corresponding to each label
train_data.y = np.float32(np.eye(10))[train_data.y]
val_data.y = np.float32(np.eye(10))[val_data.y]
test_data.y = np.float32(np.eye(10))[test_data.y]
# TODO: convert the data to range [-1,1]
# reshape the data into image size(#images, channels, height, width).
# Each row contains an image in the original dataset
train_data.X = np.reshape(train_data.X, (-1, 1, 28, 28))
val_data.X = np.reshape(val_data.X, (-1, 1, 28, 28))
test_data.X = np.reshape(test_data.X, (-1, 1, 28, 28))
# convert to [-1 1] range
train_data.X = train_data.X * 2.0 - 1.0
val_data.X = val_data.X * 2.0 - 1.0
test_data.X = test_data.X * 2.0 - 1.0
elif(dataset.lower() == 'cifar10'):
print('Loading CIFAR-10 dataset from pylearn2')
train_set_size = int(_DATASET_SIZE['cifar10'] * train_percent)
train_data = CIFAR10(which_set='train', start=0, stop=train_set_size)
val_data = CIFAR10(which_set='train', start=train_set_size, stop=50000)
test_data = CIFAR10(which_set='test')
# convert labels into 1D array
train_data.y = np.hstack(train_data.y)
val_data.y = np.hstack(val_data.y)
test_data.y = np.hstack(test_data.y)
# create 10 dimensional vector corresponding to each label
train_data.y = np.float32(np.eye(10))[train_data.y]
val_data.y = np.float32(np.eye(10))[val_data.y]
test_data.y = np.float32(np.eye(10))[test_data.y]
# TODO: convert the data to range [-1,1]
# reshape the data into image size(#images, channels, height, width).
# Each row contains an image in the original dataset
train_data.X = np.reshape(train_data.X, (-1, 3, 32, 32))
val_data.X = np.reshape(val_data.X, (-1, 3, 32, 32))
test_data.X = np.reshape(test_data.X, (-1, 3, 32, 32))
# convert to [-1 1] range
train_data.X = train_data.X * (2.0/255) - 1.0
val_data.X = val_data.X * (2.0/255) - 1.0
test_data.X = test_data.X * (2.0/255) - 1.0
elif(dataset.lower() == 'svhn'):
train_data = SVHN(which_set= 'splitted_train', axes= ['b', 'c', 0, 1])
val_data = SVHN(which_set= 'valid', axes= ['b', 'c', 0, 1])
test_data = SVHN(which_set= 'test', axes= ['b', 'c', 0, 1])
# convert labels into 1D array
train_data.y = np.hstack(train_data.y)
val_data.y = np.hstack(val_data.y)
test_data.y = np.hstack(test_data.y)
# create 10 dimensional vector corresponding to each label
train_data.y = np.float32(np.eye(10))[train_data.y]
val_data.y = np.float32(np.eye(10))[val_data.y]
test_data.y = np.float32(np.eye(10))[test_data.y]
# convert to [-1, 1] range
train_data.X = np.reshape(np.subtract(np.multiply(2.0/255, train_data.X), 1.0), (-1, 3, 32, 32))
val_data.X = np.reshape(np.subtract(np.multiply(2.0/255, val_data.X), 1.0), (-1, 3, 32, 32))
test_data.X = np.reshape(np.subtract(np.multiply(2.0/255, test_data.X), 1.0), (-1, 3, 32, 32))
else:
print('This dataset is not supported. Only MNIST and CIFAR-10 are supported as of now.')
raise ValueError('Dataset is not supported')
print('Trainset shape = ', train_data.X.shape, train_data.y.shape)
print('Valset shape = ', val_data.X.shape, val_data.y.shape)
print('Testset shape = ', test_data.X.shape, test_data.y.shape)
return train_data.X, val_data.X, test_data.X, train_data.y, val_data.y, test_data.y
def setUp(self):
skip_if_no_data()
self.train = MNIST(which_set = 'train')
self.test = MNIST(which_set = 'test')
cols = 10
from pylearn2.models.dbm import load_matlab_dbm
model = load_matlab_dbm('joint_trained_dbm_interm.mat', num_chains = rows * cols)
dbm = model
from theano import function
import theano.tensor as T
sample_func = function([],updates = model.get_sampling_updates())
render_func = function([],T.nnet.sigmoid(T.dot(dbm.H_chains[0],dbm.W[0].T)+dbm.bias_vis))
from pylearn2.datasets.mnist import MNIST
dataset = MNIST(which_set = 'train')
X = dataset.get_batch_design(rows*cols)
model.V_chains.set_value(X)
for i in xrange(200):
print i
sample_func()
from pylearn2.gui.patch_viewer import make_viewer
pv = make_viewer(dataset.adjust_for_viewer(render_func()))
except IndexError:
print "Usage: predict.py <model file> <test file> <output file>"
quit()
try:
model = serial.load(model_path)
except Exception, e:
print model_path + "doesn't seem to be a valid model path, I got this error when trying to load it: "
print e
# dataset = yaml_parse.load( model.dataset_yaml_src )
# dataset = dataset.get_test_set()
# or maybe specify test in yaml
dataset = MNIST(path=test_path, one_hot=True)
# use smallish batches to avoid running out of memory
batch_size = 100
model.set_batch_size(batch_size)
# dataset must be multiple of batch size of some batches will have
# different sizes. theano convolution requires a hard-coded batch size
m = dataset.X.shape[0]
extra = batch_size - m % batch_size
assert (m + extra) % batch_size == 0
import numpy as np
if extra > 0:
dataset.X = np.concatenate((dataset.X, np.zeros((extra, dataset.X.shape[1]), dtype=dataset.X.dtype)), axis=0)
assert dataset.X.shape[0] % batch_size == 0
from pylearn2.datasets.mnist import MNIST
import os, sys
import cv2
import numpy as np
test_imgs = MNIST(which_set='test', center=False)
test_imgs.X = np.reshape(test_imgs.X, (-1, 1, 28, 28))
dst_dir = os.path.join(os.getcwd(), 'mnist-testset')
if(not os.path.isdir(dst_dir)):
os.mkdir(dst_dir)
print ('No of test images = {:d}'.format(test_imgs.X.shape[0]))
print type(test_imgs.y[0,0])
for im in range(test_imgs.X.shape[0]):
img_name = os.path.join(dst_dir, 'mnist_test_img_'+str(im)+'.pgm')
print('Writing {:s}'.format(img_name))
cv2.imwrite(img_name, 255*test_imgs.X[im, 0, :, :])
with open(os.path.join(os.getcwd(), 'mnist_test_img_list.csv'), 'w') as lf:
for im in range(test_imgs.y.shape[0]):
lf.write('mnist_test_img_{:s}.pgm, {:d}\n'.format(str(im), int(test_imgs.y[im, :])))
print("LR_start = "+str(LR_start))
LR_fin = 0.0000003
print("LR_fin = "+str(LR_fin))
LR_decay = (LR_fin/LR_start)**(1./num_epochs)
print("LR_decay = "+str(LR_decay))
# BTW, LR decay might good for the BN moving average...
save_path = "mnist_parameters.npz"
print("save_path = "+str(save_path))
shuffle_parts = 1
print("shuffle_parts = "+str(shuffle_parts))
print('Loading MNIST dataset...')
train_set = MNIST(which_set= 'train', start=0, stop = 50000, center = False)
valid_set = MNIST(which_set= 'train', start=50000, stop = 60000, center = False)
test_set = MNIST(which_set= 'test', center = False)
# bc01 format
# Inputs in the range [-1,+1]
# print("Inputs in the range [-1,+1]")
train_set.X = 2* train_set.X.reshape(-1, 1, 28, 28) - 1.
valid_set.X = 2* valid_set.X.reshape(-1, 1, 28, 28) - 1.
test_set.X = 2* test_set.X.reshape(-1, 1, 28, 28) - 1.
# flatten targets
train_set.y = np.hstack(train_set.y)
valid_set.y = np.hstack(valid_set.y)
test_set.y = np.hstack(test_set.y)
nhid2 = 500 mf_iter = 10 batch_size = 100 lr = .1 lr_decay = 1.0001 min_lr = 5e-5 init_tv = 2. tv_mult = 1.001 max_tv = 10. l1wd = 0. l2wd = 0. l3wd = 0. sp_coeff = .000 sp_targ = .1 dataset = MNIST(which_set = 'train', one_hot = True) base_topo = dataset.get_topological_view() transformed_dataset = MNIST(which_set = 'train', one_hot = True) transformed_topo = base_topo[:,remove_border:-remove_border,remove_border:-remove_border,:].copy() transformed_dataset.set_topological_view(transformed_topo) transformed_topo X = sharedX(transformed_dataset.X) rng = np.random.RandomState([2012,7,24]) W1 = rng.uniform(-irange1,irange1,(nvis,nhid1)) b1 = np.zeros((nhid1,))-1.
from pylearn2.costs.cost import MethodCost
from pylearn2.datasets.mnist import MNIST
from pylearn2.models.mlp import MLP, Sigmoid, Softmax
from pylearn2.train import Train
from pylearn2.training_algorithms.sgd import SGD
from pylearn2.training_algorithms.learning_rule import Momentum, MomentumAdjustor
from pylearn2.termination_criteria import EpochCounter
train_set = MNIST(which_set='train', start=0, stop=50000)
valid_set = MNIST(which_set='train', start=50000, stop=60000)
test_set = MNIST(which_set='test')
model = MLP(nvis=784,
layers=[Sigmoid(layer_name='h', dim=500, irange=0.01),
Softmax(layer_name='y', n_classes=10, irange=0.01)])
algorithm = SGD(batch_size=100, learning_rate=0.01,
learning_rule=Momentum(init_momentum=0.5),
monitoring_dataset={'train': train_set,
'valid': valid_set,
'test': test_set},
cost=MethodCost('cost_from_X'),
termination_criterion=EpochCounter(10))
train = Train(dataset=train_set, model=model, algorithm=algorithm,
save_path="mnist_example.pkl", save_freq=1,
extensions=[MomentumAdjustor(start=5, saturate=6,
final_momentum=0.95)])
train.main_loop()
def test_ais():
"""
Test ais computation by comparing the output of estimate_likelihood to
Russ's code's output for the same parameters.
"""
try:
trainset = MNIST(which_set='train')
testset = MNIST(which_set='test')
except NoDataPathError:
raise SkipTest("PYLEARN2_DATA_PATH environment variable not defined")
nvis = 784
nhid = 20
# Random initialization of RBM parameters
numpy.random.seed(98734)
w_hid = 10 * numpy.cast[theano.config.floatX](numpy.random.randn(nvis,
nhid))
b_vis = 10 * numpy.cast[theano.config.floatX](numpy.random.randn(nvis))
b_hid = 10 * numpy.cast[theano.config.floatX](numpy.random.randn(nhid))
# Initialization of RBM
visible_layer = BinaryVector(nvis)
hidden_layer = BinaryVectorMaxPool(detector_layer_dim=nhid, pool_size=1,
layer_name='h', irange=0.1)
rbm = DBM(100, visible_layer, [hidden_layer], 1)
rbm.visible_layer.set_biases(b_vis)
rbm.hidden_layers[0].set_weights(w_hid)
rbm.hidden_layers[0].set_biases(b_hid)
rbm.nvis = nvis
rbm.nhid = nhid
# Compute real logz and associated train_ll and test_ll using rbm_tools
v_sample = T.matrix('v_sample')
h_sample = T.matrix('h_sample')
W = theano.shared(rbm.hidden_layers[0].get_weights())
hbias = theano.shared(rbm.hidden_layers[0].get_biases())
vbias = theano.shared(rbm.visible_layer.get_biases())
wx_b = T.dot(v_sample, W) + hbias
vbias_term = T.dot(v_sample, vbias)
hidden_term = T.sum(T.log(1 + T.exp(wx_b)), axis=1)
free_energy_v = -hidden_term - vbias_term
free_energy_v_fn = theano.function(inputs=[v_sample],
outputs=free_energy_v)
wh_c = T.dot(h_sample, W.T) + vbias
hbias_term = T.dot(h_sample, hbias)
visible_term = T.sum(T.log(1 + T.exp(wh_c)), axis=1)
free_energy_h = -visible_term - hbias_term
free_energy_h_fn = theano.function(inputs=[h_sample],
outputs=free_energy_h)
real_logz = rbm_tools.compute_log_z(rbm, free_energy_h_fn)
real_ais_train_ll = -rbm_tools.compute_nll(rbm,
trainset.get_design_matrix(),
real_logz, free_energy_v_fn)
real_ais_test_ll = -rbm_tools.compute_nll(rbm, testset.get_design_matrix(),
real_logz, free_energy_v_fn)
# Compute train_ll, test_ll and logz using dbm_metrics
train_ll, test_ll, logz = dbm_metrics.estimate_likelihood([W],
[vbias, hbias],
trainset,
testset,
pos_mf_steps=100)
assert (real_logz - logz) < 2.0
assert (real_ais_train_ll - train_ll) < 2.0
assert (real_ais_test_ll - test_ll) < 2.0
num_pieces = 1,
kernel_shape = (4, 4),
pool_shape = (1, 1),
pool_stride=(1, 1),
irange = 0.05)
deconv = Deconv(layer_name = 'deconv',
num_channels = 1,
kernel_shape = (4, 4),
irange = 0.05)
mlp = MLP(input_space =input_space,
layers = [conv, deconv])
mlp.layers[1].transformer._filters.set_value(mlp.layers[0].transformer._filters.get_value())
x = input_space.get_theano_batch()
out = mlp.fprop(x)
f = theano.function([x], out)
data = MNIST('test')
data_specs = (input_space, 'features')
iter = data.iterator(mode = 'sequential', batch_size = 2, data_specs = data_specs)
pv = patch_viewer.PatchViewer((10, 10), (28, 28))
for item in iter:
res = f(item)
pv.add_patch(item[0,:,:,0])
pv.add_patch(res[0,:,:,0])
pv.show()
break
def main():
data_name, model, pretrain, test, filter_channel, \
filter_size, activation, sparse_coeff, is_tied_weights, \
is_linear, do_zca, do_scale, do_maxpoool, n_epochs, batch_size = ProcessCommandLine()
print '... Loading data and parameters'
print 'dataset: ', data_name
print 'filter_channel=', filter_channel
print 'filter_size=', filter_size
print 'activation=', activation
print 'sparsity coefficient=', sparse_coeff
print 'Max pooling=', do_maxpoool
print 'tied weight=', is_tied_weights
print 'linear CAE=', is_linear
print 'ZCA whitening=', do_zca
print 'Scale image=', do_scale
print 'Batch size=', batch_size
print 'number of epoch=', n_epochs
# batch_size = 100 # number of images in each batch
n_epochs = 100 # number of experiment epochs
learning_rate = 0.1 # learning rate of SGD
# filter_channel = 16 # number of feature maps in ConvAE
# dataset = 'data/mnist.pkl.gz' # address of data
# rng = np.random.RandomState(23455) # random generator
# filter_size = 11
# n_images = 20
# sparse_coeff = 1
if sparse_coeff == 0:
model_type = 'cae'
else:
model_type = 'spcae'
model_save_name = model_type+'_'+data_name+'_'+ \
'[fn=%d,fs=%d,sp=%.3f,maxpool=%s,tied=%s,act=%s,linear=%s,ZCA=%s,scale=%s]' \
% (filter_channel, filter_size, sparse_coeff, do_maxpoool, is_tied_weights,
activation, is_linear, do_zca, do_scale)
results_dir = model_save_name+'/'
# results_dir = data_name+'_'+model_name+'/'
if not os.path.isdir(results_dir):
os.mkdir(results_dir)
if data_name == 'smallNORB':
### load smallNORB train set ###
# norb = SmallNORB('train', True)
# data_x = norb.adjust_for_viewer(norb.X)
# data_y = norb.y
# pickle.dump((data_x, data_y),open('smallNORB.pkl','wb'), -1)
# results_dir = 'smallNORB_scae/'
# results_dir = 'smallNORB_'+model_name+'/'
# if not os.path.isdir(results_dir):
# os.mkdir(results_dir)
# f = open('smallNORB.pkl', 'r')
# data, data_y = pickle.load(f)
# _, feat = data.shape
# f.close()
# train = NORB(which_norb='small', which_set='train')
# ipdb.set_trace()
# window = preprocessing.CentralWindow(window_shape=(64,64))
# train.apply_preprocessor(preprocessor=window)
# train.X = train.X.astype('float32')
# zca = preprocessing.ZCA()
# train.apply_preprocessor(preprocessor=zca, can_fit=True)
# _, feat = train.X.shape
# data_x = train.X[:, :feat/2]
norb = sio.loadmat('smallNORB_matlab/smallNORB_train_32x32.mat')
train_x = norb['trainData'].transpose(1, 0)
if do_zca:
zca = zca_whitening.ZCA()
zca.fit(train_x)
train_x = zca.transform(train_x)
if do_scale:
min_max_scaler = MinMaxScaler()
train_x_T = min_max_scaler.fit_transform(train_x.T)
train_x = train_x_T.T
data_x = train_x
idx = random.shuffle(range(data_x.shape[0]))
data_x = data_x[idx, :][0,:,:]
# ipdb.set_trace()
im_channel = 1
im_height = 32
im_width = 32
elif data_name == 'cifar':
### load cifar10 ###
# results_dir = 'cifar'+model_name+'/'
# data_x = pickle.load(open('cifar10.pkl', 'r'))
train = CIFAR10('train', gcn=55.)
im_channel = 3
im_height = 32
im_width = 32
# min_max_scaler = MinMaxScaler()
# data_x = min_max_scaler.fit_transform(data_x)
# data_x = cifar10.X
# elif data_name == 'cifarw':
# ### load cifar10 ###
# # results_dir = 'cifar'+model_name+'/'
# # data_x = pickle.load(open('cifar10_whitened.pkl', 'r'))
# train = CIFAR10('train', gcn=55.)
# # zca = preprocessing.ZCA()
# # cifar10.apply_preprocessor(preprocessor=zca, can_fit=True)
# im_channel = 3
# im_height = 32
# im_width = 32
# min_max_scaler = MinMaxScaler()
# data_x = min_max_scaler.fit_transform(cifar10.X)
# data_x = cifar10.X
# ipdb.set_trace()
elif data_name == 'svhn':
f = open('svhn.pkl', 'r')
svhn = pickle.load(f)
f.close()
im_channel = 3
im_height = 32
im_width = 32
train_x, train_y = svhn[0]
test_x, test_y = svhn[1]
extra_x, extra_y = svhn[2]
train_x = train_x.transpose([3, 2, 0, 1])
test_x = test_x.transpose([3, 2, 0, 1])
extra_x = extra_x.transpose([3, 2, 0, 1])
# Scale to [0,1]
channel_scale_factor = train_x.max(axis=(2, 3)).astype('float32')
train_x_scaled = train_x/channel_scale_factor.reshape(channel_scale_factor.shape[0], im_channel, 1, 1)
data_x = train_x_scaled.reshape(train_x.shape[0], im_channel*im_height*im_width)
elif data_name == 'mnist':
# f = open('mnist.pkl', 'r')
# mnist = pickle.load(f)
# f.close()
# train_x, train_y = mnist[0]
# valid_x, valid_y = mnist[1]
# test_x, test_y = mnist[2]
# data_x = train_x
train = MNIST('train')
# zca = preprocessing.ZCA()
# train.apply_preprocessor(preprocessor=zca, can_fit=True)
# ipdb.set_trace()
# min_max_scaler = MinMaxScaler()
# data_x = min_max_scaler.fit_transform(train.X)
# data_x = train.X
# data_y = train.y
im_channel = 1
im_height = 28
im_width = 28
elif data_name == 'bmnist':
train = BinarizedMNIST(which_set='train')
# zca = preprocessing.ZCA()
# train.apply_preprocessor(preprocessor=zca, can_fit=True)
# ipdb.set_trace()
# min_max_scaler = MinMaxScaler()
# data_x = min_max_scaler.fit_transform(train.X)
# data_x = train.X
# data_y = train.y
im_channel = 1
im_height = 28
im_width = 28
if do_zca and data_name not in ['smallNORB', 'svhn']:
zca = preprocessing.ZCA()
train.apply_preprocessor(preprocessor=zca, can_fit=True)
data_x = train.X
pass
if do_scale and data_name not in ['smallNORB', 'svhn']:
min_max_scaler = MinMaxScaler()
data_x_T = min_max_scaler.fit_transform(train.X.T)
data_x = data_x_T.T
pass
if not do_zca and not do_scale and data_name not in ['smallNORB', 'svhn']:
data_x = train.X
# if data_name not in ['smallNORB']:
# data_x = train.X
n_samples, n_feat = data_x.shape
data_x = data_x.reshape((n_samples, im_channel, im_height, im_width))
if data_name == 'mnist':
data_x = data_x.transpose(0,1,3,2)
train_set_x = theano.shared(np.asarray(data_x, dtype=np.float32), borrow=True)
# image_shp = (batch_size, im_channel, data_x.shape[2], data_x.shape[3])
image_shp = (batch_size, im_channel, im_height, im_width)
filter_shp = (filter_channel, im_channel, filter_size, filter_size)
print 'building model'
cae1 = CAE(image_shape=image_shp,
data=train_set_x,
filter_shape=filter_shp,
poolsize=(2, 2),
sparse_coeff=sparse_coeff,
activation=activation,
do_max_pool=do_maxpoool,
tied_weight=is_tied_weights,
is_linear=is_linear)
print 'model built'
sys.stdout.flush()
if model:
cae1.load(model)
pass
if pretrain:
do_pretraining_cae(data_name=data_name,
model=cae1,
save_name=model_save_name,
image_shape=image_shp,
result_dir=results_dir,
max_epoch=n_epochs)
elif test:
do_visualize(data_name=data_name,
model=cae1,
result_dir=results_dir)
# MAIN
# stochastic learning rate works with continuous weights
# multilayer does not work so far
# discrete weights + continuous bias do work for one layer
# batch and activations -> only update the most wrong weight
if __name__ == "__main__":
print 'Beginning of the program'
start_time = time.clock()
print 'Loading the dataset'
train_set = MNIST(which_set= 'train', start=0, stop = 50000, center = True)
valid_set = MNIST(which_set= 'train', start=50000, stop = 60000, center = True)
test_set = MNIST(which_set= 'test', center = True)
# for both datasets, onehot the target
train_set.y = np.float32(onehot(train_set.y))
valid_set.y = np.float32(onehot(valid_set.y))
test_set.y = np.float32(onehot(test_set.y))
train_set.y = 2* train_set.y - 1.
valid_set.y = 2* valid_set.y - 1.
test_set.y = 2* test_set.y - 1.
# print train_set.X
# print np.shape(train_set.X)
# print np.max(train_set.X)
job_name = 'cfa_test' from pylearn2.datasets.mnist import MNIST from pylearn2.pca import CovEigPCA import theano.tensor as T from theano import function from models import expand from pylearn2.utils import serial import time import SkyNet import gc import numpy as N print 'Loading MNIST train set' t1 = time.time() X = MNIST(which_set='train').get_design_matrix() t2 = time.time() print(t2 - t1), ' seconds' num_examples, input_dim = X.shape print 'Training PCA with %d dimensions' % pca_dim t1 = time.time() pca_model = CovEigPCA(num_components=pca_dim) pca_model.train(X) t2 = time.time() print(t2 - t1), ' seconds' print 'Compiling theano PCA function' t1 = time.time() pca_input = T.matrix()
print("batch_size = "+str(batch_size))
# MLP parameters
num_units = 4096
print("num_units = "+str(num_units))
n_hidden_layers = 3
print("n_hidden_layers = "+str(n_hidden_layers))
# kernel = "baseline"
kernel = "xnor"
# kernel = "theano"
print("kernel = "+ kernel)
print('Loading MNIST dataset...')
test_set = MNIST(which_set= 'test', center = False)
# Inputs in the range [-1,+1]
test_set.X = 2* test_set.X.reshape(-1, 784) - 1.
# flatten targets
test_set.y = test_set.y.reshape(-1)
print('Building the MLP...')
# Prepare Theano variables for inputs and targets
input = T.matrix('inputs')
target = T.vector('targets')
mlp = lasagne.layers.InputLayer(shape=(None, 784),input_var=input)
# Input layer is not binary -> use baseline kernel in first hidden layer
mlp = binary_ops.DenseLayer(
def setUp(self):
skip_if_no_data()
# TODO: the one_hot=True is only necessary because one_hot=False is
# broken, remove it after one_hot=False is fixed.
self.train = MNIST(which_set = 'train', one_hot=True)
self.test = MNIST(which_set = 'test', one_hot=True)
#gets
#doing epoch 3999
# TRAIN acc [ 0.99998333]
# terminal velocity multiplier: 10.0
# learning rate: 0.0670400485022
# test acc: [array(0.9832)]
from pylearn2.datasets.mnist import MNIST
from theano.printing import Print
import numpy as np
from cDBM import cDBM
import sys
name = sys.argv[0].replace('.py','')
remove_border = 3
dataset = MNIST(which_set = 'train', one_hot = True)
topo = dataset.get_topological_view()
topo = topo[:,remove_border:-remove_border,remove_border:-remove_border,:]
dataset.set_topological_view(topo)
rng = np.random.RandomState([2012,07,24])
irange1 = .05
irange2 = .05
irange3 = .05
nvis = (28-2*remove_border)**2
nclass = 10
nhid1 = 500
nhid2 = 500
mf_iter = 10
batch_size = 100
lr = .1
return result
# MAIN
if __name__ == "__main__":
print 'Beginning of the program'
start_time = time.clock()
print 'Loading the dataset'
dataset = sys.argv[1]
if dataset == "PI_MNIST" or dataset == "MNIST":
train_set = MNIST(which_set= 'train',start=0, stop = 50000)#, center = True)
valid_set = MNIST(which_set= 'train',start=50000, stop = 60000)#, center = True)
test_set = MNIST(which_set= 'test')#, center = True)
# for both datasets, onehot the target
train_set.y = np.float32(onehot(train_set.y))
valid_set.y = np.float32(onehot(valid_set.y))
test_set.y = np.float32(onehot(test_set.y))
elif dataset == "CIFAR10":
preprocessor = serial.load("${PYLEARN2_DATA_PATH}/cifar10/pylearn2_gcn_whitened/preprocessor.pkl")
train_set = ZCA_Dataset(
preprocessed_dataset=serial.load("${PYLEARN2_DATA_PATH}/cifar10/pylearn2_gcn_whitened/train.pkl"),
preprocessor = preprocessor,
start=0, stop = 45000)
print("batch_size = "+str(batch_size))
# MLP parameters
num_units = 4096
print("num_units = "+str(num_units))
n_hidden_layers = 3
print("n_hidden_layers = "+str(n_hidden_layers))
# kernel = "baseline"
kernel = "xnor"
# kernel = "theano"
print("kernel = "+ kernel)
print('Loading MNIST dataset...')
test_set = MNIST(which_set= 'test', center = False)
# Inputs in the range [-1,+1]
test_set.X = 2* test_set.X.reshape(-1, 784) - 1.
# flatten targets
test_set.y = test_set.y.reshape(-1)
print('Building the MLP...')
# Prepare Theano variables for inputs and targets
input = T.matrix('inputs')
target = T.vector('targets')
mlp = lasagne.layers.InputLayer(shape=(None, 784),input_var=input)
# Input layer is not binary -> use baseline kernel in first hidden layer
mlp = binary_ops.DenseLayer(
from pylearn2.termination_criteria import MonitorBased
from pylearn2.train_extensions import best_params
from pylearn2.utils import serial
from theano import function
from theano import tensor as T
import numpy as np
import os
h0 = mlp.Sigmoid(layer_name='h0', dim=500, sparse_init=15)
ylayer = mlp.Softmax(layer_name='y', n_classes=10, irange=0)
layers = [h0, ylayer]
model = mlp.MLP(layers, nvis=784)
train = MNIST('train', one_hot=1, start=0, stop=50000)
valid = MNIST('train', one_hot=1, start=50000, stop=60000)
test = MNIST('test', one_hot=1, start=0, stop=10000)
monitoring = dict(valid=valid)
termination = MonitorBased(channel_name="valid_y_misclass")
extensions = [best_params.MonitorBasedSaveBest(channel_name="valid_y_misclass",
save_path="train_best.pkl")]
algorithm = bgd.BGD(batch_size=10000, line_search_mode = 'exhaustive', conjugate = 1,
monitoring_dataset = monitoring, termination_criterion = termination)
save_path = "train_best.pkl"
if os.path.exists(save_path):
model = serial.load(save_path)
else:
print 'Running training'
train_job = Train(train, model, algorithm, extensions=extensions, save_path="train.pkl", save_freq=1)
train_job.main_loop()
def main():
#import pdb
#pdb.set_trace()
###################
#BUILD THE DATASET#
###################
print 'build the dataset'
train_set = MNIST(which_set='train', one_hot=False)
test_set = MNIST(which_set='test', one_hot=False)
train_setX, valid_setX = split(train_set.X, [50000], axis=0)
train_sety, valid_sety = split(train_set.y, [50000], axis=0)
#import pdb
#pdb.set_trace()
##################
#BUILD THE LAYERS#
##################
print 'build the layers'
input_size = len(train_setX[0])
h1 = NoisyRELU(prev_layer_size=input_size, this_layer_size=2000, noise_factor=1, threshold=5)
output_layer = Softmax(prev_layer_size=h1.this_layer_size, this_layer_size=10,
W_range=[0,0], b_range=[0,0])
#y_layer = Sigmoid(prev_layer_size=h2.this_layer_size, this_layer_size=[10,1])
# import pdb
# pdb.set_trace()
#, 0.1, 0.5, 0.001, 0.005
plots = []
legends = []
batches = []
errors = []
for learning_rate in [0.01, 0.05]:
print 'build the model'
print 'learning rate', learning_rate
mlp = MLP(input_size=input_size,
layers=[h1, output_layer],
train_set=[train_setX, train_sety],
valid_set=[valid_setX, valid_sety],
test_set=[test_set.X, test_set.y],
error_function=loglikehood,
batch_size=20,
learning_rate=learning_rate)
print 'start training'
mlp.train()
#p = plt.plot(mlp.epoch, mlp.valid_error)
#plots.append(p)
batches.append(mlp.epoch)
errors.append(mlp.valid_error)
legends.append(learning_rate)
with open('batches.pkl', 'wb') as bat:
cPickle.dump(batches, bat)
with open('errors.pkl', 'wb') as err:
cPickle.dump(errors, err)
with open('legends.pkl', 'wb') as leg:
cPickle.dump(legends, leg)
# Old hyperparameters
binary_training=False
stochastic_training=False
binary_test=False
stochastic_test=False
if BinaryConnect == True:
binary_training=True
if stochastic == True:
stochastic_training=True
else:
binary_test=True
print 'Loading the dataset'
train_set = MNIST(which_set= 'train', start=0, stop = train_set_size, center = True)
valid_set = MNIST(which_set= 'train', start=50000, stop = 60000, center = True)
test_set = MNIST(which_set= 'test', center = True)
# bc01 format
train_set.X = train_set.X.reshape(train_set_size,1,28,28)
valid_set.X = valid_set.X.reshape(10000,1,28,28)
test_set.X = test_set.X.reshape(10000,1,28,28)
# flatten targets
train_set.y = np.hstack(train_set.y)
valid_set.y = np.hstack(valid_set.y)
test_set.y = np.hstack(test_set.y)
# Onehot the targets
train_set.y = np.float32(np.eye(10)[train_set.y])
