#running this with local mean sub resulted in it getting about 70.98% test acc #added local division by variance. irange was .05 before but this resulted in # optimization not going anywhere for local division by variance. # bumped it up to .17 (weight norm about 1 on average) to make things initially clear the threshold # this resulted in it hovering around 71.8 test acc from pylearn2.datasets.cifar10 import CIFAR10 import time import warnings from theano.printing import Print import numpy as np from pylearn2.utils import serial from theano.gof.op import get_debug_values from theano.printing import min_informative_str dataset = CIFAR10(which_set='train', one_hot=True, gcn=55.) rng = np.random.RandomState([2012, 07, 24]) irange = 1. nvis = 784 nclass = 10 nhid = 500 mf_iter = 10 batch_size = 10 lr = .01 momentum = 1. / 20. from pylearn2.utils import sharedX import theano.tensor as T X = dataset.X
layer = int(layer_str)
model = serial.load(model_path)
# Get access to the intermediate layers of the augmented DBM
if hasattr(model, 'super_dbm'):
model = model.super_dbm
model.set_batch_size(m)
if hasattr(model, 'dataset_yaml_src'):
dataset = yaml_parse.load(model.dataset_yaml_src)
else:
from pylearn2.datasets.cifar10 import CIFAR10
dataset = CIFAR10(which_set='test', gcn=55.)
X = sharedX(dataset.get_batch_topo(m))
H_hat = model.mf(X)
H_hat = H_hat[layer]
H_hat
p, h = model.hidden_layers[layer].state_to_b01c(H_hat)
f = function([], [p, h])
while True:
p, h = f()
LR_start = 0.001
print("LR_start = "+str(LR_start))
LR_fin = 0.0000003
print("LR_fin = "+str(LR_fin))
LR_decay = (LR_fin/LR_start)**(1./num_epochs)
print("LR_decay = "+str(LR_decay))
# BTW, LR decay might good for the BN moving average...
train_set_size = 45000
print("train_set_size = "+str(train_set_size))
shuffle_parts = 1
print("shuffle_parts = "+str(shuffle_parts))
print('Loading CIFAR-10 dataset...')
train_set = CIFAR10(which_set="train", start=0, stop=train_set_size)
valid_set = CIFAR10(which_set="train", start=train_set_size, stop=50000)
test_set = CIFAR10(which_set="test")
# bc01 format
# Inputs in the range [-1,+1]
# print("Inputs in the range [-1,+1]")
train_set.X = np.reshape(
np.subtract(np.multiply(2./255., train_set.X), 1.), (-1, 3, 32, 32))
valid_set.X = np.reshape(
np.subtract(np.multiply(2./255., valid_set.X), 1.), (-1, 3, 32, 32))
test_set.X = np.reshape(
np.subtract(np.multiply(2./255., test_set.X), 1.), (-1, 3, 32, 32))
# flatten targets
train_set.y = np.hstack(train_set.y)
def main():
data_name, model, pretrain, test, filter_channel, \
filter_size, activation, sparse_coeff, is_tied_weights, \
is_linear, do_zca, do_scale, do_maxpoool, n_epochs, batch_size = ProcessCommandLine()
print '... Loading data and parameters'
print 'dataset: ', data_name
print 'filter_channel=', filter_channel
print 'filter_size=', filter_size
print 'activation=', activation
print 'sparsity coefficient=', sparse_coeff
print 'Max pooling=', do_maxpoool
print 'tied weight=', is_tied_weights
print 'linear CAE=', is_linear
print 'ZCA whitening=', do_zca
print 'Scale image=', do_scale
print 'Batch size=', batch_size
print 'number of epoch=', n_epochs
# batch_size = 100 # number of images in each batch
n_epochs = 100 # number of experiment epochs
learning_rate = 0.1 # learning rate of SGD
# filter_channel = 16 # number of feature maps in ConvAE
# dataset = 'data/mnist.pkl.gz' # address of data
# rng = np.random.RandomState(23455) # random generator
# filter_size = 11
# n_images = 20
# sparse_coeff = 1
if sparse_coeff == 0:
model_type = 'cae'
else:
model_type = 'spcae'
model_save_name = model_type+'_'+data_name+'_'+ \
'[fn=%d,fs=%d,sp=%.3f,maxpool=%s,tied=%s,act=%s,linear=%s,ZCA=%s,scale=%s]' \
% (filter_channel, filter_size, sparse_coeff, do_maxpoool, is_tied_weights,
activation, is_linear, do_zca, do_scale)
results_dir = model_save_name + '/'
# results_dir = data_name+'_'+model_name+'/'
if not os.path.isdir(results_dir):
os.mkdir(results_dir)
if data_name == 'smallNORB':
### load smallNORB train set ###
# norb = SmallNORB('train', True)
# data_x = norb.adjust_for_viewer(norb.X)
# data_y = norb.y
# pickle.dump((data_x, data_y),open('smallNORB.pkl','wb'), -1)
# results_dir = 'smallNORB_scae/'
# results_dir = 'smallNORB_'+model_name+'/'
# if not os.path.isdir(results_dir):
# os.mkdir(results_dir)
# f = open('smallNORB.pkl', 'r')
# data, data_y = pickle.load(f)
# _, feat = data.shape
# f.close()
# train = NORB(which_norb='small', which_set='train')
# ipdb.set_trace()
# window = preprocessing.CentralWindow(window_shape=(64,64))
# train.apply_preprocessor(preprocessor=window)
# train.X = train.X.astype('float32')
# zca = preprocessing.ZCA()
# train.apply_preprocessor(preprocessor=zca, can_fit=True)
# _, feat = train.X.shape
# data_x = train.X[:, :feat/2]
norb = sio.loadmat('smallNORB_matlab/smallNORB_train_32x32.mat')
train_x = norb['trainData'].transpose(1, 0)
if do_zca:
zca = zca_whitening.ZCA()
zca.fit(train_x)
train_x = zca.transform(train_x)
if do_scale:
min_max_scaler = MinMaxScaler()
train_x_T = min_max_scaler.fit_transform(train_x.T)
train_x = train_x_T.T
data_x = train_x
idx = random.shuffle(range(data_x.shape[0]))
data_x = data_x[idx, :][0, :, :]
# ipdb.set_trace()
im_channel = 1
im_height = 32
im_width = 32
elif data_name == 'cifar':
### load cifar10 ###
# results_dir = 'cifar'+model_name+'/'
# data_x = pickle.load(open('cifar10.pkl', 'r'))
train = CIFAR10('train', gcn=55.)
im_channel = 3
im_height = 32
im_width = 32
# min_max_scaler = MinMaxScaler()
# data_x = min_max_scaler.fit_transform(data_x)
# data_x = cifar10.X
# elif data_name == 'cifarw':
# ### load cifar10 ###
# # results_dir = 'cifar'+model_name+'/'
# # data_x = pickle.load(open('cifar10_whitened.pkl', 'r'))
# train = CIFAR10('train', gcn=55.)
# # zca = preprocessing.ZCA()
# # cifar10.apply_preprocessor(preprocessor=zca, can_fit=True)
# im_channel = 3
# im_height = 32
# im_width = 32
# min_max_scaler = MinMaxScaler()
# data_x = min_max_scaler.fit_transform(cifar10.X)
# data_x = cifar10.X
# ipdb.set_trace()
elif data_name == 'svhn':
f = open('svhn.pkl', 'r')
svhn = pickle.load(f)
f.close()
im_channel = 3
im_height = 32
im_width = 32
train_x, train_y = svhn[0]
test_x, test_y = svhn[1]
extra_x, extra_y = svhn[2]
train_x = train_x.transpose([3, 2, 0, 1])
test_x = test_x.transpose([3, 2, 0, 1])
extra_x = extra_x.transpose([3, 2, 0, 1])
# Scale to [0,1]
channel_scale_factor = train_x.max(axis=(2, 3)).astype('float32')
train_x_scaled = train_x / channel_scale_factor.reshape(
channel_scale_factor.shape[0], im_channel, 1, 1)
data_x = train_x_scaled.reshape(train_x.shape[0],
im_channel * im_height * im_width)
elif data_name == 'mnist':
# f = open('mnist.pkl', 'r')
# mnist = pickle.load(f)
# f.close()
# train_x, train_y = mnist[0]
# valid_x, valid_y = mnist[1]
# test_x, test_y = mnist[2]
# data_x = train_x
train = MNIST('train')
# zca = preprocessing.ZCA()
# train.apply_preprocessor(preprocessor=zca, can_fit=True)
# ipdb.set_trace()
# min_max_scaler = MinMaxScaler()
# data_x = min_max_scaler.fit_transform(train.X)
# data_x = train.X
# data_y = train.y
im_channel = 1
im_height = 28
im_width = 28
elif data_name == 'bmnist':
train = BinarizedMNIST(which_set='train')
# zca = preprocessing.ZCA()
# train.apply_preprocessor(preprocessor=zca, can_fit=True)
# ipdb.set_trace()
# min_max_scaler = MinMaxScaler()
# data_x = min_max_scaler.fit_transform(train.X)
# data_x = train.X
# data_y = train.y
im_channel = 1
im_height = 28
im_width = 28
if do_zca and data_name not in ['smallNORB', 'svhn']:
zca = preprocessing.ZCA()
train.apply_preprocessor(preprocessor=zca, can_fit=True)
data_x = train.X
pass
if do_scale and data_name not in ['smallNORB', 'svhn']:
min_max_scaler = MinMaxScaler()
data_x_T = min_max_scaler.fit_transform(train.X.T)
data_x = data_x_T.T
pass
if not do_zca and not do_scale and data_name not in ['smallNORB', 'svhn']:
data_x = train.X
# if data_name not in ['smallNORB']:
# data_x = train.X
n_samples, n_feat = data_x.shape
data_x = data_x.reshape((n_samples, im_channel, im_height, im_width))
if data_name == 'mnist':
data_x = data_x.transpose(0, 1, 3, 2)
train_set_x = theano.shared(np.asarray(data_x, dtype=np.float32),
borrow=True)
# image_shp = (batch_size, im_channel, data_x.shape[2], data_x.shape[3])
image_shp = (batch_size, im_channel, im_height, im_width)
filter_shp = (filter_channel, im_channel, filter_size, filter_size)
print 'building model'
cae1 = CAE(image_shape=image_shp,
data=train_set_x,
filter_shape=filter_shp,
poolsize=(2, 2),
sparse_coeff=sparse_coeff,
activation=activation,
do_max_pool=do_maxpoool,
tied_weight=is_tied_weights,
is_linear=is_linear)
print 'model built'
sys.stdout.flush()
if model:
cae1.load(model)
pass
if pretrain:
do_pretraining_cae(data_name=data_name,
model=cae1,
save_name=model_save_name,
image_shape=image_shp,
result_dir=results_dir,
max_epoch=n_epochs)
elif test:
do_visualize(data_name=data_name, model=cae1, result_dir=results_dir)
exit(-1)
top_dir = os.environ['CRAFT_BNN_ROOT']
params_dir = top_dir + '/params'
# BinaryOut
activation = hardware_net.SignTheano
print("activation = sign(x)")
no_bias = True
print("no_bias = " + str(no_bias))
# BinaryConnect
H = 1.
print('Loading CIFAR-10 dataset...')
test_set = CIFAR10(which_set="test")
print("Test set size = " + str(len(test_set.X)))
test_instances = 1
print("Using instances 0 .. " + str(test_instances))
# bc01 format
# Inputs in the range [-1,+1]
test_set.X = np.reshape(
np.subtract(np.multiply(2. / 255., test_set.X), 1.), (-1, 3, 32, 32))
# flatten targets
test_set.y = np.hstack(test_set.y)
# Onehot the targets
test_set.y = np.float32(np.eye(10)[test_set.y])
# for hinge loss
test_set.y = 2 * test_set.y - 1.
from pylearn2.utils import serial
from pylearn2.datasets.cifar10 import CIFAR10
from numpy.linalg import lstsq
import numpy as np
import sys
print 'Loading labels'
y = CIFAR10(which_set='train', one_hot=True).y
ignore, features_path = sys.argv
print 'Loading features'
X = serial.load(features_path)
def train(X, y):
print 'Solving'
return lstsq(X, y)[0]
X_train = X[0:40000, :]
y_train = y[0:40000, :]
W = train(X_train, y_train)
def acc(W, X, y):
Z = np.dot(X, W)
y_hat = np.argmax(Z, axis=1)
y = np.argmax(y, axis=1)
acc = (y_hat == y).mean()
import theano
import theano.tensor as T
import lasagne
import binary_net
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Testing Script')
parser.add_argument('--dataset', '-d', default="cifar10", help='dataset to use cifar10, cifar100, mnist')
parser.add_argument('--model', '-m', default="cnv", help='model to use resnet, lenet, inception, cnv')
args = parser.parse_args()
batch_size = 10000
if args.dataset == 'cifar10':
print('Loading CIFAR-10 dataset...')
from pylearn2.datasets.cifar10 import CIFAR10
test_set = CIFAR10(which_set="test", start=0, stop = batch_size)
classes = 10
test_set.X = np.reshape(np.subtract(np.multiply(2./255,test_set.X),1.),(-1,3,32,32))
elif args.dataset == 'cifar100':
print('Loading CIFAR-100 dataset...')
from pylearn2.datasets.cifar100 import CIFAR100
test_set = CIFAR100(which_set="test", start=0, stop = batch_size)
classes = 100
test_set.X = np.reshape(np.subtract(np.multiply(2./255,test_set.X),1.),(-1,3,32,32))
elif args.dataset == 'mnist':
print('Loading MNIST dataset...')
from pylearn2.datasets.mnist import MNIST
test_set = MNIST(which_set="test", start=0, stop = batch_size)
classes = 10