def main():
dataset_file = 'mnist.npz'
if not isfile(dataset_file):
downloadMnist(dataset_file)
data = dict(np.load(dataset_file))
data['x_tr'] = data['x_tr'] * 2.0 - 1.0
data['x_va'] = data['x_va'] * 2.0 - 1.0
data['x_te'] = data['x_te'] * 2.0 - 1.0
data['t_tr'] = data['t_tr'].astype(np.int32)
data['t_va'] = data['t_va'].astype(np.int32)
data['t_te'] = data['t_te'].astype(np.int32)
rng = np.random.RandomState()
srng = RandomStreamsGPU(rng.randint(1, 2147462579, size=(6,)))
# Setup data loaders
train_generator = DefaultDataLoader(data['x_tr'], data['t_tr'], 100, rng=rng)
validation_generator = DefaultDataLoader(data['x_va'], data['t_va'], 100)
test_generator = DefaultDataLoader(data['x_te'], data['t_te'], 100)
# Load real-valued model parameters for initialization
init_model_file = 'mnist_pi_model_ternary_tanh.npz'
if isfile(init_model_file):
initial_parameters = dict(np.load(init_model_file))
print 'Loading initial parameters from \'%s\'' % (init_model_file)
print 'Parameters:', [e for e in initial_parameters]
else:
raise Exception('Cannot find initial model \'%s\'' % (init_model_file))
# Create model
global parameters
layer, parameters = getMnistPIModel(initial_parameters, rng, srng)
# Do optimization
print layer.getMessage()
cbErrVaDecreased = lambda : cbValidationErrorDecreased()
global p_vals
optimizeNetwork(layer,
loader_tr=train_generator,
loader_va=validation_generator,
loader_te=test_generator,
optimization_algorithm='adam',
step_size=1e-3,
step_size_discrete=1e-2,
step_size_scale_fn={'type' : 'plateau',
'monitor' : 'ce_va',
'cooldown' : 50,
'patience' : 10,
'factor' : 0.5},
n_epochs=500,
do_bn_updates_after_epoch=True,
callback_validation_error_decreased=[(cbErrVaDecreased, [])])
# Store model parameters. The model parameters of the best model according to the validation error are now in
# p_vals.
model_file = 'mnist_pi_model_ternary_sign_from_tanh.npz'
print 'Optimization finished. Storing model parameters to ''%s''' % model_file
np.savez_compressed(model_file, **p_vals)
def loadDataset(dataset, pca=None):
'''
Loads the given data set and returns its training set, validation set, and
test set. If the given file is not found on the file system, it is
downloaded from the internet first. Optionally, PCA is performed to reduce
the input features to fewer dimensions and to perform whitening.
dataset: The data set to be loaded. 'mnist' and 'mnist_basic' will be
downloaded if they are not found on the file system. Other data sets must
be located in a file named '<dataset>.npz'
pca: The number of dimensions to which the input features should be reduced
using PCA or None if no PCA should be performed
'''
if dataset == 'mnist':
if not isfile('mnist.npz'):
downloadMnist('mnist.npz')
elif dataset == 'mnist_basic':
if not isfile('mnist_basic.npz'):
downloadMnistBasic('mnist_basic.npz')
with np.load(dataset + '.npz') as data:
x_tr_np = data['x_tr_np']
t_tr_np = data['t_tr_np']
x_va_np = data['x_va_np']
t_va_np = data['t_va_np']
x_te_np = data['x_te_np']
t_te_np = data['t_te_np']
if pca is not None:
x_tr_np, x_va_np, x_te_np = pcaTransform(x_tr_np, x_va_np, x_te_np, pca)
return x_tr_np, t_tr_np, x_va_np, t_va_np, x_te_np, t_te_np
def main():
dataset_file = 'mnist.npz'
if not isfile(dataset_file):
downloadMnist(dataset_file)
data = dict(np.load(dataset_file))
data['x_tr'] = data['x_tr'] * 2.0 - 1.0
data['x_va'] = data['x_va'] * 2.0 - 1.0
data['x_te'] = data['x_te'] * 2.0 - 1.0
data['t_tr'] = data['t_tr'].astype(np.int32)
data['t_va'] = data['t_va'].astype(np.int32)
data['t_te'] = data['t_te'].astype(np.int32)
rng = np.random.RandomState()
srng = RandomStreamsGPU(rng.randint(1, 2147462579, size=(6, )))
# Setup data loaders
train_generator = DefaultDataLoader(data['x_tr'],
data['t_tr'],
100,
rng=rng)
validation_generator = DefaultDataLoader(data['x_va'], data['t_va'], 100)
test_generator = DefaultDataLoader(data['x_te'], data['t_te'], 100)
# Create model
global parameters
layer, parameters = getMnistPIModel(rng, srng)
# Do optimization
print layer.getMessage()
cbErrVaDecreased = lambda: cbValidationErrorDecreased()
global p_vals
optimizeNetwork(layer,
loader_tr=train_generator,
loader_va=validation_generator,
loader_te=test_generator,
optimization_algorithm='adam',
step_size=1e-3,
step_size_scale_fn={
'type': 'plateau',
'monitor': 'ce_va',
'cooldown': 150,
'patience': 25,
'factor': 0.5
},
n_epochs=1000,
callback_validation_error_decreased=[(cbErrVaDecreased, [])
])
# Store model parameters. The model parameters of the best model according to the validation error are now in
# p_vals.
model_file = 'mnist_pi_model_real.npz'
print 'Optimization finished. Storing model parameters to ' '%s' '' % model_file
np.savez_compressed(model_file, **p_vals)
def loadDataset(dataset, pca=None, n_labeled=50000, n_unlabeled=0):
'''
Loads the given data set and returns its training set, validation set, and
test set. If the given file is not found on the file system, it is
downloaded from the internet first. Optionally, PCA is performed to reduce
the input features to fewer dimensions and to perform whitening. The number
of labeled and unlabeled training samples can be varied. Larger values of
n_labeled result in a strict superset of labeled samples. For fixed
n_labeled samples, larger values of n_unlabeled result in a strict superset
of unlabeled samples. A fixed seed for the random number generator is used
to obtain the same data sets used for the experiments in [1].
dataset: The data set to be loaded. 'mnist' and 'mnist_basic' will be
downloaded if they are not found on the file system. Other data sets must
be located in a file named '<dataset>.npz'
pca: The number of dimensions to which the input features should be reduced
using PCA or None if no PCA should be performed
n_labeled: The number of labeled training samples in the data set.
n_unlabeled: The number of unlabeled training samples in the data set. Note
that if n_labeled and n_unlabeled do not sum up to the number of total
samples, some samples are removed from the data set.
[1] Roth W., Peharz R., Tschiatschek S., Pernkopf F., Hybrid generative-
discriminative training of Gaussian mixture models, Pattern Recognition
Letters 2018, (accepted)
'''
if dataset == 'mnist':
if not isfile('mnist.npz'):
downloadMnist('mnist.npz')
elif dataset == 'mnist_basic':
if not isfile('mnist_basic.npz'):
downloadMnistBasic('mnist_basic.npz')
with np.load(dataset + '.npz') as data:
x_tr_np = data['x_tr_np']
t_tr_np = data['t_tr_np']
x_va_np = data['x_va_np']
t_va_np = data['t_va_np']
x_te_np = data['x_te_np']
t_te_np = data['t_te_np']
if pca is not None:
x_tr_np, x_va_np, x_te_np = pcaTransform(x_tr_np, x_va_np, x_te_np,
pca)
# Generate random labeled/unlabeled data. The labels are removed as follows:
# (1) Generate a random permutation of size N_tr
# (2) The first n_labeled samples of the random permutation are indices of
# labeled samples.
# (3) The next n_unlabeled samples of the random permutation are indices of
# unlabeled samples. Note that some samples are excluded if n_labeled+
# n_unlabeled is not equal to the total number of samples.
# (4) The first n_labeled+n_unlabeled samples of the random permutation are
# the used samples. These indices are sorted to preserve the ordering of
# the original data set.
if n_labeled != x_tr_np.shape[0]:
rng_ssl = np.random.RandomState(1)
randperm = rng_ssl.permutation(np.arange(x_tr_np.shape[0]))
idx_used = np.sort(randperm[:n_labeled + n_unlabeled])
idx_unlabeled = randperm[n_labeled:n_labeled + n_unlabeled]
t_tr_np[idx_unlabeled] = -1 # set to unlabeled
t_tr_np = t_tr_np[idx_used]
x_tr_np = x_tr_np[idx_used[:, None], np.arange(x_tr_np.shape[1])]
return x_tr_np, t_tr_np, x_va_np, t_va_np, x_te_np, t_te_np