def main(argv):
parser = ArgumentParser(argv[0], description=__doc__)
parser.add_argument('model', type=str)
parser.add_argument('--data', '-d', type=str, default='data/BSDS300_8x8.mat')
parser.add_argument('--num_data', '-N', type=int, default=1000000)
args = parser.parse_args(argv[1:])
# load data
data = loadmat(args.data)['patches_test']
if args.num_data > 0 and args.num_data < data.shape[0]:
# select random subset of data
data = data[random_select(args.num_data, data.shape[0])]
print 'Transforming data...'
# transform data
data_all = [
data,
transform(data, True, False, False),
transform(data, False, True, False),
transform(data, True, True, False),
transform(data, False, False, True),
transform(data, True, False, True),
transform(data, False, True, True),
transform(data, True, True, True)]
# each entry corresponds to a different model/transformation
loglik = [0.] * len(data_all)
# load model
experiment = Experiment(args.model)
models = experiment['models'] # models for the different pixels
preconditioners = experiment['preconditioners']
if len(models) != data.shape[1]:
print 'Wrong number of models!'
return 0
for i, model in enumerate(models):
for n, data in enumerate(data_all):
inputs, outputs = data.T[:i], data.T[[i]]
# this sums over pixels
if isinstance(model, MoGSM):
loglik[n] = loglik[n] + model.loglikelihood(outputs)
else:
loglik[n] = loglik[n] + model.loglikelihood(*preconditioners[i](inputs, outputs)) \
+ preconditioners[i].logjacobian(inputs, outputs)
print '{0}/{1}'.format(i + 1, data.shape[1])
# each row of loglik is a different model/transformation, each column a different data point
loglik = logsumexp(loglik, 0) - log(len(loglik))
print '{0:.4f} [nat]'.format(mean(loglik))
return 0
def main(argv):
parser = ArgumentParser(argv[0], description=__doc__)
parser.add_argument('model', type=str)
parser.add_argument('--data', '-d', type=str, default='data/BSDS300_8x8.mat')
parser.add_argument('--num_data', '-N', type=int, default=1000000)
parser.add_argument('--batch_size', '-B', type=int, default=100000)
args = parser.parse_args(argv[1:])
data_test = loadmat(args.data)['patches_test']
dim = data_test.shape[1]
# reconstruct patches
patch_size = int(sqrt(data_test.shape[1] + 1) + .5)
data_test = hstack([data_test, -sum(data_test, 1)[:, None]])
data_test = data_test.reshape(-1, patch_size, patch_size)
if args.num_data > 0 and data_test.shape[0] > args.num_data:
data_test = data_test[random_select(args.num_data, data_test.shape[0])]
model = Experiment(args.model)['model']
# container for average log-likelihoods of batches
logliks = []
try:
for b in range(0, data_test.shape[0], args.batch_size):
# select batch
batch = data_test[b:b + args.batch_size]
# compute average log-likelihood of different models
loglik = []
loglik.append(model.loglikelihood(batch))
loglik.append(model.loglikelihood(transform(batch, True, False, False)))
loglik.append(model.loglikelihood(transform(batch, False, True, False)))
loglik.append(model.loglikelihood(transform(batch, True, True, False)))
loglik.append(model.loglikelihood(transform(batch, False, False, True)))
loglik.append(model.loglikelihood(transform(batch, True, False, True)))
loglik.append(model.loglikelihood(transform(batch, False, True, True)))
loglik.append(model.loglikelihood(transform(batch, True, True, True)))
# compute average log-likelihood of mixture model
loglik = logsumexp(loglik, 0) - log(len(loglik))
logliks.append(loglik.mean())
print '{0:.3f} [nat]'.format(mean(logliks))
except KeyboardInterrupt:
pass
import pdb
pdb.set_trace()
def main(argv):
parser = ArgumentParser(argv[0], description=__doc__)
parser.add_argument('model', type=str)
parser.add_argument('--data', '-d', type=str, default='data/deadleaves_test.mat')
parser.add_argument('--patch_size', '-p', type=int, default=64,
help='Images are split into patches of this size and evaluated separately.')
parser.add_argument('--fraction', '-f', type=float, default=1.,
help='Only use a fraction of the data for a faster estimate.')
parser.add_argument('--mode', '-q', type=str, default='CPU', choices=['CPU', 'GPU'])
parser.add_argument('--device', '-D', type=int, default=0)
parser.add_argument('--stride', '-S', type=int, default=1)
args = parser.parse_args(argv[1:])
# select CPU or GPU for caffe
if args.mode.upper() == 'GPU':
caffe.set_mode_gpu()
caffe.set_device(args.device)
else:
caffe.set_mode_cpu()
# load data
data = loadmat(args.data)['data']
# load model
experiment = Experiment(args.model)
model = experiment['model']
if isinstance(model, PatchRIDE):
if args.patch_size != model.num_rows or args.patch_size != model.num_cols:
print 'Model is for {0}x{1} patches but data is {2}x{2}.'.format(
model.num_rows, model.num_cols, args.patch_size)
return 0
# apply model to data
logloss = []
for i in range(0, data.shape[1] - args.patch_size + 1, args.patch_size * args.stride):
for j in range(0, data.shape[2] - args.patch_size + 1, args.patch_size * args.stride):
# select random subset
idx = random_select(int(ceil(args.fraction * data.shape[0]) + .5), data.shape[0])
logloss.append(
model.evaluate(
data[:, i:i + args.patch_size, j:j + args.patch_size][idx]))
loglik_avg = -mean(logloss)
loglik_err = std(logloss, ddof=1) / sqrt(len(logloss)) if len(logloss) > 1 else inf
print 'Avg. log-likelihood: {0:.5f} +- {1:.5f} [bit/px]'.format(loglik_avg, loglik_err)
return 0
def main(argv):
experiment = Experiment()
parser = ArgumentParser(argv[0], description=__doc__)
parser.add_argument('--model', '-m', type=str, required=True)
parser.add_argument('--data',
'-d',
type=str,
default='data/deadleaves_train.mat')
parser.add_argument('--width', '-W', type=int, default=512)
parser.add_argument('--height', '-H', type=int, default=512)
parser.add_argument('--crop', '-C', type=int, default=16)
parser.add_argument('--log', '-L', type=int, default=0)
parser.add_argument('--output',
'-o',
type=str,
default='results/sample.png')
args = parser.parse_args(argv[1:])
images = loadmat(args.data)['data']
vmin = percentile(images, 0.02)
vmax = percentile(images, 98.)
experiment = Experiment(args.model)
img = empty([args.height + args.crop, args.width + 2 * args.crop])
img.ravel()[:] = images.ravel()[random_select(img.size, images.size)]
img = sample_image(img,
experiment['model'],
experiment['input_mask'],
experiment['output_mask'],
experiment['preconditioner'],
min_value=vmin,
max_value=vmax)
if args.log:
# linearize and gamma-correct
img = power(exp(img), .45)
vmin = power(exp(vmin), .45)
vmax = power(exp(vmax), .45)
imwrite(
args.output,
imformat(img[args.crop:, args.crop:-args.crop],
vmin=vmin,
vmax=vmax,
symmetric=False))
savez('sample.npz', sample=img)
return 0
def _precondition(self, inputs, outputs=None):
"""
Remove any correlations within and between inputs and outputs.
"""
shape = inputs.shape
if outputs is None:
if self.preconditioner is None:
raise RuntimeError('No preconditioning possible.')
inputs = inputs.reshape(-1, inputs.shape[-1]).T
inputs = self.preconditioner(inputs)
inputs = inputs.T.reshape(*shape)
return inputs
else:
inputs = inputs.reshape(-1, inputs.shape[-1]).T
outputs = outputs.reshape(-1, outputs.shape[-1]).T
# avoids memory issues
MAX_SAMPLES = 5000000
if self.preconditioner is None:
inputs_ = inputs
if self._indicators:
# half of the inputs are indicators, don't preprocess them
inputs_ = inputs.copy()
inputs_[inputs.shape[0] // 2:] = randn(
inputs.shape[0] // 2, *inputs.shape[1:])
if inputs.shape[1] > MAX_SAMPLES:
idx = random_select(MAX_SAMPLES, inputs.shape[1])
self.preconditioner = WhiteningPreconditioner(
inputs_[:, idx], outputs[:, idx])
else:
self.preconditioner = WhiteningPreconditioner(
inputs_, outputs)
# precondition
for b in range(0, inputs.shape[1], MAX_SAMPLES):
inputs[:, b:b + MAX_SAMPLES], outputs[:, b:b + MAX_SAMPLES] = \
self.preconditioner(inputs[:, b:b + MAX_SAMPLES], outputs[:, b:b + MAX_SAMPLES])
inputs = inputs.T.reshape(*shape)
outputs = outputs.T.reshape(shape[0], shape[1], shape[2], -1)
return inputs, outputs
def _precondition(self, inputs, outputs=None):
"""
Remove any correlations within and between inputs and outputs.
"""
shape = inputs.shape
if outputs is None:
if self.preconditioner is None:
raise RuntimeError('No preconditioning possible.')
inputs = inputs.reshape(-1, inputs.shape[-1]).T
inputs = self.preconditioner(inputs)
inputs = inputs.T.reshape(*shape)
return inputs
else:
inputs = inputs.reshape(-1, inputs.shape[-1]).T
outputs = outputs.reshape(-1, outputs.shape[-1]).T
# avoids memory issues
MAX_SAMPLES = 5000000
if self.preconditioner is None:
inputs_ = inputs
if self._indicators:
# half of the inputs are indicators, don't preprocess them
inputs_ = inputs.copy()
inputs_[inputs.shape[0] // 2:] = randn(inputs.shape[0] // 2, *inputs.shape[1:])
if inputs.shape[1] > MAX_SAMPLES:
idx = random_select(MAX_SAMPLES, inputs.shape[1])
self.preconditioner = WhiteningPreconditioner(inputs_[:, idx], outputs[:, idx])
else:
self.preconditioner = WhiteningPreconditioner(inputs_, outputs)
# precondition
for b in range(0, inputs.shape[1], MAX_SAMPLES):
inputs[:, b:b + MAX_SAMPLES], outputs[:, b:b + MAX_SAMPLES] = \
self.preconditioner(inputs[:, b:b + MAX_SAMPLES], outputs[:, b:b + MAX_SAMPLES])
inputs = inputs.T.reshape(*shape)
outputs = outputs.T.reshape(shape[0], shape[1], shape[2], -1)
return inputs, outputs
def _precondition(self, inputs, outputs=None):
"""
Remove any correlations within and between inputs and outputs (conditional whitening).
@type inputs: C{ndarray}
@param inputs: pixel neighborhoods stored column-wise
@type outputs: C{ndarray}
@param outputs: output pixels stored column-wise
"""
shape = inputs.shape
if outputs is None:
if self.preconditioner is None:
raise RuntimeError('No preconditioning possible.')
inputs = inputs.reshape(-1, inputs.shape[-1]).T
inputs = self.preconditioner(inputs)
inputs = inputs.T.reshape(*shape)
return inputs
else:
inputs = inputs.reshape(-1, inputs.shape[-1]).T
outputs = outputs.reshape(-1, outputs.shape[-1]).T
# avoids memory issues
MAX_SAMPLES = 500000
if self.preconditioner is None:
if inputs.shape[1] > MAX_SAMPLES:
idx = random_select(MAX_SAMPLES, inputs.shape[1])
self.preconditioner = WhiteningPreconditioner(
inputs[:, idx], outputs[:, idx])
else:
self.preconditioner = WhiteningPreconditioner(
inputs, outputs)
for b in range(0, inputs.shape[1], MAX_SAMPLES):
inputs[:, b:b + MAX_SAMPLES], outputs[:, b:b + MAX_SAMPLES] = \
self.preconditioner(inputs[:, b:b + MAX_SAMPLES], outputs[:, b:b + MAX_SAMPLES])
inputs = inputs.T.reshape(*shape)
outputs = outputs.T.reshape(shape[0], shape[1], shape[2], -1)
return inputs, outputs
def _precondition(self, inputs, outputs=None):
"""
Remove any correlations within and between inputs and outputs (conditional whitening).
@type inputs: C{ndarray}
@param inputs: pixel neighborhoods stored column-wise
@type outputs: C{ndarray}
@param outputs: output pixels stored column-wise
"""
shape = inputs.shape
if outputs is None:
if self.preconditioner is None:
raise RuntimeError("No preconditioning possible.")
inputs = inputs.reshape(-1, inputs.shape[-1]).T
inputs = self.preconditioner(inputs)
inputs = inputs.T.reshape(*shape)
return inputs
else:
inputs = inputs.reshape(-1, inputs.shape[-1]).T
outputs = outputs.reshape(-1, outputs.shape[-1]).T
# avoids memory issues
MAX_SAMPLES = 500000
if self.preconditioner is None:
if inputs.shape[1] > MAX_SAMPLES:
idx = random_select(MAX_SAMPLES, inputs.shape[1])
self.preconditioner = WhiteningPreconditioner(inputs[:, idx], outputs[:, idx])
else:
self.preconditioner = WhiteningPreconditioner(inputs, outputs)
for b in range(0, inputs.shape[1], MAX_SAMPLES):
inputs[:, b : b + MAX_SAMPLES], outputs[:, b : b + MAX_SAMPLES] = self.preconditioner(
inputs[:, b : b + MAX_SAMPLES], outputs[:, b : b + MAX_SAMPLES]
)
inputs = inputs.T.reshape(*shape)
outputs = outputs.T.reshape(shape[0], shape[1], shape[2], -1)
return inputs, outputs
def main(argv):
experiment = Experiment()
parser = ArgumentParser(argv[0], description=__doc__)
parser.add_argument('--model', '-m', type=str, required=True)
parser.add_argument('--data', '-d', type=str, default='data/deadleaves_train.mat')
parser.add_argument('--width', '-W', type=int, default=512)
parser.add_argument('--height', '-H', type=int, default=512)
parser.add_argument('--crop', '-C', type=int, default=16)
parser.add_argument('--log', '-L', type=int, default=0)
parser.add_argument('--output', '-o', type=str, default='results/sample.png')
args = parser.parse_args(argv[1:])
images = loadmat(args.data)['data']
vmin = percentile(images, 0.02)
vmax = percentile(images, 98.)
experiment = Experiment(args.model)
img = empty([args.height + args.crop, args.width + 2 * args.crop])
img.ravel()[:] = images.ravel()[random_select(img.size, images.size)]
img = sample_image(
img,
experiment['model'],
experiment['input_mask'],
experiment['output_mask'],
experiment['preconditioner'],
min_value=vmin,
max_value=vmax)
if args.log:
# linearize and gamma-correct
img = power(exp(img), .45)
vmin = power(exp(vmin), .45)
vmax = power(exp(vmax), .45)
imwrite(args.output, imformat(img[args.crop:, args.crop:-args.crop], vmin=vmin, vmax=vmax, symmetric=False))
savez('sample.npz', sample=img)
return 0
def main(argv):
parser = ArgumentParser(argv[0], description=__doc__)
parser.add_argument('model', type=str)
parser.add_argument('--data', '-d', type=str, default='data/BSDS300_8x8.mat')
parser.add_argument('--num_data', '-N', type=int, default=1000000)
args = parser.parse_args(argv[1:])
data_test = loadmat(args.data)['patches_test']
dim = data_test.shape[1]
# reconstruct patches
patch_size = int(sqrt(data_test.shape[1] + 1) + .5)
data_test = hstack([data_test, -sum(data_test, 1)[:, None]])
data_test = data_test.reshape(-1, patch_size, patch_size)
if args.num_data > 0 and data_test.shape[0] > args.num_data:
data_test = data_test[random_select(args.num_data, data_test.shape[0])]
model = Experiment(args.model)['model']
print '{0:.3f} [nat]'.format(
-model.evaluate(data_test) * dim * log(2.))
def finetune(self,
images,
max_iter=1000,
train_means=False,
num_samples_train=500000,
num_samples_valid=100000):
"""
Train MCGSM using L-BFGS while keeping parameters of SLSTM fixed.
@type images: C{ndarray}/C{list}
@param images: an array or a list of images
"""
if images.shape[0] > num_samples_train:
images = images[random_select(num_samples_train, images.shape[0])]
print 'Preprocessing...'
inputs, outputs = self._preprocess(images)
if self.preconditioner:
print 'Preconditioning...'
# remove correlations
inputs, outputs = self._precondition(inputs, outputs)
# compute hidden unit activations
hiddens = inputs
print 'Forward...'
for l in range(self.num_layers):
self.slstm[l] = SLSTM(num_rows=hiddens.shape[1],
num_cols=hiddens.shape[2],
num_channels=hiddens.shape[3],
num_hiddens=self.num_hiddens,
batch_size=min(
[hiddens.shape[0], self.MAX_BATCH_SIZE]),
nonlinearity=self.nonlinearity,
extended=self.extended,
slstm=self.slstm[l],
verbosity=self.verbosity)
hiddens = self.slstm[l].forward(hiddens)
print 'Reshape...'
# remove bottom-right pixels (BSDS300)
hiddens = hiddens.reshape(hiddens.shape[0], -1, self.num_hiddens)
outputs = outputs.reshape(outputs.shape[0], -1, self.num_channels)
hiddens = hiddens[:, :-1]
outputs = outputs[:, :-1]
# form inputs to MCGSM
hiddens = hiddens.reshape(-1, self.num_hiddens).T
outputs = outputs.reshape(-1, self.num_channels).T
print 'Finetuning...'
if hiddens.shape[1] > num_samples_train:
num_samples_valid = min(
[num_samples_valid, hiddens.shape[1] - num_samples_train])
# select subset of data points for finetuning
idx = random_select(num_samples_train + num_samples_valid,
hiddens.shape[1])
if num_samples_valid > 0:
# split data into training and validation set
hiddens_train = asarray(hiddens[:, idx[:num_samples_train]],
order='F')
outputs_train = asarray(outputs[:, idx[:num_samples_train]],
order='F')
hiddens_valid = asarray(hiddens[:, idx[num_samples_train:]],
order='F')
outputs_valid = asarray(outputs[:, idx[num_samples_train:]],
order='F')
# finetune with early stopping based on validation performance
return self.mcgsm.train(hiddens_train,
outputs_train,
hiddens_valid,
outputs_valid,
parameters={
'verbosity': self.verbosity,
'train_means': train_means,
'max_iter': max_iter
})
else:
hiddens = asarray(hiddens[:, idx], order='F')
outputs = asarray(outputs[:, idx], order='F')
return self.mcgsm.train(hiddens,
outputs,
parameters={
'verbosity': self.verbosity,
'train_means': train_means,
'max_iter': max_iter
})
def main(argv):
parser = ArgumentParser(argv[0], description=__doc__)
parser.add_argument('model', type=str)
parser.add_argument('--data',
'-d',
type=str,
default='data/deadleaves_test.mat')
parser.add_argument(
'--patch_size',
'-p',
type=int,
default=64,
help=
'Images are split into patches of this size and evaluated separately.')
parser.add_argument('--fraction', '-f', type=float, default=1.)
parser.add_argument('--mode',
'-q',
type=str,
default='CPU',
choices=['CPU', 'GPU'])
parser.add_argument('--device', '-D', type=int, default=0)
parser.add_argument('--horizontal', '-H', type=int, default=1)
parser.add_argument('--vertical', '-V', type=int, default=1)
parser.add_argument('--transpose', '-T', type=int, default=1)
args = parser.parse_args(argv[1:])
# select CPU or GPU for caffe
if args.mode.upper() == 'GPU':
caffe.set_mode_gpu()
caffe.set_device(args.device)
else:
caffe.set_mode_cpu()
# load data
data = loadmat(args.data)['data']
if data.ndim < 4:
data = data[:, :, :, None]
# load model
experiment = Experiment(args.model)
model = experiment['model']
# apply model to data
logloss = []
def evaluate(patches):
"""
Returns a conditional log-likelihood for each pixel.
"""
# evaluate all reachable pixels
loglik = model.loglikelihood(patches)
# make sure the same pixels are evaluated with every transformation
loglik = loglik[:, :-(model.input_mask.shape[0] - 1), :]
# reshape into images x pixels
return loglik.reshape(loglik.shape[0], -1)
for i in range(0, data.shape[1] - args.patch_size + 1, args.patch_size):
for j in range(0, data.shape[2] - args.patch_size + 1,
args.patch_size):
# select random subset
idx = random_select(int(ceil(args.fraction * data.shape[0]) + .5),
data.shape[0])
patches = data[:, i:i + args.patch_size,
j:j + args.patch_size][idx]
loglik = []
loglik.append(evaluate(patches))
if args.horizontal:
loglik.append(evaluate(patches[:, :, ::-1]))
if args.vertical:
loglik.append(evaluate(patches[:, ::-1, :]))
if args.horizontal and args.vertical:
loglik.append(evaluate(patches[:, ::-1, ::-1]))
if args.transpose:
patches = transpose(patches, [0, 2, 1, 3])
loglik.append(evaluate(patches))
if args.horizontal:
loglik.append(evaluate(patches[:, :, ::-1]))
if args.vertical:
loglik.append(evaluate(patches[:, ::-1, :]))
if args.horizontal and args.vertical:
loglik.append(evaluate(patches[:, ::-1, ::-1]))
loglik = asarray(loglik)
# compute log-likelihood for each image and model by summing over pixels
num_pixels = loglik.shape[2]
loglik = loglik.sum(2) # sum over pixels
# compute log-likelihood for mixture model
loglik = logsumexp(loglik, 0) - log(loglik.shape[0])
# compute average log-loss in bit/px
logloss.append(-mean(loglik) / num_pixels / log(2.))
print 'Avg. log-likelihood: {0:.5f} [bit/px]'.format(
-mean(logloss))
return 0
def main(argv):
parser = ArgumentParser(argv[0], description=__doc__)
parser.add_argument('model', type=str)
parser.add_argument('--data', '-d', type=str, default='data/deadleaves_test.mat')
parser.add_argument('--patch_size', '-p', type=int, default=64,
help='Images are split into patches of this size and evaluated separately.')
parser.add_argument('--fraction', '-f', type=float, default=1.)
parser.add_argument('--mode', '-q', type=str, default='CPU', choices=['CPU', 'GPU'])
parser.add_argument('--device', '-D', type=int, default=0)
parser.add_argument('--horizontal', '-H', type=int, default=1)
parser.add_argument('--vertical', '-V', type=int, default=1)
parser.add_argument('--transpose', '-T', type=int, default=1)
args = parser.parse_args(argv[1:])
# select CPU or GPU for caffe
if args.mode.upper() == 'GPU':
caffe.set_mode_gpu()
caffe.set_device(args.device)
else:
caffe.set_mode_cpu()
# load data
data = loadmat(args.data)['data']
if data.ndim < 4:
data = data[:, :, :, None]
# load model
experiment = Experiment(args.model)
model = experiment['model']
# apply model to data
logloss = []
def evaluate(patches):
"""
Returns a conditional log-likelihood for each pixel.
"""
# evaluate all reachable pixels
loglik = model.loglikelihood(patches)
# make sure the same pixels are evaluated with every transformation
loglik = loglik[:, :-(model.input_mask.shape[0] - 1), :]
# reshape into images x pixels
return loglik.reshape(loglik.shape[0], -1)
for i in range(0, data.shape[1] - args.patch_size + 1, args.patch_size):
for j in range(0, data.shape[2] - args.patch_size + 1, args.patch_size):
# select random subset
idx = random_select(int(ceil(args.fraction * data.shape[0]) + .5), data.shape[0])
patches = data[:, i:i + args.patch_size, j:j + args.patch_size][idx]
loglik = []
loglik.append(evaluate(patches))
if args.horizontal:
loglik.append(evaluate(patches[:, :, ::-1]))
if args.vertical:
loglik.append(evaluate(patches[:, ::-1, :]))
if args.horizontal and args.vertical:
loglik.append(evaluate(patches[:, ::-1, ::-1]))
if args.transpose:
patches = transpose(patches, [0, 2, 1, 3])
loglik.append(evaluate(patches))
if args.horizontal:
loglik.append(evaluate(patches[:, :, ::-1]))
if args.vertical:
loglik.append(evaluate(patches[:, ::-1, :]))
if args.horizontal and args.vertical:
loglik.append(evaluate(patches[:, ::-1, ::-1]))
loglik = asarray(loglik)
# compute log-likelihood for each image and model by summing over pixels
num_pixels = loglik.shape[2]
loglik = loglik.sum(2) # sum over pixels
# compute log-likelihood for mixture model
loglik = logsumexp(loglik, 0) - log(loglik.shape[0])
# compute average log-loss in bit/px
logloss.append(-mean(loglik) / num_pixels / log(2.))
print 'Avg. log-likelihood: {0:.5f} [bit/px]'.format(-mean(logloss))
return 0
def finetune(self,
images,
max_iter=1000,
train_means=False,
num_samples_train=500000,
num_samples_valid=100000):
"""
Train MCGSM using L-BFGS while keeping parameters of spatial LSTMs fixed.
@type images: C{ndarray}/C{list}
@param images: an array or a list of images
@type max_iter: C{int}
@param max_iter: maximum number of L-BFGS iterations
@type train_means: C{bool}
@param train_means: whether or not to optimize the mean parameters of the MCGSM
@type num_samples_train: C{int}
@param num_samples_train: number of training examples extracted from images
@type num_samples_valid: C{int}
@type num_samples_valid: number of validation examples used for early stopping
@rtype: C{bool}
@return: true if training converged, false otherwise
"""
if images.shape[0] > min([200000, num_samples_train]):
images = images[random_select(min([200000, num_samples_train]),
images.shape[0])]
if self.verbosity > 0:
print 'Preprocessing...'
inputs, outputs = self._preprocess(images)
if self.preconditioner:
if self.verbosity > 0:
print 'Preconditioning...'
# remove correlations
inputs, outputs = self._precondition(inputs, outputs)
# compute hidden unit activations
hiddens = inputs
if self.verbosity > 0:
print 'Computing hidden states...'
for l in range(self.num_layers):
self.slstm[l] = SLSTM(num_rows=hiddens.shape[1],
num_cols=hiddens.shape[2],
num_channels=hiddens.shape[3],
num_hiddens=self.num_hiddens,
batch_size=min(
[hiddens.shape[0], self.MAX_BATCH_SIZE]),
nonlinearity=self.nonlinearity,
extended=self.extended,
slstm=self.slstm[l],
verbosity=self.verbosity)
hiddens = self.slstm[l].forward(hiddens)
if self.verbosity > 0:
print 'Preparing inputs and outputs...'
# form inputs to MCGSM
hiddens = hiddens.reshape(-1, self.num_hiddens).T
outputs = outputs.reshape(-1, self.num_channels).T
if hiddens.shape[1] > num_samples_train:
num_samples_valid = min(
[num_samples_valid, hiddens.shape[1] - num_samples_train])
# select subset of data points for finetuning
idx = random_select(num_samples_train + num_samples_valid,
hiddens.shape[1])
if num_samples_valid > 0:
# split data into training and validation set
hiddens_train = asarray(hiddens[:, idx[:num_samples_train]],
order='F')
outputs_train = asarray(outputs[:, idx[:num_samples_train]],
order='F')
hiddens_valid = asarray(hiddens[:, idx[num_samples_train:]],
order='F')
outputs_valid = asarray(outputs[:, idx[num_samples_train:]],
order='F')
# finetune with early stopping based on validation performance
return self.mcgsm.train(hiddens_train,
outputs_train,
hiddens_valid,
outputs_valid,
parameters={
'verbosity': self.verbosity,
'train_means': train_means,
'max_iter': max_iter
})
else:
hiddens = asarray(hiddens[:, idx], order='F')
outputs = asarray(outputs[:, idx], order='F')
if self.verbosity > 0:
print 'Finetuning...'
return self.mcgsm.train(hiddens,
outputs,
parameters={
'verbosity': self.verbosity,
'train_means': train_means,
'max_iter': max_iter
})
def finetune(self, images, max_iter=1000, train_means=False, num_samples_train=500000, num_samples_valid=100000):
"""
Train MCGSM using L-BFGS while keeping parameters of spatial LSTMs fixed.
@type images: C{ndarray}/C{list}
@param images: an array or a list of images
@type max_iter: C{int}
@param max_iter: maximum number of L-BFGS iterations
@type train_means: C{bool}
@param train_means: whether or not to optimize the mean parameters of the MCGSM
@type num_samples_train: C{int}
@param num_samples_train: number of training examples extracted from images
@type num_samples_valid: C{int}
@type num_samples_valid: number of validation examples used for early stopping
@rtype: C{bool}
@return: true if training converged, false otherwise
"""
if images.shape[0] > min([200000, num_samples_train]):
images = images[random_select(min([200000, num_samples_train]), images.shape[0])]
if self.verbosity > 0:
print "Preprocessing..."
inputs, outputs = self._preprocess(images)
if self.preconditioner:
if self.verbosity > 0:
print "Preconditioning..."
# remove correlations
inputs, outputs = self._precondition(inputs, outputs)
# compute hidden unit activations
hiddens = inputs
if self.verbosity > 0:
print "Computing hidden states..."
for l in range(self.num_layers):
self.slstm[l] = SLSTM(
num_rows=hiddens.shape[1],
num_cols=hiddens.shape[2],
num_channels=hiddens.shape[3],
num_hiddens=self.num_hiddens,
batch_size=min([hiddens.shape[0], self.MAX_BATCH_SIZE]),
nonlinearity=self.nonlinearity,
extended=self.extended,
slstm=self.slstm[l],
verbosity=self.verbosity,
)
hiddens = self.slstm[l].forward(hiddens)
if self.verbosity > 0:
print "Preparing inputs and outputs..."
# form inputs to MCGSM
hiddens = hiddens.reshape(-1, self.num_hiddens).T
outputs = outputs.reshape(-1, self.num_channels).T
if hiddens.shape[1] > num_samples_train:
num_samples_valid = min([num_samples_valid, hiddens.shape[1] - num_samples_train])
# select subset of data points for finetuning
idx = random_select(num_samples_train + num_samples_valid, hiddens.shape[1])
if num_samples_valid > 0:
# split data into training and validation set
hiddens_train = asarray(hiddens[:, idx[:num_samples_train]], order="F")
outputs_train = asarray(outputs[:, idx[:num_samples_train]], order="F")
hiddens_valid = asarray(hiddens[:, idx[num_samples_train:]], order="F")
outputs_valid = asarray(outputs[:, idx[num_samples_train:]], order="F")
# finetune with early stopping based on validation performance
return self.mcgsm.train(
hiddens_train,
outputs_train,
hiddens_valid,
outputs_valid,
parameters={"verbosity": self.verbosity, "train_means": train_means, "max_iter": max_iter},
)
else:
hiddens = asarray(hiddens[:, idx], order="F")
outputs = asarray(outputs[:, idx], order="F")
if self.verbosity > 0:
print "Finetuning..."
return self.mcgsm.train(
hiddens, outputs, parameters={"verbosity": self.verbosity, "train_means": train_means, "max_iter": max_iter}
)
def main(argv):
parser = ArgumentParser(argv[0], description=__doc__)
parser.add_argument('model', type=str)
parser.add_argument('--data',
'-d',
type=str,
default='data/BSDS300_8x8.mat')
parser.add_argument('--num_data', '-N', type=int, default=1000000)
args = parser.parse_args(argv[1:])
# load data
data = loadmat(args.data)['patches_test']
if args.num_data > 0 and args.num_data < data.shape[0]:
# select random subset of data
data = data[random_select(args.num_data, data.shape[0])]
print 'Transforming data...'
# transform data
data_all = [
data,
transform(data, True, False, False),
transform(data, False, True, False),
transform(data, True, True, False),
transform(data, False, False, True),
transform(data, True, False, True),
transform(data, False, True, True),
transform(data, True, True, True)
]
# each entry corresponds to a different model/transformation
loglik = [0.] * len(data_all)
# load model
experiment = Experiment(args.model)
models = experiment['models'] # models for the different pixels
preconditioners = experiment['preconditioners']
if len(models) != data.shape[1]:
print 'Wrong number of models!'
return 0
for i, model in enumerate(models):
for n, data in enumerate(data_all):
inputs, outputs = data.T[:i], data.T[[i]]
# this sums over pixels
if isinstance(model, MoGSM):
loglik[n] = loglik[n] + model.loglikelihood(outputs)
else:
loglik[n] = loglik[n] + model.loglikelihood(*preconditioners[i](inputs, outputs)) \
+ preconditioners[i].logjacobian(inputs, outputs)
print '{0}/{1}'.format(i + 1, data.shape[1])
# each row of loglik is a different model/transformation, each column a different data point
loglik = logsumexp(loglik, 0) - log(len(loglik))
print '{0:.4f} [nat]'.format(mean(loglik))
return 0
def preprocess(data, i, N): if N > 0 and N < data.shape[0]: # select subset of data idx = random_select(N, data.shape[0]) return data[idx, :i].T, data[idx, i][None, :] return data.T[:i], data.T[[i]]
def main(argv):
parser = ArgumentParser(argv[0], description=__doc__)
parser.add_argument('--patch_size', '-p', type=int, default=[8, 10, 12, 14, 16, 18, 20, 22], nargs='+')
parser.add_argument('--row_multiplier', '-R', type=int, default=[1], nargs='+',
help='Can be used to train on elongated patches.')
parser.add_argument('--col_multiplier', '-C', type=int, default=[1], nargs='+',
help='Can be used to train on elongated patches.')
parser.add_argument('--num_patches', '-P', type=int, default=None,
help='If given, subsample training data.')
parser.add_argument('--num_valid', '-V', type=int, default=0,
help='Number of training images used for validation error based early stopping.')
parser.add_argument('--finetune', '-F', type=int, default=[1], nargs='+',
help='Indicate iterations in which to finetune MCGSM with L-BFGS.')
parser.add_argument('--learning_rate', '-l', type=float, default=[1., .5, .1, .05, .01, 0.005, 0.001, 0.0005], nargs='+')
parser.add_argument('--momentum', '-m', type=float, default=[.9], nargs='+')
parser.add_argument('--batch_size', '-B', type=int, default=[50], nargs='+')
parser.add_argument('--nb_size', '-b', type=int, default=5,
help='Size of the causal neighborhood of pixels.')
parser.add_argument('--num_hiddens', '-n', type=int, default=64)
parser.add_argument('--num_components', '-c', type=int, default=32)
parser.add_argument('--num_scales', '-s', type=int, default=4)
parser.add_argument('--num_features', '-f', type=int, default=32)
parser.add_argument('--num_epochs', '-e', type=int, default=[1], nargs='+')
parser.add_argument('--precondition', '-Q', type=int, default=1)
parser.add_argument('--method', '-M', type=str, default=['SGD'], nargs='+')
parser.add_argument('--data', '-d', type=str, default='data/deadleaves_train.mat')
parser.add_argument('--noise', '-N', type=float, default=None,
help='Standard deviation of Gaussian noise added to data before training (as fraction of data standard deviation).')
parser.add_argument('--model', '-I', type=str, default='',
help='Start with this model as initialization. Other flags will be ignored.')
parser.add_argument('--add_layer', '-a', type=int, default=[0], nargs='+')
parser.add_argument('--train_top_layer', '-T', type=int, default=[0], nargs='+')
parser.add_argument('--train_means', '-S', type=int, default=[0], nargs='+')
parser.add_argument('--mode', '-q', type=str, default='CPU', choices=['CPU', 'GPU'])
parser.add_argument('--device', '-D', type=int, default=0)
parser.add_argument('--augment', '-A', type=int, default=1,
help='Increase training set size by transforming data.')
parser.add_argument('--overlap', '-O', type=int, default=[1], nargs='+')
parser.add_argument('--output', '-o', type=str, default='results/deadleaves/')
parser.add_argument('--patch_model', type=int, default=0,
help='Train a patch-based model instead of a stochastic process.')
parser.add_argument('--extended', '-X', type=int, default=0,
help='Use extended version of spatial LSTM.')
parser.add_argument('--multiscale', '-Y', type=int, default=0,
help='Apply recurrent image model to multiscale representation of images.')
parser.add_argument('--color', '-Z', type=int, default=0,
help='Use separate models to model color and grayscale values.')
args = parser.parse_args(argv[1:])
experiment = Experiment()
if args.mode.upper() == 'GPU':
caffe.set_mode_gpu()
caffe.set_device(args.device)
else:
caffe.set_mode_cpu()
# load data
if args.data.lower()[-4:] in ['.gif', '.png', '.jpg', 'jpeg']:
data = imread(args.data)[None]
data += rand(*data.shape)
else:
data = loadmat(args.data)['data']
if args.augment > 0:
data = vstack([data, data[:, :, ::-1]])
if args.augment > 1:
data = vstack([data, data[:, ::-1, :]])
if args.noise is not None:
# add noise as a means for regularization
data += randn(*data.shape) * (std(data, ddof=1) / args.noise)
if args.num_valid > 0:
if args.num_valid >= data.shape[0]:
print 'Cannot use {0} for validation, there are only {1} training images.'.format(
args.num_valid, data.shape[0])
return 1
# select subset for validation
idx = random_select(args.num_valid, data.shape[0])
data_valid = data[idx]
data = asarray([image for i, image in enumerate(data) if i not in idx])
print '{0} training images'.format(data.shape[0])
print '{0} validation images'.format(data_valid.shape[0])
patches_valid = []
patch_size = min([64, data.shape[1], data.shape[2]])
for i in range(0, data_valid.shape[1] - patch_size + 1, patch_size):
for j in range(0, data_valid.shape[2] - patch_size + 1, patch_size):
patches_valid.append(data_valid[:, i:i + patch_size, j:j + patch_size])
patches_valid = vstack(patches_valid)
if args.model:
# load pretrained model
results = Experiment(args.model)
model = results['model']
loss = [results['loss']]
if args.patch_model and not isinstance(model, PatchRIDE):
model = PatchRIDE(
model=model,
num_rows=args.patch_size[0],
num_cols=args.patch_size[0])
else:
# create recurrent image model
if args.patch_model:
model = PatchRIDE(
num_rows=args.patch_size[0],
num_cols=args.patch_size[0],
num_channels=data.shape[-1] if data.ndim > 3 else 1,
num_hiddens=args.num_hiddens,
nb_size=args.nb_size,
num_components=args.num_components,
num_scales=args.num_scales,
num_features=args.num_features,
model_class=ColorRIDE if args.color else RIDE)
if args.extended:
print 'Extended patch model not supported.'
return 0
if args.multiscale:
print 'Multiscale patch model not supported.'
return 0
else:
if args.multiscale:
if data.ndim > 3 and data.shape[-1] > 1:
print 'Multiscale color model not supported.'
return 0
model = MultiscaleRIDE(
num_hiddens=args.num_hiddens,
nb_size=args.nb_size,
num_components=args.num_components,
num_scales=args.num_scales,
num_features=args.num_features,
extended=args.extended > 0)
elif args.color:
if data.ndim < 4 or data.shape[-1] != 3:
print 'These images don\'t look like RGB images.'
return 0
model = ColorRIDE(
num_hiddens=args.num_hiddens,
nb_size=args.nb_size,
num_components=args.num_components,
num_scales=args.num_scales,
num_features=args.num_features,
extended=args.extended > 0)
else:
model = RIDE(
num_channels=data.shape[-1] if data.ndim > 3 else 1,
num_hiddens=args.num_hiddens,
nb_size=args.nb_size,
num_components=args.num_components,
num_scales=args.num_scales,
num_features=args.num_features,
extended=args.extended > 0)
loss = []
# compute initial performance
loss_valid = []
if args.num_valid > 0:
print 'Computing validation loss...'
loss_valid.append(model.evaluate(patches_valid))
model_copy = deepcopy(model)
for k, patch_size in enumerate(args.patch_size):
if args.multiscale:
patch_size *= 2
if k < len(args.add_layer):
for _ in range(args.add_layer[k]):
# add spatial LSTM to the network
model.add_layer()
# extract patches of given patch size
patches = []
row_size = patch_size * args.row_multiplier[k % len(args.row_multiplier)]
col_size = patch_size * args.col_multiplier[k % len(args.col_multiplier)]
if isinstance(model, PatchRIDE):
model.num_rows = row_size
model.num_cols = col_size
for i in range(0, data.shape[1] - row_size + 1, row_size / args.overlap[k % len(args.overlap)]):
for j in range(0, data.shape[2] - col_size + 1, col_size / args.overlap[k % len(args.overlap)]):
patches.append(data[:, i:i + row_size, j:j + col_size])
patches = vstack(patches)
# randomize order of patches
if args.num_patches is not None and args.num_patches < len(patches):
patches = patches[random_select(args.num_patches, len(patches))]
else:
patches = patches[permutation(len(patches))]
# determine batch size
if args.method[k % len(args.method)].upper() == 'SFO':
num_batches = int(max([25, sqrt(patches.shape[0]) / 5.]))
batch_size = patches.shape[0] // num_batches
else:
batch_size = args.batch_size[k % len(args.batch_size)]
if batch_size < 1:
raise RuntimeError('Too little data.')
print 'Patch size: {0}x{1}'.format(row_size, col_size)
print 'Number of patches: {0}'.format(patches.shape[0])
print 'Batch size: {0}'.format(batch_size)
# train recurrent image model
print 'Training...'
loss.append(
model.train(patches,
batch_size=batch_size,
method=args.method[k % len(args.method)],
num_epochs=args.num_epochs[k % len(args.num_epochs)],
learning_rate=args.learning_rate[k % len(args.learning_rate)],
momentum=args.momentum[k % len(args.momentum)],
precondition=args.precondition > 0,
train_top_layer=args.train_top_layer[k % len(args.train_top_layer)] > 0,
train_means=args.train_means[k % len(args.train_means)] > 0))
if args.finetune[k % len(args.finetune)]:
print 'Finetuning...'
model.finetune(patches, num_samples_train=1000000, max_iter=500)
if args.num_valid > 0:
print 'Computing validation loss...'
loss_valid.append(model.evaluate(patches_valid))
if loss_valid[-1] > loss_valid[-2]:
print 'Performance got worse. Stopping optimization.'
model = model_copy
break
print 'Copying model...'
model_copy = deepcopy(model)
experiment['batch_size'] = batch_size
experiment['args'] = args
experiment['model'] = model
experiment['loss_valid'] = loss_valid
experiment['loss'] = hstack(loss) if len(loss) > 0 else []
experiment.save(os.path.join(args.output, 'rim.{0}.{1}.xpck'))
return 0
def test_random_select(self): # select all elements self.assertTrue(set(random_select(8, 8)) == set(range(8))) # n should be larger than k self.assertRaises(Exception, random_select, 10, 4)
def main(argv):
parser = ArgumentParser(argv[0], description=__doc__)
parser.add_argument('--patch_size',
'-p',
type=int,
default=[8, 10, 12, 14, 16, 18, 20, 22],
nargs='+')
parser.add_argument('--row_multiplier',
'-R',
type=int,
default=[1],
nargs='+',
help='Can be used to train on elongated patches.')
parser.add_argument('--col_multiplier',
'-C',
type=int,
default=[1],
nargs='+',
help='Can be used to train on elongated patches.')
parser.add_argument('--num_patches',
'-P',
type=int,
default=None,
help='If given, subsample training data.')
parser.add_argument(
'--num_valid',
'-V',
type=int,
default=0,
help=
'Number of training images used for validation error based early stopping.'
)
parser.add_argument(
'--finetune',
'-F',
type=int,
default=[1],
nargs='+',
help='Indicate iterations in which to finetune MCGSM with L-BFGS.')
parser.add_argument('--learning_rate',
'-l',
type=float,
default=[1., .5, .1, .05, .01, 0.005, 0.001, 0.0005],
nargs='+')
parser.add_argument('--momentum',
'-m',
type=float,
default=[.9],
nargs='+')
parser.add_argument('--batch_size',
'-B',
type=int,
default=[50],
nargs='+')
parser.add_argument('--nb_size',
'-b',
type=int,
default=5,
help='Size of the causal neighborhood of pixels.')
parser.add_argument('--num_hiddens', '-n', type=int, default=64)
parser.add_argument('--num_components', '-c', type=int, default=32)
parser.add_argument('--num_scales', '-s', type=int, default=4)
parser.add_argument('--num_features', '-f', type=int, default=32)
parser.add_argument('--num_epochs', '-e', type=int, default=[1], nargs='+')
parser.add_argument('--precondition', '-Q', type=int, default=1)
parser.add_argument('--method', '-M', type=str, default=['SGD'], nargs='+')
parser.add_argument('--data',
'-d',
type=str,
default='data/deadleaves_train.mat')
parser.add_argument(
'--noise',
'-N',
type=float,
default=None,
help=
'Standard deviation of Gaussian noise added to data before training (as fraction of data standard deviation).'
)
parser.add_argument(
'--model',
'-I',
type=str,
default='',
help=
'Start with this model as initialization. Other flags will be ignored.'
)
parser.add_argument('--add_layer', '-a', type=int, default=[0], nargs='+')
parser.add_argument('--train_top_layer',
'-T',
type=int,
default=[0],
nargs='+')
parser.add_argument('--train_means',
'-S',
type=int,
default=[0],
nargs='+')
parser.add_argument('--mode',
'-q',
type=str,
default='CPU',
choices=['CPU', 'GPU'])
parser.add_argument('--device', '-D', type=int, default=0)
parser.add_argument(
'--augment',
'-A',
type=int,
default=1,
help='Increase training set size by transforming data.')
parser.add_argument('--overlap', '-O', type=int, default=[1], nargs='+')
parser.add_argument('--output',
'-o',
type=str,
default='results/deadleaves/')
parser.add_argument(
'--patch_model',
type=int,
default=0,
help='Train a patch-based model instead of a stochastic process.')
parser.add_argument('--extended',
'-X',
type=int,
default=0,
help='Use extended version of spatial LSTM.')
parser.add_argument(
'--multiscale',
'-Y',
type=int,
default=0,
help=
'Apply recurrent image model to multiscale representation of images.')
parser.add_argument(
'--color',
'-Z',
type=int,
default=0,
help='Use separate models to model color and grayscale values.')
args = parser.parse_args(argv[1:])
experiment = Experiment()
if args.mode.upper() == 'GPU':
caffe.set_mode_gpu()
caffe.set_device(args.device)
else:
caffe.set_mode_cpu()
# load data
if args.data.lower()[-4:] in ['.gif', '.png', '.jpg', 'jpeg']:
data = imread(args.data)[None]
data += rand(*data.shape)
else:
data = loadmat(args.data)['data']
if args.augment > 0:
data = vstack([data, data[:, :, ::-1]])
if args.augment > 1:
data = vstack([data, data[:, ::-1, :]])
if args.noise is not None:
# add noise as a means for regularization
data += randn(*data.shape) * (std(data, ddof=1) / args.noise)
if args.num_valid > 0:
if args.num_valid >= data.shape[0]:
print 'Cannot use {0} for validation, there are only {1} training images.'.format(
args.num_valid, data.shape[0])
return 1
# select subset for validation
idx = random_select(args.num_valid, data.shape[0])
data_valid = data[idx]
data = asarray([image for i, image in enumerate(data) if i not in idx])
print '{0} training images'.format(data.shape[0])
print '{0} validation images'.format(data_valid.shape[0])
patches_valid = []
patch_size = min([64, data.shape[1], data.shape[2]])
for i in range(0, data_valid.shape[1] - patch_size + 1, patch_size):
for j in range(0, data_valid.shape[2] - patch_size + 1,
patch_size):
patches_valid.append(data_valid[:, i:i + patch_size,
j:j + patch_size])
patches_valid = vstack(patches_valid)
if args.model:
# load pretrained model
results = Experiment(args.model)
model = results['model']
loss = [results['loss']]
if args.patch_model and not isinstance(model, PatchRIDE):
model = PatchRIDE(model=model,
num_rows=args.patch_size[0],
num_cols=args.patch_size[0])
else:
# create recurrent image model
if args.patch_model:
model = PatchRIDE(
num_rows=args.patch_size[0],
num_cols=args.patch_size[0],
num_channels=data.shape[-1] if data.ndim > 3 else 1,
num_hiddens=args.num_hiddens,
nb_size=args.nb_size,
num_components=args.num_components,
num_scales=args.num_scales,
num_features=args.num_features,
model_class=ColorRIDE if args.color else RIDE)
if args.extended:
print 'Extended patch model not supported.'
return 0
if args.multiscale:
print 'Multiscale patch model not supported.'
return 0
else:
if args.multiscale:
if data.ndim > 3 and data.shape[-1] > 1:
print 'Multiscale color model not supported.'
return 0
model = MultiscaleRIDE(num_hiddens=args.num_hiddens,
nb_size=args.nb_size,
num_components=args.num_components,
num_scales=args.num_scales,
num_features=args.num_features,
extended=args.extended > 0)
elif args.color:
if data.ndim < 4 or data.shape[-1] != 3:
print 'These images don\'t look like RGB images.'
return 0
model = ColorRIDE(num_hiddens=args.num_hiddens,
nb_size=args.nb_size,
num_components=args.num_components,
num_scales=args.num_scales,
num_features=args.num_features,
extended=args.extended > 0)
else:
model = RIDE(
num_channels=data.shape[-1] if data.ndim > 3 else 1,
num_hiddens=args.num_hiddens,
nb_size=args.nb_size,
num_components=args.num_components,
num_scales=args.num_scales,
num_features=args.num_features,
extended=args.extended > 0)
loss = []
# compute initial performance
loss_valid = []
if args.num_valid > 0:
print 'Computing validation loss...'
loss_valid.append(model.evaluate(patches_valid))
model_copy = deepcopy(model)
for k, patch_size in enumerate(args.patch_size):
if args.multiscale:
patch_size *= 2
if k < len(args.add_layer):
for _ in range(args.add_layer[k]):
# add spatial LSTM to the network
model.add_layer()
# extract patches of given patch size
patches = []
row_size = patch_size * args.row_multiplier[k %
len(args.row_multiplier)]
col_size = patch_size * args.col_multiplier[k %
len(args.col_multiplier)]
if isinstance(model, PatchRIDE):
model.num_rows = row_size
model.num_cols = col_size
for i in range(0, data.shape[1] - row_size + 1,
row_size / args.overlap[k % len(args.overlap)]):
for j in range(0, data.shape[2] - col_size + 1,
col_size / args.overlap[k % len(args.overlap)]):
patches.append(data[:, i:i + row_size, j:j + col_size])
patches = vstack(patches)
# randomize order of patches
if args.num_patches is not None and args.num_patches < len(patches):
patches = patches[random_select(args.num_patches, len(patches))]
else:
patches = patches[permutation(len(patches))]
# determine batch size
if args.method[k % len(args.method)].upper() == 'SFO':
num_batches = int(max([25, sqrt(patches.shape[0]) / 5.]))
batch_size = patches.shape[0] // num_batches
else:
batch_size = args.batch_size[k % len(args.batch_size)]
if batch_size < 1:
raise RuntimeError('Too little data.')
print 'Patch size: {0}x{1}'.format(row_size, col_size)
print 'Number of patches: {0}'.format(patches.shape[0])
print 'Batch size: {0}'.format(batch_size)
# train recurrent image model
print 'Training...'
loss.append(
model.train(
patches,
batch_size=batch_size,
method=args.method[k % len(args.method)],
num_epochs=args.num_epochs[k % len(args.num_epochs)],
learning_rate=args.learning_rate[k % len(args.learning_rate)],
momentum=args.momentum[k % len(args.momentum)],
precondition=args.precondition > 0,
train_top_layer=args.train_top_layer[k %
len(args.train_top_layer)]
> 0,
train_means=args.train_means[k % len(args.train_means)] > 0))
if args.finetune[k % len(args.finetune)]:
print 'Finetuning...'
model.finetune(patches, num_samples_train=1000000, max_iter=500)
if args.num_valid > 0:
print 'Computing validation loss...'
loss_valid.append(model.evaluate(patches_valid))
if loss_valid[-1] > loss_valid[-2]:
print 'Performance got worse. Stopping optimization.'
model = model_copy
break
print 'Copying model...'
model_copy = deepcopy(model)
experiment['batch_size'] = batch_size
experiment['args'] = args
experiment['model'] = model
experiment['loss_valid'] = loss_valid
experiment['loss'] = hstack(loss) if len(loss) > 0 else []
experiment.save(os.path.join(args.output, 'rim.{0}.{1}.xpck'))
return 0
def preprocess(data, i, N):
if N > 0 and N < data.shape[0]:
# select subset of data
idx = random_select(N, data.shape[0])
return data[idx, :i].T, data[idx, i][None, :]
return data.T[:i], data.T[[i]]
def test_random_select(self):
# select all elements
self.assertTrue(set(random_select(8, 8)) == set(range(8)))
# n should be larger than k
self.assertRaises(Exception, random_select, 10, 4)
def finetune(self, images,
max_iter=1000,
train_means=False,
num_samples_train=500000,
num_samples_valid=100000):
"""
Train MCGSM using L-BFGS while keeping parameters of SLSTM fixed.
@type images: C{ndarray}/C{list}
@param images: an array or a list of images
"""
if images.shape[0] > num_samples_train:
images = images[random_select(num_samples_train, images.shape[0])]
print 'Preprocessing...'
inputs, outputs = self._preprocess(images)
if self.preconditioner:
print 'Preconditioning...'
# remove correlations
inputs, outputs = self._precondition(inputs, outputs)
# compute hidden unit activations
hiddens = inputs
print 'Forward...'
for l in range(self.num_layers):
self.slstm[l] = SLSTM(
num_rows=hiddens.shape[1],
num_cols=hiddens.shape[2],
num_channels=hiddens.shape[3],
num_hiddens=self.num_hiddens,
batch_size=min([hiddens.shape[0], self.MAX_BATCH_SIZE]),
nonlinearity=self.nonlinearity,
extended=self.extended,
slstm=self.slstm[l],
verbosity=self.verbosity)
hiddens = self.slstm[l].forward(hiddens)
print 'Reshape...'
# remove bottom-right pixels (BSDS300)
hiddens = hiddens.reshape(hiddens.shape[0], -1, self.num_hiddens)
outputs = outputs.reshape(outputs.shape[0], -1, self.num_channels)
hiddens = hiddens[:, :-1]
outputs = outputs[:, :-1]
# form inputs to MCGSM
hiddens = hiddens.reshape(-1, self.num_hiddens).T
outputs = outputs.reshape(-1, self.num_channels).T
print 'Finetuning...'
if hiddens.shape[1] > num_samples_train:
num_samples_valid = min([num_samples_valid, hiddens.shape[1] - num_samples_train])
# select subset of data points for finetuning
idx = random_select(num_samples_train + num_samples_valid, hiddens.shape[1])
if num_samples_valid > 0:
# split data into training and validation set
hiddens_train = asarray(hiddens[:, idx[:num_samples_train]], order='F')
outputs_train = asarray(outputs[:, idx[:num_samples_train]], order='F')
hiddens_valid = asarray(hiddens[:, idx[num_samples_train:]], order='F')
outputs_valid = asarray(outputs[:, idx[num_samples_train:]], order='F')
# finetune with early stopping based on validation performance
return self.mcgsm.train(
hiddens_train, outputs_train,
hiddens_valid, outputs_valid,
parameters={
'verbosity': self.verbosity,
'train_means': train_means,
'max_iter': max_iter})
else:
hiddens = asarray(hiddens[:, idx], order='F')
outputs = asarray(outputs[:, idx], order='F')
return self.mcgsm.train(hiddens, outputs, parameters={
'verbosity': self.verbosity,
'train_means': train_means,
'max_iter': max_iter})
def main(argv):
parser = ArgumentParser(argv[0], description=__doc__,
formatter_class=lambda prog: HelpFormatter(prog, max_help_position=10, width=120))
parser.add_argument('dataset', type=str, nargs='+',
help='Dataset(s) used for training.')
parser.add_argument('output', type=str,
help='Directory or file where trained models will be stored.')
parser.add_argument('--num_components', '-c', type=int, default=3,
help='Number of components used in STM model (default: %(default)d).')
parser.add_argument('--num_features', '-f', type=int, default=2,
help='Number of quadratic features used in STM model (default: %(default)d).')
parser.add_argument('--num_models', '-m', type=int, default=4,
help='Number of models trained (predictions will be averaged across models, default: %(default)d).')
parser.add_argument('--keep_all', '-k', type=int, default=1,
help='If set to 0, only the best model of all trained models is kept (default: %(default)d).')
parser.add_argument('--finetune', '-n', type=int, default=0,
help='If set to 1, enables another finetuning step which is performed after training (default: %(default)d).')
parser.add_argument('--num_train', '-t', type=int, default=0,
help='If specified, a (random) subset of cells is used for training.')
parser.add_argument('--num_valid', '-s', type=int, default=0,
help='If specified, a (random) subset of cells will be used for early stopping based on validation error.')
parser.add_argument('--var_explained', '-e', type=float, default=95.,
help='Controls the degree of dimensionality reduction of fluorescence windows (default: %(default).0f).')
parser.add_argument('--window_length', '-w', type=float, default=1000.,
help='Length of windows extracted from calcium signal for prediction (in milliseconds, default: %(default).0f).')
parser.add_argument('--regularize', '-r', type=float, default=0.,
help='Amount of parameter regularization (filters are regularized for smoothness, default: %(default).1f).')
parser.add_argument('--preprocess', '-p', type=int, default=0,
help='If the data is not already preprocessed, this can be used to do it.')
parser.add_argument('--verbosity', '-v', type=int, default=1)
args, _ = parser.parse_known_args(argv[1:])
experiment = Experiment()
if not args.dataset:
print 'You have to specify at least 1 dataset.'
return 0
data = []
for dataset in args.dataset:
with open(dataset) as handle:
data = data + load(handle)
if args.preprocess:
data = preprocess(data, args.verbosity)
if 'cell_num' not in data[0]:
# no cell number is given, assume traces correspond to cells
for k, entry in enumerate(data):
entry['cell_num'] = k
# collect cell ids
cell_ids = unique([entry['cell_num'] for entry in data])
# pick cells for training
if args.num_train > 0:
training_cells = random_select(args.num_train, len(cell_ids))
else:
# use all cells for training
training_cells = range(len(cell_ids))
models = train([entry for entry in data if entry['cell_num'] in training_cells],
num_valid=args.num_valid,
num_models=args.num_models,
var_explained=args.var_explained,
window_length=args.window_length,
keep_all=args.keep_all,
finetune=args.finetune,
model_parameters={
'num_components': args.num_components,
'num_features': args.num_features},
training_parameters={
'verbosity': 1},
regularize=args.regularize,
verbosity=args.verbosity)
experiment['args'] = args
experiment['training_cells'] = training_cells
experiment['models'] = models
if os.path.isdir(args.output):
experiment.save(os.path.join(args.output, 'model.xpck'))
else:
experiment.save(args.output)
return 0
def main(argv):
parser = ArgumentParser(argv[0], description=__doc__)
parser.add_argument('model', type=str)
parser.add_argument('--data',
'-d',
type=str,
default='data/BSDS300_8x8.mat')
parser.add_argument('--num_data', '-N', type=int, default=1000000)
parser.add_argument('--batch_size', '-B', type=int, default=100000)
args = parser.parse_args(argv[1:])
data_test = loadmat(args.data)['patches_test']
dim = data_test.shape[1]
# reconstruct patches
patch_size = int(sqrt(data_test.shape[1] + 1) + .5)
data_test = hstack([data_test, -sum(data_test, 1)[:, None]])
data_test = data_test.reshape(-1, patch_size, patch_size)
if args.num_data > 0 and data_test.shape[0] > args.num_data:
data_test = data_test[random_select(args.num_data, data_test.shape[0])]
model = Experiment(args.model)['model']
# container for average log-likelihoods of batches
logliks = []
try:
for b in range(0, data_test.shape[0], args.batch_size):
# select batch
batch = data_test[b:b + args.batch_size]
# compute average log-likelihood of different models
loglik = []
loglik.append(model.loglikelihood(batch))
loglik.append(
model.loglikelihood(transform(batch, True, False, False)))
loglik.append(
model.loglikelihood(transform(batch, False, True, False)))
loglik.append(
model.loglikelihood(transform(batch, True, True, False)))
loglik.append(
model.loglikelihood(transform(batch, False, False, True)))
loglik.append(
model.loglikelihood(transform(batch, True, False, True)))
loglik.append(
model.loglikelihood(transform(batch, False, True, True)))
loglik.append(
model.loglikelihood(transform(batch, True, True, True)))
# compute average log-likelihood of mixture model
loglik = logsumexp(loglik, 0) - log(len(loglik))
logliks.append(loglik.mean())
print '{0:.3f} [nat]'.format(mean(logliks))
except KeyboardInterrupt:
pass
import pdb
pdb.set_trace()
