def test_generate_maks(self): # make sure masks don't overlap input_mask, output_mask = generate_masks(7, 1) self.assertFalse(any(input_mask & output_mask)) input_mask, output_mask = generate_masks(8, 2) self.assertFalse(any(input_mask & output_mask)) input_mask, output_mask = generate_masks(7, 1, [1, 0]) self.assertFalse(any(input_mask & output_mask)) input_mask, output_mask = generate_masks([3, 7, 7], 3, [1, 0, 0]) self.assertFalse(any(input_mask & output_mask))
def test_generate_maks(self):
# make sure masks don't overlap
input_mask, output_mask = generate_masks(7, 1)
self.assertFalse(any(input_mask & output_mask))
input_mask, output_mask = generate_masks(8, 2)
self.assertFalse(any(input_mask & output_mask))
input_mask, output_mask = generate_masks(7, 1, [1, 0])
self.assertFalse(any(input_mask & output_mask))
input_mask, output_mask = generate_masks([3, 7, 7], 3, [1, 0, 0])
self.assertFalse(any(input_mask & output_mask))
def main(argv):
# load image and turn into grayscale
img = rgb2ycc(imread('media/newyork.png'))
# generate masks for grayscale and color model, respectively
input_mask0, output_mask0 = generate_masks(7, 1)
input_mask1, output_mask1 = generate_masks([5, 7, 7], 1, [1, 0, 0])
# train model
model0, pre0 = train_model(img[:, :, 0], input_mask0, output_mask0)
model1, pre1 = train_model(img, input_mask1, output_mask1)
# synthesize a new image
img_sample = img.copy()
# sample intensities
img_sample[:, :, 0] = sample_image(img_sample[:, :, 0], model0,
input_mask0, output_mask0, pre0)
# sample color
img_sample = sample_image(img_sample, model1, input_mask1, output_mask1,
pre1)
# convert back to RGB and enforce constraints
img_sample = ycc2rgb(img_sample)
imwrite('newyork_sample.png', img_sample, vmin=0, vmax=255)
# save model
with open('image_model.pck', 'wb') as handle:
dump(
{
'model0': model0,
'model1': model1,
'input_mask0': input_mask0,
'input_mask1': input_mask1,
'output_mask0': output_mask0,
'output_mask1': output_mask1
}, handle, 1)
return 0
def test_patchmcgsm_stationary(self):
xmask = ones([2, 2], dtype='bool')
ymask = zeros([2, 2], dtype='bool')
xmask[-1, -1] = False
ymask[-1, -1] = True
model = PatchMCGSM(3,
3,
xmask,
ymask,
model=MCGSM(sum(xmask), 1, 2, 2))
data = randn(4, 10000)
model.initialize(data)
model.train(data,
parameters={
'verbosity': 0,
'max_iter': 10,
'stationary': True,
'treshold': 1e-4
})
self.assertTrue(all(model[0, 2].predictors == model[0, 1].predictors))
self.assertFalse(all(model[1, 0].predictors == model[0, 1].predictors))
self.assertTrue(all(model[1, 2].weights == model[1, 1].weights))
self.assertTrue(all(model[1, 2].features == model[1, 1].features))
self.assertTrue(all(model[1, 2].scales == model[1, 1].scales))
self.assertTrue(all(model[1, 2].priors == model[1, 1].priors))
xmask, ymask = generate_masks(3)
model = PatchMCGSM(3,
3,
xmask,
ymask,
model=MCGSM(sum(xmask), 1, 2, 2))
data = randn(4, 10000)
model.initialize(data)
model.train(data,
parameters={
'verbosity': 0,
'max_iter': 10,
'stationary': True,
'treshold': 1e-4
})
self.assertTrue(all(model[0, 2].weights == model[0, 1].weights))
self.assertTrue(all(model[2, 0].features == model[1, 0].features))
self.assertTrue(all(model[2, 2].scales == model[1, 2].scales))
def main(argv):
# load image and turn into grayscale
img = rgb2ycc(imread('media/newyork.png'))
# generate masks for grayscale and color model, respectively
input_mask0, output_mask0 = generate_masks(7, 1)
input_mask1, output_mask1 = generate_masks([5, 7, 7], 1, [1, 0, 0])
# train model
model0, pre0 = train_model(img[:, :, 0], input_mask0, output_mask0)
model1, pre1 = train_model(img, input_mask1, output_mask1)
# synthesize a new image
img_sample = img.copy()
# sample intensities
img_sample[:, :, 0] = sample_image(
img_sample[:, :, 0], model0, input_mask0, output_mask0, pre0)
# sample color
img_sample = sample_image(
img_sample, model1, input_mask1, output_mask1, pre1)
# convert back to RGB and enforce constraints
img_sample = ycc2rgb(img_sample)
imwrite('newyork_sample.png', img_sample, vmin=0, vmax=255)
# save model
with open('image_model.pck', 'wb') as handle:
dump({
'model0': model0,
'model1': model1,
'input_mask0': input_mask0,
'input_mask1': input_mask1,
'output_mask0': output_mask0,
'output_mask1': output_mask1}, handle, 1)
return 0
def __init__(self,
num_channels=1,
num_hiddens=10,
num_components=4,
num_scales=4,
num_features=16,
num_layers=1,
nb_size=3,
nonlinearity='TanH',
verbosity=1,
extended=False,
input_mask=None,
output_mask=None):
self.verbosity = verbosity
self.num_channels = num_channels
self.num_hiddens = num_hiddens
self.num_layers = num_layers
self.nonlinearity = nonlinearity
self.extended = extended
self.input_mask, self.output_mask = generate_masks([nb_size] *
num_channels)
if input_mask:
self.input_mask = input_mask
if output_mask:
self.output_mask = output_mask
self.num_channels = sum(self.output_mask)
self.slstm = [None] * num_layers
self.mcgsm = MCGSM(dim_in=num_hiddens,
dim_out=num_channels,
num_components=num_components,
num_scales=num_scales,
num_features=num_features)
self.preconditioner = None
# see PatchRIDE
self._indicators = False
def __init__(self, num_channels=1, num_hiddens=10, num_components=4, num_scales=4, num_features=16, num_layers=1, nb_size=3, nonlinearity='TanH', verbosity=1, extended=False, input_mask=None, output_mask=None): self.verbosity = verbosity self.num_channels = num_channels self.num_hiddens = num_hiddens self.num_layers = num_layers self.nonlinearity = nonlinearity self.extended = extended self.input_mask, self.output_mask = generate_masks([nb_size] * num_channels) if input_mask: self.input_mask = input_mask if output_mask: self.output_mask = output_mask self.num_channels = sum(self.output_mask) self.slstm = [None] * num_layers self.mcgsm = MCGSM( dim_in=num_hiddens, dim_out=num_channels, num_components=num_components, num_scales=num_scales, num_features=num_features) self.preconditioner = None # see PatchRIDE self._indicators = False
def test_patchmcgsm_stationary(self):
xmask = ones([2, 2], dtype='bool')
ymask = zeros([2, 2], dtype='bool')
xmask[-1, -1] = False
ymask[-1, -1] = True
model = PatchMCGSM(3, 3, xmask, ymask, model=MCGSM(sum(xmask), 1, 2, 2))
data = randn(4, 10000)
model.initialize(data)
model.train(data, parameters={
'verbosity': 0,
'max_iter': 10,
'stationary': True,
'treshold': 1e-4})
self.assertTrue(all(model[0, 2].predictors == model[0, 1].predictors))
self.assertFalse(all(model[1, 0].predictors == model[0, 1].predictors))
self.assertTrue(all(model[1, 2].weights == model[1, 1].weights))
self.assertTrue(all(model[1, 2].features == model[1, 1].features))
self.assertTrue(all(model[1, 2].scales == model[1, 1].scales))
self.assertTrue(all(model[1, 2].priors == model[1, 1].priors))
xmask, ymask = generate_masks(3)
model = PatchMCGSM(3, 3, xmask, ymask, model=MCGSM(sum(xmask), 1, 2, 2))
data = randn(4, 10000)
model.initialize(data)
model.train(data, parameters={
'verbosity': 0,
'max_iter': 10,
'stationary': True,
'treshold': 1e-4})
self.assertTrue(all(model[0, 2].weights == model[0, 1].weights))
self.assertTrue(all(model[2, 0].features == model[1, 0].features))
self.assertTrue(all(model[2, 2].scales == model[1, 2].scales))
def __init__(self, num_channels=[1], nb_size=5, **kwargs):
"""
Create RIDE models.
If C{num_channels} is C{[1, 2]}, for example, then the first channel of
a three-channel image is modeled by one RIDE, and the next two channels are
modeled conditioned on the first channel.
Additional arguments will be passed on to the constructor of L{RIDE}.
@type num_channels: C{list}
@param num_channels: a number of channels for each RIDE model
@type nb_size: C{int}
@param nb_size: controls the neighborhood of pixels read from an image
"""
self.models = []
self.num_channels = num_channels
n_sum = 0
for i, n in enumerate(num_channels):
# generate masks
input_mask, output_mask = generate_masks(
input_size=[nb_size] * (n_sum + n),
output_size=1,
observed=[1] * n_sum + [0] * n)
kwargs['num_channels'] = n
kwargs['input_mask'] = input_mask
kwargs['output_mask'] = output_mask
self.models.append(RIDE(**kwargs))
n_sum += n
# double-check that all masks have the same size
self._match_masks()
def __init__(self, num_channels=[1], nb_size=5, **kwargs):
"""
Create RIDE models.
If C{num_channels} is C{[1, 2]}, for example, then the first channel of
a three-channel image is modeled by one RIDE, and the next two channels are
modeled conditioned on the first channel.
Additional arguments will be passed on to the constructor of L{RIDE}.
@type num_channels: C{list}
@param num_channels: a number of channels for each RIDE model
@type nb_size: C{int}
@param nb_size: controls the neighborhood of pixels read from an image
"""
self.models = []
self.num_channels = num_channels
n_sum = 0
for i, n in enumerate(num_channels):
# generate masks
input_mask, output_mask = generate_masks(
input_size=[nb_size] * (n_sum + n),
output_size=1,
observed=[1] * n_sum + [0] * n)
kwargs['num_channels'] = n
kwargs['input_mask'] = input_mask
kwargs['output_mask'] = output_mask
self.models.append(RIDE(**kwargs))
n_sum += n
# double-check that all masks have the same size
self._match_masks()
def main(argv):
experiment = Experiment()
parser = ArgumentParser(argv[0], description=__doc__)
parser.add_argument('--data', '-d', type=str, default='data/vanhateren_deq2_train.mat')
parser.add_argument('--num_data', '-N', type=int, default=1000000)
parser.add_argument('--num_valid', '-V', type=int, default=200000)
parser.add_argument('--input_size', '-i', type=int, default=9)
parser.add_argument('--max_iter', '-I', type=int, default=3000)
parser.add_argument('--num_components', '-c', type=int, default=128)
parser.add_argument('--num_features', '-f', type=int, default=48)
parser.add_argument('--num_scales', '-s', type=int, default=4)
parser.add_argument('--verbosity', '-v', type=int, default=1)
parser.add_argument('--output', '-o', type=str, default='results/vanhateren_deq2/mcgsm.{0}.{1}.xpck')
args = parser.parse_args(argv[1:])
### DATA HANDLING
if args.verbosity > 0:
print 'Loading data...'
# load data
images = loadmat(args.data)['data']
# define causal neighborhood
input_mask, output_mask = generate_masks(input_size=args.input_size, output_size=1)
# extract causal neighborhoods
num_samples = int((args.num_data + args.num_valid) / images.shape[0] + .9)
def extract(image):
return generate_data_from_image(
image, input_mask, output_mask, num_samples)
inputs, outputs = zip(*mapp(extract, images))
inputs, outputs = hstack(inputs), hstack(outputs)
inputs_train = inputs[:, :args.num_data]
outputs_train = outputs[:, :args.num_data]
inputs_valid = inputs[:, args.num_data:]
outputs_valid = outputs[:, args.num_data:]
if inputs_valid.size < 100:
print 'Not enough data for validation.'
inputs_valid = None
outputs_valid = None
### MODEL TRAINING
if args.verbosity > 0:
print 'Preconditioning...'
preconditioner = WhiteningPreconditioner(inputs_train, outputs_train)
inputs_train, outputs_train = preconditioner(inputs_train, outputs_train)
if inputs_valid is not None:
inputs_valid, outputs_valid = preconditioner(inputs_valid, outputs_valid)
# free memory
del inputs
del outputs
if args.verbosity > 0:
print 'Training model...'
model = MCGSM(
dim_in=inputs_train.shape[0],
dim_out=outputs_train.shape[0],
num_components=args.num_components,
num_features=args.num_features,
num_scales=args.num_scales)
def callback(i, mcgsm):
experiment['args'] = args
experiment['model'] = mcgsm
experiment['preconditioner'] = preconditioner
experiment['input_mask'] = input_mask
experiment['output_mask'] = output_mask
experiment.save(args.output)
model.train(
inputs_train, outputs_train,
inputs_valid, outputs_valid,
parameters={
'verbosity': args.verbosity,
'cb_iter': 500,
'callback': callback,
'max_iter': args.max_iter})
### SAVE RESULTS
experiment['args'] = args
experiment['model'] = model
experiment['preconditioner'] = preconditioner
experiment['input_mask'] = input_mask
experiment['output_mask'] = output_mask
experiment.save(args.output)
return 0
def main(argv):
experiment = Experiment()
parser = ArgumentParser(argv[0], description=__doc__)
parser.add_argument('--data',
'-d',
type=str,
default='data/vanhateren_deq2_train.mat')
parser.add_argument('--num_data', '-N', type=int, default=1000000)
parser.add_argument('--num_valid', '-V', type=int, default=200000)
parser.add_argument('--input_size', '-i', type=int, default=9)
parser.add_argument('--max_iter', '-I', type=int, default=3000)
parser.add_argument('--num_components', '-c', type=int, default=128)
parser.add_argument('--num_features', '-f', type=int, default=48)
parser.add_argument('--num_scales', '-s', type=int, default=4)
parser.add_argument('--verbosity', '-v', type=int, default=1)
parser.add_argument('--output',
'-o',
type=str,
default='results/vanhateren_deq2/mcgsm.{0}.{1}.xpck')
args = parser.parse_args(argv[1:])
### DATA HANDLING
if args.verbosity > 0:
print 'Loading data...'
# load data
images = loadmat(args.data)['data']
# define causal neighborhood
input_mask, output_mask = generate_masks(input_size=args.input_size,
output_size=1)
# extract causal neighborhoods
num_samples = int((args.num_data + args.num_valid) / images.shape[0] + .9)
def extract(image):
return generate_data_from_image(image, input_mask, output_mask,
num_samples)
inputs, outputs = zip(*mapp(extract, images))
inputs, outputs = hstack(inputs), hstack(outputs)
inputs_train = inputs[:, :args.num_data]
outputs_train = outputs[:, :args.num_data]
inputs_valid = inputs[:, args.num_data:]
outputs_valid = outputs[:, args.num_data:]
if inputs_valid.size < 100:
print 'Not enough data for validation.'
inputs_valid = None
outputs_valid = None
### MODEL TRAINING
if args.verbosity > 0:
print 'Preconditioning...'
preconditioner = WhiteningPreconditioner(inputs_train, outputs_train)
inputs_train, outputs_train = preconditioner(inputs_train, outputs_train)
if inputs_valid is not None:
inputs_valid, outputs_valid = preconditioner(inputs_valid,
outputs_valid)
# free memory
del inputs
del outputs
if args.verbosity > 0:
print 'Training model...'
model = MCGSM(dim_in=inputs_train.shape[0],
dim_out=outputs_train.shape[0],
num_components=args.num_components,
num_features=args.num_features,
num_scales=args.num_scales)
def callback(i, mcgsm):
experiment['args'] = args
experiment['model'] = mcgsm
experiment['preconditioner'] = preconditioner
experiment['input_mask'] = input_mask
experiment['output_mask'] = output_mask
experiment.save(args.output)
model.train(inputs_train,
outputs_train,
inputs_valid,
outputs_valid,
parameters={
'verbosity': args.verbosity,
'cb_iter': 500,
'callback': callback,
'max_iter': args.max_iter
})
### SAVE RESULTS
experiment['args'] = args
experiment['model'] = model
experiment['preconditioner'] = preconditioner
experiment['input_mask'] = input_mask
experiment['output_mask'] = output_mask
experiment.save(args.output)
return 0
def __init__(
self,
num_channels=1,
num_hiddens=10,
num_components=8,
num_scales=4,
num_features=16,
num_layers=1,
nb_size=5,
nonlinearity="TanH",
verbosity=1,
extended=False,
input_mask=None,
output_mask=None,
):
"""
@type num_channels: C{int}
@param num_channels: dimensionality of each pixel
@type num_hiddens: C{int}
@param num_hiddens: number of LSTM units in each spatial LSTM layer
@type num_components: C{int}
@param num_components: number of mixture components used by the MCGSM
@type num_scales: C{int}
@param num_scales: number of scales used by the MCGSM
@type num_features: C{int}
@param num_features: number of quadratic features used by the MCGSM
@type num_layers: C{int}
@param num_layers: number of layers of spatial LSTM units
@type nb_size: C{int}
@param nb_size: controls the neighborhood of pixels read from an image
@type nonlinearity: C{str}
@param nonlinearity: nonlinearity used by spatial LSTM (e.g., TanH, ReLU)
@type verbosity: C{int}
@param verbosity: controls how much information is printed during training, etc.
@type extended: C{bool}
@param extended: use previous memory states as additional inputs to LSTM (more parameters)
@type input_mask C{ndarray}
@param input_mask: Boolean mask used to define custom input neighborhood of pixels
@type output_mask C{ndarray}
@param output_mask: determines the position of the output pixel relative to the neighborhood
"""
self.verbosity = verbosity
self.num_channels = num_channels
self.num_hiddens = num_hiddens
self.num_layers = num_layers
self.nonlinearity = nonlinearity
self.extended = extended
self.input_mask, self.output_mask = generate_masks([nb_size] * num_channels)
if input_mask is not None:
self.input_mask = input_mask
if output_mask is not None:
self.output_mask = output_mask
self.num_channels = sum(self.output_mask)
self.slstm = [None] * num_layers
self.mcgsm = MCGSM(
dim_in=self.num_hiddens,
dim_out=self.num_channels,
num_components=num_components,
num_scales=num_scales,
num_features=num_features,
)
self.preconditioner = None
def __init__(self,
num_channels=1,
num_hiddens=10,
num_components=8,
num_scales=4,
num_features=16,
num_layers=1,
nb_size=5,
nonlinearity='TanH',
verbosity=1,
extended=False,
input_mask=None,
output_mask=None):
"""
@type num_channels: C{int}
@param num_channels: dimensionality of each pixel
@type num_hiddens: C{int}
@param num_hiddens: number of LSTM units in each spatial LSTM layer
@type num_components: C{int}
@param num_components: number of mixture components used by the MCGSM
@type num_scales: C{int}
@param num_scales: number of scales used by the MCGSM
@type num_features: C{int}
@param num_features: number of quadratic features used by the MCGSM
@type num_layers: C{int}
@param num_layers: number of layers of spatial LSTM units
@type nb_size: C{int}
@param nb_size: controls the neighborhood of pixels read from an image
@type nonlinearity: C{str}
@param nonlinearity: nonlinearity used by spatial LSTM (e.g., TanH, ReLU)
@type verbosity: C{int}
@param verbosity: controls how much information is printed during training, etc.
@type extended: C{bool}
@param extended: use previous memory states as additional inputs to LSTM (more parameters)
@type input_mask C{ndarray}
@param input_mask: Boolean mask used to define custom input neighborhood of pixels
@type output_mask C{ndarray}
@param output_mask: determines the position of the output pixel relative to the neighborhood
"""
self.verbosity = verbosity
self.num_channels = num_channels
self.num_hiddens = num_hiddens
self.num_layers = num_layers
self.nonlinearity = nonlinearity
self.extended = extended
self.input_mask, self.output_mask = generate_masks([nb_size] *
num_channels)
if input_mask is not None:
self.input_mask = input_mask
if output_mask is not None:
self.output_mask = output_mask
self.num_channels = sum(self.output_mask)
self.slstm = [None] * num_layers
self.mcgsm = MCGSM(dim_in=self.num_hiddens,
dim_out=self.num_channels,
num_components=num_components,
num_scales=num_scales,
num_features=num_features)
self.preconditioner = None
