class Embedder(Initializable):
"""
Linear Embedding Brick
Parameters
----------
dim_in: :class:`int`
Dimensionality of the input
dim_out: :class:`int`
Dimensionality of the output
output_type: :class:`str`
fc for fully connected. conv for convolutional
"""
def __init__(self, dim_in, dim_out, output_type='fc', **kwargs):
self.dim_in = dim_in
self.dim_out = dim_out
self.output_type = output_type
self.linear = Linear(dim_in, dim_out, name='embed_layer')
children = [self.linear]
kwargs.setdefault('children', []).extend(children)
super(Embedder, self).__init__(**kwargs)
@application(inputs=['y'], outputs=['outputs'])
def apply(self, y):
embedding = self.linear.apply(y)
if self.output_type == 'fc':
return embedding
if self.output_type == 'conv':
return embedding.reshape((-1, embedding.shape[-1], 1, 1))
def get_dim(self, name):
if self.output_type == 'fc':
return self.linear.get_dim(name)
if self.output_type == 'conv':
return (self.linear.get_dim(name), 1, 1)
class Qlinear(Initializable):
"""
brick to handle the intermediate layer of an Autoencoder.
In this brick a simple linear mix is performed (a kind of PCA.)
"""
def __init__(self, input_dim, output_dim, **kwargs):
super(Qlinear, self).__init__(**kwargs)
self.mean_transform = Linear(
name=self.name+'_mean',
input_dim=input_dim, output_dim=output_dim,
weights_init=self.weights_init, biases_init=self.biases_init,
use_bias=True)
self.children = [self.mean_transform]
def get_dim(self, name):
if name == 'input':
return self.mean_transform.get_dim('input')
elif name == 'output':
return self.mean_transform.get_dim('output')
else:
raise ValueError
@application(inputs=['x'], outputs=['z', 'kl_term'])
def sample(self, x):
"""Sampling is trivial in this case
"""
mean = self.mean_transform.apply(x)
z = mean
# Calculate KL
batch_size = x.shape[0]
kl = T.zeros((batch_size,),dtype=floatX)
return z, kl
@application(inputs=['x'], outputs=['z'])
def mean_z(self, x):
return self.mean_transform.apply(x)
class Embedder(Initializable):
"""
Linear Embedding Brick
Parameters
----------
dim_in: :class:`int`
Dimensionality of the input
dim_out: :class:`int`
Dimensionality of the output
output_type: :class:`str`
fc for fully connected. conv for convolutional
"""
def __init__(self, dim_in, dim_out, output_type='fc', **kwargs):
self.dim_in = dim_in
self.dim_out = dim_out
self.output_type = output_type
self.linear = Linear(dim_in, dim_out, name='embed_layer')
children = [self.linear]
kwargs.setdefault('children', []).extend(children)
super(Embedder, self).__init__(**kwargs)
@application(inputs=['y'], outputs=['outputs'])
def apply(self, y):
embedding = self.linear.apply(y)
if self.output_type == 'fc':
return embedding
if self.output_type == 'conv':
return embedding.reshape((-1, embedding.shape[-1], 1, 1))
def get_dim(self, name):
if self.output_type == 'fc':
return self.linear.get_dim(name)
if self.output_type == 'conv':
return (self.linear.get_dim(name), 1, 1)
class Qsampler(Initializable, Random):
def __init__(self, input_dim, output_dim, **kwargs):
super(Qsampler, self).__init__(**kwargs)
self.prior_mean = 0.
self.prior_log_sigma = 0.
self.mean_transform = Linear(
name=self.name+'_mean',
input_dim=input_dim, output_dim=output_dim,
weights_init=self.weights_init, biases_init=self.biases_init,
use_bias=True)
self.log_sigma_transform = Linear(
name=self.name+'_log_sigma',
input_dim=input_dim, output_dim=output_dim,
weights_init=self.weights_init, biases_init=self.biases_init,
use_bias=True)
self.children = [self.mean_transform, self.log_sigma_transform]
def get_dim(self, name):
if name == 'input':
return self.mean_transform.get_dim('input')
elif name == 'output':
return self.mean_transform.get_dim('output')
else:
raise ValueError
@application(inputs=['x', 'u'], outputs=['z', 'kl_term'])
def sample(self, x, u):
"""Return a samples and the corresponding KL term
Parameters
----------
x :
Returns
-------
z : tensor.matrix
Samples drawn from Q(z|x)
kl : tensor.vector
KL(Q(z|x) || P_z)
"""
mean = self.mean_transform.apply(x)
log_sigma = self.log_sigma_transform.apply(x)
# Sample from mean-zeros std.-one Gaussian
#u = self.theano_rng.normal(
# size=mean.shape,
# avg=0., std=1.)
# ... and scale/translate samples
z = mean + tensor.exp(log_sigma) * u
# Calculate KL
kl = (
self.prior_log_sigma - log_sigma
+ 0.5 * (
tensor.exp(2 * log_sigma) + (mean - self.prior_mean) ** 2
) / tensor.exp(2 * self.prior_log_sigma)
- 0.5
).sum(axis=-1)
return z, kl
#@application(inputs=['n_samples'])
@application(inputs=['u'], outputs=['z_prior'])
def sample_from_prior(self, u):
"""Sample z from the prior distribution P_z.
Parameters
----------
u : tensor.matrix
gaussian random source
Returns
-------
z : tensor.matrix
samples
"""
z_dim = self.mean_transform.get_dim('output')
# Sample from mean-zeros std.-one Gaussian
#u = self.theano_rng.normal(
# size=(n_samples, z_dim),
# avg=0., std=1.)
# ... and scale/translate samples
z = self.prior_mean + tensor.exp(self.prior_log_sigma) * u
#z.name("z_prior")
return z
class NoisyLinear(Initializable, Feedforward, Random):
"""Linear transformation sent through a learned noisy channel.
Parameters
----------
input_dim : int
The dimension of the input. Required by :meth:`~.Brick.allocate`.
output_dim : int
The dimension of the output. Required by :meth:`~.Brick.allocate`.
num_pieces : int
The number of linear functions. Required by
:meth:`~.Brick.allocate`.
"""
@lazy(allocation=['input_dim', 'output_dim', 'noise_batch_size'])
def __init__(self,
input_dim,
output_dim,
noise_batch_size,
prior_mean=0,
prior_noise_level=0,
**kwargs):
self.linear = Linear()
self.mask = Linear(name='mask')
children = [self.linear, self.mask]
kwargs.setdefault('children', []).extend(children)
super(NoisyLinear, self).__init__(**kwargs)
self.input_dim = input_dim
self.output_dim = output_dim
self.noise_batch_size = noise_batch_size
self.prior_mean = prior_mean
self.prior_noise_level = prior_noise_level
def _push_allocation_config(self):
self.linear.input_dim = self.input_dim
self.linear.output_dim = self.output_dim
self.mask.input_dim = self.output_dim
self.mask.output_dim = self.output_dim
def _allocate(self):
N = shared_floatx_zeros((self.noise_batch_size, self.output_dim),
name='N')
add_role(N, NOISE)
self.parameters.append(N)
@application(inputs=['input_'], outputs=['output'])
def apply(self, input_, application_call):
"""Apply the linear transformation followed by masking with noise.
Parameters
----------
input_ : :class:`~tensor.TensorVariable`
The input on which to apply the transformations
Returns
-------
output : :class:`~tensor.TensorVariable`
The transformed input
"""
pre_noise = self.linear.apply(input_)
noise_level = (self.prior_noise_level -
tensor.clip(self.mask.apply(pre_noise), -16, 16))
noise_level = copy_and_tag_noise(noise_level, self, LOG_SIGMA,
'log_sigma')
# Allow incomplete batches by just taking the noise that is needed
# noise = Print('noise')(self.parameters[0][:noise_level.shape[0], :])
noise = self.parameters[0][:noise_level.shape[0], :]
# noise = Print('noise')(self.theano_rng.normal(noise_level.shape))
kl = (self.prior_noise_level - noise_level + 0.5 *
(tensor.exp(2 * noise_level) +
(pre_noise - self.prior_mean)**2) /
tensor.exp(2 * self.prior_noise_level) - 0.5)
application_call.add_auxiliary_variable(kl, roles=[NITS], name='nits')
return pre_noise + tensor.exp(noise_level) * noise
def get_dim(self, name):
if name == 'input_':
return self.linear.get_dim(name)
if name == 'output':
return self.linear.get_dim(name)
if name == 'nits':
return self.linear.get_dim('output')
return super(NoisyLinear, self).get_dim(name)
class Qsampler(Initializable, Random):
def __init__(self, input_dim, output_dim, **kwargs):
super(Qsampler, self).__init__(**kwargs)
self.prior_mean = 0.
self.prior_log_sigma = 0.
self.mean_transform = Linear(name=self.name + '_mean',
input_dim=input_dim,
output_dim=output_dim,
weights_init=self.weights_init,
biases_init=self.biases_init,
use_bias=True)
self.log_sigma_transform = Linear(name=self.name + '_log_sigma',
input_dim=input_dim,
output_dim=output_dim,
weights_init=self.weights_init,
biases_init=self.biases_init,
use_bias=True)
self.children = [self.mean_transform, self.log_sigma_transform]
def get_dim(self, name):
if name == 'input':
return self.mean_transform.get_dim('input')
elif name == 'output':
return self.mean_transform.get_dim('output')
else:
raise ValueError
@application(inputs=['x', 'u'], outputs=['z', 'kl_term'])
def sample(self, x, u):
"""Return a samples and the corresponding KL term
Parameters
----------
x :
Returns
-------
z : tensor.matrix
Samples drawn from Q(z|x)
kl : tensor.vector
KL(Q(z|x) || P_z)
"""
mean = self.mean_transform.apply(x)
log_sigma = self.log_sigma_transform.apply(x)
# Sample from mean-zeros std.-one Gaussian
#u = self.theano_rng.normal(
# size=mean.shape,
# avg=0., std=1.)
# ... and scale/translate samples
z = mean + tensor.exp(log_sigma) * u
# Calculate KL
kl = (self.prior_log_sigma - log_sigma + 0.5 *
(tensor.exp(2 * log_sigma) + (mean - self.prior_mean)**2) /
tensor.exp(2 * self.prior_log_sigma) - 0.5).sum(axis=-1)
return z, kl
#@application(inputs=['n_samples'])
@application(inputs=['u'], outputs=['z_prior'])
def sample_from_prior(self, u):
"""Sample z from the prior distribution P_z.
Parameters
----------
u : tensor.matrix
gaussian random source
Returns
-------
z : tensor.matrix
samples
"""
z_dim = self.mean_transform.get_dim('output')
# Sample from mean-zeros std.-one Gaussian
#u = self.theano_rng.normal(
# size=(n_samples, z_dim),
# avg=0., std=1.)
# ... and scale/translate samples
z = self.prior_mean + tensor.exp(self.prior_log_sigma) * u
#z.name("z_prior")
return z
class NoisyLinear(Initializable, Feedforward, Random):
"""Linear transformation sent through a learned noisy channel.
Parameters
----------
input_dim : int
The dimension of the input. Required by :meth:`~.Brick.allocate`.
output_dim : int
The dimension of the output. Required by :meth:`~.Brick.allocate`.
num_pieces : int
The number of linear functions. Required by
:meth:`~.Brick.allocate`.
"""
@lazy(allocation=['input_dim', 'output_dim', 'noise_batch_size'])
def __init__(self, input_dim, output_dim, noise_batch_size,
prior_mean=0, prior_noise_level=0, **kwargs):
self.linear = Linear()
self.mask = Linear(name='mask')
children = [self.linear, self.mask]
kwargs.setdefault('children', []).extend(children)
super(NoisyLinear, self).__init__(**kwargs)
self.input_dim = input_dim
self.output_dim = output_dim
self.noise_batch_size = noise_batch_size
self.prior_mean = prior_mean
self.prior_noise_level = prior_noise_level
def _push_allocation_config(self):
self.linear.input_dim = self.input_dim
self.linear.output_dim = self.output_dim
self.mask.input_dim = self.output_dim
self.mask.output_dim = self.output_dim
def _allocate(self):
N = shared_floatx_zeros(
(self.noise_batch_size, self.output_dim), name='N')
add_role(N, NOISE)
self.parameters.append(N)
@application(inputs=['input_'], outputs=['output'])
def apply(self, input_, application_call):
"""Apply the linear transformation followed by masking with noise.
Parameters
----------
input_ : :class:`~tensor.TensorVariable`
The input on which to apply the transformations
Returns
-------
output : :class:`~tensor.TensorVariable`
The transformed input
"""
pre_noise = self.linear.apply(input_)
noise_level = (self.prior_noise_level
- tensor.clip(self.mask.apply(pre_noise), -16, 16))
noise_level = copy_and_tag_noise(
noise_level, self, LOG_SIGMA, 'log_sigma')
# Allow incomplete batches by just taking the noise that is needed
# noise = Print('noise')(self.parameters[0][:noise_level.shape[0], :])
noise = self.parameters[0][:noise_level.shape[0], :]
# noise = Print('noise')(self.theano_rng.normal(noise_level.shape))
kl = (
self.prior_noise_level - noise_level
+ 0.5 * (
tensor.exp(2 * noise_level)
+ (pre_noise - self.prior_mean) ** 2
) / tensor.exp(2 * self.prior_noise_level)
- 0.5
)
application_call.add_auxiliary_variable(kl, roles=[NITS], name='nits')
return pre_noise + tensor.exp(noise_level) * noise
def get_dim(self, name):
if name == 'input_':
return self.linear.get_dim(name)
if name == 'output':
return self.linear.get_dim(name)
if name == 'nits':
return self.linear.get_dim('output')
return super(NoisyLinear, self).get_dim(name)
class Qsampler(Initializable, Random):
def __init__(self, input_dim, output_dim, **kwargs):
super(Qsampler, self).__init__(**kwargs)
self.mean_transform = Linear(
name=self.name+'_mean',
input_dim=input_dim, output_dim=output_dim,
weights_init=self.weights_init, biases_init=self.biases_init,
use_bias=True)
self.logvar_transform = Linear(
name=self.name+'_logvar',
input_dim=input_dim, output_dim=output_dim,
weights_init=self.weights_init, biases_init=self.biases_init,
use_bias=True)
self.children = [self.mean_transform, self.logvar_transform]
return
def get_dim(self, name):
if name == 'input':
return self.mean_transform.get_dim('input')
elif name == 'output':
return self.mean_transform.get_dim('output')
else:
raise ValueError
return
@application(inputs=['x', 'u'], outputs=['z', 'all_klds'])
def sample(self, x, u):
"""
Return samples and the corresponding KL term
Parameters
----------
x : input features for estimating mean and log-variance
u : standard Normal samples to scale and shift
Returns
-------
z : tensor.matrix
Samples drawn from Q(z|x)
kl : tensor.vector
KL(Q(z|x) || P_z)
"""
mean = self.mean_transform.apply(x)
logvar = self.logvar_transform.apply(x)
# ... and scale/translate samples
z = mean + tensor.exp(0.5 * logvar) * u
# Calculate KL
all_klds = gaussian_kld(mean, logvar, 0.0, 0.0)
return z, all_klds
@application(inputs=['u'], outputs=['z_prior'])
def sample_from_prior(self, u):
"""
Sample z from the prior distribution P_z.
Parameters
----------
u : tensor.matrix
gaussian random source
Returns
-------
z : tensor.matrix
samples
"""
z_prior = u
return z_prior
