def log_likelihood_lower_bound(self, x):
x_sequence = tensor.tile(x.dimshuffle('x', 0, 1), (self.T, 1, 1))
rval = self.apply(x_sequence)
c_states, mu_phi, log_sigma_phi = rval[0], rval[-2], rval[-1]
prior_mu = self.prior_mu.dimshuffle('x', 'x', 0)
prior_log_sigma = self.prior_log_sigma.dimshuffle('x', 'x', 0)
kl_term = (
prior_log_sigma - log_sigma_phi +
0.5 * (
tensor.exp(2 * log_sigma_phi) + (mu_phi - prior_mu) ** 2
) / tensor.exp(2 * prior_log_sigma) - 0.5).sum(axis=2).sum(axis=0)
kl_term.name = 'kl_term'
reconstruction_term = - (
x * tensor.nnet.softplus(-c_states[-1]) +
(1 - x) * tensor.nnet.softplus(c_states[-1])).sum(axis=1)
reconstruction_term.name = 'reconstruction_term'
log_likelihood_lower_bound = reconstruction_term - kl_term
log_likelihood_lower_bound.name = 'log_likelihood_lower_bound'
annotation = Annotation()
annotation.add_auxiliary_variable(kl_term, name='kl_term')
annotation.add_auxiliary_variable(-reconstruction_term,
name='reconstruction_term')
add_annotation(log_likelihood_lower_bound, annotation)
return log_likelihood_lower_bound
def add_auxiliary_variable(self, variable, roles=None, name=None):
if name:
variable.name = _variable_name(
self.application.brick.name, self.application.name, name)
variable.tag.name = name
name = None
add_annotation(variable, self.application.brick)
return super(ApplicationCall, self).add_auxiliary_variable(
variable, roles, name)
def add_auxiliary_variable(self, variable, roles=None, name=None):
if name:
variable.name = _variable_name(self.brick.name,
self.application.name, name)
variable.tag.name = name
name = None
add_annotation(variable, self.brick)
return super(ApplicationCall,
self).add_auxiliary_variable(variable, roles, name)
def copy_and_tag_noise(variable, brick, role, name):
"""Helper method to copy a variable and annotate it."""
copy = variable.copy()
# Theano name
copy.name = "{}_apply_{}".format(brick.name, name)
add_annotation(copy, brick)
# Blocks name
copy.tag.name = name
add_role(copy, role)
return copy
def copy_and_tag_noise(variable, brick, role, name):
"""Helper method to copy a variable and annotate it."""
copy = variable.copy()
# Theano name
copy.name = "{}_apply_{}".format(brick.name, name)
add_annotation(copy, brick)
# Blocks name
copy.tag.name = name
add_role(copy, role)
return copy
def apply(self, *args, **kwargs):
out = self._apply(*args, **kwargs)
# ====== add roles ====== #
tmp = out
if not isinstance(tmp, (tuple, list)):
tmp = [out]
for o in tmp:
add_role(o, OUTPUT)
add_annotation(o, self)
# return outputs
return out
def copy_and_tag(variable, role, name):
"""Helper method to copy a variable and annotate it."""
copy = variable.copy()
# Theano name
copy.name = _variable_name(brick.name, self.name, name)
add_annotation(copy, brick)
add_annotation(copy, call)
# Blocks name
copy.tag.name = name
add_role(copy, role)
return copy
def copy_and_tag(variable, role, name):
"""Helper method to copy a variable and annotate it."""
copy = variable.copy()
# Theano name
copy.name = _variable_name(brick.name, self.name, name)
add_annotation(copy, brick)
add_annotation(copy, call)
# Blocks name
copy.tag.name = name
add_role(copy, role)
return copy
def get_mean_logsigma(self, x):
b_mean = 0. if not hasattr(self, 'b_mean') else self.b_mean
b_logsigma = 0. if not hasattr(self, 'b_logsigma') else self.b_logsigma
mean = self.activation(K.dot(x, self.W_mean) + b_mean)
logsigma = self.activation(K.dot(x, self.W_logsigma) + b_logsigma)
mean.name = 'variational_mean'
logsigma.name = 'variational_logsigma'
add_role(mean, VARIATIONAL_MEAN)
add_annotation(mean, self)
add_role(logsigma, VARIATIONAL_LOGSIGMA)
add_annotation(logsigma, self)
return mean, logsigma
def _allocate(self):
input_dim = ((self.input_dim, )
if not isinstance(self.input_dim, collections.Sequence)
else self.input_dim)
broadcastable = (tuple(False for _ in input_dim)
if self.broadcastable is None else self.broadcastable)
if len(input_dim) != len(broadcastable):
raise ValueError("input_dim and broadcastable must be same length")
var_dim = tuple(
1 if broadcast else dim
for dim, broadcast in equizip(input_dim, broadcastable))
broadcastable = broadcastable
# "gamma", from the Ioffe & Szegedy manuscript.
self.scale = shared_floatx_nans(var_dim,
name='batch_norm_scale',
broadcastable=broadcastable)
# "beta", from the Ioffe & Szegedy manuscript.
self.shift = shared_floatx_nans(var_dim,
name='batch_norm_shift',
broadcastable=broadcastable)
add_role(self.scale, BATCH_NORM_SCALE_PARAMETER)
add_role(self.shift, BATCH_NORM_SHIFT_PARAMETER)
self.parameters.append(self.scale)
self.parameters.append(self.shift)
# These aren't technically parameters, in that they should not be
# learned using the same cost function as other model parameters.
self.population_mean = shared_floatx_zeros(
((self.n_iter, ) if self.n_iter else ()) + var_dim,
name='population_mean',
broadcastable=((False, ) if self.n_iter else ()) + broadcastable)
self.population_stdev = shared_floatx(
numpy.ones(((self.n_iter, ) if self.n_iter else ()) + var_dim),
name='population_stdev',
broadcastable=((False, ) if self.n_iter else ()) + broadcastable)
add_role(self.population_mean, BATCH_NORM_POPULATION_MEAN)
add_role(self.population_stdev, BATCH_NORM_POPULATION_STDEV)
# Normally these would get annotated by an AnnotatingList, but they
# aren't in self.parameters.
add_annotation(self.population_mean, self)
add_annotation(self.population_stdev, self)
def _allocate(self):
input_dim = ((self.input_dim,)
if not isinstance(self.input_dim, collections.Sequence)
else self.input_dim)
broadcastable = (tuple(False for _ in input_dim)
if self.broadcastable is None else self.broadcastable)
if len(input_dim) != len(broadcastable):
raise ValueError("input_dim and broadcastable must be same length")
var_dim = tuple(1 if broadcast else dim for dim, broadcast in
equizip(input_dim, broadcastable))
broadcastable = broadcastable
# "gamma", from the Ioffe & Szegedy manuscript.
self.scale = shared_floatx_nans(var_dim, name='batch_norm_scale',
broadcastable=broadcastable)
# "beta", from the Ioffe & Szegedy manuscript.
self.shift = shared_floatx_nans(var_dim, name='batch_norm_shift',
broadcastable=broadcastable)
add_role(self.scale, BATCH_NORM_SCALE_PARAMETER)
add_role(self.shift, BATCH_NORM_SHIFT_PARAMETER)
self.parameters.append(self.scale)
self.parameters.append(self.shift)
# These aren't technically parameters, in that they should not be
# learned using the same cost function as other model parameters.
self.population_mean = shared_floatx_zeros(((self.n_iter,) if self.n_iter else ()) + var_dim,
name='population_mean',
broadcastable=((False,) if self.n_iter else ()) + broadcastable)
self.population_stdev = shared_floatx(numpy.ones(((self.n_iter,) if self.n_iter else ()) + var_dim),
name='population_stdev',
broadcastable=((False,) if self.n_iter else ()) + broadcastable)
add_role(self.population_mean, BATCH_NORM_POPULATION_MEAN)
add_role(self.population_stdev, BATCH_NORM_POPULATION_STDEV)
# Normally these would get annotated by an AnnotatingList, but they
# aren't in self.parameters.
add_annotation(self.population_mean, self)
add_annotation(self.population_stdev, self)
def _set(self, key, value):
if isinstance(value, Variable):
add_role(value, PARAMETER)
add_annotation(value, self.brick)
def annotate_update(self, update, tag_to):
a = Annotation()
for (var, up) in update:
a.updates[var] = up
add_annotation(tag_to, a)
def _setitem(self, key, value):
if isinstance(value, Variable):
add_role(value, PARAMETER)
add_annotation(value, self.brick)
def annotate_update(self, update, tag_to):
a = Annotation()
for (var, up) in update:
a.updates[var] = up
add_annotation(tag_to, a)
def _add_role_and_annotate(var, role, annotations=()):
"""Add a role and zero or more annotations to a variable."""
add_role(var, role)
for annotation in annotations:
add_annotation(var, annotation)
def annotated_statistic(self, var):
add_annotation(var, self)
var.tag.batch_normalization_brick = self
return var
def create_params(self, spec, shape, name, roles=[], annotations=[]):
if not isinstance(roles, (tuple, list)):
roles = [roles]
if not isinstance(annotations, (tuple, list)):
annotations = [annotations]
shape = tuple(shape) # convert to tuple if needed
if any(d <= 0 for d in shape):
raise ValueError((
"Cannot create param with a non-positive shape dimension. "
"Tried to create param with shape=%r, name=%r") %
(shape, name))
#####################################
# 1. Shared variable, just check the shape.
if K.is_shared_variable(spec):
spec_shape = K.eval(K.shape(spec))
if shape is None:
shape = spec_shape
elif tuple(shape) != tuple(spec_shape):
self.raise_arguments('Given variable has different shape '
'from requirement, %s != %s' %
(str(spec_shape), str(shape)))
#####################################
# 2. expression, we can only check number of dimension.
elif K.is_variable(spec):
# We cannot check the shape here, Theano expressions (even shared
# variables) do not have a fixed compile-time shape. We can check the
# dimensionality though.
# Note that we cannot assign a name here. We could assign to the
# `name` attribute of the variable, but the user may have already
# named the variable and we don't want to override this.
if shape is not None and K.ndim(spec) != len(shape):
self.raise_arguments("parameter variable has %d dimensions, "
"should be %d" % (spec.ndim, len(shape)))
#####################################
# 3. numpy ndarray, create shared variable wraper for it.
elif isinstance(spec, np.ndarray):
if shape is not None and spec.shape != shape:
raise RuntimeError("parameter array has shape %s, should be "
"%s" % (spec.shape, shape))
spec = K.variable(spec, name=name)
#####################################
# 4. initializing function.
elif hasattr(spec, '__call__'):
arr = spec(shape)
if K.is_shared_variable(arr):
spec = arr
elif K.is_variable(arr) and K.ndim(arr) == len(shape):
spec = arr
elif isinstance(arr, np.ndarray):
spec = K.variable(arr, name=name)
#####################################
# 5. Exception.
else:
raise RuntimeError("cannot initialize parameters: 'spec' is not "
"a numpy array, a Theano expression, or a "
"callable")
# ====== create and return params ====== #
for i in roles:
if isinstance(i, VariableRole):
add_role(spec, i)
for i in annotations:
if isinstance(i, Annotation):
add_annotation(spec, i)
spec.name = name
# return actual variable or expression
for i, j in enumerate(self._paramters): # override other parameters with same name
if j.name == name:
self._paramters[i] = spec
if spec not in self._paramters:
self._paramters.append(spec)
return spec
def _add_role_and_annotate(var, role, annotations=()):
"""Add a role and zero or more annotations to a variable."""
add_role(var, role)
for annotation in annotations:
add_annotation(var, annotation)
