def fprop(self,
state_below,
temp=numpy.float32(1),
use_noise=True,
additional_inputs=None,
no_noise_bias=False):
"""
Forward pass through the cost layer.
:type state_below: tensor or layer
:param state_below: The theano expression (or groundhog layer)
representing the input of the cost layer
:type temp: float or tensor scalar
:param temp: scalar representing the temperature that should be used
when sampling from the output distribution
:type use_noise: bool
:param use_noise: flag. If true, noise is used when computing the
output of the model
:type no_noise_bias: bool
:param no_noise_bias: flag, stating if weight noise should be added
to the bias as well, or only to the weights
"""
if self.rank_n_approx:
if self.weight_noise and use_noise and self.noise_params:
emb_val = self.rank_n_activ(
utils.dot(state_below, self.W_em1 + self.nW_em1))
emb_val = TT.dot(emb_val, self.W_em2 + self.nW_em2)
else:
emb_val = self.rank_n_activ(utils.dot(state_below, self.W_em1))
emb_val = TT.dot(emb_val, self.W_em2)
else:
if self.weight_noise and use_noise and self.noise_params:
emb_val = utils.dot(state_below, self.W_em + self.nW_em)
else:
emb_val = utils.dot(state_below, self.W_em)
if additional_inputs:
if use_noise and self.noise_params:
for inp, weight, noise_weight in zip(
additional_inputs, self.additional_weights,
self.noise_additional_weights):
emb_val += utils.dot(inp, (noise_weight + weight))
else:
for inp, weight in zip(additional_inputs,
self.additional_weights):
emb_val += utils.dot(inp, weight)
self.preactiv = emb_val
if self.weight_noise and use_noise and self.noise_params and \
not no_noise_bias:
emb_val = utils.softmax(temp * (emb_val + self.b_em + self.nb_em))
else:
emb_val = utils.softmax(temp * (emb_val + self.b_em))
self.out = emb_val
self.state_below = state_below
self.model_output = emb_val
return emb_val
def fprop(self,
state_below,
temp=numpy.float32(1),
use_noise=True,
additional_inputs=None,
no_noise_bias=False):
"""
Forward pass through the cost layer.
:type state_below: tensor or layer
:param state_below: The theano expression (or groundhog layer)
representing the input of the cost layer
:type temp: float or tensor scalar
:param temp: scalar representing the temperature that should be used
when sampling from the output distribution
:type use_noise: bool
:param use_noise: flag. If true, noise is used when computing the
output of the model
:type no_noise_bias: bool
:param no_noise_bias: flag, stating if weight noise should be added
to the bias as well, or only to the weights
"""
if self.rank_n_approx:
if self.weight_noise and use_noise and self.noise_params:
emb_val = self.rank_n_activ(utils.dot(state_below,
self.W_em1+self.nW_em1))
emb_val = TT.dot(emb_val, self.W_em2 + self.nW_em2)
else:
emb_val = self.rank_n_activ(utils.dot(state_below, self.W_em1))
emb_val = TT.dot(emb_val, self.W_em2)
else:
if self.weight_noise and use_noise and self.noise_params:
emb_val = utils.dot(state_below, self.W_em + self.nW_em)
else:
emb_val = utils.dot(state_below, self.W_em)
if additional_inputs:
if use_noise and self.noise_params:
for inp, weight, noise_weight in zip(
additional_inputs, self.additional_weights,
self.noise_additional_weights):
emb_val += utils.dot(inp, (noise_weight + weight))
else:
for inp, weight in zip(additional_inputs, self.additional_weights):
emb_val += utils.dot(inp, weight)
self.preactiv = emb_val
if self.weight_noise and use_noise and self.noise_params and \
not no_noise_bias:
emb_val = utils.softmax(temp * (emb_val + self.b_em + self.nb_em))
else:
emb_val = utils.softmax(temp * (emb_val + self.b_em))
self.out = emb_val
self.state_below = state_below
self.model_output = emb_val
return emb_val
def fprop(self,
state_below,
temp=numpy.float32(1),
use_noise=True,
additional_inputs=None,
no_noise_bias=False,
target=None,
full_softmax=True):
"""
Forward pass through the cost layer.
:type state_below: tensor or layer
:param state_below: The theano expression (or groundhog layer)
representing the input of the cost layer
:type temp: float or tensor scalar
:param temp: scalar representing the temperature that should be used
when sampling from the output distribution
:type use_noise: bool
:param use_noise: flag. If true, noise is used when computing the
output of the model
:type no_noise_bias: bool
:param no_noise_bias: flag, stating if weight noise should be added
to the bias as well, or only to the weights
"""
if not full_softmax:
assert target is not None, 'target must be given'
if self.rank_n_approx:
if self.weight_noise and use_noise and self.noise_params:
emb_val = self.rank_n_activ(utils.dot(state_below,
self.W_em1 + self.nW_em1))
nW_em = self.nW_em2
else:
emb_val = self.rank_n_activ(utils.dot(state_below, self.W_em1))
W_em = self.W_em2
else:
W_em = self.W_em
if self.weight_noise:
nW_em = self.nW_em
emb_val = state_below
if full_softmax:
if self.weight_noise and use_noise and self.noise_params:
emb_val = TT.dot(emb_val, W_em + nW_em)
else:
emb_val = TT.dot(emb_val, W_em)
if additional_inputs:
if use_noise and self.noise_params:
for inp, weight, noise_weight in zip(
additional_inputs, self.additional_weights,
self.noise_additional_weights):
emb_val += utils.dot(inp, (noise_weight + weight))
else:
for inp, weight in zip(additional_inputs, self.additional_weights):
emb_val += utils.dot(inp, weight)
if self.weight_noise and use_noise and self.noise_params and \
not no_noise_bias:
emb_val = temp * (emb_val + self.b_em + self.nb_em)
else:
emb_val = temp * (emb_val + self.b_em)
else:
W_em = W_em[:, target]
if self.weight_noise:
nW_em = nW_em[:, target]
W_em += nW_em
if emb_val.ndim == 3:
emb_val = emb_val.reshape([emb_val.shape[0] * emb_val.shape[1], emb_val.shape[2]])
emb_val = (W_em.T * emb_val).sum(1) + self.b_em[target]
if self.weight_noise and use_noise:
emb_val += self.nb_em[target]
emb_val = temp * emb_val
self.preactiv = emb_val
if full_softmax:
emb_val = utils.softmax(emb_val)
else:
emb_val = TT.nnet.sigmoid(emb_val)
self.out = emb_val
self.state_below = state_below
self.model_output = emb_val
return emb_val
def fprop(self,
state_below,
temp=numpy.float32(1),
use_noise=True,
additional_inputs=None,
no_noise_bias=False,
target=None,
full_softmax=True):
"""
Forward pass through the cost layer.
:type state_below: tensor or layer
:param state_below: The theano expression (or groundhog layer)
representing the input of the cost layer
:type temp: float or tensor scalar
:param temp: scalar representing the temperature that should be used
when sampling from the output distribution
:type use_noise: bool
:param use_noise: flag. If true, noise is used when computing the
output of the model
:type no_noise_bias: bool
:param no_noise_bias: flag, stating if weight noise should be added
to the bias as well, or only to the weights
"""
if not full_softmax:
assert target != None, 'target must be given'
if self.rank_n_approx:
if self.weight_noise and use_noise and self.noise_params:
emb_val = self.rank_n_activ(
utils.dot(state_below, self.W_em1 + self.nW_em1))
nW_em = self.nW_em2
else:
emb_val = self.rank_n_activ(utils.dot(state_below, self.W_em1))
W_em = self.W_em2
else:
W_em = self.W_em
if self.weight_noise:
nW_em = self.nW_em
emb_val = state_below
if full_softmax:
if self.weight_noise and use_noise and self.noise_params:
emb_val = TT.dot(emb_val, W_em + nW_em)
else:
emb_val = TT.dot(emb_val, W_em)
if additional_inputs:
if use_noise and self.noise_params:
for inp, weight, noise_weight in zip(
additional_inputs, self.additional_weights,
self.noise_additional_weights):
emb_val += utils.dot(inp, (noise_weight + weight))
else:
for inp, weight in zip(additional_inputs,
self.additional_weights):
emb_val += utils.dot(inp, weight)
if self.weight_noise and use_noise and self.noise_params and \
not no_noise_bias:
emb_val = temp * (emb_val + self.b_em + self.nb_em)
else:
emb_val = temp * (emb_val + self.b_em)
else:
W_em = W_em[:, target]
if self.weight_noise:
nW_em = nW_em[:, target]
W_em += nW_em
if emb_val.ndim == 3:
emb_val = emb_val.reshape(
[emb_val.shape[0] * emb_val.shape[1], emb_val.shape[2]])
emb_val = (W_em.T * emb_val).sum(1) + self.b_em[target]
if self.weight_noise and use_noise:
emb_val += self.nb_em[target]
emb_val = temp * emb_val
self.preactiv = emb_val
if full_softmax:
emb_val = utils.softmax(emb_val)
else:
emb_val = TT.nnet.sigmoid(emb_val)
self.out = emb_val
self.state_below = state_below
self.model_output = emb_val
return emb_val
