def normalize(input_, output_dim):
if normalization == 'off':
return input_, None, None
#normed = tensor.clip(normed, -3., 3.)
#out = G * normed + B
#params = [G, B]
if normalization == 'bn1':
output = (input_ - input_.mean(
axis=0, keepdims=True)) / (input_.std(
axis=0, keepdims=True) + 1E-6)
return output, None, None
if normalization == 'bn2':
M = shared_floatx_nans((output_dim,))
M.name = 'M'
S = shared_floatx_nans((output_dim,))
S.name = 'S'
#M = input_.mean(axis=0, keepdims=True)
#S = input_.std(axis=0, keepdims=True)
output = (input_ - M) / (S + 1E-6)
return output, M, S
def _allocate(self):
self.W_state = shared_floatx_nans((self.dim, 4*self.dim),
name='W_state')
self.W_cell_to_in = shared_floatx_nans((self.dim,),
name='W_cell_to_in')
self.W_cell_to_forget = shared_floatx_nans((self.dim,),
name='W_cell_to_forget')
self.W_cell_to_out = shared_floatx_nans((self.dim,),
name='W_cell_to_out')
# The underscore is required to prevent collision with
# the `initial_state` application method
self.initial_state_ = shared_floatx_zeros((self.dim,),
name="initial_state")
self.initial_cells = shared_floatx_zeros((self.dim,),
name="initial_cells")
add_role(self.W_state, WEIGHT)
add_role(self.W_cell_to_in, WEIGHT)
add_role(self.W_cell_to_forget, WEIGHT)
add_role(self.W_cell_to_out, WEIGHT)
add_role(self.initial_state_, INITIAL_STATE)
add_role(self.initial_cells, INITIAL_STATE)
self.parameters = [
self.W_state, self.W_cell_to_in, self.W_cell_to_forget,
self.W_cell_to_out, self.initial_state_, self.initial_cells]
def _allocate(self):
self.W_patch = shared_floatx_nans(
(np.prod(self.patch_shape), 4 * self.dim), name='W_input')
self.W_state = shared_floatx_nans((self.dim, 4 * self.dim),
name='W_state')
self.W_cell_to_in = shared_floatx_nans((self.dim, ),
name='W_cell_to_in')
self.W_cell_to_forget = shared_floatx_nans((self.dim, ),
name='W_cell_to_forget')
self.W_cell_to_out = shared_floatx_nans((self.dim, ),
name='W_cell_to_out')
# The underscore is required to prevent collision with
# the `initial_state` application method
self.initial_state_ = shared_floatx_zeros((self.dim, ),
name="initial_state")
self.initial_cells = shared_floatx_zeros((self.dim, ),
name="initial_cells")
add_role(self.W_state, WEIGHT)
add_role(self.W_patch, WEIGHT)
add_role(self.W_cell_to_in, WEIGHT)
add_role(self.W_cell_to_forget, WEIGHT)
add_role(self.W_cell_to_out, WEIGHT)
add_role(self.initial_state_, INITIAL_STATE)
add_role(self.initial_cells, INITIAL_STATE)
self.parameters = [
self.W_state, self.W_cell_to_in, self.W_cell_to_forget,
self.W_patch, self.W_cell_to_out, self.initial_state_,
self.initial_cells
]
def _allocate(self):
self.W_patch = shared_floatx_nans((np.prod(self.patch_shape) + 4, 4 * self.dim), name="W_patch")
self.b = shared_floatx_nans((4 * self.dim,), name="b")
self.W_state = shared_floatx_nans((self.dim, 4 * self.dim), name="W_state")
# The underscore is required to prevent collision with
# the `initial_state` application method
self.initial_state_ = shared_floatx_zeros((self.dim,), name="initial_state")
self.initial_cells = shared_floatx_zeros((self.dim,), name="initial_cells")
self.initial_location = shared_floatx_zeros((2,), name="initial_location")
self.initial_scale = shared_floatx_zeros((2,), name="initial_scale")
add_role(self.W_state, WEIGHT)
add_role(self.b, BIAS)
add_role(self.W_patch, WEIGHT)
add_role(self.initial_state_, INITIAL_STATE)
add_role(self.initial_cells, INITIAL_STATE)
add_role(self.initial_location, INITIAL_STATE)
add_role(self.initial_scale, INITIAL_STATE)
self.parameters = [
self.W_state,
self.W_patch,
self.b,
self.initial_state_,
self.initial_cells,
self.initial_location,
self.initial_scale,
]
def _allocate(self):
W = shared_floatx_nans((self.length, self.dim), name='W_lookup')
self.parameters.append(W)
add_role(W, WEIGHT)
b = shared_floatx_nans((self.dim,), name='b_lookup')
self.parameters.append(b)
add_role(b, BIAS)
def _allocate(self):
W = shared_floatx_nans((self.length, self.dim), name='W_lookup')
self.parameters.append(W)
add_role(W, WEIGHT)
b = shared_floatx_nans((self.dim, ), name='b_lookup')
self.parameters.append(b)
add_role(b, BIAS)
def _allocate(self):
self.padding = shared_floatx_nans((self.dim, ), name='padding')
add_role(self.padding, WEIGHT)
self.location_component = shared_floatx_nans(
(self.max_length, self.dim), name='loc_comp')
add_role(self.location_component, WEIGHT)
self.parameters = [self.padding, self.location_component]
def _allocate(self):
self.W_state = shared_floatx_nans((self.dim, 4*self.dim),
name='W_state')
self.W_cell_to_in = shared_floatx_nans((self.dim,),
name='W_cell_to_in')
self.W_cell_to_forget = shared_floatx_nans((self.dim,),
name='W_cell_to_forget')
self.W_cell_to_out = shared_floatx_nans((self.dim,),
name='W_cell_to_out')
# The underscore is required to prevent collision with
# the `initial_state` application method
self.initial_state_ = shared_floatx_zeros((self.dim,),
name="initial_state")
self.initial_cells = shared_floatx_zeros((self.dim,),
name="initial_cells")
add_role(self.W_state, WEIGHT)
add_role(self.W_cell_to_in, WEIGHT)
add_role(self.W_cell_to_forget, WEIGHT)
add_role(self.W_cell_to_out, WEIGHT)
add_role(self.initial_state_, INITIAL_STATE)
add_role(self.initial_cells, INITIAL_STATE)
# gamma
gamma_val = 0.1 * numpy.ones((self.dim), dtype=config.floatX)
self.gamma = shared(name='gamma', value=gamma_val)
add_role(self.gamma, PARAMETER)
# beta
beta_val = numpy.zeros((self.dim), dtype=config.floatX)
self.beta = shared(name='beta', value=beta_val)
add_role(self.beta, PARAMETER)
self.parameters = [
self.W_state, self.W_cell_to_in, self.W_cell_to_forget,
self.W_cell_to_out, self.initial_state_, self.initial_cells,
self.gamma, self.beta]
def __allocate(self, input_dim, output_dim0, output_dim1):
W = shared_floatx_nans((input_dim, output_dim0, output_dim1), name='W')
add_role(W, WEIGHT)
self.parameters.append(W)
self.add_auxiliary_variable(W.norm(2), name='W_norm')
b = shared_floatx_nans((output_dim0, output_dim1), name='b')
add_role(b, BIAS)
self.parameters.append(b)
def _allocate(self):
super(Convolutional, self)._allocate()
log_gamma = shared_floatx_nans((self.num_filters, ), name='log_gamma')
beta = shared_floatx_nans((self.num_filters, ), name='beta')
self.parameters.append(log_gamma)
self.parameters.append(beta)
add_role(log_gamma, WEIGHT)
add_role(beta, BIAS)
def _allocate(self):
self.parameters.append(shared_floatx_nans((self.dim, self.dim),
name='state_to_state'))
self.parameters.append(shared_floatx_nans((self.dim, 2 * self.dim),
name='state_to_gates'))
for i in range(2):
if self.parameters[i]:
add_role(self.parameters[i], WEIGHT)
def _allocate(self):
super(Linear, self)._allocate()
log_gamma = shared_floatx_nans((self.output_dim, ), name='log_gamma')
beta = shared_floatx_nans((self.output_dim, ), name='beta')
self.parameters.append(log_gamma)
self.parameters.append(beta)
add_role(log_gamma, WEIGHT)
add_role(beta, BIAS)
def _allocate(self):
W = shared_floatx_nans((self.n_out, self.dwin * self.vector_size),
name='W')
b = shared_floatx_nans((self.n_out, ), name='b')
add_role(b, BIAS)
add_role(W, WEIGHT)
self.parameters.append(W)
self.parameters.append(b)
self.mlp.allocate()
def _allocate(self):
W = shared_floatx_nans((self.input_dim, self.attention_dim), name='W')
add_role(W, WEIGHT)
self.parameters.append(W)
self.add_auxiliary_variable(W.norm(2), name='W_norm')
if self.use_bias:
b = shared_floatx_nans((1, ), name='b')
add_role(b, BIAS)
self.parameters.append(b)
def _allocate(self):
W = shared_floatx_nans((self.input_dim, self.output_dim), name='W')
add_role(W, WEIGHT)
self.parameters.append(W)
self.add_auxiliary_variable(W.norm(2), name='W_norm')
if self.use_bias:
b = shared_floatx_nans((self.output_dim,), name='b')
add_role(b, BIAS)
self.parameters.append(b)
self.add_auxiliary_variable(b.norm(2), name='b_norm')
def _allocate(self):
W = shared_floatx_nans((self.num_filters, self.input_dim,
self.filter_length, 1), name='W')
add_role(W, FILTER)
self.params.append(W)
if self.use_bias:
b = shared_floatx_nans((self.num_filters, ), name='b')
add_role(b, BIAS)
self.params.append(b)
def _allocate(self):
W = shared_floatx_nans((self.input_dim, self.output_dim), name='W')
add_role(W, WEIGHT)
self.parameters.append(W)
self.add_auxiliary_variable(W.norm(2), name='W_norm')
if getattr(self, 'use_bias', True):
b = shared_floatx_nans((self.output_dim, ), name='b')
add_role(b, BIAS)
self.parameters.append(b)
self.add_auxiliary_variable(b.norm(2), name='b_norm')
def _allocate(self):
self.parameters.append(shared_floatx_nans((self.dim, self.dim),
name='state_to_state'))
self.parameters.append(shared_floatx_nans((self.dim, 2 * self.dim),
name='state_to_gates'))
self.parameters.append(shared_floatx_zeros((self.dim,),
name="initial_state"))
for i in range(2):
if self.parameters[i]:
add_role(self.parameters[i], WEIGHT)
add_role(self.parameters[2], INITIAL_STATE)
def _allocate(self):
W = shared_floatx_nans(
(self.num_filters, self.input_dim, self.filter_length, 1),
name='W')
add_role(W, FILTER)
self.params.append(W)
if self.use_bias:
b = shared_floatx_nans((self.num_filters, ), name='b')
add_role(b, BIAS)
self.params.append(b)
def _allocate(self):
W = shared_floatx_nans(
(self.num_filters, self.num_channels) + self.filter_size, name='W')
add_role(W, FILTER)
self.params.append(W)
self.add_auxiliary_variable(W.norm(2), name='W_norm')
if self.use_bias:
b = shared_floatx_nans(self.get_dim('output'), name='b')
add_role(b, BIAS)
self.params.append(b)
self.add_auxiliary_variable(b.norm(2), name='b_norm')
def _allocate(self):
W = shared_floatx_nans((self.num_filters, self.num_channels) +
self.filter_size, name='W')
add_role(W, FILTERS)
self.params.append(W)
self.add_auxiliary_variable(W.norm(2), name='W_norm')
if self.use_bias:
b = shared_floatx_nans(self.get_dim('output'), name='b')
add_role(b, BIASES)
self.params.append(b)
self.add_auxiliary_variable(b.norm(2), name='b_norm')
def _allocate(self):
self.parameters.append(shared_floatx_nans((self.dim, self.dim),
name='state_to_state'))
self.parameters.append(shared_floatx_nans((self.dim, 2 * self.dim),
name='state_to_gates'))
self.parameters.append(shared_floatx_zeros((self.dim,),
name="initial_state"))
for i in range(2):
if self.parameters[i]:
add_role(self.parameters[i], WEIGHT)
add_role(self.parameters[2], INITIAL_STATE)
def _allocate(self):
self.Wh = shared_floatx_nans((self.dim, self.dim), name='Wh')
self.Wi = shared_floatx_nans((self.input_dim, self.dim), name='Wi')
self.b = shared_floatx_nans((1,self.dim), name='b')
self.params.append(self.Wi)
self.params.append(self.Wh)
self.params.append(self.b)
add_role(self.Wi, WEIGHT)
add_role(self.Wh, WEIGHT)
add_role(self.b, BIAS)
def _allocate(self):
self.parameters.append(shared_floatx_nans((self.dim, self.dim),
name='state_to_state'))
add_role(self.parameters[-1], WEIGHT)
self.parameters.append(shared_floatx_nans((self.dim, 2 * self.dim),
name='state_to_gates'))
add_role(self.parameters[-1], WEIGHT)
self.parameters.append(shared_floatx_nans((self.dim,),
name="initial_state"))
add_role(self.parameters[-1], INITIAL_STATE)
def _allocate(self):
W = shared_floatx_nans((self.num_filters, self.num_channels) + self.filter_size, name="W")
add_role(W, FILTER)
self.parameters.append(W)
self.add_auxiliary_variable(W.norm(2), name="W_norm")
if self.use_bias:
if self.shared_bias:
b = shared_floatx_nans(self.num_filters, name="b")
else:
b = shared_floatx_nans(self.get_dim("output"), name="b")
add_role(b, BIAS)
self.parameters.append(b)
self.add_auxiliary_variable(b.norm(2), name="b_norm")
def _allocate(self):
W = shared_floatx_nans((self.visible_dim, self.hidden_dim), name='Wrbm')
add_role(W, WEIGHT)
self.params.append(W)
self.add_auxiliary_variable(W.norm(2), name='W_norm')
bv = shared_floatx_nans((self.visible_dim,), name='bv')
add_role(bv, BIAS)
self.params.append(bv)
self.add_auxiliary_variable(bv.norm(2), name='bv_norm')
bh = shared_floatx_nans((self.hidden_dim,), name='bh')
add_role(bh, BIAS)
self.params.append(bh)
self.add_auxiliary_variable(bh.norm(2), name='bh_norm')
def _allocate(self):
self.W_ss = shared_floatx_nans((self.dim, 4*self.dim), name='W_ss')
self.W_is = shared_floatx_nans((self.dim,), name='W_is')
# The underscore is required to prevent collision with
# the `initial_state` application method
self.initial_state_ = shared_floatx_zeros((self.dim,),
name="initial_state")
add_role(self.W_ss, WEIGHT)
add_role(self.W_is, WEIGHT)
add_role(self.initial_state_, INITIAL_STATE)
self.parameters = [
self.W_ss, self.W_is, self.initial_state_]
def _allocate(self):
self.W_state = shared_floatx_nans((self.dim, 4 * self.dim),
name='W_state')
self.W_cell_to_in = shared_floatx_nans((self.dim, ),
name='W_cell_to_in')
self.W_cell_to_forget = shared_floatx_nans((self.dim, ),
name='W_cell_to_forget')
self.W_cell_to_out = shared_floatx_nans((self.dim, ),
name='W_cell_to_out')
# The underscore is required to prevent collision with
# the `initial_state` application method
self.initial_state_ = shared_floatx_zeros((self.dim, ),
name="initial_state")
self.initial_cells = shared_floatx_zeros((self.dim, ),
name="initial_cells")
add_role(self.W_state, WEIGHT)
add_role(self.W_cell_to_in, WEIGHT)
add_role(self.W_cell_to_forget, WEIGHT)
add_role(self.W_cell_to_out, WEIGHT)
add_role(self.initial_state_, INITIAL_STATE)
add_role(self.initial_cells, INITIAL_STATE)
# layer_norm params
scale_add = 0.0
scale_mul = 1.0
self.b1 = scale_add * (shared_floatx_zeros(
(4 * self.dim), name="b1") + 1)
self.b2 = scale_add * (shared_floatx_zeros(
(4 * self.dim), name="b2") + 1)
self.b3 = scale_add * (shared_floatx_zeros(
(1 * self.dim), name="b3") + 1)
self.s1 = scale_mul * (shared_floatx_zeros(
(4 * self.dim), name="s1") + 1)
self.s2 = scale_mul * (shared_floatx_zeros(
(4 * self.dim), name="s2") + 1)
self.s3 = scale_mul * (shared_floatx_zeros(
(1 * self.dim), name="s3") + 1)
# biases
add_role(self.b1, WEIGHT)
add_role(self.b2, WEIGHT)
add_role(self.b3, WEIGHT)
add_role(self.s1, WEIGHT)
add_role(self.s2, WEIGHT)
add_role(self.s3, WEIGHT)
self.parameters = [
self.W_state, self.W_cell_to_in, self.W_cell_to_forget,
self.W_cell_to_out, self.initial_state_, self.initial_cells,
self.b1, self.b2, self.b3, self.s1, self.s2, self.s3
]
def _allocate(self):
parameter_shape = [1] + [1 if broadcast else dim for dim, broadcast
in zip(self.shape, self.broadcastable)]
self.population_stats = dict(
(stat, self.annotated_statistic(
shared_floatx_nans(parameter_shape,
name="population_%s" % stat)))
for stat in self.stats)
self.gamma = shared_floatx_nans(parameter_shape, name='gamma')
self.beta = shared_floatx_nans(parameter_shape, name='beta')
add_role(self.gamma, WEIGHT)
add_role(self.beta, BIAS)
self.parameters.append(self.gamma)
self.parameters.append(self.beta)
def _allocate(self):
self.params.append(shared_floatx_nans((self.dim, self.dim),
name='state_to_state'))
self.params.append(shared_floatx_nans((self.dim, self.dim),
name='state_to_update'))
self.params.append(shared_floatx_nans((self.dim, self.dim),
name='state_to_reset'))
self.params.append(shared_floatx_zeros((self.dim,),
name="initial_state"))
for i in range(3):
if self.params[i]:
add_role(self.params[i], WEIGHT)
add_role(self.params[3], INITIAL_STATE)
def _allocate(self):
W = shared_floatx_nans((self.visible_dim, self.hidden_dim),
name='Wrbm')
add_role(W, WEIGHT)
self.params.append(W)
self.add_auxiliary_variable(W.norm(2), name='W_norm')
bv = shared_floatx_nans((self.visible_dim, ), name='bv')
add_role(bv, BIAS)
self.params.append(bv)
self.add_auxiliary_variable(bv.norm(2), name='bv_norm')
bh = shared_floatx_nans((self.hidden_dim, ), name='bh')
add_role(bh, BIAS)
self.params.append(bh)
self.add_auxiliary_variable(bh.norm(2), name='bh_norm')
def _allocate(self):
self.params.append(
shared_floatx_nans((self.dim, self.dim), name='state_to_state'))
self.params.append(
shared_floatx_nans((self.dim, self.dim), name='state_to_update'))
self.params.append(
shared_floatx_nans((self.dim, self.dim), name='state_to_reset'))
self.params.append(
shared_floatx_zeros((self.dim, ), name="initial_state"))
for i in range(3):
if self.params[i]:
add_role(self.params[i], WEIGHT)
add_role(self.params[3], INITIAL_STATE)
def _allocate(self):
self.W_rz = shared_floatx_nans((self.dim, 2 * self.dim),
name='W_state')
self.W_htilde = shared_floatx_nans((self.dim, self.dim),
name='W_state')
# The underscore is required to prevent collision with
# the `initial_state` application method
self.initial_state_ = shared_floatx_zeros((self.dim,),
name="initial_state")
add_role(self.W_rz, WEIGHT)
add_role(self.W_htilde, WEIGHT)
add_role(self.initial_state_, INITIAL_STATE)
#self.parameters = [self.W_state, self.initial_state_, self.initial_cells]
self.parameters = [self.W_rz, self.W_htilde, self.initial_state_]
def _allocate(self):
c_dim = self.get_dim('c')
zm_dim = self.get_dim('z_mix')
# self.c_0 provides the initial state of the canvas
self.c_0 = shared_floatx_nans((c_dim,), name='c_0')
# self.zm_mean provides the mean of z_mix
self.zm_mean = shared_floatx_nans((zm_dim,), name='zm_mean')
# self.zm_logvar provides the logvar of z_mix
self.zm_logvar = shared_floatx_nans((zm_dim,), name='zm_logvar')
add_role(self.c_0, PARAMETER)
add_role(self.zm_mean, PARAMETER)
add_role(self.zm_logvar, PARAMETER)
# add the theano shared variables to our parameter lists
self.params.extend([ self.c_0, self.zm_mean, self.zm_logvar ])
return
def _allocate(self):
parameter_shape = [1 if broadcast else dim
for dim, broadcast in zip(self.shape, self.broadcastable)]
self.beta = shared_floatx_nans(parameter_shape, name='beta')
add_role(self.beta, BIAS)
self.parameters.append(self.beta)
self.add_auxiliary_variable(self.beta.norm(2), name='beta_norm')
def _allocate(self):
"""Allocates the single parameter of this brick: the initial
element on the stack.
"""
self.parameters.append(
shared_floatx_nans((1, self.stack_dim), name='init_stack'))
add_role(self.parameters[-1], INITIAL_STATE)
def _allocate(self):
"""In addition to the GRU parameters ``state_to_state`` and
``state_to_gates``, add the initial state if the search
strategy is "constant".
"""
self.parameters.append(shared_floatx_nans((self.dim, self.dim),
name='state_to_state'))
self.parameters.append(shared_floatx_nans((self.dim, 2 * self.dim),
name='state_to_gates'))
for i in range(2):
if self.parameters[i]:
add_role(self.parameters[i], WEIGHT)
if self.init_strategy == 'constant':
self.parameters.append(shared_floatx_zeros((self.dim,),
name="initial_state"))
add_role(self.parameters[2], INITIAL_STATE)
def _allocate(self):
parameter_shape = [1 if broadcast else dim
for dim, broadcast in zip(self.shape, self.broadcastable)]
self.b = shared_floatx_nans(parameter_shape, name='b')
add_role(self.b, BIAS)
self.parameters.append(self.b)
self.add_auxiliary_variable(self.b.norm(2), name='b_norm')
def _allocate(self):
self.W_state = shared_floatx_nans((self.dim, 4*self.dim),
name='W_state')
self.W_cell_to_in = shared_floatx_nans((self.dim,),
name='W_cell_to_in')
self.W_cell_to_forget = shared_floatx_nans((self.dim,),
name='W_cell_to_forget')
self.W_cell_to_out = shared_floatx_nans((self.dim,),
name='W_cell_to_out')
add_role(self.W_state, WEIGHT)
add_role(self.W_cell_to_in, WEIGHT)
add_role(self.W_cell_to_forget, WEIGHT)
add_role(self.W_cell_to_out, WEIGHT)
self.params = [self.W_state, self.W_cell_to_in, self.W_cell_to_forget,
self.W_cell_to_out]
def _allocate(self):
parameter_shape = [1 if broadcast else dim
for dim, broadcast in zip(self.shape, self.broadcastable)]
self.gamma = shared_floatx_nans(parameter_shape, name='gamma')
add_role(self.gamma, WEIGHT)
self.parameters.append(self.gamma)
self.add_auxiliary_variable(self.gamma.norm(2), name='gamma_norm')
def _allocate(self):
c_dim = self.get_dim('c')
self.c_0 = shared_floatx_nans((c_dim,), name='c_0')
add_role(self.c_0, PARAMETER)
# add the theano shared variables to our parameter lists
self.params.extend([ self.c_0 ])
return
def _allocate(self):
self.parameters.append(shared_floatx_nans((self.dim, self.dim),
name="W"))
add_role(self.parameters[0], WEIGHT)
self.parameters.append(shared_floatx_zeros((self.dim,),
name="initial_state"))
add_role(self.parameters[1], INITIAL_STATE)
def _allocate(self):
parameter_shape = [1 if broadcast else dim
for dim, broadcast in zip(self.shape, self.broadcastable)]
self.population_stats = dict(
(stat, shared_floatx_nans(parameter_shape,
name="population_%s" % stat))
for stat in self.stats)
def _allocate(self):
self.parameters.append(
shared_floatx_nans((self.dim, self.dim), name="W"))
add_role(self.parameters[0], WEIGHT)
self.parameters.append(
shared_floatx_zeros((self.dim, ), name="initial_state"))
add_role(self.parameters[1], INITIAL_STATE)
def _allocate(self):
"""In addition to the GRU parameters ``state_to_state`` and
``state_to_gates``, add the initial state if the search
strategy is "constant".
"""
self.parameters.append(
shared_floatx_nans((self.dim, self.dim), name='state_to_state'))
self.parameters.append(
shared_floatx_nans((self.dim, 2 * self.dim),
name='state_to_gates'))
for i in range(2):
if self.parameters[i]:
add_role(self.parameters[i], WEIGHT)
if self.init_strategy == 'constant':
self.parameters.append(
shared_floatx_zeros((self.dim, ), name="initial_state"))
add_role(self.parameters[2], INITIAL_STATE)
def _allocate(self):
for i in range(self.n_tables):
self.tables[i].allocate()
self.mlp.allocate()
W = shared_floatx_nans((sum(self.n_mot), sum(self.vect_size)),
name='W')
add_role(W, WEIGHT)
self.parameters.append(W)
def _allocate(self):
self.W_state = shared_floatx_nans((self.dim, 4 * self.dim),
name='W_state')
self.W_cell_to_in = shared_floatx_nans((self.dim, ),
name='W_cell_to_in')
self.W_cell_to_forget = shared_floatx_nans((self.dim, ),
name='W_cell_to_forget')
self.W_cell_to_out = shared_floatx_nans((self.dim, ),
name='W_cell_to_out')
add_role(self.W_state, WEIGHT)
add_role(self.W_cell_to_in, WEIGHT)
add_role(self.W_cell_to_forget, WEIGHT)
add_role(self.W_cell_to_out, WEIGHT)
self.parameters = [
self.W_state, self.W_cell_to_in, self.W_cell_to_forget,
self.W_cell_to_out
]
def _allocate(self):
parameter_shape = [
1 if broadcast else dim
for dim, broadcast in zip(self.shape, self.broadcastable)
]
self.w = shared_floatx_nans(parameter_shape, name='w')
add_role(self.w, WEIGHT)
self.parameters.append(self.w)
self.add_auxiliary_variable(self.w.norm(2), name='w_norm')
def _allocate(self):
parameter_shape = [
1 if broadcast else dim
for dim, broadcast in zip(self.shape, self.broadcastable)
]
self.population_stats = dict(
(stat,
shared_floatx_nans(parameter_shape, name="population_%s" % stat))
for stat in self.stats)
def _allocate(self):
input_dim = ((self.input_dim, )
if not isinstance(self.input_dim, collections.Sequence)
else self.input_dim)
broadcastable = (tuple(False for _ in input_dim)
if self.broadcastable is None else self.broadcastable)
if len(input_dim) != len(broadcastable):
raise ValueError("input_dim and broadcastable must be same length")
var_dim = tuple(
1 if broadcast else dim
for dim, broadcast in equizip(input_dim, broadcastable))
broadcastable = broadcastable
# "gamma", from the Ioffe & Szegedy manuscript.
self.scale = shared_floatx_nans(var_dim,
name='batch_norm_scale',
broadcastable=broadcastable)
# "beta", from the Ioffe & Szegedy manuscript.
self.shift = shared_floatx_nans(var_dim,
name='batch_norm_shift',
broadcastable=broadcastable)
add_role(self.scale, BATCH_NORM_SCALE_PARAMETER)
add_role(self.shift, BATCH_NORM_SHIFT_PARAMETER)
self.parameters.append(self.scale)
self.parameters.append(self.shift)
# These aren't technically parameters, in that they should not be
# learned using the same cost function as other model parameters.
self.population_mean = shared_floatx_zeros(
((self.n_iter, ) if self.n_iter else ()) + var_dim,
name='population_mean',
broadcastable=((False, ) if self.n_iter else ()) + broadcastable)
self.population_stdev = shared_floatx(
numpy.ones(((self.n_iter, ) if self.n_iter else ()) + var_dim),
name='population_stdev',
broadcastable=((False, ) if self.n_iter else ()) + broadcastable)
add_role(self.population_mean, BATCH_NORM_POPULATION_MEAN)
add_role(self.population_stdev, BATCH_NORM_POPULATION_STDEV)
# Normally these would get annotated by an AnnotatingList, but they
# aren't in self.parameters.
add_annotation(self.population_mean, self)
add_annotation(self.population_stdev, self)
def _allocate(self):
if not isinstance(self.input_dim, collections.Iterable)\
or len(self.input_dim) != 3:
raise ValueError("`input_dim` on Convolutional(%s) "
"should be iterable and have a inputdim of 3. "
"Got %s"
%(self.name, str(self.input_dim)))
num_channels = self.input_dim[0]
W = shared_floatx_nans((self.num_filters, num_channels) +
self.filter_size, name='W')
add_role(W, FILTER)
self.parameters.append(W)
self.add_auxiliary_variable(W.norm(2), name='W_norm')
if self.use_bias:
b = shared_floatx_nans((self.num_filters,), name='b')
add_role(b, BIAS)
self.parameters.append(b)
self.add_auxiliary_variable(b.norm(2), name='b_norm')
def _allocate(self):
input_dim = ((self.input_dim,)
if not isinstance(self.input_dim, collections.Sequence)
else self.input_dim)
broadcastable = (tuple(False for _ in input_dim)
if self.broadcastable is None else self.broadcastable)
if len(input_dim) != len(broadcastable):
raise ValueError("input_dim and broadcastable must be same length")
var_dim = tuple(1 if broadcast else dim for dim, broadcast in
equizip(input_dim, broadcastable))
broadcastable = broadcastable
# "gamma", from the Ioffe & Szegedy manuscript.
self.scale = shared_floatx_nans(var_dim, name='batch_norm_scale',
broadcastable=broadcastable)
# "beta", from the Ioffe & Szegedy manuscript.
self.shift = shared_floatx_nans(var_dim, name='batch_norm_shift',
broadcastable=broadcastable)
add_role(self.scale, BATCH_NORM_SCALE_PARAMETER)
add_role(self.shift, BATCH_NORM_SHIFT_PARAMETER)
self.parameters.append(self.scale)
self.parameters.append(self.shift)
# These aren't technically parameters, in that they should not be
# learned using the same cost function as other model parameters.
self.population_mean = shared_floatx_zeros(((self.n_iter,) if self.n_iter else ()) + var_dim,
name='population_mean',
broadcastable=((False,) if self.n_iter else ()) + broadcastable)
self.population_stdev = shared_floatx(numpy.ones(((self.n_iter,) if self.n_iter else ()) + var_dim),
name='population_stdev',
broadcastable=((False,) if self.n_iter else ()) + broadcastable)
add_role(self.population_mean, BATCH_NORM_POPULATION_MEAN)
add_role(self.population_stdev, BATCH_NORM_POPULATION_STDEV)
# Normally these would get annotated by an AnnotatingList, but they
# aren't in self.parameters.
add_annotation(self.population_mean, self)
add_annotation(self.population_stdev, self)
def _allocate(self):
# W
self.parameters.append(shared_floatx_nans((self.dim, self.dim), name="W"))
add_role(self.parameters[0], WEIGHT)
# gamma
gamma_val = 0.1 * numpy.ones((self.dim), dtype=config.floatX)
gamma_ = shared(name='gamma', value=gamma_val)
add_role(gamma_, PARAMETER)
self.parameters.append(gamma_)
# beta
beta_val = numpy.zeros((self.dim), dtype=config.floatX)
beta_ = shared(name='beta', value=beta_val)
add_role(beta_, PARAMETER)
self.parameters.append(beta_)
def _allocate(self):
W = shared_floatx_nans((self.num_filters, self.num_channels) +
self.filter_size, name='W')
add_role(W, FILTER)
self.parameters.append(W)
self.add_auxiliary_variable(W.norm(2), name='W_norm')
if getattr(self, 'use_bias', True):
if self.tied_biases:
b = shared_floatx_nans((self.num_filters,), name='b')
else:
# this error is raised here instead of during initializiation
# because ConvolutionalSequence may specify the image size
if self.image_size == (None, None) and not self.tied_biases:
raise ValueError('Cannot infer bias size without '
'image_size specified. If you use '
'variable image_size, you should use '
'tied_biases=True.')
b = shared_floatx_nans(self.get_dim('output'), name='b')
add_role(b, BIAS)
self.parameters.append(b)
self.add_auxiliary_variable(b.norm(2), name='b_norm')
