def __init__(self, incoming, num_units, hidden_nonlinearity,
gate_nonlinearity=LN.sigmoid, name=None,
W_init=LI.HeUniform(), b_init=LI.Constant(0.),
hidden_init=LI.Constant(0.), hidden_init_trainable=True):
if hidden_nonlinearity is None:
hidden_nonlinearity = LN.identity
if gate_nonlinearity is None:
gate_nonlinearity = LN.identity
super(GRULayer, self).__init__(incoming, name=name)
input_shape = self.input_shape[2:]
input_dim = ext.flatten_shape_dim(input_shape)
# self._name = name
# Weights for the initial hidden state
self.h0 = self.add_param(hidden_init, (num_units,), name="h0", trainable=hidden_init_trainable,
regularizable=False)
# Weights for the reset gate
self.W_xr = self.add_param(W_init, (input_dim, num_units), name="W_xr")
self.W_hr = self.add_param(W_init, (num_units, num_units), name="W_hr")
self.b_r = self.add_param(b_init, (num_units,), name="b_r", regularizable=False)
# Weights for the update gate
self.W_xu = self.add_param(W_init, (input_dim, num_units), name="W_xu")
self.W_hu = self.add_param(W_init, (num_units, num_units), name="W_hu")
self.b_u = self.add_param(b_init, (num_units,), name="b_u", regularizable=False)
# Weights for the cell gate
self.W_xc = self.add_param(W_init, (input_dim, num_units), name="W_xc")
self.W_hc = self.add_param(W_init, (num_units, num_units), name="W_hc")
self.b_c = self.add_param(b_init, (num_units,), name="b_c", regularizable=False)
self.gate_nonlinearity = gate_nonlinearity
self.num_units = num_units
self.nonlinearity = hidden_nonlinearity
def __init__(self, incoming, num_units, hidden_nonlinearity,
name=None,
W_init=LI.GlorotUniform(), b_init=LI.Constant(0.), Wi_init=LI.GlorotUniform(),
hidden_init=LI.Constant(0.), hidden_init_trainable=True, **kwargs):
if hidden_nonlinearity is None:
hidden_nonlinearity = NL.identity
super(RecurrentLayer, self).__init__(incoming, name=name)
input_shape = self.input_shape[2:]
input_dim = ext.flatten_shape_dim(input_shape)
# self._name = name
# initial hidden state
self.h0 = self.add_param(hidden_init, (num_units,), name="h0", trainable=hidden_init_trainable,
regularizable=False)
# Weights from input to hidden
self.W_xh = self.add_param(Wi_init, (input_dim, num_units), name="W_xh")
self.b_h = self.add_param(b_init, (num_units,), name="b_h", regularizable=False)
# Recurrent weights
self.W_hh = self.add_param(W_init, (num_units, num_units), name="W_hh")
self.num_units = num_units
self.nonlinearity = hidden_nonlinearity
def __init__(self, incoming, num_units, hidden_nonlinearity,
gate_nonlinearity=LN.sigmoid, name=None,
W_init=LI.GlorotUniform(), b_init=LI.Constant(0.),
hidden_init=LI.Constant(0.), hidden_init_trainable=True):
if hidden_nonlinearity is None:
hidden_nonlinearity = LN.identity
if gate_nonlinearity is None:
gate_nonlinearity = LN.identity
super(GRULayer, self).__init__(incoming, name=name)
input_shape = self.input_shape[2:]
input_dim = ext.flatten_shape_dim(input_shape)
# self._name = name
# Weights for the initial hidden state
self.h0 = self.add_param(hidden_init, (num_units,), name="h0", trainable=hidden_init_trainable,
regularizable=False)
# Weights for the reset gate
self.W_xr = self.add_param(W_init, (input_dim, num_units), name="W_xr")
self.W_hr = self.add_param(W_init, (num_units, num_units), name="W_hr")
self.b_r = self.add_param(b_init, (num_units,), name="b_r", regularizable=False)
# Weights for the update gate
self.W_xu = self.add_param(W_init, (input_dim, num_units), name="W_xu")
self.W_hu = self.add_param(W_init, (num_units, num_units), name="W_hu")
self.b_u = self.add_param(b_init, (num_units,), name="b_u", regularizable=False)
# Weights for the cell gate
self.W_xc = self.add_param(W_init, (input_dim, num_units), name="W_xc")
self.W_hc = self.add_param(W_init, (num_units, num_units), name="W_hc")
self.b_c = self.add_param(b_init, (num_units,), name="b_c", regularizable=False)
self.gate_nonlinearity = gate_nonlinearity
self.num_units = num_units
self.nonlinearity = hidden_nonlinearity
def __init__(self, incoming, num_units, hidden_nonlinearity, name=None,
W_init=SpectralRadius(density=0.2),
hidden_init=LI.Constant(0.), hidden_init_trainable=True,
Wi_init=LI.Uniform(0.5), leak_rate=0.5, **kwargs):
if hidden_nonlinearity is None:
hidden_nonlinearity = NL.identity
L.Layer.__init__(self, incoming, name=name) # skip direct parent, we'll do all that init here
input_shape = self.input_shape[2:]
input_dim = ext.flatten_shape_dim(input_shape)
# self._name = name
# initial hidden state
self.h0 = self.add_param(hidden_init, (num_units,), name="h0", trainable=hidden_init_trainable,
regularizable=False)
# Weights from input to hidden
self.W_xh = self.add_param(Wi_init, (input_dim, num_units), name="W_xh", trainable=False, regularizable=False)
# Recurrent weights
self.W_hh = self.add_param(W_init, (num_units, num_units), name="W_hh", trainable=False, regularizable=False)
self.leak_rate = leak_rate
self.num_units = num_units
self.nonlinearity = hidden_nonlinearity