def __init__(self, incoming, num_units, hidden_nonlinearity, gate_nonlinearity=LN.sigmoid, name=None, W_init=LI.HeUniform(), b_init=LI.Constant(0.), hidden_init=LI.Constant(0.), hidden_init_trainable=True): if hidden_nonlinearity is None: hidden_nonlinearity = LN.identity if gate_nonlinearity is None: gate_nonlinearity = LN.identity super(GRULayer, self).__init__(incoming, name=name) input_shape = self.input_shape[2:] input_dim = ext.flatten_shape_dim(input_shape) # self._name = name # Weights for the initial hidden state self.h0 = self.add_param(hidden_init, (num_units,), name="h0", trainable=hidden_init_trainable, regularizable=False) # Weights for the reset gate self.W_xr = self.add_param(W_init, (input_dim, num_units), name="W_xr") self.W_hr = self.add_param(W_init, (num_units, num_units), name="W_hr") self.b_r = self.add_param(b_init, (num_units,), name="b_r", regularizable=False) # Weights for the update gate self.W_xu = self.add_param(W_init, (input_dim, num_units), name="W_xu") self.W_hu = self.add_param(W_init, (num_units, num_units), name="W_hu") self.b_u = self.add_param(b_init, (num_units,), name="b_u", regularizable=False) # Weights for the cell gate self.W_xc = self.add_param(W_init, (input_dim, num_units), name="W_xc") self.W_hc = self.add_param(W_init, (num_units, num_units), name="W_hc") self.b_c = self.add_param(b_init, (num_units,), name="b_c", regularizable=False) self.gate_nonlinearity = gate_nonlinearity self.num_units = num_units self.nonlinearity = hidden_nonlinearity
def __init__(self, incoming, num_units, hidden_nonlinearity, name=None, W_init=LI.GlorotUniform(), b_init=LI.Constant(0.), Wi_init=LI.GlorotUniform(), hidden_init=LI.Constant(0.), hidden_init_trainable=True, **kwargs): if hidden_nonlinearity is None: hidden_nonlinearity = NL.identity super(RecurrentLayer, self).__init__(incoming, name=name) input_shape = self.input_shape[2:] input_dim = ext.flatten_shape_dim(input_shape) # self._name = name # initial hidden state self.h0 = self.add_param(hidden_init, (num_units,), name="h0", trainable=hidden_init_trainable, regularizable=False) # Weights from input to hidden self.W_xh = self.add_param(Wi_init, (input_dim, num_units), name="W_xh") self.b_h = self.add_param(b_init, (num_units,), name="b_h", regularizable=False) # Recurrent weights self.W_hh = self.add_param(W_init, (num_units, num_units), name="W_hh") self.num_units = num_units self.nonlinearity = hidden_nonlinearity
def __init__(self, incoming, num_units, hidden_nonlinearity, gate_nonlinearity=LN.sigmoid, name=None, W_init=LI.GlorotUniform(), b_init=LI.Constant(0.), hidden_init=LI.Constant(0.), hidden_init_trainable=True): if hidden_nonlinearity is None: hidden_nonlinearity = LN.identity if gate_nonlinearity is None: gate_nonlinearity = LN.identity super(GRULayer, self).__init__(incoming, name=name) input_shape = self.input_shape[2:] input_dim = ext.flatten_shape_dim(input_shape) # self._name = name # Weights for the initial hidden state self.h0 = self.add_param(hidden_init, (num_units,), name="h0", trainable=hidden_init_trainable, regularizable=False) # Weights for the reset gate self.W_xr = self.add_param(W_init, (input_dim, num_units), name="W_xr") self.W_hr = self.add_param(W_init, (num_units, num_units), name="W_hr") self.b_r = self.add_param(b_init, (num_units,), name="b_r", regularizable=False) # Weights for the update gate self.W_xu = self.add_param(W_init, (input_dim, num_units), name="W_xu") self.W_hu = self.add_param(W_init, (num_units, num_units), name="W_hu") self.b_u = self.add_param(b_init, (num_units,), name="b_u", regularizable=False) # Weights for the cell gate self.W_xc = self.add_param(W_init, (input_dim, num_units), name="W_xc") self.W_hc = self.add_param(W_init, (num_units, num_units), name="W_hc") self.b_c = self.add_param(b_init, (num_units,), name="b_c", regularizable=False) self.gate_nonlinearity = gate_nonlinearity self.num_units = num_units self.nonlinearity = hidden_nonlinearity
def __init__(self, incoming, num_units, hidden_nonlinearity, name=None, W_init=SpectralRadius(density=0.2), hidden_init=LI.Constant(0.), hidden_init_trainable=True, Wi_init=LI.Uniform(0.5), leak_rate=0.5, **kwargs): if hidden_nonlinearity is None: hidden_nonlinearity = NL.identity L.Layer.__init__(self, incoming, name=name) # skip direct parent, we'll do all that init here input_shape = self.input_shape[2:] input_dim = ext.flatten_shape_dim(input_shape) # self._name = name # initial hidden state self.h0 = self.add_param(hidden_init, (num_units,), name="h0", trainable=hidden_init_trainable, regularizable=False) # Weights from input to hidden self.W_xh = self.add_param(Wi_init, (input_dim, num_units), name="W_xh", trainable=False, regularizable=False) # Recurrent weights self.W_hh = self.add_param(W_init, (num_units, num_units), name="W_hh", trainable=False, regularizable=False) self.leak_rate = leak_rate self.num_units = num_units self.nonlinearity = hidden_nonlinearity