def __init__(self,
num_units,
alpha,
sigma_rec=0,
activation='softplus',
w_rec_init='diag',
rng=None,
reuse=None,
name=None):
super(LeakyRNNCellSeparateInput, self).__init__(_reuse=reuse,
name=name)
# Inputs must be 2-dimensional.
# self.input_spec = base_layer.InputSpec(ndim=2)
self._num_units = num_units
self._w_rec_init = w_rec_init
self._reuse = reuse
if activation == 'softplus':
self._activation = tf.nn.softplus
self._w_in_start = 1.0
self._w_rec_start = 0.5
elif activation == 'relu':
self._activation = tf.nn.relu
self._w_in_start = 1.0
self._w_rec_start = 0.5
else:
raise ValueError('Unknown activation')
self._alpha = alpha
self._sigma = np.sqrt(2 / alpha) * sigma_rec
if rng is None:
self.rng = np.random.RandomState()
else:
self.rng = rng
# Generate initialization matrix
n_hidden = self._num_units
if self._w_rec_init == 'diag':
w_rec0 = self._w_rec_start * np.eye(n_hidden)
elif self._w_rec_init == 'randortho':
w_rec0 = self._w_rec_start * tools.gen_ortho_matrix(n_hidden,
rng=self.rng)
elif self._w_rec_init == 'randgauss':
w_rec0 = (self._w_rec_start * self.rng.randn(n_hidden, n_hidden) /
np.sqrt(n_hidden))
else:
raise ValueError
self.w_rnn0 = w_rec0
self._initializer = tf.constant_initializer(w_rec0, dtype=tf.float32)
def __init__(self,
num_units,
n_input,
alpha,
sigma_rec=0,
activation='softplus',
w_rec_init='diag',
rng=None,
reuse=None,
name=None):
super(LeakyRNNCell, self).__init__(_reuse=reuse, name=name)
# Inputs must be 2-dimensional.
# self.input_spec = base_layer.InputSpec(ndim=2)
self._num_units = num_units
self._w_rec_init = w_rec_init
self._reuse = reuse
if activation == 'softplus':
self._activation = lambda x: F.softplus(x)
self._w_in_start = 1.0
self._w_rec_start = 0.5
elif activation == 'tanh':
self._activation = lambda x: F.tanh(x)
self._w_in_start = 1.0
self._w_rec_start = 1.0
elif activation == 'relu':
self._activation = lambda x: F.relu(x)
self._w_in_start = 1.0
self._w_rec_start = 0.5
elif activation == 'power':
self._activation = lambda x: torch.square(F.relu(x))
self._w_in_start = 1.0
self._w_rec_start = 0.01
elif activation == 'retanh':
self._activation = lambda x: F.tanh(F.relu(x))
self._w_in_start = 1.0
self._w_rec_start = 0.5
else:
raise ValueError('Unknown activation')
self._alpha = alpha
self._sigma = np.sqrt(2 / alpha) * sigma_rec
if rng is None:
self.rng = np.random.RandomState()
else:
self.rng = rng
# Generate initialization matrix
n_hidden = self._num_units
w_in0 = (self.rng.randn(n_input, n_hidden) / np.sqrt(n_input) *
self._w_in_start)
if self._w_rec_init == 'diag':
w_rec0 = self._w_rec_start * np.eye(n_hidden)
elif self._w_rec_init == 'randortho':
w_rec0 = self._w_rec_start * tools.gen_ortho_matrix(n_hidden,
rng=self.rng)
elif self._w_rec_init == 'randgauss':
w_rec0 = (self._w_rec_start * self.rng.randn(n_hidden, n_hidden) /
np.sqrt(n_hidden))
matrix0 = np.concatenate((w_in0, w_rec0), axis=0)
self.w_rnn0 = matrix0
nn.init.constant_(self._initializer, matrix0, dtype=torch.float32)