def __init__(self,
state_size,
use_reset_gate=True,
activation='tanh',
weight_initializer='xavier_normal',
use_bias=True,
bias_initializer='zeros',
z_bias_initializer='zeros',
reset_who='s',
dropout=0.0,
zoneout=0.0,
**kwargs):
"""
:param reset_who: in ('x', 'y')
'x': a_h = W_h * (h_{t-1} \odot r_t)
'y': a_h = r_t \odot (W_h * h_{t-1})
\hat{h}_t = \varphi(Wx*x + a_h + b)
in which r_t is the reset gate at time step t,
\odot is the Hadamard product, W_h is the hidden-to-hidden matrix
"""
# Call parent's constructor
CellBase.__init__(self, activation, weight_initializer, use_bias,
bias_initializer, **kwargs)
# Specific attributes
self._state_size = checker.check_positive_integer(state_size)
self._use_reset_gate = checker.check_type(use_reset_gate, bool)
self._z_bias_initializer = initializers.get(z_bias_initializer)
self._dropout_rate = checker.check_type(dropout, float)
self._zoneout_rate = checker.check_type(zoneout, float)
assert reset_who in ('s', 'a')
self._reset_who = reset_who
def __init__(self,
configs,
activation='tanh',
weight_initializer='xavier_normal',
use_bias=True,
bias_initializer='zeros',
reverse=False,
use_reset_gate=False,
reset_who='s',
shunt_output=False,
gate_output=False,
**kwargs):
"""
:param configs: a list or tuple of tuples with format (size, num, delta)
or a string with format `S1xM1xD1+S2xM2xD2+...`
"""
# Call parent's constructor
CellBase.__init__(self, activation, weight_initializer, use_bias,
bias_initializer, **kwargs)
# Specific attributes
self._reverse = checker.check_type(reverse, bool)
self._use_reset_gate = checker.check_type(use_reset_gate, bool)
self._shunt_output = checker.check_type(shunt_output, bool)
self._gate_output = checker.check_type(gate_output, bool)
self._groups = self._get_groups(configs)
self._state_size = self._get_total_size(self._groups)
assert reset_who in ('a', 's')
self._reset_who = reset_who
def __init__(
self,
state_size,
activation='tanh',
weight_initializer='xavier_normal',
use_bias=True,
couple_fi=False,
cell_bias_initializer='zeros',
input_bias_initializer='zeros',
output_bias_initializer='zeros',
forget_bias_initializer='zeros',
use_output_activation=True,
**kwargs):
# Call parent's constructor
CellBase.__init__(self, activation, weight_initializer,
use_bias, cell_bias_initializer, **kwargs)
# Specific attributes
self._state_size = checker.check_positive_integer(state_size)
self._input_bias_initializer = initializers.get(input_bias_initializer)
self._output_bias_initializer = initializers.get(output_bias_initializer)
self._forget_bias_initializer = initializers.get(forget_bias_initializer)
self._couple_fi = checker.check_type(couple_fi, bool)
self._use_output_activation = checker.check_type(
use_output_activation, bool)
def __init__(self,
configs,
activation='tanh',
weight_initializer='xavier_normal',
use_bias=True,
bias_initializer='zeros',
reverse=False,
use_reset_gate=False,
dropout=0.0,
layer_normalization=False,
**kwargs):
"""
:param configs: a list or tuple of tuples with format (size, num, delta)
or a string with format `S1xM1xD1+S2xM2xD2+...`
"""
# Call parent's constructor
CellBase.__init__(self, activation, weight_initializer, use_bias,
bias_initializer, layer_normalization, **kwargs)
# Specific attributes
self._reverse = checker.check_type(reverse, bool)
self._use_reset_gate = checker.check_type(use_reset_gate, bool)
self._groups = self._get_groups(configs)
self._state_size = self._get_total_size(self._groups)
self._dropout_rate = checker.check_type(dropout, float)
assert 0 <= dropout < 1
# matrices for SOG v1
self._D = None
self._S = None
def __init__(self,
fast_size,
fast_layers,
slow_size,
hyper_kernel,
activation='tanh',
weight_initializer='xavier_normal',
use_bias=True,
bias_initializer='zeros',
input_dropout=0.0,
output_dropout=0.0,
forget_bias=0,
**kwargs):
# Call parent's constructor
CellBase.__init__(self, activation, weight_initializer, use_bias,
bias_initializer, **kwargs)
self.kernel_key = checker.check_type(hyper_kernel, str)
# Specific attributes
self._fast_size = checker.check_positive_integer(fast_size)
self._fast_layers = checker.check_positive_integer(fast_layers)
self._slow_size = checker.check_positive_integer(slow_size)
self._hyper_kernel = self._get_hyper_kernel(hyper_kernel,
do=th.rec_dropout,
forget_bias=forget_bias)
self._input_do = checker.check_type(input_dropout, float)
self._output_do = checker.check_type(output_dropout, float)
def __init__(self,
configs,
factoring_dim=None,
psi_config=None,
activation='tanh',
weight_initializer='xavier_normal',
use_bias=True,
bias_initializer='zeros',
reverse=False,
**kwargs):
"""
:param psi_config: e.g. 's:xs+g;xs', 's:x'
"""
# Call parent's constructor
CellBase.__init__(self, activation, weight_initializer, use_bias,
bias_initializer, **kwargs)
# Specific attributes
self._reverse = checker.check_type(reverse, bool)
self._groups = self._get_groups(configs)
self._state_size = self._get_total_size(self._groups)
if factoring_dim is None: factoring_dim = self._state_size
self._fd = checker.check_positive_integer(factoring_dim)
if not psi_config: psi_config = 's:x'
self._psi_string = checker.check_type(psi_config, str)
self._psi_config = self._parse_psi_string()
def __init__(self, length, **kwargs): # Call parent's constructor CellBase.__init__(self, **kwargs) # Specific attributes self._length = checker.check_positive_integer(length) self._input_shape = hub.conveyor_input_shape # conveyor_input_shape must be provided in xx_core.py module assert isinstance(self._input_shape, list) self._state_shape = [self._length] + self._input_shape
def __init__(
self,
temporal_configs,
output_size=None,
spatial_configs=None,
temporal_reverse=False,
spatial_reverse=False,
temporal_activation='tanh',
spatial_activation='tanh',
weight_initializer='xavier_normal',
use_bias=True,
bias_initializer='zeros',
**kwargs):
"""
:param output_size:
Denote y as cell output, s as state
(1) output_size is 0 or None
y = new_s
(2) output_size is not None
y = neuron(x, prev_s, ...)
"""
# Call parent's constructor
CellBase.__init__(self, temporal_activation, weight_initializer,
use_bias, bias_initializer, **kwargs)
self._temporal_activation = self._activation
self._spatial_activation = spatial_activation
# Specific attributes
self._temporal_groups = self._get_groups(temporal_configs)
self._state_size = self._get_total_size(self._temporal_groups)
# Set spatial groups
self._output_size = None if output_size == 0 else output_size
self._spatial_groups = []
if spatial_configs is not None:
output_dim = (self._output_size if self._output_size is not None
else self._state_size)
# Set spatial_groups
if spatial_configs == 'default':
# Check output size
num_groups = output_dim // 2
assert num_groups * 2 == output_dim
self._spatial_groups = [(2, num_groups, 1)]
else:
assert isinstance(spatial_configs, str) and len(spatial_configs) > 0
self._spatial_groups = self._get_groups(spatial_configs)
total_size = self._get_total_size(self._spatial_groups)
# Check output dim
assert output_dim == total_size
self._reverse_t = checker.check_type(temporal_reverse, bool)
self._reverse_s = checker.check_type(spatial_reverse, bool)
def __init__(self,
state_size,
activation='tanh',
weight_initializer='xavier_normal',
use_bias=True,
cell_bias_initializer='zeros',
**kwargs):
# Call parent's constructor
CellBase.__init__(self, activation, weight_initializer, use_bias,
cell_bias_initializer, **kwargs)
# Specific attributes
self._state_size = checker.check_positive_integer(state_size)
def __init__(self,
configs,
activation='tanh',
weight_initializer='xavier_normal',
use_bias=True,
bias_initializer='zeros',
use_reset_gate=False,
**kwargs):
# Call parent's constructor
CellBase.__init__(self, activation, weight_initializer, use_bias,
bias_initializer, **kwargs)
Mixer.__init__(self, configs)
# Specific attributes
self._use_reset_gate = checker.check_type(use_reset_gate, bool)
self._state_size = self.total_size
def __init__(self,
gam_config,
head_size,
state_size,
num_layers=1,
kernel='gru',
activation='tanh',
gam_dropout=0.0,
rhn_dropout=0.0,
use_reset_gate=True,
**kwargs):
# Call parent's constructor
CellBase.__init__(self, activation=activation, **kwargs)
GAM.__init__(self, gam_config, head_size)
# Own attributes
self._gam_dropout = checker.check_gate(gam_dropout)
self._rhn_dropout = checker.check_gate(rhn_dropout)
self._state_size = checker.check_positive_integer(state_size)
self._num_layers = checker.check_positive_integer(num_layers)
self._kernel_key = checker.check_type(kernel, str)
self._highway_op = self._get_hyper_kernel(kernel, do=self._rhn_dropout)
self._use_reset_gate = checker.check_type(use_reset_gate, bool)
def __init__(
self,
state_size,
num_layers,
hyper_kernel,
activation='tanh',
weight_initializer='xavier_normal',
use_bias=True,
bias_initializer='zeros',
forget_bias=0,
**kwargs):
# Call parent's constructor
CellBase.__init__(self, activation, weight_initializer, use_bias,
bias_initializer, **kwargs)
self.kernel_key = checker.check_type(hyper_kernel, str)
# Specific attributes
self._state_size = checker.check_positive_integer(state_size)
self._num_layers = checker.check_positive_integer(num_layers)
self._hyper_kernel = self._get_hyper_kernel(
hyper_kernel, do=th.rec_dropout, ln=self._layer_normalization,
forget_bias=forget_bias)
def __init__(self,
configs,
hyper_kernel,
hyper_dim,
signal_size,
activation='tanh',
weight_initializer='xavier_normal',
use_bias=True,
bias_initializer='zeros',
reverse=False,
use_reset_gate=False,
**kwargs):
"""
:param configs: a list or tuple of tuples with format (size, num, delta)
or a string with format `S1xM1xD1+S2xM2xD2+...`
"""
# Call parent's constructor
CellBase.__init__(self, activation, weight_initializer, use_bias,
bias_initializer, **kwargs)
# NeuronArray's constructor will not be called (which is not elegant)
# Specific attributes
self._reverse = checker.check_type(reverse, bool)
self._use_reset_gate = checker.check_type(use_reset_gate, bool)
self._groups = self._get_groups(configs)
self._state_size = self._get_total_size(self._groups)
self.kernel_key = checker.check_type(hyper_kernel, str)
self._hyper_kernel = self._get_hyper_kernel(hyper_kernel)
self._hyper_dim = checker.check_positive_integer(hyper_dim)
self._signal_size = checker.check_positive_integer(signal_size)
# Assigned during linking
self._gate_signals = None
assert not self._use_reset_gate