def __init__(
self,
state_size,
activation='tanh',
use_bias=True,
weight_initializer='xavier_normal',
bias_initializer='zeros',
**kwargs):
"""
:param state_size: state size: positive int
:param activation: activation: string or callable
:param use_bias: whether to use bias
:param weight_initializer: weight initializer identifier
:param bias_initializer: bias initializer identifier
"""
# Call parent's constructor
RNet.__init__(self, self.net_name)
# Attributes
self._state_size = state_size
self._activation = activations.get(activation, **kwargs)
self._use_bias = checker.check_type(use_bias, bool)
self._weight_initializer = initializers.get(weight_initializer)
self._bias_initializer = initializers.get(bias_initializer)
self._output_scale = state_size
def __init__(self,
output_dim,
memory_units=None,
mem_config=None,
use_mem_wisely=False,
weight_regularizer=None,
**kwargs):
# Call parent's constructor
RNet.__init__(self, self.net_name)
# Attributes
self.output_dim = output_dim
self.memory_units = (self.MemoryUnit.parse_units(mem_config)
if memory_units is None else memory_units)
self.memory_units = [mu for mu in self.memory_units if mu.size > 0]
checker.check_type(self.memory_units, Ham.MemoryUnit)
self._state_size = sum([mu.size for mu in self.memory_units])
self._activation = activations.get('tanh', **kwargs)
self._use_mem_wisely = use_mem_wisely
self._truncate = kwargs.get('truncate', False)
self._weight_regularizer = regularizers.get(weight_regularizer,
**kwargs)
# self._use_global_reg = kwargs.get('global_reg', False)
self._kwargs = kwargs
def __init__(
self,
state_size,
activation='tanh',
use_bias=True,
weight_initializer='xavier_uniform',
bias_initializer='zeros',
input_gate=True,
output_gate=True,
forget_gate=True,
with_peepholes=False,
**kwargs):
"""
:param state_size: state size: positive int
:param activation: activation: string or callable
:param use_bias: whether to use bias
:param weight_initializer: weight initializer identifier
:param bias_initializer: bias initializer identifier
"""
# Call parent's constructor
RNet.__init__(self, BasicLSTMCell.net_name)
# Attributes
self._state_size = state_size
self._activation = activations.get(activation, **kwargs)
self._use_bias = checker.check_type(use_bias, bool)
self._weight_initializer = initializers.get(weight_initializer)
self._bias_initializer = initializers.get(bias_initializer)
self._input_gate = checker.check_type(input_gate, bool)
self._output_gate = checker.check_type(output_gate, bool)
self._forget_gate = checker.check_type(forget_gate, bool)
self._with_peepholes = checker.check_type(with_peepholes, bool)
self._output_scale = state_size
def __init__(self,
state_size,
periods=None,
activation='tanh',
use_bias=True,
weight_initializer='xavier_uniform',
bias_initializer='zeros',
**kwargs):
"""
:param state_size: State size
:param periods: a list of integers. If not provided, periods will be set
to a default exponential series {2^{i-1}}_{i=0}^{state_size}
"""
# Call parent's constructor
RNet.__init__(self, ClockworkRNN.net_name)
# Attributes
self._state_size = checker.check_positive_integer(state_size)
self._periods = self._get_periods(periods, **kwargs)
self._activation = activations.get(activation, **kwargs)
self._use_bias = checker.check_type(use_bias, bool)
self._weight_initializer = initializers.get(weight_initializer)
self._bias_initializer = initializers.get(bias_initializer)
# modules = [(start_index, size, period)+]
self._modules = []
self._init_modules(**kwargs)
def __init__(self, identifier, set_logits=False):
p = Parser.parse(identifier)
self._id = p.name
self.abbreviation = (p.name if isinstance(identifier, six.string_types)
else identifier.__name__)
self.full_name = self.abbreviation
self._activation = activations.get(identifier)
self._set_logits = set_logits
def masked_neurons(x,
num,
scope,
activation=None,
s=None,
x_mask=None,
s_mask=None,
use_bias=True,
weight_initializer='glorot_normal',
bias_initializer='zeros',
**kwargs):
# Sanity check
assert isinstance(x, tf.Tensor)
# Get activation and initializers
if activation is not None: activation = activations.get(activation)
weight_initializer = initializers.get(weight_initializer)
bias_initializer = initializers.get(bias_initializer)
def matmul(x, y):
batch_matmul = len(x.shape) == len(y.shape) - 1
if batch_matmul: x = tf.expand_dims(x, axis=1)
assert len(x.shape) == len(y.shape)
output = tf.matmul(x, y)
if batch_matmul: output = tf.squeeze(output, axis=1)
return output
def get_weights(tensor, name, mask=None):
shape = [get_dimension(tensor), num]
if mask is None: return get_variable(name, shape, weight_initializer)
else: return get_masked_weights(name, shape, weight_initializer, mask)
def forward():
# x -> y
Wx = get_weights(x, 'Wx', x_mask)
# .. do matrix multiplication
net_y = matmul(x, Wx)
# s -> y if s exists
if s is not None:
assert isinstance(s, tf.Tensor)
Ws = get_weights(s, 'Ws', s_mask)
# .. add s * Ws to net_y
net_y = tf.add(net_y, matmul(s, Ws))
# Add bias if necessary
if use_bias:
b = get_bias('bias', num, bias_initializer)
net_y = tf.nn.bias_add(net_y, b)
# Activate if necessary
if activation is not None: return activation(net_y)
else: return net_y
with tf.variable_scope(scope):
y = forward()
# Return
return y
def eta(num, activation, *tensors, weight_initializer='glorot_normal',
use_bias=True, bias_initializer='zeros'):
"""
* should be used inside a variable scope
"""
a = psi(num, *tensors, weight_initializer, use_bias, bias_initializer)
if activation is not None:
activation = activations.get(activation)
a = activation(a)
return a
def __init__(self, state_size, mem_fc=True, **kwargs):
# Call parent's constructor
RNet.__init__(self, self.net_name)
# Attributes
self._state_size = state_size
self._activation = activations.get('tanh', **kwargs)
# self._use_bias = True
self._weight_initializer = initializers.get('xavier_normal')
self._bias_initializer = initializers.get('zeros')
self._output_scale = state_size
self._fully_connect_memories = mem_fc
def _get_neurons_fast(self, x, h):
# Calculate net inputs
net_f_tilde, net_i_tilde_bar, net_f, net_i, net_o, net_c_hat = self.neurons(
x, h, scope='net_inputs', num_or_size_splits=6)
# Activate
cumax = activations.get('cumax')
f_tilde = cumax(net_f_tilde)
i_tilde_bar = cumax(net_i_tilde_bar)
f = tf.nn.sigmoid(net_f)
i = tf.nn.sigmoid(net_i)
o = tf.nn.sigmoid(net_o)
c_hat = self._activation(net_c_hat)
return f_tilde, i_tilde_bar, f, i, o, c_hat
def __init__(
self,
state_size,
activation='tanh',
weight_initializer='xavier_normal',
input_gate=True,
forget_gate=True,
output_gate=True,
use_g_bias=True,
g_bias_initializer='zeros',
use_i_bias=True,
i_bias_initializer='zeros',
use_f_bias=True,
f_bias_initializer='zeros',
use_o_bias=True,
o_bias_initializer='zeros',
output_as_mem=True,
fully_connect_memory=True,
activate_memory=True,
truncate_grad=False,
**kwargs):
# Call parent's constructor
RNet.__init__(self, self.net_name)
# Attributes
self._state_size = state_size
self._input_gate = checker.check_type(input_gate, bool)
self._forget_gate = checker.check_type(forget_gate, bool)
self._output_gate = checker.check_type(output_gate, bool)
self._activation = activations.get(activation, **kwargs)
self._weight_initializer = initializers.get(weight_initializer)
self._use_g_bias = checker.check_type(use_g_bias, bool)
self._g_bias_initializer = initializers.get(g_bias_initializer)
self._use_i_bias = checker.check_type(use_i_bias, bool)
self._i_bias_initializer = initializers.get(i_bias_initializer)
self._use_f_bias = checker.check_type(use_f_bias, bool)
self._f_bias_initializer = initializers.get(f_bias_initializer)
self._use_o_bias = checker.check_type(use_o_bias, bool)
self._o_bias_initializer = initializers.get(o_bias_initializer)
self._activate_mem = checker.check_type(activate_memory, bool)
self._truncate_grad = checker.check_type(truncate_grad, bool)
self._fc_memory = checker.check_type(fully_connect_memory, bool)
self._output_as_mem = checker.check_type(output_as_mem, bool)
self._kwargs = kwargs
def __init__(self,
output_dim,
neurons_per_amu=3,
activation='tanh',
use_bias=True,
weight_initializer='xavier_uniform',
bias_initializer='zeros',
**kwargs):
# Call parent's constructor
RNet.__init__(self, AMU.net_name)
# Attributes
self._output_dim = output_dim
self._neurons_per_amu = neurons_per_amu
self._activation = activations.get(activation, **kwargs)
self._use_bias = checker.check_type(use_bias, bool)
self._weight_initializer = initializers.get(weight_initializer)
self._bias_initializer = initializers.get(bias_initializer)
self._output_scale = output_dim
def __init__(self,
activation=None,
weight_initializer='xavier_normal',
use_bias=False,
bias_initializer='zeros',
layer_normalization=False,
weight_dropout=0.0,
**kwargs):
if activation: activation = activations.get(activation)
self._activation = activation
self._weight_initializer = initializers.get(weight_initializer)
self._use_bias = checker.check_type(use_bias, bool)
self._bias_initializer = initializers.get(bias_initializer)
self._layer_normalization = checker.check_type(layer_normalization,
bool)
self._gain_initializer = initializers.get(
kwargs.get('gain_initializer', 'ones'))
self._normalize_each_psi = kwargs.pop('normalize_each_psi', False)
self._weight_dropout = checker.check_type(weight_dropout, float)
assert 0 <= self._weight_dropout < 1
self._nb_kwargs = kwargs
def __init__(self, identifier, **kwargs):
self.abbreviation = (identifier if isinstance(
identifier, six.string_types) else identifier.__name__)
self.full_name = self.abbreviation
self._activation = activations.get(identifier, **kwargs)
def __init__(
self,
state_size,
cell_activation='sigmoid', # g
cell_activation_range=(-2, 2),
memory_activation='sigmoid', # h
memory_activation_range=(-1, 1),
weight_initializer='random_uniform',
weight_initial_range=(-0.1, 0.1),
use_cell_bias=False,
cell_bias_initializer='random_uniform',
cell_bias_init_range=(-0.1, 0.1),
use_in_bias=True,
in_bias_initializer='zeros',
use_out_bias=True,
out_bias_initializer='zeros',
truncate=True,
forward_gate=True,
**kwargs):
# Call parent's constructor
RNet.__init__(self, OriginalLSTMCell.net_name)
# Set state size
self._state_size = state_size
# Set activation
# .. In LSTM98, cell activation is referred to as 'g',
# .. while memory activation is 'h' and gate activation is 'f'
self._cell_activation = activations.get(
cell_activation, range=cell_activation_range)
self._memory_activation = activations.get(
memory_activation, range=memory_activation_range)
self._gate_activation = activations.get('sigmoid')
# Set weight and bias configs
self._weight_initializer = initializers.get(
weight_initializer, range=weight_initial_range)
self._use_cell_bias = use_cell_bias
self._cell_bias_initializer = initializers.get(
cell_bias_initializer, range=cell_bias_init_range)
self._use_in_bias = use_in_bias
self._in_bias_initializer = initializers.get(in_bias_initializer)
self._use_out_bias = use_out_bias
self._out_bias_initializer = initializers.get(out_bias_initializer)
if kwargs.get('rule97', False):
self._cell_bias_initializer = self._weight_initializer
self._in_bias_initializer = self._weight_initializer
# Additional options
self._truncate = truncate
self._forward_gate = forward_gate
# ...
self._num_splits = 3
self._output_scale = state_size
self._h_size = (state_size * self._num_splits if self._forward_gate else
state_size)
# TODO: BETA
self.compute_gradients = self.truncated_rtrl
def neurons(num,
external_input,
activation=None,
memory=None,
fc_memory=True,
scope=None,
use_bias=True,
truncate=False,
num_or_size_splits=None,
weight_initializer='glorot_uniform',
bias_initializer='zeros',
weight_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
**kwargs):
"""Analogous to tf.keras.layers.Dense"""
# Get activation, initializers and regularizers
if activation is not None: activation = activations.get(activation)
weight_initializer = initializers.get(weight_initializer)
bias_initializer = initializers.get(bias_initializer)
weight_regularizer = regularizers.get(weight_regularizer)
bias_regularizer = regularizers.get(bias_regularizer)
activity_regularizer = regularizers.get(activity_regularizer)
# a. Check prune configs
if 'prune_frac' in kwargs.keys():
x_prune_frac, s_prune_frac = (kwargs['prune_frac'], ) * 2
else:
x_prune_frac = kwargs.get('x_prune_frac', 0)
s_prune_frac = kwargs.get('s_prune_frac', 0)
prune_is_on = hub.pruning_rate_fc > 0.0 and x_prune_frac + s_prune_frac > 0
# b. Check sparse configs
x_heads = kwargs.get('x_heads', 0)
s_heads = kwargs.get('s_heads', 0)
sparse_is_on = x_heads + s_heads > 0
# :: Decide to concatenate or not considering a and b
# .. a
if memory is None: should_concate = False
elif prune_is_on: should_concate = x_prune_frac == s_prune_frac
else: should_concate = fc_memory
# .. b
should_concate = should_concate and not sparse_is_on
#
separate_memory_neurons = memory is not None and not should_concate
def get_weights(name, tensor, p_frac, heads):
shape = [get_dimension(tensor), num]
if prune_is_on and p_frac > 0:
assert heads == 0
return get_weights_to_prune(name, shape, weight_initializer,
p_frac)
elif heads > 0:
return _get_sparse_weights(shape[0],
shape[1],
heads,
use_bit_max=True,
coef_initializer=weight_initializer)
else:
return get_variable(name, shape, weight_initializer)
def forward():
# Prepare a weight list for potential regularizer calculation
weight_list = []
# Get x
x = (tf.concat([external_input, memory], axis=1, name='x_concat_s')
if should_concate else external_input)
# - Calculate net input for x
# .. get weights
name = 'Wx' if separate_memory_neurons else 'W'
Wx = get_weights(name, x, x_prune_frac, x_heads)
weight_list.append(Wx)
# .. append weights to context, currently only some extractors will use it
context.weights_list.append(Wx)
# .. do matrix multiplication
net_y = get_matmul(truncate)(x, Wx)
# - Calculate net input for memory and add to net_y if necessary
if separate_memory_neurons:
if not fc_memory:
assert not (prune_is_on and s_prune_frac > 0)
memory_dim = get_dimension(memory)
assert memory_dim == num
Ws = get_variable('Ws', [1, num], weight_initializer)
net_s = get_multiply(truncate)(memory, Ws)
else:
assert prune_is_on or sparse_is_on
Ws = get_weights('Ws', memory, s_prune_frac, s_heads)
net_s = get_matmul(truncate)(memory, Ws)
# Append Ws to weight list and add net_s to net_y
weight_list.append(Ws)
net_y = tf.add(net_y, net_s)
# - Add bias if necessary
b = None
if use_bias:
b = get_bias('bias', num, bias_initializer)
net_y = tf.nn.bias_add(net_y, b)
# - Activate and return
if callable(activation): net_y = activation(net_y)
return net_y, weight_list, b
if scope is not None:
with tf.variable_scope(scope):
y, W_list, b = forward()
else:
y, W_list, b = forward()
# Add regularizer if necessary
if callable(weight_regularizer):
context.add_loss_tensor(
tf.add_n([weight_regularizer(w) for w in W_list]))
if callable(bias_regularizer) and b is not None:
context.add_loss_tensor(bias_regularizer(b))
if callable(activity_regularizer):
context.add_loss_tensor(activity_regularizer(y))
# Split if necessary
if num_or_size_splits is not None:
return tf.split(y, num_or_size_splits=num_or_size_splits, axis=1)
return y
