def __init__(self,
output_dim,
force_real=False,
use_bias=True,
weight_initializer='xavier_uniform',
bias_initializer='zeros',
weight_regularizer=None,
bias_regularizer=None,
**kwargs):
if not np.isscalar(output_dim):
raise TypeError('!! output_dim must be a scalar, not {}'.format(
type(output_dim)))
self._output_dim = output_dim
self._force_real = force_real
self._use_bias = use_bias
self._weight_initializer = initializers.get(weight_initializer)
self._bias_initializer = initializers.get(bias_initializer)
self._weight_regularizer = regularizers.get(weight_regularizer,
**kwargs)
self._bias_regularizer = regularizers.get(bias_regularizer, **kwargs)
self.weights = None
self.biases = None
self.neuron_scale = [output_dim]
def __init__(self,
output_dim,
memory_units=None,
mem_config=None,
use_mem_wisely=False,
weight_regularizer=None,
**kwargs):
# Call parent's constructor
RNet.__init__(self, self.net_name)
# Attributes
self.output_dim = output_dim
self.memory_units = (self.MemoryUnit.parse_units(mem_config)
if memory_units is None else memory_units)
self.memory_units = [mu for mu in self.memory_units if mu.size > 0]
checker.check_type(self.memory_units, Ham.MemoryUnit)
self._state_size = sum([mu.size for mu in self.memory_units])
self._activation = activations.get('tanh', **kwargs)
self._use_mem_wisely = use_mem_wisely
self._truncate = kwargs.get('truncate', False)
self._weight_regularizer = regularizers.get(weight_regularizer,
**kwargs)
# self._use_global_reg = kwargs.get('global_reg', False)
self._kwargs = kwargs
def get_global_regularizer(self): if not self.use_global_regularizer: return None from tframe import regularizers return regularizers.get(self.regularizer)
def neurons(num,
external_input,
activation=None,
memory=None,
fc_memory=True,
scope=None,
use_bias=True,
truncate=False,
num_or_size_splits=None,
weight_initializer='glorot_uniform',
bias_initializer='zeros',
weight_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
**kwargs):
"""Analogous to tf.keras.layers.Dense"""
# Get activation, initializers and regularizers
if activation is not None: activation = activations.get(activation)
weight_initializer = initializers.get(weight_initializer)
bias_initializer = initializers.get(bias_initializer)
weight_regularizer = regularizers.get(weight_regularizer)
bias_regularizer = regularizers.get(bias_regularizer)
activity_regularizer = regularizers.get(activity_regularizer)
# a. Check prune configs
if 'prune_frac' in kwargs.keys():
x_prune_frac, s_prune_frac = (kwargs['prune_frac'], ) * 2
else:
x_prune_frac = kwargs.get('x_prune_frac', 0)
s_prune_frac = kwargs.get('s_prune_frac', 0)
prune_is_on = hub.pruning_rate_fc > 0.0 and x_prune_frac + s_prune_frac > 0
# b. Check sparse configs
x_heads = kwargs.get('x_heads', 0)
s_heads = kwargs.get('s_heads', 0)
sparse_is_on = x_heads + s_heads > 0
# :: Decide to concatenate or not considering a and b
# .. a
if memory is None: should_concate = False
elif prune_is_on: should_concate = x_prune_frac == s_prune_frac
else: should_concate = fc_memory
# .. b
should_concate = should_concate and not sparse_is_on
#
separate_memory_neurons = memory is not None and not should_concate
def get_weights(name, tensor, p_frac, heads):
shape = [get_dimension(tensor), num]
if prune_is_on and p_frac > 0:
assert heads == 0
return get_weights_to_prune(name, shape, weight_initializer,
p_frac)
elif heads > 0:
return _get_sparse_weights(shape[0],
shape[1],
heads,
use_bit_max=True,
coef_initializer=weight_initializer)
else:
return get_variable(name, shape, weight_initializer)
def forward():
# Prepare a weight list for potential regularizer calculation
weight_list = []
# Get x
x = (tf.concat([external_input, memory], axis=1, name='x_concat_s')
if should_concate else external_input)
# - Calculate net input for x
# .. get weights
name = 'Wx' if separate_memory_neurons else 'W'
Wx = get_weights(name, x, x_prune_frac, x_heads)
weight_list.append(Wx)
# .. append weights to context, currently only some extractors will use it
context.weights_list.append(Wx)
# .. do matrix multiplication
net_y = get_matmul(truncate)(x, Wx)
# - Calculate net input for memory and add to net_y if necessary
if separate_memory_neurons:
if not fc_memory:
assert not (prune_is_on and s_prune_frac > 0)
memory_dim = get_dimension(memory)
assert memory_dim == num
Ws = get_variable('Ws', [1, num], weight_initializer)
net_s = get_multiply(truncate)(memory, Ws)
else:
assert prune_is_on or sparse_is_on
Ws = get_weights('Ws', memory, s_prune_frac, s_heads)
net_s = get_matmul(truncate)(memory, Ws)
# Append Ws to weight list and add net_s to net_y
weight_list.append(Ws)
net_y = tf.add(net_y, net_s)
# - Add bias if necessary
b = None
if use_bias:
b = get_bias('bias', num, bias_initializer)
net_y = tf.nn.bias_add(net_y, b)
# - Activate and return
if callable(activation): net_y = activation(net_y)
return net_y, weight_list, b
if scope is not None:
with tf.variable_scope(scope):
y, W_list, b = forward()
else:
y, W_list, b = forward()
# Add regularizer if necessary
if callable(weight_regularizer):
context.add_loss_tensor(
tf.add_n([weight_regularizer(w) for w in W_list]))
if callable(bias_regularizer) and b is not None:
context.add_loss_tensor(bias_regularizer(b))
if callable(activity_regularizer):
context.add_loss_tensor(activity_regularizer(y))
# Split if necessary
if num_or_size_splits is not None:
return tf.split(y, num_or_size_splits=num_or_size_splits, axis=1)
return y
