def __init__( self, state_size, activation='tanh', use_bias=True, weight_initializer='xavier_normal', bias_initializer='zeros', **kwargs): """ :param state_size: state size: positive int :param activation: activation: string or callable :param use_bias: whether to use bias :param weight_initializer: weight initializer identifier :param bias_initializer: bias initializer identifier """ # Call parent's constructor RNet.__init__(self, self.net_name) # Attributes self._state_size = state_size self._activation = activations.get(activation, **kwargs) self._use_bias = checker.check_type(use_bias, bool) self._weight_initializer = initializers.get(weight_initializer) self._bias_initializer = initializers.get(bias_initializer) self._output_scale = state_size
def __init__(self, output_dim, memory_units=None, mem_config=None, use_mem_wisely=False, weight_regularizer=None, **kwargs): # Call parent's constructor RNet.__init__(self, self.net_name) # Attributes self.output_dim = output_dim self.memory_units = (self.MemoryUnit.parse_units(mem_config) if memory_units is None else memory_units) self.memory_units = [mu for mu in self.memory_units if mu.size > 0] checker.check_type(self.memory_units, Ham.MemoryUnit) self._state_size = sum([mu.size for mu in self.memory_units]) self._activation = activations.get('tanh', **kwargs) self._use_mem_wisely = use_mem_wisely self._truncate = kwargs.get('truncate', False) self._weight_regularizer = regularizers.get(weight_regularizer, **kwargs) # self._use_global_reg = kwargs.get('global_reg', False) self._kwargs = kwargs
def __init__( self, state_size, activation='tanh', use_bias=True, weight_initializer='xavier_uniform', bias_initializer='zeros', input_gate=True, output_gate=True, forget_gate=True, with_peepholes=False, **kwargs): """ :param state_size: state size: positive int :param activation: activation: string or callable :param use_bias: whether to use bias :param weight_initializer: weight initializer identifier :param bias_initializer: bias initializer identifier """ # Call parent's constructor RNet.__init__(self, BasicLSTMCell.net_name) # Attributes self._state_size = state_size self._activation = activations.get(activation, **kwargs) self._use_bias = checker.check_type(use_bias, bool) self._weight_initializer = initializers.get(weight_initializer) self._bias_initializer = initializers.get(bias_initializer) self._input_gate = checker.check_type(input_gate, bool) self._output_gate = checker.check_type(output_gate, bool) self._forget_gate = checker.check_type(forget_gate, bool) self._with_peepholes = checker.check_type(with_peepholes, bool) self._output_scale = state_size
def __init__(self, state_size, periods=None, activation='tanh', use_bias=True, weight_initializer='xavier_uniform', bias_initializer='zeros', **kwargs): """ :param state_size: State size :param periods: a list of integers. If not provided, periods will be set to a default exponential series {2^{i-1}}_{i=0}^{state_size} """ # Call parent's constructor RNet.__init__(self, ClockworkRNN.net_name) # Attributes self._state_size = checker.check_positive_integer(state_size) self._periods = self._get_periods(periods, **kwargs) self._activation = activations.get(activation, **kwargs) self._use_bias = checker.check_type(use_bias, bool) self._weight_initializer = initializers.get(weight_initializer) self._bias_initializer = initializers.get(bias_initializer) # modules = [(start_index, size, period)+] self._modules = [] self._init_modules(**kwargs)
def __init__(self, identifier, set_logits=False): p = Parser.parse(identifier) self._id = p.name self.abbreviation = (p.name if isinstance(identifier, six.string_types) else identifier.__name__) self.full_name = self.abbreviation self._activation = activations.get(identifier) self._set_logits = set_logits
def masked_neurons(x, num, scope, activation=None, s=None, x_mask=None, s_mask=None, use_bias=True, weight_initializer='glorot_normal', bias_initializer='zeros', **kwargs): # Sanity check assert isinstance(x, tf.Tensor) # Get activation and initializers if activation is not None: activation = activations.get(activation) weight_initializer = initializers.get(weight_initializer) bias_initializer = initializers.get(bias_initializer) def matmul(x, y): batch_matmul = len(x.shape) == len(y.shape) - 1 if batch_matmul: x = tf.expand_dims(x, axis=1) assert len(x.shape) == len(y.shape) output = tf.matmul(x, y) if batch_matmul: output = tf.squeeze(output, axis=1) return output def get_weights(tensor, name, mask=None): shape = [get_dimension(tensor), num] if mask is None: return get_variable(name, shape, weight_initializer) else: return get_masked_weights(name, shape, weight_initializer, mask) def forward(): # x -> y Wx = get_weights(x, 'Wx', x_mask) # .. do matrix multiplication net_y = matmul(x, Wx) # s -> y if s exists if s is not None: assert isinstance(s, tf.Tensor) Ws = get_weights(s, 'Ws', s_mask) # .. add s * Ws to net_y net_y = tf.add(net_y, matmul(s, Ws)) # Add bias if necessary if use_bias: b = get_bias('bias', num, bias_initializer) net_y = tf.nn.bias_add(net_y, b) # Activate if necessary if activation is not None: return activation(net_y) else: return net_y with tf.variable_scope(scope): y = forward() # Return return y
def eta(num, activation, *tensors, weight_initializer='glorot_normal', use_bias=True, bias_initializer='zeros'): """ * should be used inside a variable scope """ a = psi(num, *tensors, weight_initializer, use_bias, bias_initializer) if activation is not None: activation = activations.get(activation) a = activation(a) return a
def __init__(self, state_size, mem_fc=True, **kwargs): # Call parent's constructor RNet.__init__(self, self.net_name) # Attributes self._state_size = state_size self._activation = activations.get('tanh', **kwargs) # self._use_bias = True self._weight_initializer = initializers.get('xavier_normal') self._bias_initializer = initializers.get('zeros') self._output_scale = state_size self._fully_connect_memories = mem_fc
def _get_neurons_fast(self, x, h): # Calculate net inputs net_f_tilde, net_i_tilde_bar, net_f, net_i, net_o, net_c_hat = self.neurons( x, h, scope='net_inputs', num_or_size_splits=6) # Activate cumax = activations.get('cumax') f_tilde = cumax(net_f_tilde) i_tilde_bar = cumax(net_i_tilde_bar) f = tf.nn.sigmoid(net_f) i = tf.nn.sigmoid(net_i) o = tf.nn.sigmoid(net_o) c_hat = self._activation(net_c_hat) return f_tilde, i_tilde_bar, f, i, o, c_hat
def __init__( self, state_size, activation='tanh', weight_initializer='xavier_normal', input_gate=True, forget_gate=True, output_gate=True, use_g_bias=True, g_bias_initializer='zeros', use_i_bias=True, i_bias_initializer='zeros', use_f_bias=True, f_bias_initializer='zeros', use_o_bias=True, o_bias_initializer='zeros', output_as_mem=True, fully_connect_memory=True, activate_memory=True, truncate_grad=False, **kwargs): # Call parent's constructor RNet.__init__(self, self.net_name) # Attributes self._state_size = state_size self._input_gate = checker.check_type(input_gate, bool) self._forget_gate = checker.check_type(forget_gate, bool) self._output_gate = checker.check_type(output_gate, bool) self._activation = activations.get(activation, **kwargs) self._weight_initializer = initializers.get(weight_initializer) self._use_g_bias = checker.check_type(use_g_bias, bool) self._g_bias_initializer = initializers.get(g_bias_initializer) self._use_i_bias = checker.check_type(use_i_bias, bool) self._i_bias_initializer = initializers.get(i_bias_initializer) self._use_f_bias = checker.check_type(use_f_bias, bool) self._f_bias_initializer = initializers.get(f_bias_initializer) self._use_o_bias = checker.check_type(use_o_bias, bool) self._o_bias_initializer = initializers.get(o_bias_initializer) self._activate_mem = checker.check_type(activate_memory, bool) self._truncate_grad = checker.check_type(truncate_grad, bool) self._fc_memory = checker.check_type(fully_connect_memory, bool) self._output_as_mem = checker.check_type(output_as_mem, bool) self._kwargs = kwargs
def __init__(self, output_dim, neurons_per_amu=3, activation='tanh', use_bias=True, weight_initializer='xavier_uniform', bias_initializer='zeros', **kwargs): # Call parent's constructor RNet.__init__(self, AMU.net_name) # Attributes self._output_dim = output_dim self._neurons_per_amu = neurons_per_amu self._activation = activations.get(activation, **kwargs) self._use_bias = checker.check_type(use_bias, bool) self._weight_initializer = initializers.get(weight_initializer) self._bias_initializer = initializers.get(bias_initializer) self._output_scale = output_dim
def __init__(self, activation=None, weight_initializer='xavier_normal', use_bias=False, bias_initializer='zeros', layer_normalization=False, weight_dropout=0.0, **kwargs): if activation: activation = activations.get(activation) self._activation = activation self._weight_initializer = initializers.get(weight_initializer) self._use_bias = checker.check_type(use_bias, bool) self._bias_initializer = initializers.get(bias_initializer) self._layer_normalization = checker.check_type(layer_normalization, bool) self._gain_initializer = initializers.get( kwargs.get('gain_initializer', 'ones')) self._normalize_each_psi = kwargs.pop('normalize_each_psi', False) self._weight_dropout = checker.check_type(weight_dropout, float) assert 0 <= self._weight_dropout < 1 self._nb_kwargs = kwargs
def __init__(self, identifier, **kwargs): self.abbreviation = (identifier if isinstance( identifier, six.string_types) else identifier.__name__) self.full_name = self.abbreviation self._activation = activations.get(identifier, **kwargs)
def __init__( self, state_size, cell_activation='sigmoid', # g cell_activation_range=(-2, 2), memory_activation='sigmoid', # h memory_activation_range=(-1, 1), weight_initializer='random_uniform', weight_initial_range=(-0.1, 0.1), use_cell_bias=False, cell_bias_initializer='random_uniform', cell_bias_init_range=(-0.1, 0.1), use_in_bias=True, in_bias_initializer='zeros', use_out_bias=True, out_bias_initializer='zeros', truncate=True, forward_gate=True, **kwargs): # Call parent's constructor RNet.__init__(self, OriginalLSTMCell.net_name) # Set state size self._state_size = state_size # Set activation # .. In LSTM98, cell activation is referred to as 'g', # .. while memory activation is 'h' and gate activation is 'f' self._cell_activation = activations.get( cell_activation, range=cell_activation_range) self._memory_activation = activations.get( memory_activation, range=memory_activation_range) self._gate_activation = activations.get('sigmoid') # Set weight and bias configs self._weight_initializer = initializers.get( weight_initializer, range=weight_initial_range) self._use_cell_bias = use_cell_bias self._cell_bias_initializer = initializers.get( cell_bias_initializer, range=cell_bias_init_range) self._use_in_bias = use_in_bias self._in_bias_initializer = initializers.get(in_bias_initializer) self._use_out_bias = use_out_bias self._out_bias_initializer = initializers.get(out_bias_initializer) if kwargs.get('rule97', False): self._cell_bias_initializer = self._weight_initializer self._in_bias_initializer = self._weight_initializer # Additional options self._truncate = truncate self._forward_gate = forward_gate # ... self._num_splits = 3 self._output_scale = state_size self._h_size = (state_size * self._num_splits if self._forward_gate else state_size) # TODO: BETA self.compute_gradients = self.truncated_rtrl
def neurons(num, external_input, activation=None, memory=None, fc_memory=True, scope=None, use_bias=True, truncate=False, num_or_size_splits=None, weight_initializer='glorot_uniform', bias_initializer='zeros', weight_regularizer=None, bias_regularizer=None, activity_regularizer=None, **kwargs): """Analogous to tf.keras.layers.Dense""" # Get activation, initializers and regularizers if activation is not None: activation = activations.get(activation) weight_initializer = initializers.get(weight_initializer) bias_initializer = initializers.get(bias_initializer) weight_regularizer = regularizers.get(weight_regularizer) bias_regularizer = regularizers.get(bias_regularizer) activity_regularizer = regularizers.get(activity_regularizer) # a. Check prune configs if 'prune_frac' in kwargs.keys(): x_prune_frac, s_prune_frac = (kwargs['prune_frac'], ) * 2 else: x_prune_frac = kwargs.get('x_prune_frac', 0) s_prune_frac = kwargs.get('s_prune_frac', 0) prune_is_on = hub.pruning_rate_fc > 0.0 and x_prune_frac + s_prune_frac > 0 # b. Check sparse configs x_heads = kwargs.get('x_heads', 0) s_heads = kwargs.get('s_heads', 0) sparse_is_on = x_heads + s_heads > 0 # :: Decide to concatenate or not considering a and b # .. a if memory is None: should_concate = False elif prune_is_on: should_concate = x_prune_frac == s_prune_frac else: should_concate = fc_memory # .. b should_concate = should_concate and not sparse_is_on # separate_memory_neurons = memory is not None and not should_concate def get_weights(name, tensor, p_frac, heads): shape = [get_dimension(tensor), num] if prune_is_on and p_frac > 0: assert heads == 0 return get_weights_to_prune(name, shape, weight_initializer, p_frac) elif heads > 0: return _get_sparse_weights(shape[0], shape[1], heads, use_bit_max=True, coef_initializer=weight_initializer) else: return get_variable(name, shape, weight_initializer) def forward(): # Prepare a weight list for potential regularizer calculation weight_list = [] # Get x x = (tf.concat([external_input, memory], axis=1, name='x_concat_s') if should_concate else external_input) # - Calculate net input for x # .. get weights name = 'Wx' if separate_memory_neurons else 'W' Wx = get_weights(name, x, x_prune_frac, x_heads) weight_list.append(Wx) # .. append weights to context, currently only some extractors will use it context.weights_list.append(Wx) # .. do matrix multiplication net_y = get_matmul(truncate)(x, Wx) # - Calculate net input for memory and add to net_y if necessary if separate_memory_neurons: if not fc_memory: assert not (prune_is_on and s_prune_frac > 0) memory_dim = get_dimension(memory) assert memory_dim == num Ws = get_variable('Ws', [1, num], weight_initializer) net_s = get_multiply(truncate)(memory, Ws) else: assert prune_is_on or sparse_is_on Ws = get_weights('Ws', memory, s_prune_frac, s_heads) net_s = get_matmul(truncate)(memory, Ws) # Append Ws to weight list and add net_s to net_y weight_list.append(Ws) net_y = tf.add(net_y, net_s) # - Add bias if necessary b = None if use_bias: b = get_bias('bias', num, bias_initializer) net_y = tf.nn.bias_add(net_y, b) # - Activate and return if callable(activation): net_y = activation(net_y) return net_y, weight_list, b if scope is not None: with tf.variable_scope(scope): y, W_list, b = forward() else: y, W_list, b = forward() # Add regularizer if necessary if callable(weight_regularizer): context.add_loss_tensor( tf.add_n([weight_regularizer(w) for w in W_list])) if callable(bias_regularizer) and b is not None: context.add_loss_tensor(bias_regularizer(b)) if callable(activity_regularizer): context.add_loss_tensor(activity_regularizer(y)) # Split if necessary if num_or_size_splits is not None: return tf.split(y, num_or_size_splits=num_or_size_splits, axis=1) return y