def __init__(self, axis=-1, momentum=0.99, epsilon=1e-3, center=True, scale=True, beta_initializer='zeros', gamma_initializer='ones', moving_mean_initializer='zeros', moving_variance_initializer='ones', beta_regularizer=None, gamma_regularizer=None, beta_constraint=None, gamma_constraint=None, renorm=False, renorm_clipping=None, renorm_momentum=0.99, fused=None, trainable=True, virtual_batch_size=None, adjustment=None, name=None, **kwargs): super(BatchNormalization, self).__init__( name=name, trainable=trainable, **kwargs) if isinstance(axis, list): self.axis = axis[:] else: self.axis = axis self.momentum = momentum self.epsilon = epsilon self.center = center self.scale = scale self.beta_initializer = initializers.get(beta_initializer) self.gamma_initializer = initializers.get(gamma_initializer) self.moving_mean_initializer = initializers.get(moving_mean_initializer) self.moving_variance_initializer = initializers.get( moving_variance_initializer) self.beta_regularizer = regularizers.get(beta_regularizer) self.gamma_regularizer = regularizers.get(gamma_regularizer) self.beta_constraint = constraints.get(beta_constraint) self.gamma_constraint = constraints.get(gamma_constraint) self.renorm = renorm self.virtual_batch_size = virtual_batch_size self.adjustment = adjustment if fused is None: fused = True self.supports_masking = True self.fused = fused self._bessels_correction_test_only = True if renorm: renorm_clipping = renorm_clipping or {} keys = ['rmax', 'rmin', 'dmax'] if set(renorm_clipping) - set(keys): raise ValueError('renorm_clipping %s contains keys not in %s' % (renorm_clipping, keys)) self.renorm_clipping = renorm_clipping self.renorm_momentum = renorm_momentum
def __init__(self, filters, kernel_size, strides=1, padding='valid', data_format=None, activation=None, use_bias=True, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, **kwargs): super(LocallyConnected1D, self).__init__(**kwargs) self.filters = filters self.kernel_size = conv_utils.normalize_tuple(kernel_size, 1, 'kernel_size') self.strides = conv_utils.normalize_tuple(strides, 1, 'strides') self.padding = conv_utils.normalize_padding(padding) if self.padding != 'valid': raise ValueError('Invalid border mode for LocallyConnected1D ' '(only "valid" is supported): ' + padding) self.data_format = conv_utils.normalize_data_format(data_format) self.activation = activations.get(activation) self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) self.bias_initializer = initializers.get(bias_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.activity_regularizer = regularizers.get(activity_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.bias_constraint = constraints.get(bias_constraint) self.input_spec = InputSpec(ndim=3)
def __init__(self, units, activation=None, use_bias=True, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, **kwargs): if 'input_shape' not in kwargs and 'input_dim' in kwargs: kwargs['input_shape'] = (kwargs.pop('input_dim'),) super(Dense, self).__init__( activity_regularizer=regularizers.get(activity_regularizer), **kwargs) self.units = int(units) self.activation = activations.get(activation) self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) self.bias_initializer = initializers.get(bias_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.bias_constraint = constraints.get(bias_constraint) self.supports_masking = True self.input_spec = InputSpec(min_ndim=2)
def __init__(self, axis=-1, epsilon=1e-3, center=True, scale=True, beta_initializer='zeros', gamma_initializer='ones', beta_regularizer=None, gamma_regularizer=None, beta_constraint=None, gamma_constraint=None, trainable=True, name=None, **kwargs): super(LayerNormalization, self).__init__( name=name, trainable=trainable, **kwargs) if isinstance(axis, (list, tuple)): self.axis = axis[:] elif isinstance(axis, int): self.axis = axis else: raise ValueError('Expected an int or a list/tuple of ints for the ' 'argument \'axis\', but received instead: %s' % axis) self.epsilon = epsilon self.center = center self.scale = scale self.beta_initializer = initializers.get(beta_initializer) self.gamma_initializer = initializers.get(gamma_initializer) self.beta_regularizer = regularizers.get(beta_regularizer) self.gamma_regularizer = regularizers.get(gamma_regularizer) self.beta_constraint = constraints.get(beta_constraint) self.gamma_constraint = constraints.get(gamma_constraint) self.supports_masking = True
def __init__(self, units, activation='tanh', recurrent_activation='hard_sigmoid', kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', unit_forget_bias=True, kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, return_sequences=False, return_state=False, go_backwards=False, stateful=False, time_major=False, **kwargs): super(RNN, self).__init__(**kwargs) # pylint: disable=bad-super-call self.units = units cell_spec = collections.namedtuple('cell', ['state_size', 'output_size']) self.cell = cell_spec( state_size=(self.units, self.units), output_size=self.units) self.activation = activations.get(activation) self.recurrent_activation = activations.get(recurrent_activation) self.kernel_initializer = initializers.get(kernel_initializer) self.recurrent_initializer = initializers.get(recurrent_initializer) self.bias_initializer = initializers.get(bias_initializer) self.unit_forget_bias = unit_forget_bias self.kernel_regularizer = regularizers.get(kernel_regularizer) self.recurrent_regularizer = regularizers.get(recurrent_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.activity_regularizer = regularizers.get(activity_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.recurrent_constraint = constraints.get(recurrent_constraint) self.bias_constraint = constraints.get(bias_constraint) self.return_sequences = return_sequences self.return_state = return_state self.go_backwards = go_backwards self.stateful = stateful self.time_major = time_major self._num_constants = None self._num_inputs = None self._states = None self.input_spec = [InputSpec(ndim=3)] self.state_spec = [ InputSpec(shape=(None, dim)) for dim in (self.units, self.units) ]
def __init__(self, filters, kernel_size, strides=(1, 1), padding='valid', data_format=None, dilation_rate=(1, 1), activation='tanh', recurrent_activation='hard_sigmoid', use_bias=True, kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', unit_forget_bias=True, kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, dropout=0., recurrent_dropout=0., **kwargs): super(ConvLSTM2DCell, self).__init__(**kwargs) self.filters = filters self.kernel_size = conv_utils.normalize_tuple(kernel_size, 2, 'kernel_size') self.strides = conv_utils.normalize_tuple(strides, 2, 'strides') self.padding = conv_utils.normalize_padding(padding) self.data_format = conv_utils.normalize_data_format(data_format) self.dilation_rate = conv_utils.normalize_tuple(dilation_rate, 2, 'dilation_rate') self.activation = activations.get(activation) self.recurrent_activation = activations.get(recurrent_activation) self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) self.recurrent_initializer = initializers.get(recurrent_initializer) self.bias_initializer = initializers.get(bias_initializer) self.unit_forget_bias = unit_forget_bias self.kernel_regularizer = regularizers.get(kernel_regularizer) self.recurrent_regularizer = regularizers.get(recurrent_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.recurrent_constraint = constraints.get(recurrent_constraint) self.bias_constraint = constraints.get(bias_constraint) self.dropout = min(1., max(0., dropout)) self.recurrent_dropout = min(1., max(0., recurrent_dropout)) self.state_size = (self.filters, self.filters) self._dropout_mask = None self._recurrent_dropout_mask = None
def __init__(self, input_dim, output_dim, embeddings_initializer='uniform', embeddings_regularizer=None, activity_regularizer=None, embeddings_constraint=None, mask_zero=False, input_length=None, **kwargs): if 'input_shape' not in kwargs: if input_length: kwargs['input_shape'] = (input_length,) else: kwargs['input_shape'] = (None,) dtype = kwargs.pop('dtype', K.floatx()) super(Embedding, self).__init__(dtype=dtype, **kwargs) self.input_dim = input_dim self.output_dim = output_dim self.embeddings_initializer = initializers.get(embeddings_initializer) self.embeddings_regularizer = regularizers.get(embeddings_regularizer) self.activity_regularizer = regularizers.get(activity_regularizer) self.embeddings_constraint = constraints.get(embeddings_constraint) self.mask_zero = mask_zero self.supports_masking = mask_zero self.input_length = input_length
def __init__(self, norm_axis=None, params_axis=-1, epsilon=1e-12, center=True, scale=True, beta_initializer='zeros', gamma_initializer='ones', beta_regularizer=None, gamma_regularizer=None, beta_constraint=None, gamma_constraint=None, trainable=True, name=None, **kwargs): super(LayerNormalization, self).__init__( name=name, trainable=trainable, **kwargs) if isinstance(norm_axis, list): self.norm_axis = norm_axis[:] elif isinstance(norm_axis, int): self.norm_axis = norm_axis elif norm_axis is None: self.norm_axis = None else: raise TypeError('norm_axis must be int or list or None, type given: %s' % type(norm_axis)) if isinstance(params_axis, list): self.params_axis = params_axis[:] elif isinstance(params_axis, int): self.params_axis = params_axis else: raise TypeError('params_axis must be int or list, type given: %s' % type(params_axis)) self.epsilon = epsilon self.center = center self.scale = scale self.beta_initializer = initializers.get(beta_initializer) self.gamma_initializer = initializers.get(gamma_initializer) self.beta_regularizer = regularizers.get(beta_regularizer) self.gamma_regularizer = regularizers.get(gamma_regularizer) self.beta_constraint = constraints.get(beta_constraint) self.gamma_constraint = constraints.get(gamma_constraint) self.supports_masking = True
def __init__(self, units, kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', unit_forget_bias=True, kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, return_sequences=False, return_state=False, go_backwards=False, stateful=False, **kwargs): self.units = units cell_spec = collections.namedtuple('cell', 'state_size') self._cell = cell_spec(state_size=(self.units, self.units)) super(CuDNNLSTM, self).__init__( return_sequences=return_sequences, return_state=return_state, go_backwards=go_backwards, stateful=stateful, **kwargs) self.kernel_initializer = initializers.get(kernel_initializer) self.recurrent_initializer = initializers.get(recurrent_initializer) self.bias_initializer = initializers.get(bias_initializer) self.unit_forget_bias = unit_forget_bias self.kernel_regularizer = regularizers.get(kernel_regularizer) self.recurrent_regularizer = regularizers.get(recurrent_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.activity_regularizer = regularizers.get(activity_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.recurrent_constraint = constraints.get(recurrent_constraint) self.bias_constraint = constraints.get(bias_constraint)
def __init__(self, alpha_initializer='zeros', alpha_regularizer=None, alpha_constraint=None, shared_axes=None, **kwargs): super(PReLU, self).__init__(**kwargs) self.supports_masking = True self.alpha_initializer = initializers.get(alpha_initializer) self.alpha_regularizer = regularizers.get(alpha_regularizer) self.alpha_constraint = constraints.get(alpha_constraint) if shared_axes is None: self.shared_axes = None elif not isinstance(shared_axes, (list, tuple)): self.shared_axes = [shared_axes] else: self.shared_axes = list(shared_axes)
def __init__(self, axis=-1, momentum=0.99, epsilon=1e-3, center=True, scale=True, beta_initializer='zeros', gamma_initializer='ones', moving_mean_initializer='zeros', moving_variance_initializer='ones', beta_regularizer=None, gamma_regularizer=None, beta_constraint=None, gamma_constraint=None, renorm=False, renorm_clipping=None, renorm_momentum=0.99, fused=None, trainable=True, virtual_batch_size=None, adjustment=None, name=None, **kwargs): super(BatchNormalization, self).__init__(name=name, trainable=trainable, **kwargs) if isinstance(axis, list): self.axis = axis[:] else: self.axis = axis self.momentum = momentum self.epsilon = epsilon self.center = center self.scale = scale self.beta_initializer = initializers.get(beta_initializer) self.gamma_initializer = initializers.get(gamma_initializer) self.moving_mean_initializer = initializers.get( moving_mean_initializer) self.moving_variance_initializer = initializers.get( moving_variance_initializer) self.beta_regularizer = regularizers.get(beta_regularizer) self.gamma_regularizer = regularizers.get(gamma_regularizer) self.beta_constraint = constraints.get(beta_constraint) self.gamma_constraint = constraints.get(gamma_constraint) self.renorm = renorm self.virtual_batch_size = virtual_batch_size self.adjustment = adjustment if fused is None: fused = True self.supports_masking = True self.fused = fused self._bessels_correction_test_only = True if renorm: renorm_clipping = renorm_clipping or {} keys = ['rmax', 'rmin', 'dmax'] if set(renorm_clipping) - set(keys): raise ValueError('renorm_clipping %s contains keys not in %s' % (renorm_clipping, keys)) self.renorm_clipping = renorm_clipping self.renorm_momentum = renorm_momentum
def __init__(self, filters, kernel_size, octave=2, ratio_out=0.5, strides=1, dilation_rate=1, activation=None, use_bias=True, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, **kwargs): super(OctaveConv1D, self).__init__(**kwargs) self.filters = filters self.kernel_size = kernel_size self.octave = octave self.ratio_out = ratio_out self.strides = strides self.dilation_rate = dilation_rate self.activation = activations.get(activation) self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) self.bias_initializer = initializers.get(bias_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.activity_regularizer = regularizers.get(activity_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.bias_constraint = constraints.get(bias_constraint) self.filters_low = int(filters * self.ratio_out) self.filters_high = filters - self.filters_low self.conv_high_to_high, self.conv_low_to_high = None, None if self.filters_high > 0: self.conv_high_to_high = self._init_conv( self.filters_high, name='{}-Conv1D-HH'.format(self.name)) self.conv_low_to_high = self._init_conv(self.filters_high, name='{}-Conv1D-LH'.format( self.name)) self.conv_low_to_low, self.conv_high_to_low = None, None if self.filters_low > 0: self.conv_low_to_low = self._init_conv(self.filters_low, name='{}-Conv1D-HL'.format( self.name)) self.conv_high_to_low = self._init_conv(self.filters_low, name='{}-Conv1D-LL'.format( self.name)) self.pooling = AveragePooling1D( pool_size=self.octave, padding='valid', name='{}-AveragePooling1D'.format(self.name), ) self.up_sampling = UpSampling1D( size=self.octave, name='{}-UpSampling1D'.format(self.name), )
def __init__(self, axis=-1, momentum=0.99, epsilon=1e-3, center=True, scale=True, beta_initializer='zeros', gamma_initializer='ones', moving_mean_initializer='zeros', moving_variance_initializer='ones', beta_regularizer=None, gamma_regularizer=None, beta_constraint=None, gamma_constraint=None, renorm=False, renorm_clipping=None, renorm_momentum=0.99, fused=None, trainable=True, virtual_batch_size=None, adjustment=None, name=None, **kwargs): super(BatchNormalizationBase, self).__init__( name=name, trainable=trainable, **kwargs) if isinstance(axis, list): self.axis = axis[:] elif isinstance(axis, int): self.axis = axis else: raise TypeError('axis must be int or list, type given: %s' % type(self.axis)) self.momentum = momentum self.epsilon = epsilon self.center = center self.scale = scale self.beta_initializer = initializers.get(beta_initializer) self.gamma_initializer = initializers.get(gamma_initializer) self.moving_mean_initializer = initializers.get(moving_mean_initializer) self.moving_variance_initializer = initializers.get( moving_variance_initializer) self.beta_regularizer = regularizers.get(beta_regularizer) self.gamma_regularizer = regularizers.get(gamma_regularizer) self.beta_constraint = constraints.get(beta_constraint) self.gamma_constraint = constraints.get(gamma_constraint) self.renorm = renorm self.virtual_batch_size = virtual_batch_size self.adjustment = adjustment if self._USE_V2_BEHAVIOR: if fused: self._raise_if_fused_cannot_be_used() # We leave fused as None if self._fused_can_be_used()==True, since we # still may set it to False in self.build() if the input rank is not 4. elif fused is None and not self._fused_can_be_used(): fused = False elif fused is None: fused = True self.supports_masking = True self.fused = fused self._bessels_correction_test_only = True if renorm: renorm_clipping = renorm_clipping or {} keys = ['rmax', 'rmin', 'dmax'] if set(renorm_clipping) - set(keys): raise ValueError('renorm_clipping %s contains keys not in %s' % (renorm_clipping, keys)) self.renorm_clipping = renorm_clipping self.renorm_momentum = renorm_momentum
def __init__( self, feature_units, attn_heads=1, attn_heads_reduction="concat", # {"concat", "average"} dropout_rate=0.5, activation="relu", use_bias=True, kernel_initializer="glorot_uniform", bias_initializer="zeros", attn_kernel_initializer="glorot_uniform", kernel_regularizer=None, bias_regularizer=None, attn_kernel_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, attn_kernel_constraint=None, attention=True, **kwargs): if attn_heads_reduction not in {"concat", "average"}: raise ValueError("Possbile reduction methods: concat, average") self.F_ = feature_units # Number of output features (F" in the paper) self.attn_heads = attn_heads # Number of attention heads (K in the paper) self.attn_heads_reduction = attn_heads_reduction # Eq. 5 and 6 in the paper self.dropout_rate = dropout_rate # Internal dropout rate self.activation = activations.get(activation) # Eq. 4 in the paper self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) self.bias_initializer = initializers.get(bias_initializer) self.attn_kernel_initializer = initializers.get( attn_kernel_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.attn_kernel_regularizer = regularizers.get( attn_kernel_regularizer) self.activity_regularizer = regularizers.get(activity_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.bias_constraint = constraints.get(bias_constraint) self.attn_kernel_constraint = constraints.get(attn_kernel_constraint) self.supports_masking = False self.attention = attention # Populated by build() self.kernels = [] # Layer kernels for attention heads self.biases = [] # Layer biases for attention heads self.attn_kernels = [] # Attention kernels for attention heads if attn_heads_reduction == "concat": # Output will have shape (..., K * F") self.output_dim = self.F_ * self.attn_heads else: # Output will have shape (..., F") self.output_dim = self.F_ super(GraphAttentionLayer, self).__init__(**kwargs)
def __init__(self, time_scale, size_per_head, num_attention_heads, num_memory_slots, use_relative_position=True, activation='tanh', recurrent_activation='hard_sigmoid', mlp_activation='gelu', forget_bias=1.0, input_bias=0.0, sigma_bias=-4.6, kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', attention_initializer='truncated_normal', mlp_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, recurrent_regularizer=None, attention_regularizer=None, mlp_regularizer=None, bias_regularizer=None, kernel_constraint=None, recurrent_constraint=None, attention_constraint=None, mlp_constraint=None, bias_constraint=None, dropout=0., recurrent_dropout=0., **kwargs): super(TmpHierRMCRNNCell, self).__init__(**kwargs) self.time_scale = time_scale self.units = num_attention_heads * size_per_head self.num_memory_slots = num_memory_slots self.num_attention_heads = num_attention_heads self.size_per_head = size_per_head self.use_relative_position = use_relative_position self.activation = wrap_activations_get(activation) self.recurrent_activation = wrap_activations_get(recurrent_activation) self.mlp_activation = wrap_activations_get(mlp_activation) self.forget_bias = forget_bias self.input_bias = input_bias self.sigma_bias = sigma_bias self.kernel_initializer = initializers.get(kernel_initializer) self.recurrent_initializer = initializers.get(recurrent_initializer) self.attention_initializer = initializers.get(attention_initializer) self.mlp_initializer = initializers.get(mlp_initializer) self.bias_initializer = initializers.get(bias_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.recurrent_regularizer = regularizers.get(recurrent_regularizer) self.attention_regularizer = regularizers.get(attention_regularizer) self.mlp_regularizer = regularizers.get(mlp_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.recurrent_constraint = constraints.get(recurrent_constraint) self.attention_constraint = constraints.get(attention_constraint) self.mlp_constraint = constraints.get(mlp_constraint) self.bias_constraint = constraints.get(bias_constraint) self.dropout = min(1., max(0., dropout)) self.recurrent_dropout = min(1., max(0., recurrent_dropout)) # tuple(_ListWrapper) was silently dropping list content in at least 2.7.10, # and fixed after 2.7.16. Converting the state_size to wrapper around # NoDependency(), so that the base_layer.__setattr__ will not convert it to # ListWrapper. Down the stream, self.states will be a list since it is # generated from nest.map_structure with list, and tuple(list) will work # properly. self.state_size = [ 1, 2 * (4 + self.num_memory_slots) * self.units] self.output_size = self.units
def __init__(self, units, activation='tanh', recurrent_activation='hard_sigmoid', attention_activation='tanh', use_bias=True, kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', attention_initializer='orthogonal', bias_initializer='zeros', unit_forget_bias=True, kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, activity_regularizer=None, attention_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, attention_constraint=None, dropout=0., recurrent_dropout=0., return_attention=False, implementation=1, **kwargs): super(AttentionLSTMCell, self).__init__(**kwargs) self.input_spec = [InputSpec(ndim=3)] self.units = units self.activation = activations.get(activation) self.recurrent_activation = activations.get(recurrent_activation) self.attention_activation = activations.get(attention_activation) self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) self.recurrent_initializer = initializers.get(recurrent_initializer) self.attention_initializer = initializers.get(attention_initializer) self.bias_initializer = initializers.get(bias_initializer) self.unit_forget_bias = unit_forget_bias self.kernel_regularizer = regularizers.get(kernel_regularizer) self.recurrent_regularizer = regularizers.get(recurrent_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.activity_regularizer = regularizers.get(activity_regularizer) self.attention_regularizer = regularizers.get(attention_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.recurrent_constraint = constraints.get(recurrent_constraint) self.bias_constraint = constraints.get(bias_constraint) self.attention_constraint = constraints.get(attention_constraint) self.dropout = min(1., max(0., dropout)) self.recurrent_dropout = min(1., max(0., recurrent_dropout)) self.return_attention = return_attention self._dropout_mask = None self._recurrent_dropout_mask = None self.implementation = implementation self.state_spec = [ InputSpec(shape=(None, self.units)), InputSpec(shape=(None, self.units)) ] self.state_size = (self.units, self.units)
def __init__(self, axis=-1, momentum=0.99, epsilon=1e-3, center=True, scale=True, beta_initializer='zeros', gamma_initializer='ones', moving_mean_initializer='zeros', moving_variance_initializer='ones', beta_regularizer=None, gamma_regularizer=None, beta_constraint=None, gamma_constraint=None, renorm=False, renorm_clipping=None, renorm_momentum=0.99, fused=None, trainable=True, virtual_batch_size=None, adjustment=None, name=None, **kwargs): super(BatchNormalizationV2, self).__init__(name=name, trainable=trainable, **kwargs) if isinstance(axis, list): self.axis = axis[:] elif isinstance(axis, int): self.axis = axis else: raise TypeError('axis must be int or list, type given: %s' % type(self.axis)) self.momentum = momentum self.epsilon = epsilon self.center = center self.scale = scale self.beta_initializer = initializers.get(beta_initializer) self.gamma_initializer = initializers.get(gamma_initializer) self.moving_mean_initializer = initializers.get( moving_mean_initializer) self.moving_variance_initializer = initializers.get( moving_variance_initializer) self.beta_regularizer = regularizers.get(beta_regularizer) self.gamma_regularizer = regularizers.get(gamma_regularizer) self.beta_constraint = constraints.get(beta_constraint) self.gamma_constraint = constraints.get(gamma_constraint) self.renorm = renorm self.virtual_batch_size = virtual_batch_size self.adjustment = adjustment if self._USE_V2_BEHAVIOR: if fused: self._raise_if_fused_cannot_be_used() # We leave fused as None if self._fused_can_be_used()==True, since we # still may set it to False in self.build() if the input rank is not 4. elif fused is None and not self._fused_can_be_used(): fused = False elif fused is None: fused = True self.supports_masking = True self.fused = fused self._bessels_correction_test_only = True if renorm: renorm_clipping = renorm_clipping or {} keys = ['rmax', 'rmin', 'dmax'] if set(renorm_clipping) - set(keys): raise ValueError('renorm_clipping %s contains keys not in %s' % (renorm_clipping, keys)) self.renorm_clipping = renorm_clipping self.renorm_momentum = renorm_momentum
def __init__(self, n_slots, n_heads, head_size, n_blocks=1, n_layers=3, activation='tanh', recurrent_activation='hard_sigmoid', forget_bias=1.0, kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, **kwargs): """Initialization method. Args: n_slots (int): Number of memory slots. n_heads (int): Number of attention heads. head_size (int): Size of each attention head. n_blocks (int): Number of feed-forward networks. n_layers (int): Amout of layers per feed-forward network. activation (str): Output activation function. recurrent_activation (str): Recurrent step activation function. forget_bias (float): Forget gate bias values. kernel_initializer (str): Kernel initializer function. recurrent_initializer (str): Recurrent kernel initializer function. bias_initializer (str): Bias initializer function. kernel_regularizer (str): Kernel regularizer function. recurrent_regularizer (str): Recurrent kernel regularizer function. bias_regularizer (str): Bias regularizer function. kernel_constraint (str): Kernel constraint function. recurrent_constraint (str): Recurrent kernel constraint function. bias_constraint (str): Bias constraint function. """ # Overrides its parent class with any custom arguments if needed super(RelationalMemoryCell, self).__init__(**kwargs) # Number of memory slots and their sizes self.n_slots = n_slots self.slot_size = n_heads * head_size # Number of attention heads and their sizes self.n_heads = n_heads self.head_size = head_size # Number of feed-forward network blocks and their sizes self.n_blocks = n_blocks self.n_layers = n_layers # Activation functions self.activation = activations.get(activation) self.recurrent_activation = activations.get(recurrent_activation) # Forget gate bias value self.forget_bias = forget_bias # `W`, `U` and `b` initializers self.kernel_initializer = initializers.get(kernel_initializer) self.recurrent_initializer = initializers.get(recurrent_initializer) self.bias_initializer = initializers.get(bias_initializer) # `W`, `U` and `b` regularizers self.kernel_regularizer = regularizers.get(kernel_regularizer) self.recurrent_regularizer = regularizers.get(recurrent_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) # `W`, `U` and `b` constraints self.kernel_constraint = constraints.get(kernel_constraint) self.recurrent_constraint = constraints.get(recurrent_constraint) self.bias_constraint = constraints.get(bias_constraint) # Number of outputted units self.units = self.slot_size * n_slots # Number of outputted units from the gates self.n_gates = 2 * self.slot_size # Creating a layer for projecting the input self.projector = Dense(self.slot_size) # Creating the feed-forward network # It is composed by linear layers and normalization ones self.before_norm = LayerNormalization() self.linear = [ Dense(self.slot_size, activation='relu') for _ in range(n_layers) ] self.after_norm = LayerNormalization() # Creating the Multi-Head Attention layer self.attn = MultiHeadAttention(self.slot_size, self.n_heads)
def __init__(self, low_bound, sup_bound, with_sum='n', a_initializer='ones', a_regularizer=None, a_constraint=None, b_initializer='zeros', b_regularizer=None, b_constraint=None, **kwargs): if 'input_shape' not in kwargs and 'input_dim' in kwargs: kwargs['input_shape'] = (kwargs.pop('input_dim'),) super(Restrict, self).__init__( activity_regularizer=None, **kwargs) try: low_bound, sup_bound = float(low_bound), float(sup_bound) except TypeError: pass check_l, check_s = isinstance(low_bound, float), isinstance(sup_bound, float) if (check_l and (not check_s)) or (check_s and (not check_l)): raise TypeError('The input `low_bound` and `sup_bound` does not share the same float type.') elif check_l and check_s: self.arg_array = False else: check_l, check_s = isinstance(low_bound, (list, tuple)), isinstance(sup_bound, (list, tuple)) if (check_l and (not check_s)) or (check_s and (not check_l)): raise TypeError('The input `low_bound` and `sup_bound` does not share the same list/tuple type.') if len(low_bound) != len(sup_bound): raise TypeError('The input `low_bound` and `sup_bound` does not share the same length') for l,s in zip(low_bound, sup_bound): if l >= s: raise ValueError('The input `low_bound` should be less than `sup_bound`, but received ' + str(low_bound) + ' ' + str(sup_bound)) if with_sum == 'i' and len(low_bound) > 1: for l in low_bound[1:]: if l < 0: raise ValueError('When set increasing mode, each element of `low_bound[1:]` should be ' 'non-negative, but received ' + str(low_bound)) elif with_sum == 'd' and len(low_bound) > 1: for l in low_bound[:-1]: if l < 0: raise ValueError('When set increasing mode, each element of `low_bound[:-1]` should be ' 'non-negative, but received ' + str(low_bound)) if check_l and check_s: self.arg_array = True else: raise TypeError('At least one of `low_bound` and `sup_bound` has wrong type.') low_bound = np.array(low_bound, dtype=np.float) sup_bound = np.array(sup_bound, dtype=np.float) self.low_bound = low_bound self.sup_bound = sup_bound if with_sum not in ('i', 'd', 'n'): raise ValueError('The input `with_sum` only supports 3 modes: i, d, n. But received ' + str(with_sum)) self.with_sum = with_sum self.a_initializer = initializers.get(a_initializer) self.a_regularizer = regularizers.get(a_regularizer) self.a_constraint = constraints.get(a_constraint) self.b_initializer = initializers.get(b_initializer) self.b_regularizer = regularizers.get(b_regularizer) self.b_constraint = constraints.get(b_constraint) self.supports_masking = True
def __init__(self, units, projection_units, use_feedback=True, use_recurrent=True, activation='tanh', projection_activation='sigmoid', use_bias=True, use_projection_bias=True, kernel_initializer='glorot_uniform', projection_initializer='glorot_uniform', recurrent_initializer='orthogonal', feedback_initializer='orthogonal', bias_initializer='zeros', projection_bias_initializer='zeros', kernel_regularizer=None, projection_regularizer=None, recurrent_regularizer=None, feedback_regularizer=None, bias_regularizer=None, projection_bias_regularizer=None, kernel_constraint=None, projection_constraint=None, recurrent_constraint=None, feedback_constraint=None, bias_constraint=None, projection_bias_constraint=None, dropout=0., recurrent_dropout=0., **kwargs): super(Cell, self).__init__(**kwargs) self.units = units self.projection_units = projection_units self.use_feedback = use_feedback self.use_recurrent = use_recurrent self.activation = activations.get(activation) self.projection_activation = activations.get(projection_activation) self.use_bias = use_bias self.use_projection_bias = use_projection_bias self.kernel_initializer = initializers.get(kernel_initializer) self.projection_initializer = initializers.get(projection_initializer) self.recurrent_initializer = initializers.get(recurrent_initializer) self.feedback_initializer = initializers.get(feedback_initializer) self.bias_initializer = initializers.get(bias_initializer) self.projection_bias_initializer = initializers.get( projection_bias_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.projection_regularizer = regularizers.get(kernel_regularizer) self.recurrent_regularizer = regularizers.get(recurrent_regularizer) self.feedback_regularizer = regularizers.get(feedback_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.projection_bias_regularizer = regularizers.get( projection_bias_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.projection_constraint = constraints.get(projection_constraint) self.recurrent_constraint = constraints.get(recurrent_constraint) self.feedback_constraint = constraints.get(feedback_constraint) self.bias_constraint = constraints.get(bias_constraint) self.projection_bias_constraint = constraints.get( projection_bias_constraint) self.dropout = min(1., max(0., dropout)) self.recurrent_dropout = min(1., max(0., recurrent_dropout)) self.state_size = self.units self.output_size = self.units self.projection_size = self.projection_units
def __init__(self, rank, filters, kernel_size, param_reduction=0.5, form='diagonal', strides=1, padding='valid', data_format=None, dilation_rate=1, groups=1, activation=None, use_bias=True, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, trainable=True, name=None, conv_op=None, **kwargs): super(SzConv, self).__init__( trainable=trainable, name=name, activity_regularizer=regularizers.get(activity_regularizer), **kwargs) self.rank = rank if isinstance(filters, float): filters = int(filters) self.filters = filters self.groups = groups or 1 self.kernel_size = conv_utils.normalize_tuple(kernel_size, rank, 'kernel_size') self.strides = conv_utils.normalize_tuple(strides, rank, 'strides') self.padding = conv_utils.normalize_padding(padding) self.data_format = conv_utils.normalize_data_format(data_format) self.dilation_rate = conv_utils.normalize_tuple( dilation_rate, rank, 'dilation_rate') self.activation = activations.get(activation) self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) self.bias_initializer = initializers.get(bias_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.bias_constraint = constraints.get(bias_constraint) self.input_spec = InputSpec(min_ndim=self.rank + 2) self._validate_init() self._is_causal = self.padding == 'causal' self._channels_first = self.data_format == 'channels_first' self._tf_data_format = conv_utils.convert_data_format( self.data_format, self.rank + 2) self.reduction_sv = param_reduction self.form = form self.num_ones = 0 self.num_weights = 0 self.reduced_ratio = 0 self.halfbandwidth = 0
def __init__( self, units, num_layers, extract_every_n_layers=3, # Saves every n layer and return them concatenated activation=None, use_bias=True, scale=True, scf_min=0.2, scf_max=2.0, dropconnect_prob=0.00, dropout_prob=0.05, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, **kwargs): if 'input_shape' not in kwargs and 'input_dim' in kwargs: kwargs['input_shape'] = (kwargs.pop('input_dim'), ) super(DensePipe, self).__init__( activity_regularizer=regularizers.get(activity_regularizer), **kwargs) # Save params self.units = units self.num_layers = num_layers self.extract_every_n_layers = extract_every_n_layers self.activation = activations.get(activation) self.use_bias = use_bias self.dropconnect_prob = dropconnect_prob self.dropout_prob = dropout_prob self.scale = scale self.scf_min = scf_min self.scf_max = scf_max self.kernel_initializer = initializers.get(kernel_initializer) self.bias_initializer = initializers.get(bias_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.bias_constraint = constraints.get(bias_constraint) self.kwargs = kwargs # Create Dense layers self.dense_layers = [ ScaledLinear(units=self.units, activation=self.activation, scf_min=self.scf_min, scf_max=self.scf_max, scale=self.scale, dropconnect_prob=self.dropconnect_prob, kernel_initializer=self.kernel_initializer, bias_initializer=self.bias_initializer, kernel_regularizer=self.kernel_regularizer, bias_regularizer=self.bias_regularizer, kernel_constraint=self.kernel_constraint, bias_constraint=self.bias_constraint, name='dense') for layer in range(self.num_layers) ] self.dropout = tf.keras.layers.Dropout(self.dropout_prob)
def __init__( self, rank: int, head_size: int, head_count: int, kernel_size: Union[int, Tuple, List], strides: Union[int, Tuple, List], # data_format: Optional[AnyStr], dilation_rate: Union[int, Tuple, List], activation: Optional[Union[AnyStr, Callable]], use_bias: bool, kernel_initializer: Optional[Union[Dict, AnyStr, Callable]], bias_initializer: Optional[Union[Dict, AnyStr, Callable]], embeddings_initializer: Optional[Union[Dict, AnyStr, Callable]], kernel_regularizer: Optional[Union[Dict, AnyStr, Callable]], bias_regularizer: Optional[Union[Dict, AnyStr, Callable]], activity_regularizer: Optional[Union[Dict, AnyStr, Callable]], kernel_constraint: Optional[Union[Dict, AnyStr, Callable]], bias_constraint: Optional[Union[Dict, AnyStr, Callable]], trainable=True, name=None, **kwargs): activity_regularizer = regularizers.get(activity_regularizer) super(StandAloneSelfAttention, self).__init__(trainable=trainable, name=name, activity_regularizer=activity_regularizer, **kwargs) # region Utils (normalizing tuples, data format and getting initializers/regularizers/constraints) kernel_size = conv_utils.normalize_tuple(kernel_size, rank, "kernel_size") strides = conv_utils.normalize_tuple(strides, rank, "strides") # data_format = conv_utils.normalize_data_format(data_format) dilation_rate = conv_utils.normalize_tuple(dilation_rate, rank, "dilation_rate") activation = activations.get(activation) kernel_initializer = initializers.get(kernel_initializer) bias_initializer = initializers.get(bias_initializer) if embeddings_initializer == "random_normal": embeddings_initializer = initializers.initializers_v2.RandomNormal( stddev=1.0) embeddings_initializer = initializers.get(embeddings_initializer) kernel_regularizer = regularizers.get(kernel_regularizer) bias_regularizer = regularizers.get(bias_regularizer) kernel_constraint = constraints.get(kernel_constraint) bias_constraint = constraints.get(bias_constraint) # endregion # region Base attributes self.rank = rank self.head_size = head_size self.head_count = head_count self.kernel_size = kernel_size self.strides = strides # self.data_format = data_format self.dilation_rate = dilation_rate self.activation = activation self.use_bias = use_bias self.kernel_initializer = kernel_initializer self.bias_initializer = bias_initializer self.embeddings_initializer = embeddings_initializer self.kernel_regularizer = kernel_regularizer self.bias_regularizer = bias_regularizer self.kernel_constraint = kernel_constraint self.bias_constraint = bias_constraint # endregion # region Queries/Keys/Values conv layers common_parameters = { "rank": self.rank, "filters": self.filters, "kernel_size": 1, "use_bias": self.use_bias, "kernel_initializer": self.kernel_initializer, "bias_initializer": self.bias_initializer, "kernel_regularizer": self.kernel_regularizer, "activity_regularizer": self.activity_regularizer, "kernel_constraint": self.kernel_constraint, "bias_constraint": self.bias_constraint, } self.queries_layer = Conv(name="Conv_Queries{}".format(self.name), **common_parameters) self.keys_layer = Conv(name="Conv_Keys{}".format(self.name), **common_parameters) self.values_layer = Conv(name="Conv_Values{}".format(self.name), **common_parameters) # endregion # region Queries/Keys/Values unfold layers self.queries_unfold = Unfold(kernel_size=1, strides=strides, name="Unfold_Queries_{}".format( self.name)) self.keys_unfold = Unfold(kernel_size=kernel_size, strides=strides, dilation_rate=dilation_rate, padding="SAME", name="Unfold_Keys_{}".format(self.name)) self.values_unfold = Unfold(kernel_size=kernel_size, strides=strides, dilation_rate=dilation_rate, padding="SAME", name="Unfold_Values_{}".format(self.name)) # endregion # region Time/Height/Width embeddings conv_embeddings = [] for i in range(rank): dim_embeddings_size = self.filters // rank if i == 0: dim_embeddings_size += self.filters % rank dim_embeddings_shape = (dim_embeddings_size, *[1] * i, kernel_size[i], *[1] * (rank - i - 1)) dim_embeddings = self.add_weight( name="dim_{}_embeddings".format(i + 1), shape=dim_embeddings_shape, dtype=tf.float32, initializer=self.embeddings_initializer) conv_embeddings.append(dim_embeddings) self.conv_embeddings = conv_embeddings
def __init__(self, rank, lgroups, lfilters, kernel_size, strides=1, padding='valid', data_format=None, dilation_rate=1, activation=None, use_bias=True, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, trainable=True, name=None, **kwargs): super(_GroupConv, self).__init__( trainable=trainable, name=name, activity_regularizer=regularizers.get(activity_regularizer), **kwargs) self.rank = rank if rank > 2: raise ValueError( 'The quick group convolution does not support 3D or any higher dimension.' ) initRank = rank self.lgroups = lgroups self.lfilters = lfilters self.kernel_size = conv_utils.normalize_tuple(kernel_size, rank, 'kernel_size') self.strides = conv_utils.normalize_tuple(strides, rank, 'strides') self.padding = conv_utils.normalize_padding(padding) if (self.padding == 'causal' and not isinstance(self, (Conv1D, SeparableConv1D))): raise ValueError( 'Causal padding is only supported for `Conv1D` and ``SeparableConv1D`.' ) self.data_format = conv_utils.normalize_data_format(data_format) self.dilation_rate = conv_utils.normalize_tuple( dilation_rate, rank, 'dilation_rate') if rank == 1: # when rank=1, expand the tuples to 2D case. self.kernel_size = (1, *self.kernel_size) self.strides = (1, *self.strides) self.dilation_rate = (1, *self.dilation_rate) self.activation = activations.get(activation) self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) self.bias_initializer = initializers.get(bias_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.bias_constraint = constraints.get(bias_constraint) self.input_spec = InputSpec(ndim=self.rank + 2) self.group_input_dim = None self.exp_dim_pos = None
def __init__(self, filters, kernel_size, strides=(1, 1), padding='valid', data_format=None, dilation_rate=(1, 1), activation=None, use_bias=True, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, **kwargs): super(Conv2D, self).__init__( #rank=2, filters=filters, kernel_size=kernel_size, strides=strides, padding=padding, data_format=data_format, dilation_rate=dilation_rate, activation=activations.get(activation), use_bias=use_bias, kernel_initializer=initializers.get(kernel_initializer), bias_initializer=initializers.get(bias_initializer), kernel_regularizer=regularizers.get(kernel_regularizer), bias_regularizer=regularizers.get(bias_regularizer), activity_regularizer=regularizers.get(activity_regularizer), kernel_constraint=constraints.get(kernel_constraint), bias_constraint=constraints.get(bias_constraint), **kwargs) def call(self, inputs): weights = self.kernel d = tf.math.sqrt( tf.math.sum( tf.math.square(weights), axis=[1, 2, 3], keepdims=True) + 1e-8) weights = weights / d outputs = self._convolution_op(inputs, weights) if self.use_bias: if self.data_format == 'channels_first': if self.rank == 1: # nn.bias_add does not accept a 1D input tensor. bias = array_ops.reshape(self.bias, (1, self.filters, 1)) outputs += bias else: outputs = nn.bias_add(outputs, self.bias, data_format='NCHW') else: outputs = nn.bias_add(outputs, self.bias, data_format='NHWC') if self.activation is not None: return self.activation(outputs) return outputs
def __init__(self, filters, kernel_size, rank=2, strides=(1, 1), padding='valid', data_format=None, dilation_rate=(1, 1), activation=None, use_bias=True, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, trainable=True, name=None, lambda_l1=None, lambda_mask=None, shared=None, adaptive=None, from_kb=None, atten=None, mask=None, bias=None, **kwargs): super(DecomposedConv, self).__init__( trainable=trainable, name=name, activity_regularizer=regularizers.get(activity_regularizer), **kwargs) self.rank = rank self.filters = filters self.kernel_size = conv_utils.normalize_tuple(kernel_size, rank, 'kernel_size') self.strides = conv_utils.normalize_tuple(strides, rank, 'strides') self.padding = conv_utils.normalize_padding(padding) if (self.padding == 'causal' and not isinstance(self, (Conv1D, SeparableConv1D))): raise ValueError('Causal padding is only supported for `Conv1D`' 'and ``SeparableConv1D`.') self.data_format = conv_utils.normalize_data_format(data_format) self.dilation_rate = conv_utils.normalize_tuple( dilation_rate, rank, 'dilation_rate') self.activation = activations.get(activation) self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) self.bias_initializer = initializers.get(bias_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.bias_constraint = constraints.get(bias_constraint) self.input_spec = InputSpec(ndim=self.rank + 2) self.sw = shared self.aw = adaptive self.mask = mask self.bias = bias self.aw_kb = from_kb self.atten = atten self.lambda_l1 = lambda_l1 self.lambda_mask = lambda_mask
def __init__(self, units, num_experts, num_tasks, use_expert_bias=True, use_gate_bias=True, expert_activation='relu', gate_activation='softmax', expert_bias_initializer='zeros', gate_bias_initializer='zeros', expert_bias_regularizer=None, gate_bias_regularizer=None, expert_bias_constraint=None, gate_bias_constraint=None, expert_kernel_initializer='VarianceScaling', gate_kernel_initializer='VarianceScaling', expert_kernel_regularizer=None, gate_kernel_regularizer=None, expert_kernel_constraint=None, gate_kernel_constraint=None, activity_regularizer=None, **kwargs): """ Method for instantiating MMoE layer. :param units: Number of hidden units :param num_experts: Number of experts :param num_tasks: Number of tasks :param use_expert_bias: Boolean to indicate the usage of bias in the expert weights :param use_gate_bias: Boolean to indicate the usage of bias in the gate weights :param expert_activation: Activation function of the expert weights :param gate_activation: Activation function of the gate weights :param expert_bias_initializer: Initializer for the expert bias :param gate_bias_initializer: Initializer for the gate bias :param expert_bias_regularizer: Regularizer for the expert bias :param gate_bias_regularizer: Regularizer for the gate bias :param expert_bias_constraint: Constraint for the expert bias :param gate_bias_constraint: Constraint for the gate bias :param expert_kernel_initializer: Initializer for the expert weights :param gate_kernel_initializer: Initializer for the gate weights :param expert_kernel_regularizer: Regularizer for the expert weights :param gate_kernel_regularizer: Regularizer for the gate weights :param expert_kernel_constraint: Constraint for the expert weights :param gate_kernel_constraint: Constraint for the gate weights :param activity_regularizer: Regularizer for the activity :param kwargs: Additional keyword arguments for the Layer class """ # Hidden nodes parameter self.units = units self.num_experts = num_experts self.num_tasks = num_tasks # Weight parameter self.expert_kernels = None self.gate_kernels = None self.expert_kernel_initializer = initializers.get( expert_kernel_initializer) self.gate_kernel_initializer = initializers.get( gate_kernel_initializer) self.expert_kernel_regularizer = regularizers.get( expert_kernel_regularizer) self.gate_kernel_regularizer = regularizers.get( gate_kernel_regularizer) self.expert_kernel_constraint = constraints.get( expert_kernel_constraint) self.gate_kernel_constraint = constraints.get(gate_kernel_constraint) # Activation parameter self.expert_activation = activations.get(expert_activation) self.gate_activation = activations.get(gate_activation) # Bias parameter self.expert_bias = None self.gate_bias = None self.use_expert_bias = use_expert_bias self.use_gate_bias = use_gate_bias self.expert_bias_initializer = initializers.get( expert_bias_initializer) self.gate_bias_initializer = initializers.get(gate_bias_initializer) self.expert_bias_regularizer = regularizers.get( expert_bias_regularizer) self.gate_bias_regularizer = regularizers.get(gate_bias_regularizer) self.expert_bias_constraint = constraints.get(expert_bias_constraint) self.gate_bias_constraint = constraints.get(gate_bias_constraint) # Activity parameter self.activity_regularizer = regularizers.get(activity_regularizer) # Keras parameter self.input_spec = InputSpec(min_ndim=2) self.supports_masking = True super(MMoE, self).__init__(**kwargs)
def __init__(self, units, feature_units, memory_size, conv_units, n_actions, num_head=10, dropout=0.0, use_bias=True, compression_rate=4, gate_initializer='glorot_normal', gate_regularizer='l2', gate_constraint=None, state_initializer='glorot_uniform', state_constraint=None, bias_initializer='zeros', bias_regularizer='l2', bias_constraint=None, **kwargs): """OSAR - Object-Stimulated Active Repeater # Arguments units: int > 0. Number of classic units inside the layer. feature_units: int > 0. Number of output features in the layer. memory_size: int > 1. Size of the dictionary memory. Big numbers allows to store more combinations, but requires more space and speed. conv_units: int > 0. Number of convolution units in the layer. n_actions: int > 1. Number of actions in the output vector. num_heads: int > 0. Number of attention `heads`. dropout: 0.0 <= float <= 1.0. Dropout rate inside attention weights. use_bias: Boolean. Whether to use bias (dafault - True). compression_rate: int. Compression rate of the transfomer memories. gate_initializer: keras.initializer. Initializer for attention weights (default - glorot_normal). gate_regularizer: keras.regularizer. Regularizer for attention weights (default - l2). gate_constraint: keras.constraint. Constraint for attention weights (default - None). state_initializer: keras.initializer. Initializer for state matrices (default - glorot_uniform). state_constraint: keras.constraint. Constraint for state matrices (default - None). bias_initializer: keras.initializer. Initializer for biases (default - zeros). bias_regularizer: keras.regularizer. Regularizer for biases (default - l2). bias_constraint: keras.constraint. Constraint for attention weights (default - None). # Input Shape 3D tensor with shape: `(batch_size, sequence_length, units)`. # Output Shape # References - None yet """ # return_runtime is a flag for testing, which shows the real backend # implementation chosen by grappler in graph mode. self._return_runtime = kwargs.pop('return_runtime', False) super(OSAR, self).__init__(dynamic=True, **kwargs) self.units = units self.n_actions = n_actions self.conv_units = conv_units self.feature_units = feature_units self.num_head = num_head self.dropout = dropout self.use_bias = use_bias self.memory_size = memory_size self.compression_rate = compression_rate self.state_constraint = constraints.get(state_constraint) self.state_initializer = initializers.get(state_initializer) self.gate_initializer = initializers.get(gate_initializer) self.gate_regularizer = regularizers.get(gate_regularizer) self.gate_constraint = constraints.get(gate_constraint) self.bias_initializer = initializers.get(bias_initializer) self.bias_regularizer = regularizers.get(bias_regularizer) self.bias_constraint = constraints.get(bias_constraint)
def get_constraint(self, name): if name not in self.constraints: self.constraints[name] = constraints.get(None) return self.constraints[name]
def __init__( self, filters, kernel_size, feature_number, # ????feature strides=(1, 1), padding='valid', data_format=None, dilation_rate=(1, 1), activation='tanh', recurrent_activation='hard_sigmoid', conv_activation='hard_sigmoid', convolutional_type="early", use_bias=True, kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', unit_forget_bias=True, kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, dropout=0., recurrent_dropout=0., **kwargs): """ filters : A list , Specifies the number of filters in each layer, e.g. [10,10] kernel_size : A List , Same length as filters, Window size for 1D convolution e.g. [3,3] # ????feature feature_number: int , Number of multiple time series e.g 28 sensors --> 28 recurrent_activation : A str List, Specifies the tupe of activation functions """ super(ConvLSTM1DCell, self).__init__(**kwargs) self.number_of_layer = len(filters) self.out_feature_number = feature_number self.convolutional_type = convolutional_type # ============= Each layer has different parameters ====================== self.filters = filters self.conv_layer_number = len(filters) self.kernel_size = [] for index, size in enumerate(kernel_size): if self.convolutional_type[index] == "hybrid": self.kernel_size.append( conv_utils.normalize_tuple((size, 1), 2, 'kernel_size')) if self.convolutional_type[index] == "early": self.kernel_size.append( conv_utils.normalize_tuple((size, feature_number), 2, 'kernel_size')) self.out_feature_number = 1 feature_number = 1 self.recurrent_activation = [] for acti in recurrent_activation: self.recurrent_activation.append(activations.get(acti)) self.conv_activation = [] for acti in conv_activation: self.conv_activation.append(activations.get(acti)) self.state_size = (self.filters[-1], self.filters[-1]) # ============= Each layer has the same parameter ====================== self.strides = conv_utils.normalize_tuple(strides, 2, 'strides') # (1,1) self.padding = conv_utils.normalize_padding(padding) # valid self.data_format = conv_utils.normalize_data_format( data_format) # None --- -1 self.dilation_rate = conv_utils.normalize_tuple( dilation_rate, 2, 'dilation_rate') self.activation = activations.get(activation) # tanh default self.use_bias = use_bias # True self.kernel_initializer = initializers.get( kernel_initializer) # glorot_uniform self.recurrent_initializer = initializers.get( recurrent_initializer) # orthogonal self.bias_initializer = initializers.get(bias_initializer) # zeros self.unit_forget_bias = unit_forget_bias # True self.kernel_regularizer = regularizers.get(kernel_regularizer) self.recurrent_regularizer = regularizers.get(recurrent_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.recurrent_constraint = constraints.get(recurrent_constraint) self.bias_constraint = constraints.get(bias_constraint) self.dropout = min(1., max(0., dropout)) self.recurrent_dropout = min(1., max(0., recurrent_dropout))
def __init__(self, sub_units, sub_lstms, cake_activation='tanh', cake_recurrent_activation='hard_sigmoid', sub_activation='tanh', sub_recurrent_activation='hard_sigmoid', cake_use_bias=True, sub_use_bias=True, cake_kernel_initializer='glorot_uniform', cake_recurrent_initializer='orthogonal', cake_bias_initializer='zeros', sub_kernel_initializer='glorot_uniform', sub_recurrent_initializer='orthogonal', sub_bias_initializer='zeros', cake_unit_forget_bias=True, sub_unit_forget_bias=True, cake_kernel_regularizer=None, cake_recurrent_regularizer=None, cake_bias_regularizer=None, sub_kernel_regularizer=None, sub_recurrent_regularizer=None, sub_bias_regularizer=None, cake_kernel_constraint=None, cake_recurrent_constraint=None, cake_bias_constraint=None, sub_kernel_constraint=None, sub_recurrent_constraint=None, sub_bias_constraint=None, cake_dropout=0., cake_recurrent_dropout=0., sub_dropout=0., sub_recurrent_dropout=0., implementation=1, **kwargs): super(JujubeCakeCell, self).__init__(**kwargs) self.sub_units = sub_units self.sub_lstms = sub_lstms self.units = self.sub_units * self.sub_lstms self.cake_activation = activations.get(cake_activation) self.cake_recurrent_activation = activations.get( cake_recurrent_activation) self.sub_activation = activations.get(sub_activation) self.sub_recurrent_activation = activations.get( sub_recurrent_activation) self.cake_use_bias = cake_use_bias self.sub_use_bias = sub_use_bias self.cake_kernel_initializer = initializers.get( cake_kernel_initializer) self.cake_recurrent_initializer = initializers.get( cake_recurrent_initializer) self.cake_bias_initializer = initializers.get(cake_bias_initializer) self.sub_kernel_initializer = initializers.get(sub_kernel_initializer) self.sub_recurrent_initializer = initializers.get( sub_recurrent_initializer) self.sub_bias_initializer = initializers.get(sub_bias_initializer) self.cake_unit_forget_bias = cake_unit_forget_bias self.sub_unit_forget_bias = sub_unit_forget_bias self.cake_kernel_regularizer = regularizers.get( cake_kernel_regularizer) self.cake_recurrent_regularizer = regularizers.get( cake_recurrent_regularizer) self.cake_bias_regularizer = regularizers.get(cake_bias_regularizer) self.sub_kernel_regularizer = regularizers.get(sub_kernel_regularizer) self.sub_recurrent_regularizer = regularizers.get( sub_recurrent_regularizer) self.sub_bias_regularizer = regularizers.get(sub_bias_regularizer) self.cake_kernel_constraint = constraints.get(cake_kernel_constraint) self.cake_recurrent_constraint = constraints.get( cake_recurrent_constraint) self.cake_bias_constraint = constraints.get(cake_bias_constraint) self.sub_kernel_constraint = constraints.get(sub_kernel_constraint) self.sub_recurrent_constraint = constraints.get( sub_recurrent_constraint) self.sub_bias_constraint = constraints.get(sub_bias_constraint) self.cake_dropout = min(1., max(0., cake_dropout)) self.cake_recurrent_dropout = min(1., max(0., cake_recurrent_dropout)) self.sub_dropout = min(1., max(0., sub_dropout)) self.sub_recurrent_dropout = min(1., max(0., sub_recurrent_dropout)) self.implementation = implementation self.state_size = [ self.units, self.units, self.sub_units, self.sub_units ] self.sub_state_size = [self.sub_units, self.sub_units] self._dropout_mask = None self._recurrent_dropout_mask = None self._sub_dropout_mask = None self._sub_recurrent_dropout_mask = None
def __init__(self, units, activation='relu', use_bias=True, add_self_loops=False, aggregation_method='sum', graph_regularization=None, num_bases=None, kernel_initializer='glorot_uniform', kernel_coef_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, kernel_coef_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, kernel_coef_constraint=None, bias_constraint=None, **kwargs): super(GraphConv, self).__init__( activity_regularizer=regularizers.get(activity_regularizer), **kwargs) if graph_regularization not in [None, 'basis']: raise ValueError('An invalid value of `graph_regularization` has' + ' been passed to a `GraphConv` layer.') if graph_regularization is not None and num_bases is None: raise ValueError('The `num_bases` property must be set if ' + '`graph_regularization` is not None.') if graph_regularization is None and num_bases is not None: raise ValueError('The `num_bases` property must not be set if ' + '`graph_regularization` is None.') if num_bases is not None and num_bases <= 0: raise ValueError('The `num_bases` property must be a positive ' + 'integer.') self.units = int(units) self.activation = activations.get(activation) self.use_bias = use_bias self.add_self_loops = add_self_loops self.aggregation_method = aggregation_method self.graph_regularization = graph_regularization self.num_bases = num_bases self.kernel_initializer = initializers.get(kernel_initializer) self.kernel_coef_initializer = initializers.get( kernel_coef_initializer) self.bias_initializer = initializers.get(bias_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.kernel_coef_regularizer = regularizers.get( kernel_coef_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.kernel_coef_constraint = constraints.get(kernel_coef_constraint) self.bias_constraint = constraints.get(bias_constraint) self.input_spec = { NODE_FEATURES: InputSpec(min_ndim=2), EDGE_FEATURES: InputSpec(ndim=4) }
def __init__(self, filters, kernel_size, strides=(1, 1), padding='valid', data_format=None, dilation_rate=(1, 1), groups=1, activation=None, use_bias=True, dtype=DEFAULT_COMPLEX_TYPE, kernel_initializer=ComplexGlorotUniform(), bias_initializer=Zeros(), kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, **kwargs): """ :param filters: Integer, the dimensionality of the output space (i.e. the number of output filters in the convolution). :param kernel_size: An integer or tuple/list of 2 integers, specifying the height and width of the 2D convolution window. Can be a single integer to specify the same value for all spatial dimensions. :param strides: An integer or tuple/list of 2 integers, specifying the strides of the convolution along the height and width. Can be a single integer to specify the same value for all spatial dimensions. Specifying any stride value != 1 is incompatible with specifying any `dilation_rate` value != 1. :param padding: one of `"valid"` or `"same"` (case-insensitive). `"valid"` means no padding. `"same"` results in padding evenly to the left/right or up/down of the input such that output has the same height/width dimension as the input. :param data_format: A string, one of `channels_last` (default) or `channels_first`. The ordering of the dimensions in the inputs. `channels_last` corresponds to inputs with shape `(batch_size, height, width, channels)` while `channels_first` corresponds to inputs with shape `(batch_size, channels, height, width)`. It defaults to the `image_data_format` value found in your Keras config file at `~/.keras/keras.json`. If you never set it, then it will be `channels_last`. :param dilation_rate: an integer or tuple/list of 2 integers, specifying the dilation rate to use for dilated convolution. Can be a single integer to specify the same value for all spatial dimensions. Currently, specifying any `dilation_rate` value != 1 is incompatible with specifying any stride value != 1. :param groups: A positive integer specifying the number of groups in which the input is split along the channel axis. Each group is convolved separately with `filters / groups` filters. The output is the concatenation of all the `groups` results along the channel axis. Input channels and `filters` must both be divisible by `groups`. :param activation: Activation function to use. If you don't specify anything, no activation is applied. For complex :code:`dtype`, this must be a :code:`cvnn.activations` module. :param use_bias: Boolean, whether the layer uses a bias vector. :param kernel_initializer: Initializer for the `kernel` weights matrix (see `keras.initializers`). :param bias_initializer: Initializer for the bias vector (see `keras.initializers`). :param kernel_regularizer: Regularizer function applied to the `kernel` weights matrix (see `keras.regularizers`). :param bias_regularizer: Regularizer function applied to the bias vector (see `keras.regularizers`). :param activity_regularizer: Regularizer function applied to the output of the layer (its "activation") (see `keras.regularizers`). :param kernel_constraint: Constraint function applied to the kernel matrix (see `keras.constraints`). :param bias_constraint: Constraint function applied to the bias vector (see `keras.constraints`). """ super(ComplexConv2D, self).__init__( rank=2, dtype=dtype, filters=filters, kernel_size=kernel_size, strides=strides, padding=padding, data_format=data_format, dilation_rate=dilation_rate, groups=groups, activation=activations.get(activation), use_bias=use_bias, kernel_initializer=initializers.get(kernel_initializer), bias_initializer=initializers.get(bias_initializer), kernel_regularizer=regularizers.get(kernel_regularizer), bias_regularizer=regularizers.get(bias_regularizer), activity_regularizer=regularizers.get(activity_regularizer), kernel_constraint=constraints.get(kernel_constraint), bias_constraint=constraints.get(bias_constraint), **kwargs)
def __init__(self, rank, kernel_size, growth_rate, depth, output_filters=None, use_bottleneck=True, bottleneck_filters_multiplier=4, use_batch_normalization=True, data_format=None, activation="relu", use_bias=True, kernel_initializer="he_normal", bias_initializer="zeros", kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, **kwargs): if rank not in [1, 2, 3]: raise ValueError( "`rank` must be in [1, 2, 3]. Got {}".format(rank)) super(DenseBlockND, self).__init__(**kwargs) self.rank = rank self.kernel_size = conv_utils.normalize_tuple(kernel_size, rank, "kernel_size") self.output_filters = output_filters self.growth_rate = growth_rate if use_bottleneck: if (depth % 2) != 0: raise ValueError( "Depth must be a multiple of 2 when using bottlenecks. Got {}." .format(depth)) self._depth = depth // 2 if use_bottleneck else depth self.use_bottleneck = use_bottleneck self.bottleneck_filters_multiplier = bottleneck_filters_multiplier self.use_batch_normalization = use_batch_normalization self.data_format = conv_utils.normalize_data_format(data_format) self.channel_axis = -1 if self.data_format == "channels_last" else 1 self.activation = activations.get(activation) self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) self.bias_initializer = initializers.get(bias_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.activity_regularizer = regularizers.get(activity_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.bias_constraint = constraints.get(bias_constraint) self.composite_function_blocks: Optional[ List[CompositeFunctionBlock]] = None self.transition_layer = None self.input_spec = InputSpec(ndim=self.rank + 2)
def __init__(self, filters, kernel_size, cov_kernel_size=(3, 1), strides=(1, 1), padding='valid', data_format=None, dilation_rate=(1, 1), activation='tanh', recurrent_activation='hard_sigmoid', use_bias=True, kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', unit_forget_bias=True, kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, dropout=0., recurrent_dropout=0., **kwargs): super(ConvLSTM2DCell_2, self).__init__(**kwargs) self.filters = filters self.kernel_size = conv_utils.normalize_tuple(kernel_size, 2, 'kernel_size') ############################# self.cov_kernel_size = cov_kernel_size self.kernel_size_1 = conv_utils.normalize_tuple( cov_kernel_size, 2, 'kernel_size') ################################## self.strides = conv_utils.normalize_tuple(strides, 2, 'strides') self.padding = conv_utils.normalize_padding(padding) self.data_format = conv_utils.normalize_data_format(data_format) self.dilation_rate = conv_utils.normalize_tuple( dilation_rate, 2, 'dilation_rate') self.activation = activations.get(activation) self.recurrent_activation = activations.get(recurrent_activation) self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) ############################### self.cov_kernel_initializer = initializers.get(kernel_initializer) ########################## self.recurrent_initializer = initializers.get(recurrent_initializer) self.bias_initializer = initializers.get(bias_initializer) self.unit_forget_bias = unit_forget_bias self.kernel_regularizer = regularizers.get(kernel_regularizer) self.recurrent_regularizer = regularizers.get(recurrent_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.recurrent_constraint = constraints.get(recurrent_constraint) self.bias_constraint = constraints.get(bias_constraint) self.dropout = min(1., max(0., dropout)) self.recurrent_dropout = min(1., max(0., recurrent_dropout)) self.state_size = (self.filters, self.filters)
def __init__(self, filters, kernel_size, strides=(1, 1), padding='valid', data_format=None, dilation_rate=(1, 1), depth_multiplier=1, activation=None, use_bias=True, depthwise_initializer='glorot_uniform', pointwise_initializer='glorot_uniform', bias_initializer='zeros', depthwise_regularizer=None, pointwise_regularizer=None, bias_regularizer=None, activity_regularizer=None, depthwise_constraint=None, pointwise_constraint=None, bias_constraint=None, activations_datatype_size_byte=1, weights_datatype_size_byte=1, results_datatype_size_byte=4, systolic_array_height=256, systolic_array_width=256, activation_fifo_depth=8, accumulator_array_height=4096, log_file_output_dir='.', model_name='unnamed', **kwargs): super(mpusim_separable_conv2d, self).__init__( rank=2, filters=filters, kernel_size=kernel_size, strides=strides, padding=padding, data_format=data_format, dilation_rate=dilation_rate, depth_multiplier=depth_multiplier, activation=activations.get(activation), use_bias=use_bias, depthwise_initializer=initializers.get(depthwise_initializer), pointwise_initializer=initializers.get(pointwise_initializer), bias_initializer=initializers.get(bias_initializer), depthwise_regularizer=regularizers.get(depthwise_regularizer), pointwise_regularizer=regularizers.get(pointwise_regularizer), bias_regularizer=regularizers.get(bias_regularizer), activity_regularizer=regularizers.get(activity_regularizer), depthwise_constraint=constraints.get(depthwise_constraint), pointwise_constraint=constraints.get(pointwise_constraint), bias_constraint=constraints.get(bias_constraint), **kwargs) self.activations_datatype_size_byte=activations_datatype_size_byte self.weights_datatype_size_byte=weights_datatype_size_byte self.results_datatype_size_byte=results_datatype_size_byte self.systolic_array_height=systolic_array_height self.systolic_array_width=systolic_array_width self.activation_fifo_depth=activation_fifo_depth self.accumulator_array_height=accumulator_array_height self.log_file_output_dir=log_file_output_dir self.model_name=model_name
def __init__(self, units_vec, modules=None, tau_vec=1., connectivity='dense', activation='tanh', use_bias=True, kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, dropout=0., recurrent_dropout=0., **kwargs): super(CTRNNCell, self).__init__(**kwargs) self.connectivity = connectivity if isinstance(units_vec, list): self.units_vec = units_vec[:] self.modules = len(units_vec) self.units = 0 for k in range(self.modules): self.units += units_vec[k] else: self.units = units_vec if modules is not None and modules > 1: self.modules = int(modules) self.units_vec = [ units_vec // modules for k in range(self.modules) ] else: self.modules = 1 self.units_vec = [units_vec] self.connectivity = 'dense' # smallest timescale should be 1.0 if isinstance(tau_vec, list): if len(tau_vec) != self.modules: raise ValueError("vector of tau must be of same size as " "num_modules or size of vector of num_units") self.tau_vec = tau_vec[:] self.taus = array_ops.constant([[max(1., float(tau_vec[k]))] for k in range(self.modules) for n in range(self.units_vec[k])], dtype=self.dtype, shape=[self.units], name="taus") else: if self.modules > 1: self.tau_vec = [ max(1., float(tau_vec)) for k in range(self.modules) ] else: self.tau_vec = [max(1., float(tau_vec))] self.taus = array_ops.constant(max(1., tau_vec), dtype=self.dtype, shape=[self.units], name="taus") self.activation = activations.get(activation) self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) self.recurrent_initializer = initializers.get(recurrent_initializer) self.bias_initializer = initializers.get(bias_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.recurrent_regularizer = regularizers.get(recurrent_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.recurrent_constraint = constraints.get(recurrent_constraint) self.bias_constraint = constraints.get(bias_constraint) self.dropout = min(1., max(0., dropout)) self.recurrent_dropout = min(1., max(0., recurrent_dropout)) self.state_size = (self.units, self.units) self.output_size = (self.units, self.units) self._dropout_mask = None self._recurrent_dropout_mask = None
def __init__(self, units, activation='tanh', init_kern_1='RandomUniform', init_kern_2='RandomUniform', init_kern_3='RandomUniform', init_recur_kern_2='orthogonal', init_out_fb='orthogonal', init_bias='zeros', connectivity=None, reg_kern_1=None, reg_kern_3=None, reg_out_fb=None, reg_bias=None, constraint_kern_1=None, constraint_kern_3=None, constraint_recur_kern_2=None, constraint_out_fb=None, constraint_bias=None, in_dropout=None, recur_dropout=None, train_kern_1=False, train_kern_3=False, train_out_fb=False, use_out_fb=False, use_dropout_mask=False, use_recur=False, use_clock=False, clock_rate=None, data_format=None, **kwargs): super(SimpleRNNCell, self).__init__(**kwargs) self.units = units self.activation = activations.get(activation) self.init_kern_1 = initializers.get(init_kern_1) self.init_kern_2 = initializers.get(init_kern_2) self.init_kern_3 = initializers.get(init_kern_3) self.init_recur_kern_2 = initializers.get(init_recur_kern_2) self.init_out_fb = initializers.get(init_out_fb) self.init_bias = initializers.get(init_bias) if len(connectivity) is not 3: self.connectivity_1 = 1. self.connectivity_2 = 1. self.connectivity_3 = 1. else: self.connectivity_1 = min(1., max(0.,connectivity[0])) self.connectivity_2 = min(1., max(0.,connectivity[1])) self.connectivity_3 = min(1., max(0.,connectivity[2])) self.reg_kern_1 = regularizers.get(reg_kern_1) self.reg_kern_3 = regularizers.get(reg_kern_3) self.reg_out_fb = regularizers.get(reg_out_fb) self.reg_bias = regularizers.get(reg_bias) self.constraint_kern_1 = constraints.get(constraint_kern_1) self.constraint_kern_3 = constraints.get(constraint_kern_3) self.constraint_recur_kern_2 = constraints.get(constraint_recur_kern_2) self.constraint_out_fb = constraints.get(constraint_out_fb) self.constraint_bias = constraints.get(constraint_bias) self.in_dropout = min(1., max(0.,in_dropout)) self.recur_dropout = min(1., max(0.,recur_dropout)) self.train_kern_1 = train_kern_1 self.train_kern_3 = train_kern_3 self.clock = clock self.clock_rate = clock_rate self.in_dropout_mask = None self.recur_dropout_mask = None self.state_size = self.units self.output_size = self.units self.tf_data_format = self.data_format self.clock_kernel = None