def __init__(self, axis=-1, momentum=0.99, epsilon=1e-3, center=True, scale=True, beta_initializer='zeros', gamma_initializer='ones', moving_mean_initializer='zeros', moving_variance_initializer='ones', beta_regularizer=None, gamma_regularizer=None, beta_constraint=None, gamma_constraint=None, renorm=False, renorm_clipping=None, renorm_momentum=0.99, fused=None, trainable=True, virtual_batch_size=None, adjustment=None, name=None, **kwargs): super(BatchNormalization, self).__init__( name=name, trainable=trainable, **kwargs) if isinstance(axis, list): self.axis = axis[:] else: self.axis = axis self.momentum = momentum self.epsilon = epsilon self.center = center self.scale = scale self.beta_initializer = initializers.get(beta_initializer) self.gamma_initializer = initializers.get(gamma_initializer) self.moving_mean_initializer = initializers.get(moving_mean_initializer) self.moving_variance_initializer = initializers.get( moving_variance_initializer) self.beta_regularizer = regularizers.get(beta_regularizer) self.gamma_regularizer = regularizers.get(gamma_regularizer) self.beta_constraint = constraints.get(beta_constraint) self.gamma_constraint = constraints.get(gamma_constraint) self.renorm = renorm self.virtual_batch_size = virtual_batch_size self.adjustment = adjustment if fused is None: fused = True self.supports_masking = True self.fused = fused self._bessels_correction_test_only = True if renorm: renorm_clipping = renorm_clipping or {} keys = ['rmax', 'rmin', 'dmax'] if set(renorm_clipping) - set(keys): raise ValueError('renorm_clipping %s contains keys not in %s' % (renorm_clipping, keys)) self.renorm_clipping = renorm_clipping self.renorm_momentum = renorm_momentum
def __init__(self, units, activation=None, use_bias=True, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, **kwargs): if 'input_shape' not in kwargs and 'input_dim' in kwargs: kwargs['input_shape'] = (kwargs.pop('input_dim'),) super(Dense, self).__init__( activity_regularizer=regularizers.get(activity_regularizer), **kwargs) self.units = int(units) self.activation = activations.get(activation) self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) self.bias_initializer = initializers.get(bias_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.bias_constraint = constraints.get(bias_constraint) self.supports_masking = True self.input_spec = InputSpec(min_ndim=2)
def __init__(self, filters, kernel_size, strides=1, padding='valid', data_format=None, activation=None, use_bias=True, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, **kwargs): super(LocallyConnected1D, self).__init__(**kwargs) self.filters = filters self.kernel_size = conv_utils.normalize_tuple(kernel_size, 1, 'kernel_size') self.strides = conv_utils.normalize_tuple(strides, 1, 'strides') self.padding = conv_utils.normalize_padding(padding) if self.padding != 'valid': raise ValueError('Invalid border mode for LocallyConnected1D ' '(only "valid" is supported): ' + padding) self.data_format = conv_utils.normalize_data_format(data_format) self.activation = activations.get(activation) self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) self.bias_initializer = initializers.get(bias_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.activity_regularizer = regularizers.get(activity_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.bias_constraint = constraints.get(bias_constraint) self.input_spec = InputSpec(ndim=3)
def __init__(self, axis=-1, epsilon=1e-3, center=True, scale=True, beta_initializer='zeros', gamma_initializer='ones', beta_regularizer=None, gamma_regularizer=None, beta_constraint=None, gamma_constraint=None, trainable=True, name=None, **kwargs): super(LayerNormalization, self).__init__( name=name, trainable=trainable, **kwargs) if isinstance(axis, (list, tuple)): self.axis = axis[:] elif isinstance(axis, int): self.axis = axis else: raise ValueError('Expected an int or a list/tuple of ints for the ' 'argument \'axis\', but received instead: %s' % axis) self.epsilon = epsilon self.center = center self.scale = scale self.beta_initializer = initializers.get(beta_initializer) self.gamma_initializer = initializers.get(gamma_initializer) self.beta_regularizer = regularizers.get(beta_regularizer) self.gamma_regularizer = regularizers.get(gamma_regularizer) self.beta_constraint = constraints.get(beta_constraint) self.gamma_constraint = constraints.get(gamma_constraint) self.supports_masking = True
def __init__(self, units, activation='tanh', recurrent_activation='hard_sigmoid', kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', unit_forget_bias=True, kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, return_sequences=False, return_state=False, go_backwards=False, stateful=False, time_major=False, **kwargs): super(RNN, self).__init__(**kwargs) # pylint: disable=bad-super-call self.units = units cell_spec = collections.namedtuple('cell', ['state_size', 'output_size']) self.cell = cell_spec( state_size=(self.units, self.units), output_size=self.units) self.activation = activations.get(activation) self.recurrent_activation = activations.get(recurrent_activation) self.kernel_initializer = initializers.get(kernel_initializer) self.recurrent_initializer = initializers.get(recurrent_initializer) self.bias_initializer = initializers.get(bias_initializer) self.unit_forget_bias = unit_forget_bias self.kernel_regularizer = regularizers.get(kernel_regularizer) self.recurrent_regularizer = regularizers.get(recurrent_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.activity_regularizer = regularizers.get(activity_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.recurrent_constraint = constraints.get(recurrent_constraint) self.bias_constraint = constraints.get(bias_constraint) self.return_sequences = return_sequences self.return_state = return_state self.go_backwards = go_backwards self.stateful = stateful self.time_major = time_major self._num_constants = None self._num_inputs = None self._states = None self.input_spec = [InputSpec(ndim=3)] self.state_spec = [ InputSpec(shape=(None, dim)) for dim in (self.units, self.units) ]
def __init__(self, filters, kernel_size, strides=(1, 1), padding='valid', data_format=None, dilation_rate=(1, 1), activation='tanh', recurrent_activation='hard_sigmoid', use_bias=True, kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', unit_forget_bias=True, kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, dropout=0., recurrent_dropout=0., **kwargs): super(ConvLSTM2DCell, self).__init__(**kwargs) self.filters = filters self.kernel_size = conv_utils.normalize_tuple(kernel_size, 2, 'kernel_size') self.strides = conv_utils.normalize_tuple(strides, 2, 'strides') self.padding = conv_utils.normalize_padding(padding) self.data_format = conv_utils.normalize_data_format(data_format) self.dilation_rate = conv_utils.normalize_tuple(dilation_rate, 2, 'dilation_rate') self.activation = activations.get(activation) self.recurrent_activation = activations.get(recurrent_activation) self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) self.recurrent_initializer = initializers.get(recurrent_initializer) self.bias_initializer = initializers.get(bias_initializer) self.unit_forget_bias = unit_forget_bias self.kernel_regularizer = regularizers.get(kernel_regularizer) self.recurrent_regularizer = regularizers.get(recurrent_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.recurrent_constraint = constraints.get(recurrent_constraint) self.bias_constraint = constraints.get(bias_constraint) self.dropout = min(1., max(0., dropout)) self.recurrent_dropout = min(1., max(0., recurrent_dropout)) self.state_size = (self.filters, self.filters) self._dropout_mask = None self._recurrent_dropout_mask = None
def add_slot(self, var, slot_name, initializer="zeros"): """Add a new slot variable for `var`.""" if slot_name not in self._slot_names: self._slot_names.append(slot_name) var_key = _var_key(var) slot_dict = self._slots.setdefault(var_key, {}) weight = slot_dict.get(slot_name, None) if weight is None: if isinstance(initializer, six.string_types) or callable(initializer): initializer = initializers.get(initializer) weight = slot_creator.create_slot_with_initializer( var, initializer, shape=var.shape, dtype=var.dtype, name=slot_name) else: weight = slot_creator.create_slot(var, initializer, slot_name) backend.track_variable(weight) slot_dict[slot_name] = weight self._restore_slot_variable(slot_name=slot_name, variable=var, slot_variable=weight) self._weights.append(weight) return weight
def __init__(self, input_dim, output_dim, embeddings_initializer='uniform', embeddings_regularizer=None, activity_regularizer=None, embeddings_constraint=None, mask_zero=False, input_length=None, **kwargs): if 'input_shape' not in kwargs: if input_length: kwargs['input_shape'] = (input_length, ) else: kwargs['input_shape'] = (None, ) dtype = kwargs.pop('dtype', K.floatx()) super(Embedding, self).__init__(dtype=dtype, **kwargs) self.input_dim = input_dim self.output_dim = output_dim self.embeddings_initializer = initializers.get(embeddings_initializer) self.embeddings_regularizer = regularizers.get(embeddings_regularizer) self.activity_regularizer = regularizers.get(activity_regularizer) self.embeddings_constraint = constraints.get(embeddings_constraint) self.mask_zero = mask_zero self.supports_masking = mask_zero self.input_length = input_length self._can_use_graph_functions = True
def build(self, input_shape): """Build `Layer`""" input_shape = tensor_shape.TensorShape(input_shape).as_list() self.input_spec = InputSpec(shape=input_shape) if not self.layer._built: self.layer.build(input_shape) self.layer._built = False if not hasattr(self.layer, 'kernel'): raise ValueError('`WeightNorm` must wrap a layer that' ' contains a `kernel` for weights') # The kernel's filter or unit dimension is -1 self.layer_depth = int(self.layer.W.shape[-1]) self.norm_axes = list(range(self.layer.W.shape.ndims - 1)) self.layer.v = self.layer.W self.layer.g = self.layer.add_variable( name="g", shape=(self.layer_depth, ), initializer=initializers.get('ones'), dtype=self.layer.kernel.dtype, trainable=True) with ops.control_dependencies( [self.layer.g.assign(self._init_norm(self.layer.v))]): self._compute_weights() self.layer.built = True super(WeightNorm, self).build() self._built = True
def __init__(self, shape, kernel_initializer='glorot_uniform'): super(InputLayer, self).__init__() self.kernel_initializer = initializers.get(kernel_initializer) self.init_embeds = self.add_weight('embedding', shape=shape, dtype='float32', trainable=True)
def _add_weight(self, name, shape=(), dtype=None, initializer='zeros'): """Adds a weight to this loss scale. Args: name: Variable name. shape: Variable shape. dtype: The type of the variable. initializer: The initializer to use. Returns: A variable. """ if isinstance(initializer, six.string_types) or callable(initializer): initializer = initializers.get(initializer) variable = self._add_variable_with_custom_getter( name=name, shape=shape, getter=base_layer_utils.make_variable, overwrite=True, initializer=initializer, dtype=dtype, trainable=False, use_resource=True, synchronization=variables.VariableSynchronization.AUTO, # Set aggregation to NONE, as loss scaling variables should never be # aggregated. aggregation=variables.VariableAggregation.NONE) backend.track_variable(variable) return variable
def __init__(self, alpha_initializer="zeros", data_format="channels_last", **kwargs): super(ScaleBlock, self).__init__(**kwargs) self.data_format = data_format self.alpha_initializer = initializers.get(alpha_initializer)
def add_slot(self, var, slot_name, initializer="zeros"): """Add a new slot variable for `var`.""" if slot_name not in self._slot_names: self._slot_names.append(slot_name) var_key = _var_key(var) slot_dict = self._slots.setdefault(var_key, {}) weight = slot_dict.get(slot_name, None) if weight is None: if isinstance(initializer, six.string_types) or callable(initializer): initializer = initializers.get(initializer) initial_value = functools.partial( initializer, shape=var.shape, dtype=var.dtype) else: initial_value = initializer weight = tf_variables.Variable( name="%s/%s" % (var._shared_name, slot_name), # pylint: disable=protected-access dtype=var.dtype, trainable=False, initial_value=initial_value) backend.track_variable(weight) slot_dict[slot_name] = weight self._restore_slot_variable( slot_name=slot_name, variable=var, slot_variable=weight) self._weights.append(weight) return weight
def build(self, input_shape): input_dim = input_shape[-1] self.w = self.add_weight( shape=(input_dim, self.units * 4), name='kernel', initializer=initializers.get('glorot_uniform')) self.u = self.add_weight(shape=(self.units, self.units * 4), name='recurrent_kernel', initializer=initializers.get('orthogonal')) self.bias = self.add_weight(shape=(self.units * 4), name='bias', initializer=initializers.get('zeros')) self.recurrent_activation = activations.get('hard_sigmoid') self.activation = activations.get('tanh')
def __init__(self, input_dim, output_dim, embeddings_initializer='uniform', embeddings_regularizer=None, activity_regularizer=None, embeddings_constraint=None, mask_zero=False, input_length=None, **kwargs): if 'input_shape' not in kwargs: if input_length: kwargs['input_shape'] = (input_length, ) else: kwargs['input_shape'] = (None, ) dtype = kwargs.pop('dtype', K.floatx()) # We set autocast to False, as we do not want to cast floating- point inputs # to self.dtype. In call(), we cast to int32, and casting to self.dtype # before casting to int32 might cause the int32 values to be different due # to a loss of precision. kwargs['autocast'] = False super(Embedding, self).__init__(dtype=dtype, **kwargs) self.input_dim = input_dim self.output_dim = output_dim self.embeddings_initializer = initializers.get(embeddings_initializer) self.embeddings_regularizer = regularizers.get(embeddings_regularizer) self.activity_regularizer = regularizers.get(activity_regularizer) self.embeddings_constraint = constraints.get(embeddings_constraint) self.mask_zero = mask_zero self.supports_masking = mask_zero self.input_length = input_length self._supports_ragged_inputs = True
def add_slot(self, var, slot_name, initializer="zeros"): """Add a new slot variable for `var`.""" if slot_name not in self._slot_names: self._slot_names.append(slot_name) var_key = _var_key(var) slot_dict = self._slots.setdefault(var_key, {}) weight = slot_dict.get(slot_name, None) if weight is None: if isinstance(initializer, six.string_types) or callable(initializer): initializer = initializers.get(initializer) initial_value = functools.partial(initializer, shape=var.shape, dtype=var.dtype) else: initial_value = initializer strategy = distribute_ctx.get_strategy() with strategy.extended.colocate_vars_with(var): weight = tf_variables.Variable( name="%s/%s" % (var._shared_name, slot_name), # pylint: disable=protected-access dtype=var.dtype, trainable=False, initial_value=initial_value) backend.track_variable(weight) slot_dict[slot_name] = weight self._restore_slot_variable(slot_name=slot_name, variable=var, slot_variable=weight) self._weights.append(weight) return weight
def __init__(self, step_dim, ll, get_alpha=False, get_sequence=False, W_regularizer=None, b_regularizer=None, L_regularizer=None, W_constraint=None, b_constraint=None, L_constraint=None, bias=False, **kwargs): self.supports_masking = True self.init = initializers.get('glorot_uniform') self.l_init = initializers.constant(value=0.5) self.ll = ll self.W_regularizer = regularizers.get(W_regularizer) self.b_regularizer = regularizers.get(b_regularizer) self.L_regularizer = regularizers.get(L_regularizer) self.W_constraint = constraints.get(W_constraint) self.b_constraint = constraints.get(b_constraint) self.L_constraint = constraints.get(L_constraint) self.get_sequence = get_sequence self.bias = bias self.step_dim = step_dim self.features_dim = 0 self.get_alpha = get_alpha super(Test_IAN, self).__init__(**kwargs)
def __init__(self, input_dim, output_dim, embeddings_initializer='uniform', embeddings_regularizer=None, activity_regularizer=None, embeddings_constraint=None, mask_zero=False, input_length=None, **kwargs): if 'input_shape' not in kwargs: if input_length: kwargs['input_shape'] = (input_length,) else: kwargs['input_shape'] = (None,) dtype = kwargs.pop('dtype', K.floatx()) super(Embedding, self).__init__(dtype=dtype, **kwargs) self.input_dim = input_dim self.output_dim = output_dim self.embeddings_initializer = initializers.get(embeddings_initializer) self.embeddings_regularizer = regularizers.get(embeddings_regularizer) self.activity_regularizer = regularizers.get(activity_regularizer) self.embeddings_constraint = constraints.get(embeddings_constraint) self.mask_zero = mask_zero self.supports_masking = mask_zero self.input_length = input_length
def _add_weight(self, name, shape=(), dtype=None, initializer='zeros'): """Adds a weight to this loss scale. Args: name: Variable name. shape: Variable shape. dtype: The type of the variable. initializer: The initializer to use. Returns: A variable. """ if isinstance(initializer, six.string_types) or callable(initializer): initializer = initializers.get(initializer) variable = self._add_variable_with_custom_getter( name=name, shape=shape, getter=base_layer_utils.make_variable, overwrite=True, initializer=initializer, dtype=dtype, trainable=False, use_resource=True, synchronization=variables.VariableSynchronization.AUTO, # Set aggregation to NONE, as loss scaling variables should never be # aggregated. aggregation=variables.VariableAggregation.NONE) backend.track_variable(variable) return variable
def __init__(self, filters, kernel_size, strides=(1, 1), padding='valid', output_padding=None, data_format=None, dilation_rate=(1, 1), activation=None, use_bias=True, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, tied_to=None, **kwargs): super(Conv2DTranspose, self).__init__( filters=filters, kernel_size=kernel_size, strides=strides, padding=padding, data_format=data_format, dilation_rate=dilation_rate, activation=activations.get(activation), use_bias=use_bias, kernel_initializer=initializers.get(kernel_initializer), bias_initializer=initializers.get(bias_initializer), kernel_regularizer=regularizers.get(kernel_regularizer), bias_regularizer=regularizers.get(bias_regularizer), activity_regularizer=regularizers.get(activity_regularizer), kernel_constraint=constraints.get(kernel_constraint), bias_constraint=constraints.get(bias_constraint), **kwargs) self.tied_to = tied_to self.output_padding = output_padding if self.output_padding is not None: self.output_padding = conv_utils.normalize_tuple( self.output_padding, 2, 'output_padding') for stride, out_pad in zip(self.strides, self.output_padding): if out_pad >= stride: raise ValueError('Stride ' + str(self.strides) + ' must be ' 'greater than output padding ' + str(self.output_padding))
def __init__(self, filters, kernel_size, strides=1, padding="valid", output_padding=None, data_format=None, dilation_rate=1, activation=None, use_bias=True, kernel_initializer="glorot_uniform", bias_initializer="zeros", kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, **kwargs): if output_padding is not None: output_padding = (1, output_padding) else: output_padding = extract_singleton(output_padding) kernel_size = extract_singleton(kernel_size) strides = extract_singleton(strides) dilation_rate = extract_singleton(dilation_rate) super(Conv1DTranspose, self).__init__( filters=filters, kernel_size=(1, kernel_size), strides=(1, strides), padding=padding, output_padding=output_padding, data_format=data_format, dilation_rate=(1, dilation_rate), activation=activations.get(activation), use_bias=use_bias, kernel_initializer=initializers.get(kernel_initializer), bias_initializer=initializers.get(bias_initializer), kernel_regularizer=regularizers.get(kernel_regularizer), bias_regularizer=regularizers.get(bias_regularizer), activity_regularizer=regularizers.get(activity_regularizer), kernel_constraint=constraints.get(kernel_constraint), bias_constraint=constraints.get(bias_constraint), **kwargs) self._input_dim = None
def __init__(self, rank, filters, kernel_size, strides=1, padding='valid', data_format=None, dilation_rate=1, activation=None, use_bias=True, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, use_spectral_norm=False, is_training=False, trainable=True, name=None, **kwargs): super(MyKerasConv, self).__init__( trainable=trainable, name=name, activity_regularizer=regularizers.get(activity_regularizer), **kwargs) self.rank = rank self.filters = filters self.kernel_size = conv_utils.normalize_tuple(kernel_size, rank, 'kernel_size') self.strides = conv_utils.normalize_tuple(strides, rank, 'strides') self.padding = conv_utils.normalize_padding(padding) self.data_format = conv_utils.normalize_data_format(data_format) self.dilation_rate = conv_utils.normalize_tuple( dilation_rate, rank, 'dilation_rate') self.activation = activations.get(activation) self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) self.bias_initializer = initializers.get(bias_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.use_spectral_norm = use_spectral_norm self.is_training = is_training self.bias_regularizer = regularizers.get(bias_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.bias_constraint = constraints.get(bias_constraint) self.input_spec = InputSpec(ndim=self.rank + 2)
def __init__(self, rank, filters, kernel_size, strides=1, padding='valid', data_format=None, dilation_rate=1, activation=None, use_bias=True, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, trainable=True, name=None, **kwargs): super(ConvAux, self).__init__( trainable=trainable, name=name, activity_regularizer=regularizers.get(activity_regularizer), **kwargs) self.rank = rank if filters is not None and not isinstance(filters, int): filters = int(filters) self.filters = filters self.kernel_size = conv_utils.normalize_tuple(kernel_size, rank, 'kernel_size') self.strides = conv_utils.normalize_tuple(strides, rank, 'strides') self.padding = conv_utils.normalize_padding(padding) if (self.padding == 'causal'): raise ValueError('Causal padding is only supported.') self.data_format = conv_utils.normalize_data_format(data_format) self.dilation_rate = conv_utils.normalize_tuple( dilation_rate, rank, 'dilation_rate') self.activation = activations.get(activation) self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) self.bias_initializer = initializers.get(bias_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.bias_constraint = constraints.get(bias_constraint) self.input_spec = InputSpec(ndim=self.rank + 2)
def __init__( self, filters, kernel_size, strides=1, padding='valid', output_padding=None, data_format=None, activation=None, use_bias=True, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, **kwargs, ): super().__init__( filters=filters, kernel_size=kernel_size, strides=strides, padding=padding, data_format=data_format, activation=activations.get(activation), use_bias=use_bias, kernel_initializer=initializers.get(kernel_initializer), bias_initializer=initializers.get(bias_initializer), kernel_regularizer=regularizers.get(kernel_regularizer), bias_regularizer=regularizers.get(bias_regularizer), activity_regularizer=regularizers.get(activity_regularizer), kernel_constraint=constraints.get(kernel_constraint), bias_constraint=constraints.get(bias_constraint), **kwargs, ) self.output_padding = output_padding if self.output_padding is not None: self.output_padding = conv_utils.normalize_tuple( self.output_padding, 1, 'output_padding', ) for stride, out_pad in zip(self.strides, self.output_padding): if out_pad >= stride: raise ValueError( f'Stride {self.strides} must be greater than ' f'output padding {self.output_padding}', )
def __init__( self, units, activation = complex_activations.complex_amp_phase_exp, use_bias = True, kernel_initializer = complex_initializers.ComplexGlorotNormal, recurrent_initializer = complex_initializers.ComplexGlorotNormal, bias_initializer = 'zeros', kernel_regularizer = None, recurrent_regularizer = None, bias_regularizer = None, activity_regularizer = None, kernel_constraint = None, recurrent_constraint = None, bias_constraint = None, dropout = 0., recurrent_dropout = 0., return_sequences = False, return_state = False, go_backwards = False, stateful = False, unroll = False, **kwargs ): super(SimpleComplexRNNCell, self).__init__(**kwargs) self.units = units self.activation = activations.get(activation) self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) self.recurrent_initializer = initializers.get(recurrent_initializer) self.bias_initializer = initializers.get(bias_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.recurrent_regularizer = regularizers.get(recurrent_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.recurrent_constraint = constraints.get(recurrent_constraint) self.bias_constraint = constraints.get(bias_constraint) self.dropout = min(1., max(0., dropout)) self.recurrent_dropout = min(1., max(0., recurrent_dropout)) self.state_size = self.units self.output_size = self.units
def build(self, input_shape): super(PruneLowMagnitude, self).build(input_shape) weight_vars, mask_vars, threshold_vars = [], [], [] self.prunable_weights = self.layer.get_prunable_weights() # For each of the prunable weights, add mask and threshold variables for weight in self.prunable_weights: mask = self.add_variable( 'mask', shape=weight.shape, initializer=initializers.get('ones'), dtype=weight.dtype, trainable=False, aggregation=tf_variables.VariableAggregation.MEAN) threshold = self.add_variable( 'threshold', shape=[], initializer=initializers.get('zeros'), dtype=weight.dtype, trainable=False, aggregation=tf_variables.VariableAggregation.MEAN) weight_vars.append(weight) mask_vars.append(mask) threshold_vars.append(threshold) self.pruning_vars = list(zip(weight_vars, mask_vars, threshold_vars)) # Add a scalar tracking the number of updates to the wrapped layer. self.pruning_step = self.add_variable( 'pruning_step', shape=[], initializer=initializers.Constant(-1), dtype=dtypes.int64, trainable=False) def training_step_fn(): return self.pruning_step # Create a pruning object self.pruning_obj = pruning_impl.Pruning( training_step_fn=training_step_fn, pruning_vars=self.pruning_vars, pruning_schedule=self.pruning_schedule, block_size=self.block_size, block_pooling_type=self.block_pooling_type)
def _get_initializer(name): if name in ['vs_fan_avg_uniform', 'trfk_init']: return initializers.VarianceScaling(mode='fan_avg', distribution='uniform') elif name == 'vs_fan_avg_normal': return initializers.VarianceScaling(mode='fan_avg', distribution='normal') return initializers.get(name)
def __init__(self, nb_filter, n_atom_features, batch_size, init='glorot_uniform', activation='linear', dropout=None, max_deg=10, min_deg=0, **kwargs): """ Parameters ---------- nb_filter: int Number of convolutional filters. n_atom_features: int Number of features listed per atom. init: str, optional Weight initialization for filters. activation: str, optional Activation function applied after convolution. dropout: float, optional Dropout probability. max_deg: int, optional Maximum degree of atoms in molecules. min_deg: int, optional Minimum degree of atoms in molecules. """ super(GraphConv_and_gather, self).__init__(**kwargs) self.n_atom_features = n_atom_features self.init = initializers.get(init) # Set weight initialization self.activation = activations.get(activation) # Get activations self.nb_filter = nb_filter # Save number of filters self.dropout = dropout # Save dropout params self.max_deg = max_deg self.min_deg = min_deg self.batch_size = batch_size # Is there a solid explanation here? self.nb_affine = 3 * max_deg + (2 - min_deg) self.n_atom_features = n_atom_features n_atom_features = self.n_atom_features self.beta_init = initializers.get('zero') self.gamma_init = initializers.get('one') self.epsilon = 1e-5 self.momentum = 0.99
def __init__(self, units, k, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, **kwargs): super(Maxout, self).__init__(**kwargs) self.units = int(units) if not isinstance(units, int) else units self.k = int(k) if not isinstance(k, int) else k self.kernel_initializer = initializers.get(kernel_initializer) self.bias_initializer = initializers.get(bias_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer)
def test_initializer_v2_get(self): tf2_force_enabled = tf2._force_enable # pylint: disable=protected-access try: tf2.enable() rn = initializers.get('random_normal') self.assertIn('init_ops_v2', rn.__class__.__module__) finally: tf2._force_enable = tf2_force_enabled # pylint: disable=protected-access
def __init__(self, norm_axis=None, params_axis=-1, epsilon=1e-12, center=True, scale=True, beta_initializer='zeros', gamma_initializer='ones', beta_regularizer=None, gamma_regularizer=None, beta_constraint=None, gamma_constraint=None, trainable=True, name=None, **kwargs): super(LayerNormalization, self).__init__( name=name, trainable=trainable, **kwargs) if isinstance(norm_axis, list): self.norm_axis = norm_axis[:] elif isinstance(norm_axis, int): self.norm_axis = norm_axis elif norm_axis is None: self.norm_axis = None else: raise TypeError('norm_axis must be int or list or None, type given: %s' % type(norm_axis)) if isinstance(params_axis, list): self.params_axis = params_axis[:] elif isinstance(params_axis, int): self.params_axis = params_axis else: raise TypeError('params_axis must be int or list, type given: %s' % type(params_axis)) self.epsilon = epsilon self.center = center self.scale = scale self.beta_initializer = initializers.get(beta_initializer) self.gamma_initializer = initializers.get(gamma_initializer) self.beta_regularizer = regularizers.get(beta_regularizer) self.gamma_regularizer = regularizers.get(gamma_regularizer) self.beta_constraint = constraints.get(beta_constraint) self.gamma_constraint = constraints.get(gamma_constraint) self.supports_masking = True
def __init__(self, units, activation=None, use_bias=True, scale=True, scf_min=0.2, scf_max=2.0, dropconnect_prob=0.05, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, **kwargs): if 'input_shape' not in kwargs and 'input_dim' in kwargs: kwargs['input_shape'] = (kwargs.pop('input_dim'), ) super(ScaledLinear, self).__init__( activity_regularizer=regularizers.get(activity_regularizer), **kwargs) # Save params self.units = units self.activation = activations.get(activation) self.use_bias = use_bias self.dropconnect_prob = dropconnect_prob self.scale = scale self.scf_min = scf_min self.scf_max = scf_max self.kernel_initializer = initializers.get(kernel_initializer) self.bias_initializer = initializers.get(bias_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.bias_constraint = constraints.get(bias_constraint) self.kwargs = kwargs # Initialize scaling factor self.scaler = ScalingFactor(scf_min=self.scf_min, scf_max=self.scf_max, name="scaling_factor") self.supports_masking = True self.input_spec = InputSpec(min_ndim=2)
def __init__(self, norm_axis=None, params_axis=-1, epsilon=1e-12, center=True, scale=True, beta_initializer='zeros', gamma_initializer='ones', beta_regularizer=None, gamma_regularizer=None, beta_constraint=None, gamma_constraint=None, trainable=True, name=None, **kwargs): super(LayerNormalization, self).__init__( name=name, trainable=trainable, **kwargs) if isinstance(norm_axis, list): self.norm_axis = norm_axis[:] elif isinstance(norm_axis, int): self.norm_axis = norm_axis elif norm_axis is None: self.norm_axis = None else: raise TypeError('norm_axis must be int or list or None, type given: %s' % type(norm_axis)) if isinstance(params_axis, list): self.params_axis = params_axis[:] elif isinstance(params_axis, int): self.params_axis = params_axis else: raise TypeError('params_axis must be int or list, type given: %s' % type(params_axis)) self.epsilon = epsilon self.center = center self.scale = scale self.beta_initializer = initializers.get(beta_initializer) self.gamma_initializer = initializers.get(gamma_initializer) self.beta_regularizer = regularizers.get(beta_regularizer) self.gamma_regularizer = regularizers.get(gamma_regularizer) self.beta_constraint = constraints.get(beta_constraint) self.gamma_constraint = constraints.get(gamma_constraint) self.supports_masking = True
def __init__(self, units, tau=1., activation='tanh', use_bias=True, kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, dropout=0., recurrent_dropout=0., **kwargs): super(SimpleCTRNNCell, self).__init__(**kwargs) self.units = units self.tau = tau self.taus = array_ops.constant(tau, dtype=self.dtype, shape=[self.units], name="taus") self.activation = activations.get(activation) self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) self.recurrent_initializer = initializers.get(recurrent_initializer) self.bias_initializer = initializers.get(bias_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.recurrent_regularizer = regularizers.get(recurrent_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.recurrent_constraint = constraints.get(recurrent_constraint) self.bias_constraint = constraints.get(bias_constraint) self.dropout = min(1., max(0., dropout)) self.recurrent_dropout = min(1., max(0., recurrent_dropout)) self.state_size = (self.units, self.units, self.units) self.output_size = self.units self._dropout_mask = None self._recurrent_dropout_mask = None
def __init__(self, units, control_units, activation='tanh', recurrent_activation='hard_sigmoid', use_bias=True, kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', unit_forget_bias=True, kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, dropout=0., recurrent_dropout=0., implementation=1, **kwargs): super(SCLSTMCell, self).__init__(**kwargs) self.units = units self.control_units = control_units self.activation = activations.get(activation) self.recurrent_activation = activations.get(recurrent_activation) self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) self.recurrent_initializer = initializers.get(recurrent_initializer) self.bias_initializer = initializers.get(bias_initializer) self.unit_forget_bias = unit_forget_bias self.kernel_regularizer = regularizers.get(kernel_regularizer) self.recurrent_regularizer = regularizers.get(recurrent_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.recurrent_constraint = constraints.get(recurrent_constraint) self.bias_constraint = constraints.get(bias_constraint) self.dropout = min(1., max(0., dropout)) self.recurrent_dropout = min(1., max(0., recurrent_dropout)) self.implementation = implementation self.state_size = (self.units, self.units, self.control_units) self._dropout_mask = None self._recurrent_dropout_mask = None
def add_weight(self, name=None, shape=None, trainable=None, partitioner=None, initializer=None, regularizer=None, constraint=None, dtype=None, use_wnorm=False, use_resource=None, **kwargs): if initializer is not None: self.initializers[name] = initializers.get(initializer) if regularizer is not None: self.regularizers[name] = regularizers.get(regularizer) if constraint is not None: self.constraints[name] = constraints.get(constraint) _initializer = self.get_initializer(name) if use_wnorm or (self.wnorm and (name in ['kernel', 'embedding'] or name.endswith('kernel'))): if name in self.normalizers and self.normalizers[name] is not None: self.normalizers[name] = weight_normalizers.WeightNormalizer( _initializer, next_layer=self.normalizers[name]) else: self.normalizers[name] = weight_normalizers.WeightNormalizer( _initializer) if self.use_wscale: _initializer = _WscaleInitializer(_initializer, lrmul=self.lrmul) self.initializers[name] = _initializer if name in self.normalizers and self.normalizers[name] is not None: self.normalizers[name] = weight_normalizers.WscaleNormalizer( next_layer=self.normalizers[name], lrmul=self.lrmul, gain=self.gain) else: self.normalizers[name] = weight_normalizers.WscaleNormalizer( lrmul=self.lrmul, gain=self.gain) if dtype is None: dtype = self.dtype or K.floatx() weight = super(DynastesBaseLayer, self).add_weight(name=name, shape=shape, initializer=_initializer, regularizer=self.get_regularizer(name), trainable=trainable, constraint=self.get_constraint(name), partitioner=partitioner, use_resource=use_resource, dtype=dtype, **kwargs) if name in self.normalizers: if self.normalizers[name] is not None: self.normalizers[name].build(shape) self.weights_dict[name] = weight return weight
def __init__(self, num_units, gate_mod=None, ngram=False, no_feedback=False, use_peepholes=False, cell_clip=None, initializer=None, num_proj=None, proj_clip=None, num_unit_shards=None, num_proj_shards=None, forget_bias=1.0, state_is_tuple=True, layer_norm=False, activation=None, reuse=None, name=None, dtype=None, **kwargs): super(LSTMCell_mod, self).__init__(_reuse=reuse, name=name, dtype=dtype, **kwargs) print("LSTM cell mode: {0}".format(gate_mod)) # Inputs must be 2-dimensional. self.input_spec = base_layer.InputSpec(ndim=2) self._num_units = num_units self._gate_mod = gate_mod self._ngram = ngram self._no_feedback = no_feedback self._use_peepholes = use_peepholes self._cell_clip = cell_clip self._initializer = initializers.get(initializer) self._num_proj = num_proj self._proj_clip = proj_clip self._num_unit_shards = num_unit_shards self._num_proj_shards = num_proj_shards self._forget_bias = forget_bias self._state_is_tuple = state_is_tuple self._layer_norm = layer_norm if activation: self._activation = activations.get(activation) else: self._activation = math_ops.tanh if num_proj: self._state_size = (LSTMStateTuple(num_units, num_proj) if state_is_tuple else num_units + num_proj) self._output_size = num_proj else: self._state_size = (LSTMStateTuple(num_units, num_units) if state_is_tuple else 2 * num_units) self._output_size = num_units
def __init__(self, axis=-1, momentum=0.9, epsilon=1e-5, m_per_group=0, affine=True, beta_initializer='zeros', gamma_initializer='ones', moving_mean_initializer='zeros', moving_projection_initializer='identity', beta_regularizer=None, gamma_regularizer=None, beta_constraint=None, gamma_constraint=None, trainable=True, name=None, **kwargs): super(DecorrelatedBN, self).__init__(name=name, trainable=trainable, **kwargs) if isinstance(axis, int): self.axis = axis else: raise TypeError('axis must be int, type given: %s' % type(self.axis)) self.momentum = momentum self.epsilon = epsilon self.m_per_group = m_per_group self.affine = affine self.beta_initializer = initializers.get(beta_initializer) self.gamma_initializer = initializers.get(gamma_initializer) self.moving_mean_initializer = initializers.get( moving_mean_initializer) self.moving_projection_initializer = initializers.get( moving_projection_initializer) self.beta_regularizer = regularizers.get(beta_regularizer) self.gamma_regularizer = regularizers.get(gamma_regularizer) self.beta_constraint = constraints.get(beta_constraint) self.gamma_constraint = constraints.get(gamma_constraint) self.moving_means = [] self.moving_projections = [] self._trainable_var = None
def __init__(self, norm_method='std', filter_size=61, data_format=None, activation=None, use_bias=False, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, **kwargs): self.valid_modes = {'std', 'max', None, 'whole_image'} if norm_method not in self.valid_modes: raise ValueError('Invalid `norm_method`: "{}". ' 'Use one of {}.'.format(norm_method, self.valid_modes)) if 'trainable' not in kwargs: kwargs['trainable'] = False super(ImageNormalization2D, self).__init__( activity_regularizer=regularizers.get(activity_regularizer), **kwargs) self.activation = activations.get(activation) self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) self.bias_initializer = initializers.get(bias_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.bias_constraint = constraints.get(bias_constraint) self.input_spec = InputSpec(ndim=4) # hardcoded for 2D data self.filter_size = filter_size self.norm_method = norm_method self.data_format = conv_utils.normalize_data_format(data_format) if self.data_format == 'channels_first': self.channel_axis = 1 else: self.channel_axis = 3 # hardcoded for 2D data if isinstance(self.norm_method, str): self.norm_method = self.norm_method.lower()
def __init__( self, input_rows, input_dim, output_dim, support, # input_rows x input_rows activation=None, use_bias=False, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, kernel_constraint=None, bias_constraint=None, dropout=0., sparse_inputs=False, featureless=False, model='gcn', perturbation=None, **kwargs): super(GraphConvolution, self).__init__(**kwargs) self.input_rows = input_rows self.input_dim = input_dim self.output_dim = output_dim self.support = support self.activation = activation self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) self.bias_initializer = initializers.get(bias_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.bias_constraint = constraints.get(bias_constraint) self.dropout = dropout self.sparse_inputs = sparse_inputs self.featureless = featureless self.model = model self.perturbation = perturbation self.supports_masking = True self.input_spec = InputSpec(min_ndim=2)
def __init__(self, units, kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', unit_forget_bias=True, kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, return_sequences=False, return_state=False, go_backwards=False, stateful=False, **kwargs): self.units = units cell_spec = collections.namedtuple('cell', 'state_size') self._cell = cell_spec(state_size=(self.units, self.units)) super(CuDNNLSTM, self).__init__( return_sequences=return_sequences, return_state=return_state, go_backwards=go_backwards, stateful=stateful, **kwargs) self.kernel_initializer = initializers.get(kernel_initializer) self.recurrent_initializer = initializers.get(recurrent_initializer) self.bias_initializer = initializers.get(bias_initializer) self.unit_forget_bias = unit_forget_bias self.kernel_regularizer = regularizers.get(kernel_regularizer) self.recurrent_regularizer = regularizers.get(recurrent_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.activity_regularizer = regularizers.get(activity_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.recurrent_constraint = constraints.get(recurrent_constraint) self.bias_constraint = constraints.get(bias_constraint)
def __init__(self, alpha_initializer='zeros', alpha_regularizer=None, alpha_constraint=None, shared_axes=None, **kwargs): super(PReLU, self).__init__(**kwargs) self.supports_masking = True self.alpha_initializer = initializers.get(alpha_initializer) self.alpha_regularizer = regularizers.get(alpha_regularizer) self.alpha_constraint = constraints.get(alpha_constraint) if shared_axes is None: self.shared_axes = None elif not isinstance(shared_axes, (list, tuple)): self.shared_axes = [shared_axes] else: self.shared_axes = list(shared_axes)
def add_weight(self, name, shape, dtype=None, initializer="zeros", trainable=None, synchronization=tf_variables.VariableSynchronization.AUTO, aggregation=tf_variables.VariableAggregation.NONE): if dtype is None: dtype = dtypes.float32 if isinstance(initializer, six.string_types) or callable(initializer): initializer = initializers.get(initializer) if synchronization == tf_variables.VariableSynchronization.ON_READ: if trainable: raise ValueError( "Synchronization value can be set to " "VariableSynchronization.ON_READ only for non-trainable variables. " "You have specified trainable=True and " "synchronization=VariableSynchronization.ON_READ.") else: # Set trainable to be false when variable is to be synced on read. trainable = False elif trainable is None: trainable = True variable = self._add_variable_with_custom_getter( name=name, shape=shape, getter=base_layer_utils.make_variable, overwrite=True, initializer=initializer, dtype=dtype, trainable=trainable, use_resource=True, synchronization=synchronization, aggregation=aggregation) backend.track_variable(variable) return variable
def __init__(self, axis=-1, momentum=0.99, epsilon=1e-3, center=True, scale=True, beta_initializer='zeros', gamma_initializer='ones', moving_mean_initializer='zeros', moving_variance_initializer='ones', beta_regularizer=None, gamma_regularizer=None, beta_constraint=None, gamma_constraint=None, renorm=False, renorm_clipping=None, renorm_momentum=0.99, fused=None, trainable=True, virtual_batch_size=None, adjustment=None, name=None, **kwargs): super(BatchNormalizationBase, self).__init__( name=name, trainable=trainable, **kwargs) if isinstance(axis, list): self.axis = axis[:] elif isinstance(axis, int): self.axis = axis else: raise TypeError('axis must be int or list, type given: %s' % type(self.axis)) self.momentum = momentum self.epsilon = epsilon self.center = center self.scale = scale self.beta_initializer = initializers.get(beta_initializer) self.gamma_initializer = initializers.get(gamma_initializer) self.moving_mean_initializer = initializers.get(moving_mean_initializer) self.moving_variance_initializer = initializers.get( moving_variance_initializer) self.beta_regularizer = regularizers.get(beta_regularizer) self.gamma_regularizer = regularizers.get(gamma_regularizer) self.beta_constraint = constraints.get(beta_constraint) self.gamma_constraint = constraints.get(gamma_constraint) self.renorm = renorm self.virtual_batch_size = virtual_batch_size self.adjustment = adjustment if self._USE_V2_BEHAVIOR: if fused: self._raise_if_fused_cannot_be_used() # We leave fused as None if self._fused_can_be_used()==True, since we # still may set it to False in self.build() if the input rank is not 4. elif fused is None and not self._fused_can_be_used(): fused = False elif fused is None: fused = True self.supports_masking = True self.fused = fused self._bessels_correction_test_only = True if renorm: renorm_clipping = renorm_clipping or {} keys = ['rmax', 'rmin', 'dmax'] if set(renorm_clipping) - set(keys): raise ValueError('renorm_clipping %s contains keys not in %s' % (renorm_clipping, keys)) self.renorm_clipping = renorm_clipping self.renorm_momentum = renorm_momentum