def build_with_input(self, input: Union[Nested[tf.TensorSpec], Nested[tf.Tensor]], *args: Any, **kwargs: Any) -> None: bd = self.batch_dims self._input_spec = tf.nest.map_structure( lambda x: layers.InputSpec(shape=([None] * bd + x.shape[bd:]) [:x.shape.ndims], dtype=x.dtype), input) dummy_input = tf.nest.map_structure( lambda x: tf.zeros((list(self.min_batch_shape) + x.shape[bd:]) [:x.shape.ndims], x.dtype), input) if 'mask' in kwargs: kwargs['mask'] = tf.ones(self.min_batch_shape, tf.bool) kwargs['training'] = False dummy_output = super().__call__( dummy_input, *args, **kwargs) # type: ignore[misc] # mypy/issues/5887 # if isinstance(tf.nest.flatten(dummy_output)[0], tf.Tensor): if isinstance(dummy_output, tfd.Distribution): self._output_spec = layers.InputSpec( shape=[None] * len(dummy_output.batch_shape) + dummy_output.event_shape, dtype=dummy_output.dtype) else: self._output_spec = tf.nest.map_structure( lambda x: layers.InputSpec(shape=[None] * bd + x.shape[bd:], dtype=x.dtype), dummy_output) self.built_with_input = True
def __init__(self, filters, kernel_size, strides=1, padding='valid', dilation_rate=(1, 1), kernel_initializer='glorot_uniform', kernel_regularizer=None, activity_regularizer=None, kernel_constraint=None, demod=True, **kwargs): super(ModulatedConv2D, self).__init__(**kwargs) self.filters = filters self.rank = 2 self.kernel_size = kernel_size self.strides = strides self.padding = padding self.dilation_rate = dilation_rate self.kernel_initializer = initializers.get(kernel_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.demod = demod self.input_spec = [layers.InputSpec(ndim=4), layers.InputSpec(ndim=2)] self.epsilon = 1e-8
def build(self, *args, **kwargs): assert self.built == False super().build(*args, **kwargs) self.input_spec = [ self.input_spec, layers.InputSpec(self.kernel.dtype, self.kernel.get_shape()), layers.InputSpec(self.bias.dtype, self.bias.get_shape()) ]
def build_with_input(self, input: Nested[tf.Tensor], *args: Any, **kwargs: Any) -> None: bd = self.batch_dims self._input_spec = tf.nest.map_structure( lambda x: tfkl.InputSpec(shape=[None] * bd + x.shape[bd:], dtype=x.dtype), input) dummy_input = tf.nest.map_structure( lambda t: tf.zeros([2] * bd + t.shape[bd:], t.dtype), input) dummy_output = super().__call__(dummy_input, *args, **kwargs) self._output_spec = tf.nest.map_structure( lambda x: tfkl.InputSpec(shape=[None] * bd + x.shape[bd:], dtype=x.dtype), dummy_output) self.built_with_input = True
def __init__(self, units, activation=None, use_bias=True, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, tied_to=None, **kwargs): self.tied_to = tied_to if 'input_shape' not in kwargs and 'input_dim' in kwargs: kwargs['input_shape'] = (kwargs.pop('input_dim'),) super().__init__(**kwargs) self.units = units self.activation = activations.get(activation) self.use_bias = use_bias self.kernel_initializer = initializers.get(kernel_initializer) self.bias_initializer = initializers.get(bias_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.activity_regularizer = regularizers.get(activity_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.bias_constraint = constraints.get(bias_constraint) self.input_spec = layers.InputSpec(min_ndim=2) self.supports_masking = True
def build(self, input_shape): if self.data_format == 'channels_first': channel_axis = 1 else: channel_axis = -1 if input_shape[channel_axis] is None: raise ValueError('The channel dimension of the inputs ' 'should be defined. Found `None`.') input_dim = input_shape[channel_axis] kernel_shape = self.kernel_size + (input_dim, self.filters) self.kernel = self.add_weight(shape=kernel_shape, initializer=self.kernel_initializer, name='kernel', regularizer=self.kernel_regularizer, constraint=self.kernel_constraint) if self.use_bias: self.bias = self.add_weight(shape=(self.filters,), initializer=self.bias_initializer, name='bias', regularizer=self.bias_regularizer, constraint=self.bias_constraint) else: self.bias = None self.u = tf.Variable( tf.random.normal((tuple([1, self.kernel.shape.as_list()[-1]])), dtype=tf.float32) , aggregation=tf.VariableAggregation.MEAN, trainable=False) # Set input spec. self.input_spec = layers.InputSpec(ndim=self.rank + 2, axes={channel_axis: input_dim}) self.built = True
def build(self, input_shape): ndim = len(input_shape) if self.axis == 0: raise ValueError('Axis cannot be zero') if (self.axis is not None) and (ndim == 2): raise ValueError('Cannot specify axis for rank 1 tensor') self.input_spec = layers.InputSpec(ndim=ndim) if self.axis is None: shape = (1, ) else: shape = (input_shape[self.axis], ) if self.scale: self.gamma = self.add_weight(shape=shape, name='gamma', initializer=self.gamma_initializer, regularizer=self.gamma_regularizer, constraint=self.gamma_constraint) else: self.gamma = None if self.center: self.beta = self.add_weight(shape=shape, name='beta', initializer=self.beta_initializer, regularizer=self.beta_regularizer, constraint=self.beta_constraint) else: self.beta = None self.built = True
def __init__(self, pad=1, **kwargs): if type(pad) == int: self.pad = (pad, pad) else: self.pad = tuple(pad) self.input_spec = [layers.InputSpec(ndim=4)] super(ReflectionPadding2D, self).__init__(**kwargs)
def build(self, input_shape): kernalShape = self.kernel_size + (input_shape[-1], self.filters) self.kernel = self.add_weight(shape=kernalShape, initializer=self.kernel_initializer, name='kernal', regularizer=self.kernel_regularizer, constraint=self.kernel_constraint) if self.use_bias: self.bias = self.add_weight(shape=(self.filters, ), initializer=self.bias_initializer, name='bias', regularizer=self.bias_regularizer, constraint=self.bias_constraint) else: self.bias = None self.u = self.add_weight( shape=tuple([1, self.kernel.shape.as_list()[-1]]), initializer=ks.initializers.RandomNormal(0, 1), name='sn_u', trainable=False) self.input_spec = lr.InputSpec(ndim=self.rank + 2, axes={-1: input_shape[-1]}) self.build = True return
def build(self, input_shape): if self.data_format == 'channels_first': channel_axis = 1 else: channel_axis = -1 if input_shape[channel_axis] is None: raise ValueError('The channel dimension of the inputs ' 'should be defined. Found `None`.') input_dim = input_shape[channel_axis] if input_dim % self.num_group != 0: raise ValueError("The channel dimension of input tensor must divided by num_group with no remainders!") kernel_shape = self.kernel_size + (input_dim // self.num_group, self.filters) self.kernel = self.add_weight(name='kernel', shape=kernel_shape, initializer=self.kernel_initializer, regularizer=self.kernel_regularizer, constraint=self.kernel_constraint) if self.use_bias: self.bias = self.add_weight(name='bias', shape=(self.filters,), initializer=self.bias_initializer, regularizer=self.bias_regularizer, constraint=self.bias_constraint) else: self.bias = None # Set input spec. self.input_spec = layers.InputSpec(ndim=self.rank + 2, axes={channel_axis: input_dim}) self.built = True self.channel_num = input_dim
def build(self, input_shape=None): self.input_spec = layers.InputSpec(shape=input_shape) if hasattr(self.layer, 'built') and not self.layer.built: self.layer.build(input_shape) # initialise p self.p_logit = self.add_variable( name='p_logit', shape=(1, ), initializer=tf.initializers.random_uniform(minval=self.init_min, maxval=self.init_max, dtype=tf.float32), dtype=tf.float32, trainable=True) self.p = tf.nn.sigmoid(self.p_logit[0], name='dropout_rate') tf.summary.scalar('dropoutRate', self.p) tf.add_to_collection("LAYER_P", self.p) # initialise regulariser / prior KL term input_dim = int(np.prod(input_shape[1:])) weight = self.layer.kernel with tf.name_scope('dropout_regularizer'): kernel_regularizer = self.weight_regularizer * tf.reduce_sum( tf.square(weight)) / (1. - self.p) dropout_regularizer = self.p * tf.log(self.p) dropout_regularizer += (1. - self.p) * tf.log(1. - self.p) dropout_regularizer *= self.dropout_regularizer * input_dim regularizer = tf.reduce_sum(kernel_regularizer + dropout_regularizer) # Add the regularisation loss to collection. tf.add_to_collection(tf.GraphKeys.REGULARIZATION_LOSSES, regularizer)
def build(self, input_shape): dim = input_shape[self.axis] if dim is None: raise ValueError('Axis ' + str(self.axis) + ' of ' 'input tensor should have a defined dimension ' 'but the layer received an input with shape ' + str(input_shape) + '.') self.input_spec = KL.InputSpec(ndim=len(input_shape), axes={self.axis: dim}) shape = (dim, ) if self.scale: self.gamma = self.add_weight(shape=shape, name='gamma', initializer=self.gamma_initializer, regularizer=self.gamma_regularizer, constraint=self.gamma_constraint) else: self.gamma = None if self.center: self.beta = self.add_weight(shape=shape, name='beta', initializer=self.beta_initializer, regularizer=self.beta_regularizer, constraint=self.beta_constraint) else: self.beta = None self.moving_mean = self.add_weight( shape=shape, name='moving_mean', initializer=self.moving_mean_initializer, trainable=False) self.moving_variance = self.add_weight( shape=shape, name='moving_variance', initializer=self.moving_variance_initializer, trainable=False) self.mean_weights = self.add_weight( shape=(3, ), name='mean_weights', initializer=self.mean_weights_initializer, regularizer=self.mean_weights_regularizer, constraint=self.mean_weights_constraints) self.variance_weights = self.add_weight( shape=(3, ), name='variance_weights', initializer=self.variance_weights_initializer, regularizer=self.variance_weights_regularizer, constraint=self.variance_weights_constraints) self.built = True
def __init__(self, units, activation='tanh', recurrent_activation='sigmoid', use_bias=True, kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', unit_forget_bias=True, kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, dropout=0., recurrent_dropout=0., implementation=1, return_sequences=False, return_state=False, go_backwards=False, stateful=False, unroll=False, **kwargs): cell = LSTMCell(units, activation=activation, recurrent_activation=recurrent_activation, use_bias=use_bias, kernel_initializer=kernel_initializer, recurrent_initializer=recurrent_initializer, unit_forget_bias=unit_forget_bias, bias_initializer=bias_initializer, kernel_regularizer=kernel_regularizer, recurrent_regularizer=recurrent_regularizer, bias_regularizer=bias_regularizer, kernel_constraint=kernel_constraint, recurrent_constraint=recurrent_constraint, bias_constraint=bias_constraint, dropout=dropout, recurrent_dropout=recurrent_dropout, implementation=implementation, dtype=kwargs.get('dtype'), trainable=kwargs.get('trainable', True)) keras_layers.RNN.__init__(self, cell, return_sequences=return_sequences, return_state=return_state, go_backwards=go_backwards, stateful=stateful, unroll=unroll, **kwargs) self.activity_regularizer = regularizers.get(activity_regularizer) self.input_spec = [keras_layers.InputSpec(ndim=3)]
def build_with_input(self, inputs, *args, **kwargs): bd = self._batch_dims # self._input_spec = [tf.nest.map_structure( # lambda x: tfkl.InputSpec(shape=[None] * bd + x.shape[bd:], dtype=x.dtype), inputs)] dummy_input = tf.nest.map_structure( lambda t: tf.zeros([2] * bd + t.shape[bd:], t.dtype), inputs) dummy_output = super().__call__(dummy_input, *args, **kwargs) self._output_spec = tf.nest.map_structure( lambda x: tfkl.InputSpec(shape=[None] * bd + x.shape[bd:], dtype=x.dtype), dummy_output) self._built_with_input = True
def build(self, input_shape): channels = input_shape[-1] if channels is None: raise ValueError( 'Channel dimension of inputs should be defined. Found `None`.') self.input_spec = layers.InputSpec(ndim=3, axes={-1: channels}) self.make_query = layers.Dense(self.units, use_bias=False) self.make_key = layers.Dense(self.units) self.make_score = layers.Dense(1, activation='sigmoid') super(AdditiveSelfAttention, self).build([input_shape, input_shape, input_shape])
def build(self, input_shape): channels = input_shape[-1] if channels is None: raise ValueError( 'Channel dimension of inputs should be defined. Found `None`.') self.input_spec = layers.InputSpec(ndim=3, axes={-1: channels}) self.make_query = layers.Dense(channels, use_bias=False) self.att_bias = self.add_weight('bias', shape=(1, ), initializer='zeros', dtype=self.dtype, trainable=True) super(MultiplicativeSelfAttention, self).build([input_shape, input_shape, input_shape])
def __init__(self, output_dim, sparse_target=True, **kwargs): """ Args: output_dim (int): the number of labels to tag each temporal input. sparse_target (bool): whether the the ground-truth label represented in one-hot. Input shape: (batch_size, sentence length, output_dim) Output shape: (batch_size, sentence length, output_dim) """ super(CRF, self).__init__(**kwargs) self.output_dim = int(output_dim) self.sparse_target = sparse_target self.input_spec = L.InputSpec(min_ndim=3) self.supports_masking = False self.sequence_lengths = None self.transitions = None
def build(self, input_shape): assert len(input_shape) == 3 f_shape = tf.TensorShape(input_shape) input_spec = L.InputSpec(min_ndim=3, axes={-1: f_shape[-1]}) if f_shape[-1] is None: raise ValueError('The last dimension of the inputs to `CRF` ' 'should be defined. Found `None`.') if f_shape[-1] != self.output_dim: raise ValueError('The last dimension of the input shape must be equal to output' ' shape. Use a linear layer if needed.') self.input_spec = input_spec self.transitions = self.add_weight(name='transitions', shape=[self.output_dim, self.output_dim], initializer='glorot_uniform', trainable=True) self.built = True
def build(self, input_shape=None): channels = input_shape[-1] if channels is None: raise ValueError('Channel dimension of the inputs should be defined. Found `None`.') self.input_spec = layers.InputSpec(ndim=max(2, len(input_shape)), axes={-1: channels}) kernel_initializer = initializers.random_normal(mean=0., stddev=np.sqrt(1. / channels)) self.carry = layers.Dense( channels, kernel_initializer=kernel_initializer, bias_initializer=initializers.constant(-2.), activation='sigmoid') self.transform = layers.Dense( channels, kernel_initializer=kernel_initializer, activation='relu') super().build(input_shape)
def build(self, input_shape: KTensorShape): if len(input_shape) < 2: raise ValueError input_dim = input_shape[-1] self.mean_kernel = self.add_weight(shape=(input_dim, self.units), initializer=self.kernel_initializer, name='mean_kernel') self.mean_bias = self.add_weight(shape=(self.units, ), initializer=self.bias_initializer, name='mean_bias') self.std_kernel = self.add_weight(shape=(input_dim, self.units), initializer=self.kernel_initializer, name='std_kernel') self.std_bias = self.add_weight(shape=(self.units, ), initializer=self.bias_initializer, name='std_bias') self.input_spec = layers.InputSpec(min_ndim=2, axes={-1: input_dim}) super().build(input_shape)
def __init__(self, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, kernel_constraint=None, bias_constraint=None, **kwargs): self.input_spec = layers.InputSpec(ndim=3) self.supports_masking = True self.kernel_initializer = initializers.get(kernel_initializer) self.bias_initializer = initializers.get(bias_initializer) self.kernel_regularizer = regularizers.get(kernel_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) self.kernel_constraint = constraints.get(kernel_constraint) self.bias_constraint = constraints.get(bias_constraint) super(SelfAttentionWithContext, self).__init__(**kwargs)
def __init__(self, vocabulary, output_dim, normalize_unicode='NFKC', lower_case=False, zero_digits=False, max_len=None, reserved_words=None, embed_type='dense_auto', adapt_cutoff=None, adapt_factor=4, embeddings_initializer='uniform', **kwargs): super().__init__(**kwargs) self.input_spec = layers.InputSpec(min_ndim=1, max_ndim=2, dtype='string') if not isinstance(vocabulary, list) or not all(map(lambda x: isinstance(x, str), vocabulary)): raise ValueError('Expected "vocabulary" to be a list of strings') if len(vocabulary) != len(set(vocabulary)): raise ValueError('Expected "vocabulary" to contain unique values') self.vocabulary = vocabulary self.output_dim = output_dim self.normalize_unicode = normalize_unicode self.lower_case = lower_case self.zero_digits = zero_digits if max_len is not None and max_len < 3: raise ValueError('Expected "max_len" to be None or greater then 2') self.max_len = max_len if reserved_words and len(reserved_words) != len(set(reserved_words)): raise ValueError('Expected "reserved_words" to contain unique values') self.reserved_words = reserved_words if embed_type not in {'dense_auto', 'dense_cpu', 'adapt'}: raise ValueError('Expected "embed_type" to be one of "dense_auto", "dense_cpu" or "adapt"') self.embed_type = embed_type self.adapt_cutoff = adapt_cutoff self.adapt_factor = adapt_factor self.embeddings_initializer = initializers.get(embeddings_initializer) all_reserved_words = [] if reserved_words is None else [r for r in reserved_words if self.UNK_MARK != r] self._reserved_words = [self.UNK_MARK] + all_reserved_words miss_reserved_words = [m for m in self._reserved_words if m not in vocabulary] if miss_reserved_words: tf.get_logger().warning('Vocabulary missed some reserved_words values: {}. ' 'This may indicate an error in vocabulary estimation'.format(miss_reserved_words)) clean_vocab = [w for w in vocabulary if w not in self._reserved_words] self._vocabulary = self._reserved_words + clean_vocab
def __init__(self, filters, kernel_size, strides=(1, 1), padding='valid', data_format='channels_last', dilation_rate=(1, 1), num_group=1, activation=None, use_bias=True, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, **kwargs): super(GroupConv2D, self).__init__( filters=filters, kernel_size=kernel_size, strides=strides, padding=padding, data_format=data_format, dilation_rate=dilation_rate, activation=activation, use_bias=use_bias, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, kernel_regularizer=kernel_regularizer, bias_regularizer=bias_regularizer, activity_regularizer=activity_regularizer, kernel_constraint=kernel_constraint, bias_constraint=bias_constraint, **kwargs) self.num_group = num_group if self.filters % self.num_group != 0: raise ValueError("filters must divided by num_group with no remainders!") self.input_spec = layers.InputSpec(ndim=4)
def build(self, input_shape): assert len(input_shape) >= 2 input_dim = input_shape[-1] if self.tied_to is not None: self.kernel = backend.transpose(self.tied_to.kernel) self._non_trainable_weights.append(self.kernel) else: self.kernel = self.add_weight(shape=(input_dim, self.units), initializer=self.kernel_initializer, name='kernel', regularizer=self.kernel_regularizer, constraint=self.kernel_constraint) if self.use_bias: self.bias = self.add_weight(shape=(self.units,), initializer=self.bias_initializer, name='bias', regularizer=self.bias_regularizer, constraint=self.bias_constraint) else: self.bias = None self.input_spec = layers.InputSpec(min_ndim=2, axes={-1: input_dim}) self.built = True
def build(self, input_shape): channels = input_shape[-1] if channels is None: raise ValueError( 'Channel dimension of inputs should be defined. Found `None`.') self.input_spec = layers.InputSpec(ndim=3, axes={-1: channels}) self.representation = layers.Dense( channels, kernel_initializer=self.kernel_initializer, bias_initializer=self.bias_initializer, kernel_regularizer=self.kernel_regularizer, bias_regularizer=self.bias_regularizer, kernel_constraint=self.kernel_constraint, bias_constraint=self.bias_constraint) self.importance = layers.Dense( channels, use_bias=False, kernel_initializer=self.kernel_initializer, kernel_regularizer=self.kernel_regularizer, kernel_constraint=self.kernel_constraint, ) super(SelfAttentionWithContext, self).build(input_shape)
def __init__(self, padding: Tuple[int, int] = (1, 1)) -> None: super().__init__() self.padding = tuple(padding) self.input_spec = [layers.InputSpec(ndim=4)]
def __init__(self, units, **kwargs): super(AdditiveSelfAttention, self).__init__(**kwargs) self.input_spec = layers.InputSpec(ndim=3) self.units = units
def __init__(self, **kwargs): super(MultiplicativeSelfAttention, self).__init__(**kwargs) self.input_spec = layers.InputSpec(ndim=3)
def build(self, input_shape): # input_shape = (batch_size, seq_len) self.seq_len = input_shape[-1] # shape = (T, D, seq_len) self.input_spec = layers.InputSpec(min_ndim=2, axes={-1: self.seq_len})
def __init__(self, paddings=(1, 1), **kwargs): self.paddings = tuple(paddings) self.input_spec = [layers.InputSpec(ndim=4)] super(ReflectionPad2d, self).__init__(**kwargs)