def __init__(self,
units,
activation='tanh',
recurrent_activation='hard_sigmoid',
kernel_initializer='glorot_uniform',
recurrent_initializer='orthogonal',
bias_initializer='zeros',
unit_forget_bias=True,
kernel_regularizer=None,
recurrent_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
recurrent_constraint=None,
bias_constraint=None,
return_sequences=False,
return_state=False,
go_backwards=False,
stateful=False,
time_major=False,
**kwargs):
super(RNN, self).__init__(**kwargs) # pylint: disable=bad-super-call
self.units = units
cell_spec = collections.namedtuple('cell', ['state_size', 'output_size'])
self.cell = cell_spec(
state_size=(self.units, self.units), output_size=self.units)
self.activation = activations.get(activation)
self.recurrent_activation = activations.get(recurrent_activation)
self.kernel_initializer = initializers.get(kernel_initializer)
self.recurrent_initializer = initializers.get(recurrent_initializer)
self.bias_initializer = initializers.get(bias_initializer)
self.unit_forget_bias = unit_forget_bias
self.kernel_regularizer = regularizers.get(kernel_regularizer)
self.recurrent_regularizer = regularizers.get(recurrent_regularizer)
self.bias_regularizer = regularizers.get(bias_regularizer)
self.activity_regularizer = regularizers.get(activity_regularizer)
self.kernel_constraint = constraints.get(kernel_constraint)
self.recurrent_constraint = constraints.get(recurrent_constraint)
self.bias_constraint = constraints.get(bias_constraint)
self.return_sequences = return_sequences
self.return_state = return_state
self.go_backwards = go_backwards
self.stateful = stateful
self.time_major = time_major
self._num_constants = None
self._num_inputs = None
self._states = None
self.input_spec = [InputSpec(ndim=3)]
self.state_spec = [
InputSpec(shape=(None, dim)) for dim in (self.units, self.units)
]
def __init__(self,
filters,
kernel_size,
strides=(1, 1),
padding='valid',
data_format=None,
dilation_rate=(1, 1),
activation='tanh',
recurrent_activation='hard_sigmoid',
use_bias=True,
kernel_initializer='glorot_uniform',
recurrent_initializer='orthogonal',
bias_initializer='zeros',
unit_forget_bias=True,
kernel_regularizer=None,
recurrent_regularizer=None,
bias_regularizer=None,
kernel_constraint=None,
recurrent_constraint=None,
bias_constraint=None,
dropout=0.,
recurrent_dropout=0.,
**kwargs):
super(ConvLSTM2DCell, self).__init__(**kwargs)
self.filters = filters
self.kernel_size = conv_utils.normalize_tuple(kernel_size, 2, 'kernel_size')
self.strides = conv_utils.normalize_tuple(strides, 2, 'strides')
self.padding = conv_utils.normalize_padding(padding)
self.data_format = conv_utils.normalize_data_format(data_format)
self.dilation_rate = conv_utils.normalize_tuple(dilation_rate, 2,
'dilation_rate')
self.activation = activations.get(activation)
self.recurrent_activation = activations.get(recurrent_activation)
self.use_bias = use_bias
self.kernel_initializer = initializers.get(kernel_initializer)
self.recurrent_initializer = initializers.get(recurrent_initializer)
self.bias_initializer = initializers.get(bias_initializer)
self.unit_forget_bias = unit_forget_bias
self.kernel_regularizer = regularizers.get(kernel_regularizer)
self.recurrent_regularizer = regularizers.get(recurrent_regularizer)
self.bias_regularizer = regularizers.get(bias_regularizer)
self.kernel_constraint = constraints.get(kernel_constraint)
self.recurrent_constraint = constraints.get(recurrent_constraint)
self.bias_constraint = constraints.get(bias_constraint)
self.dropout = min(1., max(0., dropout))
self.recurrent_dropout = min(1., max(0., recurrent_dropout))
self.state_size = (self.filters, self.filters)
self._dropout_mask = None
self._recurrent_dropout_mask = None
def __init__(self,
units,
activation=None,
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None,
**kwargs):
if 'input_shape' not in kwargs and 'input_dim' in kwargs:
kwargs['input_shape'] = (kwargs.pop('input_dim'),)
super(Dense, self).__init__(
activity_regularizer=regularizers.get(activity_regularizer), **kwargs)
self.units = int(units)
self.activation = activations.get(activation)
self.use_bias = use_bias
self.kernel_initializer = initializers.get(kernel_initializer)
self.bias_initializer = initializers.get(bias_initializer)
self.kernel_regularizer = regularizers.get(kernel_regularizer)
self.bias_regularizer = regularizers.get(bias_regularizer)
self.kernel_constraint = constraints.get(kernel_constraint)
self.bias_constraint = constraints.get(bias_constraint)
self.supports_masking = True
self.input_spec = InputSpec(min_ndim=2)
def __init__(self,
filters,
kernel_size,
strides=1,
padding='valid',
data_format=None,
activation=None,
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None,
**kwargs):
super(LocallyConnected1D, self).__init__(**kwargs)
self.filters = filters
self.kernel_size = conv_utils.normalize_tuple(kernel_size, 1, 'kernel_size')
self.strides = conv_utils.normalize_tuple(strides, 1, 'strides')
self.padding = conv_utils.normalize_padding(padding)
if self.padding != 'valid':
raise ValueError('Invalid border mode for LocallyConnected1D '
'(only "valid" is supported): ' + padding)
self.data_format = conv_utils.normalize_data_format(data_format)
self.activation = activations.get(activation)
self.use_bias = use_bias
self.kernel_initializer = initializers.get(kernel_initializer)
self.bias_initializer = initializers.get(bias_initializer)
self.kernel_regularizer = regularizers.get(kernel_regularizer)
self.bias_regularizer = regularizers.get(bias_regularizer)
self.activity_regularizer = regularizers.get(activity_regularizer)
self.kernel_constraint = constraints.get(kernel_constraint)
self.bias_constraint = constraints.get(bias_constraint)
self.input_spec = InputSpec(ndim=3)
def step(cell_inputs, cell_states):
h_tm1 = cell_states[0] # previous memory state
c_tm1 = cell_states[1] # previous carry state
# Only use the second half of the bias weights.
_, real_bias = array_ops.split(bias, 2)
z = K.dot(cell_inputs, kernel)
z += K.dot(h_tm1, recurrent_kernel)
z = K.bias_add(z, real_bias)
z0 = z[:, :units]
z1 = z[:, units:2 * units]
z2 = z[:, 2 * units:3 * units]
z3 = z[:, 3 * units:]
i = activations.get('hard_sigmoid')(z0)
f = activations.get('hard_sigmoid')(z1)
c = f * c_tm1 + i * activations.get('tanh')(z2)
o = activations.get('hard_sigmoid')(z3)
h = o * activations.get('tanh')(c)
return h, [h, c]
def __init__(self,
num_units,
activation=None,
reuse=None,
name=None,
dtype=None,
**kwargs):
"""Initializes the parameters for an RNN cell.
Args:
num_units: int, The number of units in the RNN cell.
activation: Nonlinearity to use. Default: `tanh`. It could also be string
that is within Keras activation function names.
reuse: (optional) Python boolean describing whether to reuse variables in
an existing scope. Raises an error if not `True` and the existing scope
already has the given variables.
name: String, the name of the layer. Layers with the same name will share
weights, but to avoid mistakes we require reuse=True in such cases.
dtype: Default dtype of the layer (default of `None` means use the type of
the first input). Required when `build` is called before `call`.
**kwargs: Dict, keyword named properties for common layer attributes, like
`trainable` etc when constructing the cell from configs of get_config().
Raises:
ValueError: If the existing scope already has the given variables.
"""
super(TfLiteRNNCell, self).__init__(
_reuse=reuse, name=name, dtype=dtype, **kwargs)
# Inputs must be Rank-2.
self.input_spec = base_layer.InputSpec(ndim=2)
self._tflite_wrapper = op_hint.OpHint("UnidirectionalSequenceRnn")
self._num_units = num_units
if activation:
self._activation = activations.get(activation)
else:
self._activation = math_ops.tanh
def __init__(self,
rnn_seq_length,
activation='relu',
kernel_initializer='random_uniform',
merge_mode='concat',
output_conv_filter=None,
**kwargs):
"""
Class for Spatial RNN layer capable of learning spatial connections between pixels of an 2D image in an RNN
fashion along all four directions of up, downs,left and right. Implemented in tensorflow 2.0 with Keras API.
The RNN unit is plain RNN with ReLu activation function (default) as suggested by Li et. al. (2019).
The activation function can be chosen from activation function available from tensorflow.python.keras library.
The spatial connections will be analysed in all principal directions sweeping to right, left, down, up.
The results from spatial RNN analysis in each of principal directions would have exact same shape of input
and can be concatenated or merged together depending on "merge_mode" input parameter.
The "merge_mode" input parameter can be set to either 'concat' (default) or 'convolution'.
By default, the results for all principal directions will be concatenated together resulting in a final output
shape with 4 times number of channels as the input channels. In case of 'convolution' merge mode, the results
for all principal directions will be concatenated and then the number of channels will be converted using a
1*1 2D convolution layer. The output number of channels will be determined based on the 'output_conv_filter'
input parameter which by default is et to input shape number of channels.
The input 2D image is recommended to be square as sufficient testing with non-square input images has not been
done. When using this layer as the first layer, preceded it with an Keras "Input" layer. Should be used with
`data_format="channels_last"`. The kernel initializer and activation functions can be set using the ones
available in tensorflow.python.keras.initializers & tensorflow.python.keras.activations.
Examples:
The inputs are 5x5 RGB images with `channels_last` and the batch of 1
input_shape = (2, 5, 5, 3) # (batch, height, width, channels)
x_in = tf.keras.layers.Input((5, 5, 3))
spatial_rnn_concat = SpatialRNN2D(rnn_seq_length=4, merge_mode='concat')
spatial_rnn_merge = SpatialRNN2D(rnn_seq_length=4m, merge_mode='convolution')
y_out = spatial_rnn_concat(x_in) # output shape of (2, 5, 5, 12)
y_out = spatial_rnn_merge(x_in) # output shape of (2, 5, 5, 3)
:param rnn_seq_length: Integer, the length of pixels sequence to be analysed by RNN unit
:param activation: (relu) Activation function used after following Spatial RNN and merge convolution layers.
:param kernel_initializer: (random_uniform) Initializer for the `kernel` weights matrix.
:param merge_mode: ('concat') To concatenate or merge (by 'convolution') the result for each direction pass.
:param output_conv_filter: number of output channels in case 'convolution' merge mode is selected.
"""
super().__init__(**kwargs)
self.padding = "same"
if merge_mode not in ['concat', 'convolution']:
raise ValueError(
'Unknown merge mode: the merge mode argument can be either \'concat\' or \'convolution\'.'
)
self.merge_mode = merge_mode
self.output_conv_filter = output_conv_filter
self.seq_length = rnn_seq_length
self.activation = activations.get(activation)
self.kernel_initializer = initializers.get(kernel_initializer)
self.kernel_dic = OrderedDict()
self.kernel_switch_dic = OrderedDict({
'right': np.array([[1, 0, 0]]),
'left': np.array([[0, 0, 1]]),
'down': np.array([[1], [0], [0]]),
'up': np.array([[0], [0], [1]])
})
def __init__(self,
units,
activation='tanh',
init_kern_1='RandomUniform',
init_kern_2='RandomUniform',
init_kern_3='RandomUniform',
init_recur_kern_2='orthogonal',
init_out_fb='orthogonal',
init_bias='zeros',
connectivity=None,
reg_kern_1=None,
reg_kern_3=None,
reg_out_fb=None,
reg_bias=None,
constraint_kern_1=None,
constraint_kern_3=None,
constraint_recur_kern_2=None,
constraint_out_fb=None,
constraint_bias=None,
in_dropout=None,
recur_dropout=None,
train_kern_1=False,
train_kern_3=False,
train_out_fb=False,
use_out_fb=False,
use_dropout_mask=False,
use_recur=False,
use_clock=False,
clock_rate=None,
data_format=None,
**kwargs):
super(SimpleRNNCell, self).__init__(**kwargs)
self.units = units
self.activation = activations.get(activation)
self.init_kern_1 = initializers.get(init_kern_1)
self.init_kern_2 = initializers.get(init_kern_2)
self.init_kern_3 = initializers.get(init_kern_3)
self.init_recur_kern_2 = initializers.get(init_recur_kern_2)
self.init_out_fb = initializers.get(init_out_fb)
self.init_bias = initializers.get(init_bias)
if len(connectivity) is not 3:
self.connectivity_1 = 1.
self.connectivity_2 = 1.
self.connectivity_3 = 1.
else:
self.connectivity_1 = min(1., max(0.,connectivity[0]))
self.connectivity_2 = min(1., max(0.,connectivity[1]))
self.connectivity_3 = min(1., max(0.,connectivity[2]))
self.reg_kern_1 = regularizers.get(reg_kern_1)
self.reg_kern_3 = regularizers.get(reg_kern_3)
self.reg_out_fb = regularizers.get(reg_out_fb)
self.reg_bias = regularizers.get(reg_bias)
self.constraint_kern_1 = constraints.get(constraint_kern_1)
self.constraint_kern_3 = constraints.get(constraint_kern_3)
self.constraint_recur_kern_2 = constraints.get(constraint_recur_kern_2)
self.constraint_out_fb = constraints.get(constraint_out_fb)
self.constraint_bias = constraints.get(constraint_bias)
self.in_dropout = min(1., max(0.,in_dropout))
self.recur_dropout = min(1., max(0.,recur_dropout))
self.train_kern_1 = train_kern_1
self.train_kern_3 = train_kern_3
self.clock = clock
self.clock_rate = clock_rate
self.in_dropout_mask = None
self.recur_dropout_mask = None
self.state_size = self.units
self.output_size = self.units
self.tf_data_format = self.data_format
self.clock_kernel = None
def __init__(self, activation, **kwargs): super(Activation, self).__init__(**kwargs) self.supports_masking = True self.activation = activations.get(activation)
def __init__(self, filters, kernel_size, strides=(1, 1), padding='valid', data_format=None, dilation_rate=(1, 1),
groups=1, activation=None, use_bias=True, dtype=DEFAULT_COMPLEX_TYPE,
kernel_initializer=ComplexGlorotUniform(), bias_initializer=Zeros(),
kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None,
kernel_constraint=None, bias_constraint=None, **kwargs):
"""
:param filters: Integer, the dimensionality of the output space (i.e. the number of output filters in the convolution).
:param kernel_size: An integer or tuple/list of 2 integers, specifying the height
and width of the 2D convolution window. Can be a single integer to specify
the same value for all spatial dimensions.
:param strides: An integer or tuple/list of 2 integers, specifying the strides of
the convolution along the height and width. Can be a single integer to
specify the same value for all spatial dimensions. Specifying any stride
value != 1 is incompatible with specifying any `dilation_rate` value != 1.
:param padding: one of `"valid"` or `"same"` (case-insensitive).
`"valid"` means no padding. `"same"` results in padding evenly to
the left/right or up/down of the input such that output has the same
height/width dimension as the input.
:param data_format: A string, one of `channels_last` (default) or `channels_first`.
The ordering of the dimensions in the inputs. `channels_last` corresponds
to inputs with shape `(batch_size, height, width, channels)` while
`channels_first` corresponds to inputs with shape `(batch_size, channels,
height, width)`. It defaults to the `image_data_format` value found in
your Keras config file at `~/.keras/keras.json`. If you never set it, then
it will be `channels_last`.
:param dilation_rate: an integer or tuple/list of 2 integers, specifying the
dilation rate to use for dilated convolution. Can be a single integer to
specify the same value for all spatial dimensions. Currently, specifying
any `dilation_rate` value != 1 is incompatible with specifying any stride
value != 1.
:param groups: A positive integer specifying the number of groups in which the
input is split along the channel axis. Each group is convolved separately
with `filters / groups` filters. The output is the concatenation of all
the `groups` results along the channel axis. Input channels and `filters`
must both be divisible by `groups`.
:param activation: Activation function to use. If you don't specify anything, no activation is applied.
For complex :code:`dtype`, this must be a :code:`cvnn.activations` module.
:param use_bias: Boolean, whether the layer uses a bias vector.
:param kernel_initializer: Initializer for the `kernel` weights matrix (see `keras.initializers`).
:param bias_initializer: Initializer for the bias vector (see `keras.initializers`).
:param kernel_regularizer: Regularizer function applied to the `kernel` weights matrix (see `keras.regularizers`).
:param bias_regularizer: Regularizer function applied to the bias vector (see `keras.regularizers`).
:param activity_regularizer: Regularizer function applied to the output of the layer (its "activation") (see `keras.regularizers`).
:param kernel_constraint: Constraint function applied to the kernel matrix (see `keras.constraints`).
:param bias_constraint: Constraint function applied to the bias vector (see `keras.constraints`).
"""
super(ComplexConv2D, self).__init__(
rank=2, dtype=dtype,
filters=filters,
kernel_size=kernel_size,
strides=strides,
padding=padding,
data_format=data_format,
dilation_rate=dilation_rate,
groups=groups,
activation=activations.get(activation),
use_bias=use_bias,
kernel_initializer=initializers.get(kernel_initializer),
bias_initializer=initializers.get(bias_initializer),
kernel_regularizer=regularizers.get(kernel_regularizer),
bias_regularizer=regularizers.get(bias_regularizer),
activity_regularizer=regularizers.get(activity_regularizer),
kernel_constraint=constraints.get(kernel_constraint),
bias_constraint=constraints.get(bias_constraint),
**kwargs)
def __init__(self,
units,
relations,
heads=1,
head_aggregation=HeadAggregation.MEAN,
attention_mode=AttentionModes.ARGAT,
attention_style=AttentionStyles.SUM,
attention_units=1,
attn_use_edge_features=False,
kernel_basis_size=None,
attn_kernel_basis_size=None,
activation=None,
attn_activation=tf.nn.leaky_relu,
use_bias=False,
batch_normalisation=False,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
attn_kernel_initializer='glorot_uniform',
kernel_regularizer=None,
bias_regularizer=None,
attn_kernel_regularizer=None,
activity_regularizer=None,
feature_dropout=None,
support_dropout=None,
edge_feature_dropout=None,
name='rgat',
**kwargs):
if 'input_shape' not in kwargs and 'input_dim' in kwargs:
kwargs['input_shape'] = (kwargs.pop('input_dim'), )
super(RelationalGraphAttention, self).__init__(
activity_regularizer=regularizers.get(activity_regularizer),
name=name,
**kwargs)
self.units = int(units)
self.relations = int(relations)
self.heads = int(heads)
self.head_aggregation = HeadAggregation.validate(head_aggregation)
self.attention_mode = AttentionModes.validate(attention_mode)
self.attention_style = AttentionStyles.validate(attention_style)
self.attention_units = attention_units
self.attn_use_edge_features = attn_use_edge_features
self.kernel_basis_size = (int(kernel_basis_size)
if kernel_basis_size else None)
self.attn_kernel_basis_size = (int(attn_kernel_basis_size)
if attn_kernel_basis_size else None)
self.activation = activations.get(activation)
self.attn_activation = activations.get(attn_activation)
self.use_bias = use_bias
self.batch_normalisation = batch_normalisation
if self.batch_normalisation:
self.batch_normalisation_layer = tf.layers.BatchNormalization()
self.kernel_initializer = initializers.get(kernel_initializer)
self.bias_initializer = initializers.get(bias_initializer)
self.attn_kernel_initializer = initializers.get(
attn_kernel_initializer)
self.kernel_regularizer = regularizers.get(kernel_regularizer)
self.bias_regularizer = regularizers.get(bias_regularizer)
self.attn_kernel_regularizer = regularizers.get(
attn_kernel_regularizer)
self.feature_dropout = feature_dropout
self.support_dropout = support_dropout
self.edge_feature_dropout = edge_feature_dropout
self.supports_masking = True
self.input_spec = InputSpec(min_ndim=2)
self.dense_layer = rgat_layers.BasisDecompositionDense(
units=self.relations * self.heads * self.units,
basis_size=self.kernel_basis_size,
coefficients_size=self.relations * self.heads,
use_bias=False,
kernel_initializer=self.kernel_initializer,
kernel_regularizer=self.kernel_regularizer,
name=name + '_basis_decomposition_dense',
**kwargs)
self.attention_logits = RelationalGraphAttentionLogits(
relations=self.relations,
heads=self.heads,
attention_style=self.attention_style,
attention_units=self.attention_units,
basis_size=self.attn_kernel_basis_size,
activation=self.attn_activation,
use_edge_features=self.attn_use_edge_features,
kernel_initializer=self.attn_kernel_initializer,
kernel_regularizer=self.attn_kernel_regularizer,
feature_dropout=self.feature_dropout,
edge_feature_dropout=self.edge_feature_dropout,
batch_normalisation=self.batch_normalisation,
name="logits",
**kwargs)
if self.head_aggregation == HeadAggregation.PROJECTION:
self.projection_layer = keras_layers.Dense(
units=self.units,
use_bias=False,
kernel_initializer=self.kernel_initializer,
kernel_regularizer=self.kernel_regularizer,
name="projection",
**kwargs)
if self.batch_normalisation:
self.batch_normalisation_layer = tf.layers.BatchNormalization()
def __call__(self,
input,
forward_kernel,
backward_kernel,
activation,
layer_type,
bias=None,
**kwargs):
# Add stop gradients
input = tf.stop_gradient(input)
if bias is not None:
bias = tf.stop_gradient(bias)
if not self.update_forward:
forward_kernel = tf.stop_gradient(forward_kernel)
# Use random noise
if self.input_distribution is not None:
if self.input_distribution == 'uniform':
val = np.sqrt(12 * self.input_stddev**2) / 2
input = tf.random.uniform(tf.shape(input),
minval=-val,
maxval=val)
elif self.input_distribution == 'normal':
input = tf.random.normal(tf.shape(input),
mean=0.0,
stddev=self.input_stddev)
else:
raise ValueError
# Standardize Input
if self.center_input:
input = self._center(input)
if self.normalize_input:
input = self._normalize(input)
forward_input = tf.identity(input, name='forward_input')
backward_input = tf.identity(input, name='backward_input')
if self.batch_center_backward_input:
backward_input = self._batch_center(backward_input)
# Compute Projections
if layer_type == 'fc':
forward_projection = tf.matmul(forward_input, forward_kernel)
backward_projection = tf.matmul(backward_input, backward_kernel,
transpose_b=True)
if bias is not None:
if self.use_bias_forward:
forward_projection = tf.nn.bias_add(forward_projection, bias)
if self.use_bias_backward:
backward_projection = tf.nn.bias_add(backward_projection, bias)
elif layer_type == 'conv':
forward_projection = tf.nn.conv2d(forward_input,
forward_kernel,
**kwargs)
backward_projection = tf.nn.conv2d(backward_input,
backward_kernel,
**kwargs)
if bias is not None:
if self.use_bias_forward:
forward_projection = tf.nn.bias_add(forward_projection,
bias,
data_format='NHWC')
if self.use_bias_backward:
backward_projection = tf.nn.bias_add(backward_projection,
bias,
data_format='NHWC')
else:
raise ValueError
# Apply Activation
if self.activation_fn_override is not None:
activation = activations.get(self.activation_fn_override)
print("Overriding the default activation function with {}".format(activation))
if self.activation_forward and (activation is not None):
forward_projection = activation(forward_projection)
print("Using activation forward: {}".format(activation))
if self.activation_backward and (activation is not None):
backward_projection = activation(backward_projection)
print("Using activation backward: {}".format(activation))
# Compute Reconstruction
if layer_type == 'fc':
forward_reconstruction = tf.matmul(forward_projection, backward_kernel)
backward_reconstruction = tf.matmul(backward_projection, forward_kernel,
transpose_b=True)
elif layer_type == 'conv':
forward_reconstruction = tf.nn.conv2d_transpose(forward_projection,
backward_kernel,
tf.shape(input),
**kwargs)
backward_reconstruction = tf.nn.conv2d_transpose(backward_projection,
forward_kernel,
tf.shape(input),
**kwargs)
else:
raise ValueError
# Standardize forward output
if self.center_forward_output:
forward_projection = self._center(forward_projection)
forward_reconstruction = self._center(forward_reconstruction)
if self.normalize_forward_output:
forward_projection = self._normalize(forward_projection)
forward_reconstruction = self._normalize(forward_reconstruction)
if self.batch_center_forward_output:
forward_projection = self._batch_center(forward_projection)
forward_reconstruction = self._batch_center(forward_reconstruction)
# Standardize backward output
if self.center_backward_output:
backward_projection = self._center(backward_projection)
backward_reconstruction = self._center(backward_reconstruction)
if self.normalize_backward_output:
backward_projection = self._normalize(backward_projection)
backward_reconstruction = self._normalize(backward_reconstruction)
# Reshape backward kernel
if layer_type == 'fc':
backward_kernel = tf.transpose(backward_kernel)
elif layer_type == 'conv':
pass
else:
raise ValueError
# Add alignment loss
loss = self.regularization(input,
forward_kernel,
backward_kernel,
forward_projection,
backward_projection,
forward_reconstruction,
backward_reconstruction)
tf.add_to_collection('ALIGNMENT_LOSSES', loss)
def __init__(self,
rank,
filters,
kernel_size,
param_reduction=0.5,
form='individual',
strides=1,
padding='valid',
data_format=None,
dilation_rate=1,
groups=1,
activation=None,
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None,
trainable=True,
name=None,
conv_op=None,
**kwargs):
super(SzConv, self).__init__(
trainable=trainable,
name=name,
activity_regularizer=regularizers.get(activity_regularizer),
**kwargs)
self.rank = rank
if isinstance(filters, float):
filters = int(filters)
if filters is not None and filters < 0:
raise ValueError(f'Received a negative value for `filters`.'
f'Was expecting a positive value, got {filters}.')
self.filters = filters
self.groups = groups or 1
self.kernel_size = conv_utils.normalize_tuple(kernel_size, rank,
'kernel_size')
self.strides = conv_utils.normalize_tuple(strides, rank, 'strides')
self.padding = conv_utils.normalize_padding(padding)
self.data_format = conv_utils.normalize_data_format(data_format)
self.dilation_rate = conv_utils.normalize_tuple(
dilation_rate, rank, 'dilation_rate')
self.activation = activations.get(activation)
self.use_bias = use_bias
self.kernel_initializer = initializers.get(kernel_initializer)
self.bias_initializer = initializers.get(bias_initializer)
self.kernel_regularizer = regularizers.get(kernel_regularizer)
self.bias_regularizer = regularizers.get(bias_regularizer)
self.kernel_constraint = constraints.get(kernel_constraint)
self.bias_constraint = constraints.get(bias_constraint)
self.input_spec = InputSpec(min_ndim=self.rank + 2)
self._validate_init()
self._is_causal = self.padding == 'causal'
self._channels_first = self.data_format == 'channels_first'
self._tf_data_format = conv_utils.convert_data_format(
self.data_format, self.rank + 2)
self.reduction_sv = param_reduction
self.form = form
self.num_ones = 0
self.num_weights = 0
self.reduced_ratio = 0
self.halfbandwidth = 0
def __init__(self,
num_units,
use_peepholes=False,
cell_clip=None,
initializer=None,
num_proj=None,
proj_clip=None,
num_unit_shards=None,
num_proj_shards=None,
forget_bias=1.0,
state_is_tuple=True,
activation=None,
reuse=None,
name=None,
dtype=None,
**kwargs):
"""Initialize the parameters for an LSTM cell.
Args:
num_units: int, The number of units in the LSTM cell.
use_peepholes: bool, set True to enable diagonal/peephole connections.
cell_clip: (optional) A float value, if provided the cell state is clipped
by this value prior to the cell output activation.
initializer: (optional) The initializer to use for the weight and
projection matrices.
num_proj: (optional) int, The output dimensionality for the projection
matrices. If None, no projection is performed.
proj_clip: (optional) A float value. If `num_proj > 0` and `proj_clip` is
provided, then the projected values are clipped elementwise to within
`[-proj_clip, proj_clip]`.
num_unit_shards: Deprecated, will be removed by Jan. 2017.
Use a variable_scope partitioner instead.
num_proj_shards: Deprecated, will be removed by Jan. 2017.
Use a variable_scope partitioner instead.
forget_bias: Biases of the forget gate are initialized by default to 1
in order to reduce the scale of forgetting at the beginning of
the training. Must set it manually to `0.0` when restoring from
CudnnLSTM trained checkpoints.
state_is_tuple: If True, accepted and returned states are 2-tuples of
the `c_state` and `m_state`. If False, they are concatenated
along the column axis. This latter behavior will soon be deprecated.
activation: Activation function of the inner states. Default: `tanh`. It
could also be string that is within Keras activation function names.
reuse: (optional) Python boolean describing whether to reuse variables
in an existing scope. If not `True`, and the existing scope already has
the given variables, an error is raised.
name: String, the name of the layer. Layers with the same name will
share weights, but to avoid mistakes we require reuse=True in such
cases.
dtype: Default dtype of the layer (default of `None` means use the type
of the first input). Required when `build` is called before `call`.
**kwargs: Dict, keyword named properties for common layer attributes, like
`trainable` etc when constructing the cell from configs of get_config().
When restoring from CudnnLSTM-trained checkpoints, use
`CudnnCompatibleLSTMCell` instead.
"""
super(T_LSTMCell, self).__init__(_reuse=reuse,
name=name,
dtype=dtype,
**kwargs)
if not state_is_tuple:
logging.warn(
"%s: Using a concatenated state is slower and will soon be "
"deprecated. Use state_is_tuple=True.", self)
if num_unit_shards is not None or num_proj_shards is not None:
logging.warn(
"%s: The num_unit_shards and proj_unit_shards parameters are "
"deprecated and will be removed in Jan 2017. "
"Use a variable scope with a partitioner instead.", self)
if context.executing_eagerly() and context.num_gpus() > 0:
logging.warn(
"%s: Note that this cell is not optimized for performance. "
"Please use tf.contrib.cudnn_rnn.CudnnLSTM for better "
"performance on GPU.", self)
# Inputs must be 2-dimensional.
self.input_spec = base_layer.InputSpec(ndim=2)
self._num_units = num_units
self._use_peepholes = use_peepholes
self._cell_clip = cell_clip
self._initializer = initializers.get(initializer)
self._num_proj = num_proj
self._proj_clip = proj_clip
self._num_unit_shards = num_unit_shards
self._num_proj_shards = num_proj_shards
self._forget_bias = forget_bias
self._state_is_tuple = state_is_tuple
if activation:
self._activation = activations.get(activation)
else:
self._activation = math_ops.tanh
if num_proj:
self._state_size = (LSTMStateTuple(num_units, num_proj)
if state_is_tuple else num_units + num_proj)
self._output_size = num_proj
else:
self._state_size = (LSTMStateTuple(num_units, num_units)
if state_is_tuple else 2 * num_units)
self._output_size = num_units
def __init__(self, activation, **kwargs): super(Activation, self).__init__(**kwargs) self.supports_masking = True self.activation = activations.get(activation)
def __init__(self,
num_units,
initializer=None,
input_shape=None,
forget_bias=1.0,
trainable=True,
do_norm=False,
useDropout=False,
activation=None,
reuse=None,
name=None,
dtype=None,
**kwargs):
"""Initialize the parameters for an LSTM cell.
Args:
num_units: int, The number of units in the LSTM cell.
initializer: (optional) The initializer to use for the weight and
projection matrices.
forget_bias: Biases of the forget gate are initialized by default to 1
in order to reduce the scale of forgetting at the beginning of
the training. Must set it manually to `0.0` when restoring from
CudnnLSTM trained checkpoints.
activation: Activation function of the inner states. Default: `tanh`. It
could also be string that is within Keras activation function names.
reuse: (optional) Python boolean describing whether to reuse variables
in an existing scope. If not `True`, and the existing scope already has
the given variables, an error is raised.
name: String, the name of the layer. Layers with the same name will
share weights, but to avoid mistakes we require reuse=True in such
cases.
dtype: Default dtype of the layer (default of `None` means use the type
of the first input). Required when `build` is called before `call`.
**kwargs: Dict, keyword named properties for common layer attributes, like
`trainable` etc when constructing the cell from configs of get_config().
When restoring from CudnnLSTM-trained checkpoints, use
`CudnnCompatibleLSTMCell` instead.
"""
super(STLSTMCell, self).__init__(_reuse=reuse,
name=name,
trainable=trainable,
dtype=dtype,
**kwargs)
if context.executing_eagerly() and context.num_gpus() > 0:
logging.warn(
"%s: Note that this cell is not optimized for performance. "
"Please use tf.contrib.cudnn_rnn.CudnnLSTM for better "
"performance on GPU.", self)
# Inputs must be 2-dimensional.
self.input_spec = base_layer.InputSpec(ndim=2)
self.do_norm = do_norm
self.useDropout = useDropout
self._num_units = num_units
self._forget_bias = forget_bias
# self._initializer = initializers.get(initializer)
self._initializer = initializer
# self._state_is_tuple = state_is_tuple
if activation:
self._activation = activations.get(activation)
else:
self._activation = math_ops.tanh
self._state_size = (LSTMStateTuple(num_units, num_units))
self._output_size = num_units
if input_shape is None:
raise ValueError("Expected inputs_shape to be known")
else:
if not self.built:
self.build(input_shape)
def __init__(self, units, use_bias=True, **kwargs):
super(LstmCell, self).__init__(**kwargs)
self.activation = activations.get('tanh')
self.recurrent_activation = activations.get('hard_sigmoid')
self.units = units
self.use_bias = use_bias