def __init__(
self,
loc_initializer=tf.keras.initializers.TruncatedNormal(stddev=1e-5),
scale_initializer=tf.keras.initializers.TruncatedNormal(
mean=-3., stddev=0.1),
loc_regularizer=None,
scale_regularizer=None,
loc_constraint=None,
scale_constraint='softplus',
seed=None,
**kwargs):
"""Constructs the initializer."""
super(TrainableCauchy, self).__init__(**kwargs)
self.loc_initializer = get(loc_initializer)
self.scale_initializer = get(scale_initializer)
self.loc_regularizer = regularizers.get(loc_regularizer)
self.scale_regularizer = regularizers.get(scale_regularizer)
self.loc_constraint = constraints.get(loc_constraint)
self.scale_constraint = constraints.get(scale_constraint)
self.seed = seed
def __init__(self,
mean_initializer=tf.keras.initializers.TruncatedNormal(
stddev=1e-5),
stddev_initializer=tf.keras.initializers.TruncatedNormal(
mean=-3., stddev=0.1),
mean_regularizer=None,
stddev_regularizer=None,
mean_constraint=None,
stddev_constraint='softplus',
seed=None,
**kwargs):
"""Constructs the initializer."""
super(TrainableNormal, self).__init__(**kwargs)
self.mean_initializer = get(mean_initializer)
self.stddev_initializer = get(stddev_initializer)
self.mean_regularizer = regularizers.get(mean_regularizer)
self.stddev_regularizer = regularizers.get(stddev_regularizer)
self.mean_constraint = constraints.get(mean_constraint)
self.stddev_constraint = constraints.get(stddev_constraint)
self.seed = seed
def __init__(self,
loc_initializer='glorot_uniform',
loc_regularizer=None,
loc_constraint=None,
seed=None,
**kwargs):
"""Constructs the initializer."""
super(TrainableDeterministic, self).__init__(**kwargs)
self.loc_initializer = get(loc_initializer)
self.loc_regularizer = regularizers.get(loc_regularizer)
self.loc_constraint = constraints.get(loc_constraint)
self.seed = seed
def __init__(self,
units,
activation='tanh',
recurrent_activation='sigmoid',
use_bias=True,
kernel_initializer='trainable_normal',
recurrent_initializer='trainable_normal',
bias_initializer='zeros',
unit_forget_bias=True,
kernel_regularizer='normal_kl_divergence',
recurrent_regularizer='normal_kl_divergence',
bias_regularizer=None,
kernel_constraint=None,
recurrent_constraint=None,
bias_constraint=None,
dropout=0.,
recurrent_dropout=0.,
implementation=2,
**kwargs):
self.called_weights = False
super(LSTMCellReparameterization, self).__init__(
units=units,
activation=activation,
recurrent_activation=recurrent_activation,
use_bias=use_bias,
kernel_initializer=initializers.get(kernel_initializer),
recurrent_initializer=initializers.get(recurrent_initializer),
bias_initializer=initializers.get(bias_initializer),
unit_forget_bias=unit_forget_bias,
kernel_regularizer=regularizers.get(kernel_regularizer),
recurrent_regularizer=regularizers.get(recurrent_regularizer),
bias_regularizer=regularizers.get(bias_regularizer),
kernel_constraint=constraints.get(kernel_constraint),
recurrent_constraint=constraints.get(recurrent_constraint),
bias_constraint=constraints.get(bias_constraint),
dropout=dropout,
recurrent_dropout=recurrent_dropout,
implementation=implementation,
**kwargs)
def __init__(self,
loc_initializer=tf.keras.initializers.TruncatedNormal(
stddev=1e-5),
loc_regularizer=None,
loc_constraint=None,
seed=None,
**kwargs):
"""Constructs the initializer."""
super(TrainableDeterministic, self).__init__(**kwargs)
self.loc_initializer = get(loc_initializer)
self.loc_regularizer = regularizers.get(loc_regularizer)
self.loc_constraint = constraints.get(loc_constraint)
self.seed = seed
def __init__(self,
num_components=5,
loc_initializer=tf.keras.initializers.he_normal(),
loc_regularizer=None,
loc_constraint=None,
seed=None,
**kwargs):
"""Constructs the initializer."""
super(TrainableMixtureOfDeltas, self).__init__(**kwargs)
self.num_components = num_components
self.loc_initializer = get(loc_initializer)
self.loc_regularizer = regularizers.get(loc_regularizer)
self.loc_constraint = constraints.get(loc_constraint)
self.seed = seed
def __init__(self,
units,
activation=None,
use_bias=True,
kernel_initializer='trainable_normal',
bias_initializer='zero',
local_scale_initializer='trainable_half_cauchy',
global_scale_initializer='trainable_half_cauchy',
kernel_regularizer='normal_kl_divergence',
bias_regularizer=None,
local_scale_regularizer='half_cauchy_kl_divergence',
global_scale_regularizer=regularizers.HalfCauchyKLDivergence(
scale=1e-5),
activity_regularizer=None,
local_scale_constraint='positive',
global_scale_constraint='positive',
**kwargs):
self.local_scale_initializer = initializers.get(
local_scale_initializer)
self.global_scale_initializer = initializers.get(
global_scale_initializer)
self.local_scale_regularizer = regularizers.get(
local_scale_regularizer)
self.global_scale_regularizer = regularizers.get(
global_scale_regularizer)
self.local_scale_constraint = constraints.get(local_scale_constraint)
self.global_scale_constraint = constraints.get(global_scale_constraint)
super(DenseHierarchical, self).__init__(
units=units,
activation=activation,
use_bias=use_bias,
kernel_initializer=initializers.get(kernel_initializer),
bias_initializer=initializers.get(bias_initializer),
kernel_regularizer=regularizers.get(kernel_regularizer),
bias_regularizer=regularizers.get(bias_regularizer),
activity_regularizer=regularizers.get(activity_regularizer),
**kwargs)
def __init__(self,
stddev=1.,
mean_initializer=tf.keras.initializers.TruncatedNormal(
stddev=1e-5),
mean_regularizer=None,
mean_constraint=None,
seed=None,
**kwargs):
"""Constructs the initializer."""
super(TrainableNormalFixedStddev, self).__init__(**kwargs)
self.stddev = stddev
self.mean_initializer = get(mean_initializer)
self.mean_regularizer = regularizers.get(mean_regularizer)
self.mean_constraint = constraints.get(mean_constraint)
self.seed = seed
def __init__(self,
mean=0.,
stddev_initializer=tf.keras.initializers.TruncatedNormal(
mean=0.5413248, stddev=0.1), # mean=softplus_inverse(1.)
stddev_regularizer=None,
stddev_constraint='softplus',
scale_factor=1.,
seed=None,
**kwargs):
super(TrainableNormalKLDivergenceStdDev, self).__init__(**kwargs)
self.mean = mean
self.stddev_initializer = tf.keras.initializers.get(stddev_initializer)
self.stddev_regularizer = get(stddev_regularizer)
self.stddev_constraint = constraints.get(stddev_constraint)
self.scale_factor = scale_factor
def __init__(self,
filters,
kernel_size,
rank=1,
ensemble_size=4,
alpha_initializer='ones',
gamma_initializer='ones',
strides=(1, 1),
padding='valid',
data_format=None,
activation=None,
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None,
**kwargs):
super(Conv2DBatchEnsemble, self).__init__(**kwargs)
self.rank = rank
self.ensemble_size = ensemble_size
self.alpha_initializer = initializers.get(alpha_initializer)
self.gamma_initializer = initializers.get(gamma_initializer)
self.bias_initializer = initializers.get(bias_initializer)
self.bias_regularizer = regularizers.get(bias_regularizer)
self.bias_constraint = constraints.get(bias_constraint)
self.activation = tf.keras.activations.get(activation)
self.use_bias = use_bias
self.conv2d = tf.keras.layers.Conv2D(
filters=filters,
kernel_size=kernel_size,
strides=strides,
padding=padding,
data_format=data_format,
activation=None,
use_bias=False,
kernel_initializer=kernel_initializer,
bias_initializer=None,
kernel_regularizer=kernel_regularizer,
bias_regularizer=None,
activity_regularizer=activity_regularizer,
kernel_constraint=kernel_constraint,
bias_constraint=None)
self.filters = self.conv2d.filters
self.kernel_size = self.conv2d.kernel_size
self.data_format = self.conv2d.data_format
def __init__(self,
input_dim,
output_dim,
embeddings_initializer='trainable_normal',
embeddings_regularizer='normal_kl_divergence',
activity_regularizer=None,
embeddings_constraint=None,
mask_zero=False,
input_length=None,
**kwargs):
"""Initializes the reparameterized Bayesian embeddings layer.
Args:
input_dim: int > 0. Size of the vocabulary, i.e. maximum integer index +
1.
output_dim: int >= 0. Dimension of the dense embedding.
embeddings_initializer: Initializer for the `embeddings` matrix.
embeddings_regularizer: Regularizer function applied to the `embeddings`
matrix.
activity_regularizer: Regularizer function applied to the output of the
layer (its "activation").
embeddings_constraint: Constraint function applied to the `embeddings`
matrix.
mask_zero: Whether or not the input value 0 is a special "padding" value
that should be masked out. This is useful when using recurrent layers
which may take variable length input. If this is `True` then all
subsequent layers in the model need to support masking or an exception
will be raised. If mask_zero is set to True, as a consequence, index 0
cannot be used in the vocabulary (input_dim should equal size of
vocabulary + 1).
input_length: Length of input sequences, when it is constant. This
argument is required if you are going to connect `Flatten` then `Dense`
layers upstream (without it, the shape of the dense outputs cannot be
computed).
**kwargs: Additional keyword arguments to pass to the super class.
"""
super(EmbeddingReparameterization, self).__init__(
input_dim=input_dim,
output_dim=output_dim,
embeddings_initializer=initializers.get(embeddings_initializer),
embeddings_regularizer=regularizers.get(embeddings_regularizer),
activity_regularizer=regularizers.get(activity_regularizer),
embeddings_constraint=constraints.get(embeddings_constraint),
mask_zero=mask_zero,
input_length=input_length,
**kwargs)
def __init__(self,
kernel_size,
ensemble_size=4,
alpha_initializer='ones',
gamma_initializer='ones',
strides=(1, 1),
padding='valid',
depth_multiplier=1,
data_format=None,
activation=None,
use_bias=True,
depthwise_initializer='glorot_uniform',
bias_initializer='zeros',
depthwise_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
depthwise_constraint=None,
bias_constraint=None,
**kwargs):
super(DepthwiseConv2DBatchEnsemble, self).__init__(**kwargs)
self.ensemble_size = ensemble_size
self.alpha_initializer = initializers.get(alpha_initializer)
self.gamma_initializer = initializers.get(gamma_initializer)
self.bias_initializer = initializers.get(bias_initializer)
self.bias_regularizer = regularizers.get(bias_regularizer)
self.bias_constraint = constraints.get(bias_constraint)
self.activation = tf.keras.activations.get(activation)
self.use_bias = use_bias
self.conv2d = tf.keras.layers.DepthwiseConv2D(
kernel_size=kernel_size,
strides=strides,
padding=padding,
depth_multiplier=depth_multiplier,
data_format=data_format,
activation=None,
use_bias=False,
depthwise_initializer=depthwise_initializer,
bias_initializer=None,
depthwise_regularizer=depthwise_regularizer,
bias_regularizer=None,
activity_regularizer=activity_regularizer,
depthwise_constraint=depthwise_constraint,
bias_constraint=None)
self.kernel_size = self.conv2d.kernel_size
self.depth_multiplier = self.conv2d.depth_multiplier
self.data_format = self.conv2d.data_format
def __init__(self,
units,
rank=1,
ensemble_size=4,
activation=None,
use_bias=True,
alpha_initializer='ones',
gamma_initializer='ones',
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None,
**kwargs):
super(DenseBatchEnsemble, self).__init__(**kwargs)
self.rank = rank
self.ensemble_size = ensemble_size
self.activation = tf.keras.activations.get(activation)
self.use_bias = use_bias
self.alpha_initializer = initializers.get(alpha_initializer)
self.gamma_initializer = initializers.get(gamma_initializer)
self.bias_initializer = initializers.get(bias_initializer)
self.bias_regularizer = regularizers.get(bias_regularizer)
self.bias_constraint = constraints.get(bias_constraint)
self.dense = tf.keras.layers.Dense(
units=units,
use_bias=False,
activation=None,
kernel_initializer=kernel_initializer,
bias_initializer=None,
kernel_regularizer=kernel_regularizer,
bias_regularizer=None,
activity_regularizer=activity_regularizer,
kernel_constraint=kernel_constraint,
bias_constraint=None)
self.units = self.dense.units
def __init__(self,
units,
activation='tanh',
recurrent_activation='sigmoid',
use_bias=True,
alpha_initializer='trainable_normal',
gamma_initializer='trainable_normal',
kernel_initializer='glorot_uniform',
recurrent_alpha_initializer='trainable_normal',
recurrent_gamma_initializer='trainable_normal',
recurrent_initializer='orthogonal',
bias_initializer='zeros',
unit_forget_bias=True,
alpha_regularizer='normal_kl_divergence',
gamma_regularizer='normal_kl_divergence',
kernel_regularizer=None,
recurrent_alpha_regularizer='normal_kl_divergence',
recurrent_gamma_regularizer='normal_kl_divergence',
recurrent_regularizer=None,
bias_regularizer=None,
alpha_constraint=None,
gamma_constraint=None,
kernel_constraint=None,
recurrent_alpha_constraint=None,
recurrent_gamma_constraint=None,
recurrent_constraint=None,
bias_constraint=None,
dropout=0.,
recurrent_dropout=0.,
implementation=2,
use_additive_perturbation=False,
ensemble_size=1,
**kwargs):
"""Initializes an LSTM cell layer.
Args:
units: Positive integer, dimensionality of the output space.
activation: Activation function to use. If you pass `None`, no activation
is applied (ie. "linear" activation is `a(x) = x`).
recurrent_activation: Activation function to use for the recurrent step.
If you pass `None`, no activation is applied (ie. "linear" activation
is `a(x) = x`).
use_bias: Boolean, (default `True`), whether the layer uses a bias vector.
alpha_initializer: Initializer for the rank-1 weights vector applied to
the inputs before the `kernel` is applied.
gamma_initializer: Initializer for the rank-1 weights vector applied to
after the `kernel` is applied.
kernel_initializer: Initializer for the `kernel` weights matrix, used for
the linear transformation of the inputs.
recurrent_alpha_initializer: Initializer for the rank-1 weights vector
applied to the recurrent state before the `recurrent_kernel` is applied.
recurrent_gamma_initializer: Initializer for the rank-1 weights vector
applied after the `recurrent_kernel` is applied.
recurrent_initializer: Initializer for the `recurrent_kernel` weights
matrix, used for the linear transformation of the recurrent state.
bias_initializer: Initializer for the bias vector.
unit_forget_bias: Boolean (default `True`). If True, add 1 to the bias of
the forget gate at initialization. Setting it to true will also force
`bias_initializer="zeros"`. This is recommended in [Jozefowicz et
al.](http://www.jmlr.org/proceedings/papers/v37/jozefowicz15.pdf)
alpha_regularizer: Regularizer function applied to the `alpha` weights
vector.
gamma_regularizer: Regularizer function applied to the `gamma` weights
vector.
kernel_regularizer: Regularizer function applied to the `kernel` weights
matrix.
recurrent_alpha_regularizer: Regularizer function applied to the
`recurrent_alpha` weights vector.
recurrent_gamma_regularizer: Regularizer function applied to the
`recurrent_gamma` weights vector.
recurrent_regularizer: Regularizer function applied to the
`recurrent_kernel` weights matrix.
bias_regularizer: Regularizer function applied to the bias vector.
alpha_constraint: Constraint function applied to the `alpha` weights
vector.
gamma_constraint: Constraint function applied to the `gamma` weights
vector.
kernel_constraint: Constraint function applied to the `kernel` weights
matrix.
recurrent_alpha_constraint: Constraint function applied to the
`recurrent_alpha` weights vector.
recurrent_gamma_constraint: Constraint function applied to the
`recurrent_gamma` weights vector.
recurrent_constraint: Constraint function applied to the
`recurrent_kernel` weights matrix.
bias_constraint: Constraint function applied to the bias vector.
dropout: Float between 0 and 1. Fraction of the units to drop for the
linear transformation of the inputs.
recurrent_dropout: Float between 0 and 1. Fraction of the units to drop
for the linear transformation of the recurrent state.
implementation: Implementation mode, either 1 or 2.
Mode 1 will structure its operations as a larger number of smaller dot
products and additions, whereas mode 2 (default) will batch them into
fewer, larger operations. These modes will have different performance
profiles on different hardware and for different applications.
use_additive_perturbation: Whether or not to use additive interactions vs.
multiplicative actions.
ensemble_size: Number of ensemble members, or equivalently, number of
mixture components.
**kwargs: Any additional arguments to pass to the superclasses.
"""
self.alpha_initializer = initializers.get(alpha_initializer)
self.gamma_initializer = initializers.get(gamma_initializer)
self.recurrent_alpha_initializer = initializers.get(
recurrent_alpha_initializer)
self.recurrent_gamma_initializer = initializers.get(
recurrent_gamma_initializer)
self.alpha_regularizer = regularizers.get(alpha_regularizer)
self.gamma_regularizer = regularizers.get(gamma_regularizer)
self.recurrent_alpha_regularizer = regularizers.get(
recurrent_alpha_regularizer)
self.recurrent_gamma_regularizer = regularizers.get(
recurrent_gamma_regularizer)
self.alpha_constraint = constraints.get(alpha_constraint)
self.gamma_constraint = constraints.get(gamma_constraint)
self.recurrent_alpha_constraint = constraints.get(
recurrent_alpha_constraint)
self.recurrent_gamma_constraint = constraints.get(
recurrent_gamma_constraint)
self.use_additive_perturbation = use_additive_perturbation
self.ensemble_size = ensemble_size
self.sampled_weights = False
super().__init__(
units=units,
activation=activation,
recurrent_activation=recurrent_activation,
use_bias=use_bias,
kernel_initializer=kernel_initializer,
recurrent_initializer=recurrent_initializer,
bias_initializer=initializers.get(bias_initializer),
unit_forget_bias=unit_forget_bias,
kernel_regularizer=kernel_regularizer,
recurrent_regularizer=recurrent_regularizer,
bias_regularizer=regularizers.get(bias_regularizer),
kernel_constraint=kernel_constraint,
recurrent_constraint=recurrent_constraint,
bias_constraint=constraints.get(bias_constraint),
dropout=dropout,
recurrent_dropout=recurrent_dropout,
implementation=implementation,
**kwargs)