def _build(self, incoming, state, *args, **kwargs):
"""Gated recurrent unit (GRU) with nunits cells."""
with get_variable_scope('Gates'): # Reset gate and update gate.
weights_init = getters.get_initializer(self.weights_init)
# We start with bias of 1.0 to not reset and not update.
r, u = array_ops.split(axis=1,
num_or_size_splits=2,
value=_linear([incoming, state],
2 * self._num_units, True,
1.0, weights_init,
self.trainable, self.restore))
inner_activation = getters.get_activation(self.inner_activation)
r, u = inner_activation(r), inner_activation(u)
with get_variable_scope('Candidate'):
activation = getters.get_activation(self.activation)
c = activation(
_linear([incoming, r * state], self._num_units, True, 0.,
weights_init, self.trainable, self.restore))
new_h = u * state + (1 - u) * c
self._w, self._b = list(), list()
# Retrieve RNN Variables
with get_variable_scope(scope='Gates/Linear', reuse=True):
self._w.append(x=tf.get_variable('w'))
self._b.append(x=tf.get_variable('b'))
with get_variable_scope(scope='Candidate/Linear', reuse=True):
self._w.append(x=tf.get_variable('w'))
self._b.append(x=tf.get_variable('b'))
return new_h, new_h
def _build(self, incoming, state, *args, **kwargs):
"""Gated recurrent unit (GRU) with nunits cells."""
with get_variable_scope('Gates'): # Reset gate and update gate.
weights_init = getters.get_initializer(self.weights_init)
# We start with bias of 1.0 to not reset and not update.
r, u = array_ops.split(
axis=1, num_or_size_splits=2,
value=_linear([incoming, state], 2 * self._num_units, True, 1.0,
weights_init, self.trainable, self.restore))
inner_activation = getters.get_activation(self.inner_activation)
r, u = inner_activation(r), inner_activation(u)
with get_variable_scope('Candidate'):
activation = getters.get_activation(self.activation)
c = activation(
_linear([incoming, r * state], self._num_units, True, 0.,
weights_init, self.trainable, self.restore))
new_h = u * state + (1 - u) * c
self._w, self._b = list(), list()
# Retrieve RNN Variables
with get_variable_scope(scope='Gates/Linear', reuse=True):
self._w.append(x=tf.get_variable('w'))
self._b.append(x=tf.get_variable('b'))
with get_variable_scope(scope='Candidate/Linear', reuse=True):
self._w.append(x=tf.get_variable('w'))
self._b.append(x=tf.get_variable('b'))
return new_h, new_h
def __init__(self,
filters,
kernel_size,
strides=(1, 1),
padding='valid',
data_format=None,
dilation_rate=(1, 1),
activation='tanh',
recurrent_activation='hard_sigmoid',
use_bias=True,
kernel_initializer='glorot_uniform',
recurrent_initializer='orthogonal',
bias_initializer='zeros',
unit_forget_bias=True,
kernel_regularizer=None,
recurrent_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
recurrent_constraint=None,
bias_constraint=None,
return_sequences=False,
go_backwards=False,
stateful=False,
dropout=0.,
recurrent_dropout=0.,
**kwargs):
super(ConvLSTM2D, self).__init__(
filters,
kernel_size,
strides=strides,
padding=padding,
data_format=data_format,
dilation_rate=dilation_rate,
activation=getters.get_activation(activation),
recurrent_activation=getters.get_activation(recurrent_activation),
use_bias=use_bias,
kernel_initializer=getters.get_initializer(kernel_initializer),
recurrent_initializer=getters.get_initializer(
recurrent_initializer),
bias_initializer=getters.get_initializer(bias_initializer),
unit_forget_bias=unit_forget_bias,
kernel_regularizer=getters.get_regularizer(kernel_regularizer),
recurrent_regularizer=getters.get_regularizer(
recurrent_regularizer),
bias_regularizer=getters.get_regularizer(bias_regularizer),
activity_regularizer=getters.get_regularizer(activity_regularizer),
kernel_constraint=getters.get_regularizer(kernel_constraint),
recurrent_constraint=getters.get_regularizer(recurrent_constraint),
bias_constraint=getters.get_constraint(bias_constraint),
return_sequences=return_sequences,
go_backwards=go_backwards,
stateful=stateful,
dropout=dropout,
recurrent_dropout=recurrent_dropout,
**kwargs)
def __init__(self,
filters,
kernel_size,
strides=(1, 1),
padding='valid',
data_format=None,
dilation_rate=(1, 1),
activation=None,
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None,
**kwargs):
super(Conv2D, self).__init__(
filters=filters,
kernel_size=kernel_size,
strides=strides,
padding=padding,
data_format=data_format,
dilation_rate=dilation_rate,
activation=getters.get_activation(activation) if activation else activation,
use_bias=use_bias,
kernel_initializer=getters.get_initializer(kernel_initializer),
bias_initializer=getters.get_initializer(bias_initializer),
kernel_regularizer=getters.get_regularizer(kernel_regularizer),
bias_regularizer=getters.get_regularizer(bias_regularizer),
activity_regularizer=getters.get_regularizer(activity_regularizer),
kernel_constraint=getters.get_constraint(kernel_constraint),
bias_constraint=getters.get_constraint(bias_constraint),
**kwargs)
def _build(self, incoming, state, *args, **kwargs):
"""Long short-term memory cell (LSTM)."""
self._declare_dependencies()
activation = getters.get_activation(self.activation)
inner_activation = getters.get_activation(self.inner_activation)
# Parameters of gates are concatenated into one multiply for efficiency.
if self._state_is_tuple:
c, h = state
else:
c, h = array_ops.split(axis=1, num_or_size_splits=2, value=state)
concat = _linear([incoming, h], 4 * self._num_units, True, 0.,
self.weights_init, self.trainable, self.restore)
# i = input_gate, j = new_input, f = forget_gate, o = output_gate
i, j, f, o = array_ops.split(axis=1,
num_or_size_splits=4,
value=concat)
# apply batch normalization to inner state and gates
if self.batch_norm:
i = self._batch_norm_i(i)
j = self._batch_norm_j(j)
f = self._batch_norm_f(f)
o = self._batch_norm_o(o)
new_c = (c * inner_activation(f + self._forget_bias) +
inner_activation(i) * activation(j))
# hidden-to-hidden batch normalizaiton
if self.batch_norm:
batch_norm_new_c = self._batch_norm_c(new_c)
new_h = activation(batch_norm_new_c) * inner_activation(o)
else:
new_h = activation(new_c) * inner_activation(o)
if self._state_is_tuple:
new_state = rnn.LSTMStateTuple(new_c, new_h)
else:
new_state = tf.concat(values=[new_c, new_h], axis=1)
# Retrieve RNN Variables
with get_variable_scope(scope='Linear', reuse=True):
self._w = tf.get_variable('w')
self._b = tf.get_variable('b')
return new_h, new_state
def _build(self, incoming, state, *args, **kwargs):
"""Long short-term memory cell (LSTM)."""
self._declare_dependencies()
activation = getters.get_activation(self.activation)
inner_activation = getters.get_activation(self.inner_activation)
# Parameters of gates are concatenated into one multiply for efficiency.
if self._state_is_tuple:
c, h = state
else:
c, h = array_ops.split(axis=1, num_or_size_splits=2, value=state)
concat = _linear(
[incoming, h], 4 * self._num_units, True, 0., self.weights_init,
self.trainable, self.restore)
# i = input_gate, j = new_input, f = forget_gate, o = output_gate
i, j, f, o = array_ops.split(axis=1, num_or_size_splits=4, value=concat)
# apply batch normalization to inner state and gates
if self.batch_norm:
i = self._batch_norm_i(i)
j = self._batch_norm_j(j)
f = self._batch_norm_f(f)
o = self._batch_norm_o(o)
new_c = (c * inner_activation(f + self._forget_bias) + inner_activation(i) * activation(j))
# hidden-to-hidden batch normalizaiton
if self.batch_norm:
batch_norm_new_c = self._batch_norm_c(new_c)
new_h = activation(batch_norm_new_c) * inner_activation(o)
else:
new_h = activation(new_c) * inner_activation(o)
if self._state_is_tuple:
new_state = rnn_cell.LSTMStateTuple(new_c, new_h)
else:
new_state = tf.concat(values=[new_c, new_h], axis=1)
# Retrieve RNN Variables
with get_variable_scope(scope='Linear', reuse=True):
self._w = tf.get_variable('w')
self._b = tf.get_variable('b')
return new_h, new_state
def _build(self, incoming, *args, **kwargs):
"""
Args:
incoming: (2+)-D Tensor [samples, input dim]. If not 2D, input will be flatten.
Returns:
2D Tensor [samples, num_units].
"""
self._declare_dependencies()
input_shape = get_shape(incoming)
incoming = validate_dtype(incoming)
assert len(
input_shape) > 1, 'Incoming Tensor shape must be at least 2-D'
n_inputs = total_tensor_depth(tensor_shape=input_shape)
regularizer = getters.get_regularizer(self.regularizer,
scale=self.scale,
collect=True)
self._w = variable(name='w',
shape=[n_inputs, self.num_units],
dtype=incoming.dtype,
regularizer=regularizer,
initializer=getters.get_initializer(
self.weights_init),
trainable=self.trainable,
restore=self.restore)
track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
inference = incoming
# If input is not 2d, flatten it.
if len(input_shape) > 2:
inference = tf.reshape(tensor=inference, shape=[-1, n_inputs])
inference = tf.matmul(a=inference, b=self._w)
self._b = None
if self.bias:
self._b = variable(name='b',
shape=[self.num_units],
dtype=incoming.dtype,
initializer=getters.get_initializer(
self.bias_init),
trainable=self.trainable,
restore=self.restore)
track(self._b, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
inference = tf.nn.bias_add(value=inference, bias=self._b)
if self.activation:
inference = getters.get_activation(self.activation,
collect=True)(inference)
if self._dropout:
inference = self._dropout(inference)
track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return inference
def _build(self, inputs, state, *args, **kwargs):
"""Most basic RNN: output = new_state = activation(W * input + U * state + B)."""
weights_init = getters.get_initializer(self.weights_init)
output = getters.get_activation(self.activation)(
_linear([inputs, state], self.num_units, True, 0., weights_init,
self.trainable, self.restore))
# Retrieve RNN Variables
with get_variable_scope(name='Linear', reuse=True):
self._w = tf.get_variable(name='w')
self._b = tf.get_variable(name='b')
return output, output
def _build(self, inputs, state, *args, **kwargs):
"""Most basic RNN: output = new_state = activation(W * input + U * state + B)."""
weights_init = getters.get_initializer(self.weights_init)
output = getters.get_activation(self.activation)(
_linear([inputs, state], self.num_units, True, 0.,
weights_init, self.trainable, self.restore))
# Retrieve RNN Variables
with get_variable_scope(name='Linear', reuse=True):
self._w = tf.get_variable(name='w')
self._b = tf.get_variable(name='b')
return output, output
def __init__(self,
units,
activation='tanh',
recurrent_activation='hard_sigmoid',
use_bias=True,
kernel_initializer='glorot_uniform',
recurrent_initializer='orthogonal',
bias_initializer='zeros',
unit_forget_bias=True,
kernel_regularizer=None,
recurrent_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
recurrent_constraint=None,
bias_constraint=None,
dropout=0.,
recurrent_dropout=0.,
**kwargs):
super(LSTM, self).__init__(
units=units,
activation=getters.get_activation(activation),
recurrent_activation=getters.get_activation(recurrent_activation),
use_bias=use_bias,
kernel_initializer=getters.get_initializer(kernel_initializer),
recurrent_initializer=getters.get_initializer(
recurrent_initializer),
bias_initializer=getters.get_initializer(bias_initializer),
unit_forget_bias=unit_forget_bias,
kernel_regularizer=getters.get_regularizer(kernel_regularizer),
recurrent_regularizer=getters.get_regularizer(
recurrent_regularizer),
bias_regularizer=getters.get_regularizer(bias_regularizer),
activity_regularizer=getters.get_regularizer(activity_regularizer),
kernel_constraint=getters.get_constraint(kernel_constraint),
recurrent_constraint=getters.get_constraint(recurrent_constraint),
bias_constraint=getters.get_constraint(bias_constraint),
dropout=dropout,
recurrent_dropout=recurrent_dropout,
**kwargs)
def _build(self, incoming, *args, **kwargs):
"""
Args:
incoming: (2+)-D Tensor [samples, input dim]. If not 2D, input will be flatten.
Returns:
2D Tensor [samples, num_units].
"""
self._declare_dependencies()
input_shape = get_shape(incoming)
assert len(input_shape) > 1, 'Incoming Tensor shape must be at least 2-D'
n_inputs = total_tensor_depth(tensor_shape=input_shape)
regularizer = getters.get_regularizer(self.regularizer, scale=self.scale, collect=True)
initializer = getters.get_initializer(self.weights_init)
self._w = variable(name='w', shape=[n_inputs, self.num_units], regularizer=regularizer,
initializer=initializer, trainable=self.trainable,
restore=self.restore)
track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
self._b = variable(name='b', shape=[self.num_units],
initializer=getters.get_initializer(self.bias_init),
trainable=self.trainable, restore=self.restore)
track(self._b, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
# Weight and bias for the transform gate
self._w_t = variable(name='w_t', shape=[n_inputs, self.num_units],
regularizer=None, initializer=initializer,
trainable=self.trainable, restore=self.restore)
track(self._w_t, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
self._b_t = variable(name='b_t', shape=[self.num_units],
initializer=tf.constant_initializer(-1),
trainable=self.trainable, restore=self.restore)
track(self._b_t, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
# If input is not 2d, flatten it.
if len(input_shape) > 2:
incoming = tf.reshape(tensor=incoming, shape=[-1, n_inputs])
H = getters.get_activation(self.activation)(tf.matmul(a=incoming, b=self._w) + self._b)
T = tf.sigmoid(tf.matmul(a=incoming, b=self._w_t) + self._b_t)
if self._transform_dropout:
T = self._transform_dropout(T)
C = tf.subtract(x=1.0, y=T)
inference = tf.add(x=tf.multiply(x=H, y=T), y=tf.multiply(x=incoming, y=C))
track(inference, tf.GraphKeys.ACTIVATIONS)
track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return inference
def __init__(self,
filters,
kernel_size,
strides=(1, 1),
padding='valid',
data_format=None,
depth_multiplier=1,
activation=None,
use_bias=True,
depthwise_initializer='glorot_uniform',
pointwise_initializer='glorot_uniform',
bias_initializer='zeros',
depthwise_regularizer=None,
pointwise_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
depthwise_constraint=None,
pointwise_constraint=None,
bias_constraint=None,
**kwargs):
super(SeparableConv2D, self).__init__(
filters=filters,
kernel_size=kernel_size,
strides=strides,
padding=padding,
data_format=data_format,
depth_multiplier=depth_multiplier,
activation=getters.get_activation(activation),
use_bias=use_bias,
depthwise_initializer=getters.get_initializer(
depthwise_initializer),
pointwise_initializer=getters.get_initializer(
pointwise_initializer),
bias_initializer=getters.get_initializer(bias_initializer),
depthwise_regularizer=getters.get_regularizer(
depthwise_regularizer),
pointwise_regularizer=getters.get_regularizer(
pointwise_regularizer),
bias_regularizer=getters.get_regularizer(bias_regularizer),
activity_regularizer=getters.get_regularizer(activity_regularizer),
depthwise_constraint=getters.get_constraint(depthwise_constraint),
pointwise_constraint=getters.get_constraint(pointwise_constraint),
bias_constraint=getters.get_constraint(bias_constraint),
**kwargs)
def _build(self, incoming, *args, **kwargs):
"""
Args:
incoming: (2+)-D Tensor [samples, input dim]. If not 2D, input will be flatten.
Returns:
2D Tensor [samples, num_units].
"""
self._declare_dependencies()
input_shape = get_shape(incoming)
incoming = validate_dtype(incoming)
assert len(input_shape) > 1, 'Incoming Tensor shape must be at least 2-D'
n_inputs = total_tensor_depth(tensor_shape=input_shape)
regularizer = getters.get_regularizer(self.regularizer, scale=self.scale, collect=True)
self._w = variable(
name='w', shape=[n_inputs, self.num_units], dtype=incoming.dtype, regularizer=regularizer,
initializer=getters.get_initializer(self.weights_init), trainable=self.trainable,
restore=self.restore)
track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
inference = incoming
# If input is not 2d, flatten it.
if len(input_shape) > 2:
inference = tf.reshape(tensor=inference, shape=[-1, n_inputs])
inference = tf.matmul(a=inference, b=self._w)
self._b = None
if self.bias:
self._b = variable(name='b', shape=[self.num_units], dtype=incoming.dtype,
initializer=getters.get_initializer(self.bias_init),
trainable=self.trainable, restore=self.restore)
track(self._b, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
inference = tf.nn.bias_add(value=inference, bias=self._b)
if self.activation:
inference = getters.get_activation(self.activation, collect=True)(inference)
if self._dropout:
inference = self._dropout(inference)
track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return inference
def _build(self, incoming, *args, **kwargs):
"""
Args:
incoming: 1-D Tensor [samples]. If not 2D, input will be flatten.
Returns:
1-D Tensor [samples].
"""
input_shape = get_shape(incoming)
n_inputs = int(np.prod(a=input_shape[1:]))
initializer = tf.constant_initializer(value=np.random.randn())
self._w = variable(name='w',
shape=[n_inputs],
dtype=incoming.dtype,
initializer=initializer,
trainable=self.trainable,
restore=self.restore)
track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
inference = incoming
# If input is not 2d, flatten it.
if len(input_shape) > 1:
inference = tf.reshape(tensor=inference, shape=[-1])
inference = tf.multiply(x=inference, y=self._w)
self._b = None
if self.bias:
self._b = variable(name='b',
shape=[n_inputs],
dtype=incoming.dtype,
initializer=initializer,
trainable=self.trainable,
restore=self.restore)
inference = tf.add(inference, self._b)
track(self._b, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
if self.activation:
inference = getters.get_activation(self.activation,
collect=True)(inference)
track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return inference
def _build(self, incoming, *args, **kwargs):
"""
Args:
incoming: 1-D Tensor [samples]. If not 2D, input will be flatten.
Returns:
1-D Tensor [samples].
"""
input_shape = get_shape(incoming)
n_inputs = total_tensor_depth(tensor_shape=input_shape)
initializer = tf.constant_initializer(value=np.random.randn())
self._w = variable(name='w', shape=[n_inputs],
dtype=incoming.dtype, initializer=initializer,
trainable=self.trainable, restore=self.restore)
track(self._w, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
inference = incoming
# If input is not 2d, flatten it.
if len(input_shape) > 1:
inference = tf.reshape(tensor=inference, shape=[-1])
inference = tf.multiply(x=inference, y=self._w)
self._b = None
if self.bias:
self._b = variable(name='b', shape=[n_inputs],
dtype=incoming.dtype, initializer=initializer,
trainable=self.trainable, restore=self.restore)
inference = tf.add(inference, self._b)
track(self._b, tf.GraphKeys.LAYER_VARIABLES, self.module_name)
if self.activation:
inference = getters.get_activation(self.activation, collect=True)(inference)
track(inference, tf.GraphKeys.LAYER_TENSOR, self.module_name)
return inference
def __init__(self,
units,
activation=None,
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None,
**kwargs):
super(Dense, self).__init__(
units,
activation=getters.get_activation(activation) if activation else activation,
use_bias=use_bias,
kernel_initializer=getters.get_initializer(kernel_initializer),
bias_initializer=getters.get_initializer(bias_initializer),
kernel_regularizer=getters.get_regularizer(kernel_regularizer),
bias_regularizer=getters.get_regularizer(bias_regularizer),
activity_regularizer=getters.get_regularizer(activity_regularizer),
kernel_constraint=getters.get_constraint(kernel_constraint),
bias_constraint=getters.get_constraint(bias_constraint),
**kwargs)