def batch_normalize_with_arguments(x, arguments):
"""Applies batch normalization to x as specified in arguments.
Args:
x: A Pretty Tensor.
arguments: Either a boolean to batch_normalize or a
BatchNormalizationArguments
Returns:
x with batch normalization applied.
"""
x = prettytensor.wrap(x)
# Backwards compatibility.
if isinstance(arguments, bool):
if arguments:
return x.batch_normalize()
else:
return x
# pylint: disable=protected-access
kwargs = arguments._asdict()
defaults = prettytensor._defaults
# pylint: enable=protected-access
for arg in ('learned_moments_update_rate', 'variance_epsilon',
'scale_after_normalization'):
if kwargs.get(arg, None) is None:
if arg in defaults:
kwargs[arg] = defaults[arg]
else:
del kwargs
return x.batch_normalize(**kwargs)
def batch_normalize_with_arguments(x, arguments):
"""Applies batch normalization to x as specified in arguments.
Args:
x: A Pretty Tensor.
arguments: Either a boolean to batch_normalize or a
BatchNormalizationArguments
Returns:
x with batch normalization applied.
"""
x = prettytensor.wrap(x)
# Backwards compatibility.
if isinstance(arguments, bool):
if arguments:
return x.batch_normalize()
else:
return x
# pylint: disable=protected-access
kwargs = arguments._asdict()
defaults = prettytensor._defaults
# pylint: enable=protected-access
for arg in ('learned_moments_update_rate', 'variance_epsilon',
'scale_after_normalization'):
if kwargs.get(arg, None) is None:
if arg in defaults:
kwargs[arg] = defaults[arg]
else:
del kwargs
return x.batch_normalize(**kwargs)
def lstm_cell(input_layer, states, num_units, bias=True, peephole=True, stddev=None, init=None):
"""Long short-term memory cell (LSTM).
Args:
input_layer: The input layer.
states: The current state of the network, as
[[batch, num_units], [batch, num_units]] (c, h).
num_units: How big is the hidden state.
bias: Whether or not to use a bias.
peephole: Whether to use peephole connections as described in
http://www.jmlr.org/papers/volume3/gers02a/gers02a.pdf
stddev: Standard deviation for Gaussian initialization of parameters.
init: A tf.*Initializer that is used to initialize the variables.
Returns:
A RecurrentResult.
"""
# As a compound op, it needs to respect whether or not this is a sequential
# builder.
if input_layer.is_sequential_builder():
layer = input_layer.as_layer()
else:
layer = input_layer
c, h = [prettytensor.wrap(state, layer.bookkeeper) for state in states]
activation_input = layer.fully_connected(4 * num_units, bias=bias, activation_fn=None, stddev=stddev, init=init)
activation_h = h.fully_connected(4 * num_units, bias=False, activation_fn=None, stddev=stddev, init=init)
activation = activation_input + activation_h
# i = input_gate, j = new_input, f = forget_gate, o = output_gate
split = activation.split(1, 4)
i = split[0]
j = split[1]
f = split[2]
if bias:
# Biases of the forget gate are initialized to 1 in order to reduce the
# scale of forgetting in the beginning of the training.
f += 1.0
o = split[3]
if peephole:
# TODO(eiderman): It would be worthwhile to determine the best initialization.
i += c.diagonal_matrix_mul(stddev=stddev, init=init)
f += c.diagonal_matrix_mul(stddev=stddev, init=init)
new_c = c * f.apply(tf.sigmoid, name="f_gate") + i.apply(tf.sigmoid, name="i_gate") * j.apply(tf.tanh)
if peephole:
o += new_c.diagonal_matrix_mul(stddev=stddev, init=init)
new_h = new_c.apply(tf.tanh) * o.apply(tf.sigmoid, name="o_gate")
if input_layer.is_sequential_builder():
new_h = input_layer.set_head(input_layer)
return RecurrentResult(new_h, [new_c, new_h])
def gru_cell(input_layer,
state,
num_units,
bias=tf.zeros_initializer(),
weights=None,
phase=prettytensor.Phase.train,
parameter_modifier=parameters.identity):
"""Gated recurrent unit memory cell (GRU).
Args:
input_layer: The input layer.
state: The current state of the network. For GRUs, this is a list with
one element (tensor) of shape [batch, num_units].
num_units: How big is the hidden state.
bias: An initializer for the bias or a Tensor. No bias if set to None.
weights: An initializer for weights or a Tensor.
phase: The phase of graph construction. See `pt.Phase`.
parameter_modifier: A function to modify parameters that is applied after
creation and before use.
Returns:
A RecurrentResult.
"""
# As a compound op, it needs to respect whether or not this is a sequential
# builder.
if input_layer.is_sequential_builder():
layer = input_layer.as_layer()
else:
layer = input_layer
# We start with bias of 1.0 to not reset and not udpate.
# NB We compute activation_input and activation_state in two different ops,
# instead of concatenating them, followed by one matrix multiplication. The
# reason is that input has size [batch_size x input_size], while state has
# [ ? x state_size ], where the first dimension is 1 initially and will be
# batch_size only after the first RNN computation. We thus cannot concatenate
# input and state, and instead add the results of two fully connected ops,
# which works thanks to broadcasting, independent of state's batch size.
state = state[0]
state_pt = prettytensor.wrap(state, layer.bookkeeper)
activation_input = layer.fully_connected(
2 * num_units,
bias=None if bias is None else tf.constant_initializer(1.0),
activation_fn=None,
weights=weights,
phase=phase,
parameter_modifier=parameter_modifier)
activation_state = state_pt.fully_connected(
2 * num_units,
bias=None,
activation_fn=None,
weights=weights,
phase=phase,
parameter_modifier=parameter_modifier)
# adds batch_size x (2 * num_units) + ? x (2 * num_inputs)
activation = activation_input + activation_state
activation = activation.sigmoid()
split = activation.split(1, 2)
r = split[0]
u = split[1]
c = layer.concat(1, [r * state]).fully_connected(
num_units,
bias=bias,
activation_fn=None,
weights=weights,
phase=phase,
parameter_modifier=parameter_modifier).apply(tf.tanh)
new_h = u * state + (1 - u) * c
if input_layer.is_sequential_builder():
new_h = input_layer.set_head(input_layer)
return RecurrentResult(new_h, [new_h])
def lstm_cell(input_layer,
states,
num_units,
bias=tf.zeros_initializer(),
peephole=True,
weights=None,
phase=prettytensor.Phase.train,
parameter_modifier=parameters.identity):
"""Long short-term memory cell (LSTM).
Args:
input_layer: The input layer.
states: The current state of the network, as
[[batch, num_units], [batch, num_units]] (c, h).
num_units: How big is the hidden state.
bias: An initializer for the bias or a Tensor. No bias if set to None.
peephole: Whether to use peephole connections as described in
http://www.jmlr.org/papers/volume3/gers02a/gers02a.pdf
weights: An initializer for weights or a Tensor.
phase: The phase of graph construction. See `pt.Phase`.
parameter_modifier: A function to modify parameters that is applied after
creation and before use.
Returns:
A RecurrentResult.
"""
# As a compound op, it needs to respect whether or not this is a sequential
# builder.
if input_layer.is_sequential_builder():
layer = input_layer.as_layer()
else:
layer = input_layer
c, h = [prettytensor.wrap(state, layer.bookkeeper) for state in states]
activation_input = layer.fully_connected(
4 * num_units,
bias=bias,
activation_fn=None,
weights=weights,
parameter_modifier=parameter_modifier,
phase=phase)
activation_h = h.fully_connected(4 * num_units,
bias=None,
activation_fn=None,
weights=weights,
parameter_modifier=parameter_modifier,
phase=phase)
activation = activation_input + activation_h
# i = input_gate, j = new_input, f = forget_gate, o = output_gate
split = activation.split(1, 4)
i = split[0]
j = split[1]
f = split[2]
if bias is not None:
# Biases of the forget gate are initialized to 1 in order to reduce the
# scale of forgetting in the beginning of the training.
f += 1.
o = split[3]
if peephole:
# TODO(eiderman): It would be worthwhile to determine the best initialization.
i += c.diagonal_matrix_mul(weights=weights,
parameter_modifier=parameter_modifier,
phase=phase)
f += c.diagonal_matrix_mul(weights=weights,
parameter_modifier=parameter_modifier,
phase=phase)
f_gate = f.apply(tf.sigmoid, name='f_gate')
new_c = (c * f_gate +
i.apply(tf.sigmoid, name='i_gate') * j.apply(tf.tanh))
if peephole:
o += new_c.diagonal_matrix_mul(weights=weights,
parameter_modifier=parameter_modifier,
phase=phase)
new_h = new_c.apply(tf.tanh) * o.apply(tf.sigmoid, name='o_gate')
if input_layer.is_sequential_builder():
new_h = input_layer.set_head(input_layer)
return RecurrentResult(new_h, [new_c, new_h])
def lstm_cell(input_layer,
states,
num_units,
bias=True,
peephole=True,
stddev=None,
init=None):
"""Long short-term memory cell (LSTM).
Args:
input_layer: The input layer.
states: The current state of the network, as
[[batch, num_units], [batch, num_units]] (c, h).
num_units: How big is the hidden state.
bias: Whether or not to use a bias.
peephole: Whether to use peephole connections as described in
http://www.jmlr.org/papers/volume3/gers02a/gers02a.pdf
stddev: Standard deviation for Gaussian initialization of parameters.
init: A tf.*Initializer that is used to initialize the variables.
Returns:
A RecurrentResult.
"""
# As a compound op, it needs to respect whether or not this is a sequential
# builder.
if input_layer.is_sequential_builder():
layer = input_layer.as_layer()
else:
layer = input_layer
c, h = [prettytensor.wrap(state, layer.bookkeeper) for state in states]
activation_input = layer.fully_connected(4 * num_units,
bias=bias,
activation_fn=None,
stddev=stddev,
init=init)
activation_h = h.fully_connected(4 * num_units,
bias=False,
activation_fn=None,
stddev=stddev,
init=init)
activation = activation_input + activation_h
# i = input_gate, j = new_input, f = forget_gate, o = output_gate
split = activation.split(1, 4)
i = split[0]
j = split[1]
f = split[2]
if bias:
# Biases of the forget gate are initialized to 1 in order to reduce the
# scale of forgetting in the beginning of the training.
f += 1.
o = split[3]
if peephole:
# TODO(eiderman): It would be worthwhile to determine the best initialization.
i += c.diagonal_matrix_mul(stddev=stddev, init=init)
f += c.diagonal_matrix_mul(stddev=stddev, init=init)
new_c = (c * f.apply(tf.sigmoid, name='f_gate') +
i.apply(tf.sigmoid, name='i_gate') * j.apply(tf.tanh))
if peephole:
o += new_c.diagonal_matrix_mul(stddev=stddev, init=init)
new_h = new_c.apply(tf.tanh) * o.apply(tf.sigmoid, name='o_gate')
if input_layer.is_sequential_builder():
new_h = input_layer.set_head(input_layer)
return RecurrentResult(new_h, [new_c, new_h])
def __init__(self, input_fn, replay_size, batch_size=None):
"""Creates a ReplayableQueue that takes data from `input_fn`.
See also: `pt.train.ReplayableQueue.build_from_queue`.
Note: the shapes of the inputs must be fully defined.
Note: `input_fn` is a function instead of an input. This is because
otherwise if the input came from a queue, dependencies wouldn't be set up
properly and the data would always be dequeued. If you are providing data
from a queue, then pass in `lambda: q.dequeue_many(batch_size)`.
Args:
input_fn: A function of no arguments that returns the input as a tuple of
`Tensors`.
replay_size: The size of the replay queue.
batch_size: If provided, use this as the batch size otherwise infer it.
Raises:
ValueError: if `replay_size` is not divisible by `batch_size` or if the
shapes on the input are wrong.
"""
inputs = _make_tuple(input_fn())
for x in inputs:
x.get_shape().assert_is_fully_defined()
if batch_size is not None:
x.get_shape()[0].assert_is_compatible_with(batch_size)
else:
batch_size = x.get_shape()[0].value
dtypes = [x.dtype for x in inputs]
shapes = [x.get_shape()[1:] if x.get_shape() else () for x in inputs]
if replay_size % batch_size != 0:
raise ValueError('replay_size size (%d) must be a multiple of batch size '
'(%d)' % (replay_size, batch_size))
# Setup the flag that controls replay.
self._replay_var = tf.get_variable(
'replay',
dtype=tf.bool,
shape=[],
initializer=tf.constant_initializer(False),
trainable=False)
self._set_replay_ph = tf.placeholder(dtype=tf.bool)
self._set_replay = self._replay_var.assign(self._set_replay_ph)
self._replay_queue = tf.FIFOQueue(replay_size, dtypes, shapes)
# _fill_queue adds data to the queue and then returns whether it is full.
with tf.control_dependencies([self._replay_queue.enqueue_many(inputs)]):
self._fill_queue = tf.less(self._replay_queue.size(), replay_size)
# Dequeue all the things!
self._clear_queue = self._replay_queue.dequeue_many(
self._replay_queue.size())
def _pull_from_replay():
data_tuple = _make_tuple(self._replay_queue.dequeue_many(batch_size))
with tf.control_dependencies([self._replay_queue.enqueue_many(data_tuple)
]):
return (tf.identity(data_tuple[0]),) + data_tuple[1:]
def _pull_from_original():
return _make_tuple(input_fn())
self._output = prettytensor.wrap(
tf.cond(self._replay_var, _pull_from_replay, _pull_from_original))
def __init__(self, input_fn, replay_size, batch_size=None):
"""Creates a ReplayableQueue that takes data from `input_fn`.
See also: `pt.train.ReplayableQueue.build_from_queue`.
Note: the shapes of the inputs must be fully defined.
Note: `input_fn` is a function instead of an input. This is because
otherwise if the input came from a queue, dependencies wouldn't be set up
properly and the data would always be dequeued. If you are providing data
from a queue, then pass in `lambda: q.dequeue_many(batch_size)`.
Args:
input_fn: A function of no arguments that returns the input as a tuple of
`Tensors`.
replay_size: The size of the replay queue.
batch_size: If provided, use this as the batch size otherwise infer it.
Raises:
ValueError: if `replay_size` is not divisible by `batch_size` or if the
shapes on the input are wrong.
"""
inputs = _make_tuple(input_fn())
for x in inputs:
x.get_shape().assert_is_fully_defined()
if batch_size is not None:
x.get_shape()[0].assert_is_compatible_with(batch_size)
else:
batch_size = x.get_shape()[0].value
dtypes = [x.dtype for x in inputs]
shapes = [
x.get_shape()[1:] if x.get_shape() else () for x in inputs
]
if replay_size % batch_size != 0:
raise ValueError(
'replay_size size (%d) must be a multiple of batch size '
'(%d)' % (replay_size, batch_size))
# Setup the flag that controls replay.
self._replay_var = tf.get_variable(
'replay',
dtype=tf.bool,
shape=[],
initializer=tf.constant_initializer(False),
trainable=False)
self._set_replay_ph = tf.placeholder(dtype=tf.bool)
self._set_replay = self._replay_var.assign(self._set_replay_ph)
self._replay_queue = tf.FIFOQueue(replay_size, dtypes, shapes)
# _fill_queue adds data to the queue and then returns whether it is full.
with tf.control_dependencies([self._replay_queue.enqueue_many(inputs)
]):
self._fill_queue = tf.less(self._replay_queue.size(), replay_size)
# Dequeue all the things!
self._clear_queue = self._replay_queue.dequeue_many(
self._replay_queue.size())
def _pull_from_replay():
data_tuple = _make_tuple(
self._replay_queue.dequeue_many(batch_size))
with tf.control_dependencies(
[self._replay_queue.enqueue_many(data_tuple)]):
return (tf.identity(data_tuple[0]), ) + data_tuple[1:]
def _pull_from_original():
return _make_tuple(input_fn())
self._output = prettytensor.wrap(
tf.cond(self._replay_var, _pull_from_replay, _pull_from_original))
def gru_cell(input_layer,
state,
num_units,
bias=tf.zeros_initializer(),
weights=None,
phase=prettytensor.Phase.train,
parameter_modifier=parameters.identity):
"""Gated recurrent unit memory cell (GRU).
Args:
input_layer: The input layer.
state: The current state of the network. For GRUs, this is a list with
one element (tensor) of shape [batch, num_units].
num_units: How big is the hidden state.
bias: An initializer for the bias or a Tensor. No bias if set to None.
weights: An initializer for weights or a Tensor.
phase: The phase of graph construction. See `pt.Phase`.
parameter_modifier: A function to modify parameters that is applied after
creation and before use.
Returns:
A RecurrentResult.
"""
# As a compound op, it needs to respect whether or not this is a sequential
# builder.
if input_layer.is_sequential_builder():
layer = input_layer.as_layer()
else:
layer = input_layer
# We start with bias of 1.0 to not reset and not udpate.
# NB We compute activation_input and activation_state in two different ops,
# instead of concatenating them, followed by one matrix multiplication. The
# reason is that input has size [batch_size x input_size], while state has
# [ ? x state_size ], where the first dimension is 1 initially and will be
# batch_size only after the first RNN computation. We thus cannot concatenate
# input and state, and instead add the results of two fully connected ops,
# which works thanks to broadcasting, independent of state's batch size.
state = state[0]
state_pt = prettytensor.wrap(state, layer.bookkeeper)
activation_input = layer.fully_connected(
2 * num_units,
bias=None if bias is None else tf.constant_initializer(1.0),
activation_fn=None,
weights=weights,
phase=phase,
parameter_modifier=parameter_modifier)
activation_state = state_pt.fully_connected(
2 * num_units,
bias=None,
activation_fn=None,
weights=weights,
phase=phase,
parameter_modifier=parameter_modifier)
# adds batch_size x (2 * num_units) + ? x (2 * num_inputs)
activation = activation_input + activation_state
activation = activation.sigmoid()
split = activation.split(1, 2)
r = split[0]
u = split[1]
c = layer.concat(1, [r * state]).fully_connected(
num_units,
bias=bias,
activation_fn=None,
weights=weights,
phase=phase,
parameter_modifier=parameter_modifier).apply(tf.tanh)
new_h = u * state + (1 - u) * c
if input_layer.is_sequential_builder():
new_h = input_layer.set_head(input_layer)
return RecurrentResult(new_h, [new_h])
def lstm_cell(input_layer,
states,
num_units,
bias=tf.zeros_initializer(),
peephole=True,
weights=None,
phase=prettytensor.Phase.train,
parameter_modifier=parameters.identity):
"""Long short-term memory cell (LSTM).
Args:
input_layer: The input layer.
states: The current state of the network, as
[[batch, num_units], [batch, num_units]] (c, h).
num_units: How big is the hidden state.
bias: An initializer for the bias or a Tensor. No bias if set to None.
peephole: Whether to use peephole connections as described in
http://www.jmlr.org/papers/volume3/gers02a/gers02a.pdf
weights: An initializer for weights or a Tensor.
phase: The phase of graph construction. See `pt.Phase`.
parameter_modifier: A function to modify parameters that is applied after
creation and before use.
Returns:
A RecurrentResult.
"""
# As a compound op, it needs to respect whether or not this is a sequential
# builder.
if input_layer.is_sequential_builder():
layer = input_layer.as_layer()
else:
layer = input_layer
c, h = [prettytensor.wrap(state, layer.bookkeeper) for state in states]
activation_input = layer.fully_connected(
4 * num_units,
bias=bias,
activation_fn=None,
weights=weights,
parameter_modifier=parameter_modifier,
phase=phase)
activation_h = h.fully_connected(4 * num_units,
bias=None,
activation_fn=None,
weights=weights,
parameter_modifier=parameter_modifier,
phase=phase)
activation = activation_input + activation_h
# i = input_gate, j = new_input, f = forget_gate, o = output_gate
split = activation.split(1, 4)
i = split[0]
j = split[1]
f = split[2]
if bias is not None:
# Biases of the forget gate are initialized to 1 in order to reduce the
# scale of forgetting in the beginning of the training.
f += 1.
o = split[3]
if peephole:
# TODO(eiderman): It would be worthwhile to determine the best initialization.
i += c.diagonal_matrix_mul(weights=weights,
parameter_modifier=parameter_modifier,
phase=phase)
f += c.diagonal_matrix_mul(weights=weights,
parameter_modifier=parameter_modifier,
phase=phase)
f_gate = f.apply(tf.sigmoid, name='f_gate')
new_c = (c * f_gate + i.apply(tf.sigmoid, name='i_gate') * j.apply(tf.tanh))
if peephole:
o += new_c.diagonal_matrix_mul(weights=weights,
parameter_modifier=parameter_modifier,
phase=phase)
new_h = new_c.apply(tf.tanh) * o.apply(tf.sigmoid, name='o_gate')
if input_layer.is_sequential_builder():
new_h = input_layer.set_head(input_layer)
return RecurrentResult(new_h, [new_c, new_h])