def _create_dilation_layer(self, input_batch, layer_index, dilation,
in_channels, dilation_channels):
'''Adds a single causal dilated convolution layer.'''
weights_filter = tf.Variable(
tf.truncated_normal(
[self.filter_width, in_channels, dilation_channels],
stddev=0.2,
name="filter"))
weights_gate = tf.Variable(
tf.truncated_normal(
[self.filter_width, in_channels, dilation_channels],
stddev=0.2,
name="gate"))
conv_filter = causal_conv(input_batch, weights_filter, dilation)
conv_gate = causal_conv(input_batch, weights_gate, dilation)
out = tf.tanh(conv_filter) * tf.sigmoid(conv_gate)
weights_dense = tf.Variable(
tf.truncated_normal([1, dilation_channels, in_channels],
stddev=0.2,
name="dense"))
transformed = tf.nn.conv1d(out,
weights_dense,
stride=1,
padding="SAME",
name="dense")
layer = 'layer{}'.format(layer_index)
tf.histogram_summary(layer + '_filter', weights_filter)
tf.histogram_summary(layer + '_gate', weights_gate)
tf.histogram_summary(layer + '_dense', weights_dense)
return transformed, input_batch + transformed
def testNoTimeShift(self):
# Input to filter is a time series of values 1..10
x = np.arange(1, 11, dtype=np.float32)
# Reshape the input: shape is batch item x duration x channels = 1x10x1
x = np.reshape(x, [1, 10, 1])
# Default shape ordering for conv filter = HWIO for 2d. Since we use
# 1d, this just becomes WxIxO where:
# W = width AKA number of time steps in time series = 2
# I = input channels = 1
# O = output channels = 1
# Since the filter is size 2, for it to be identity-preserving, one
# value is 1.0, the other 0.0
filter = np.reshape(np.array([0.0, 1.0], dtype=np.float32), [2, 1, 1])
# Compute the output
out = causal_conv(x, filter, dilation=2)
with self.test_session() as sess:
result = sess.run(out)
# The shapes should be the same.
self.assertAllEqual(result.shape, x.shape)
# The output time series should be identical to the input series.
self.assertAllEqual(result, x)
def _create_causal_layer(self, input_batch, in_channels, out_channels):
with tf.name_scope('causal_layer'):
weights_filter = tf.Variable(
tf.truncated_normal(
[self.filter_width, in_channels, out_channels],
stddev=0.2,
name="filter"))
return causal_conv(input_batch, weights_filter, 1)
def _create_causal_layer(self, input_batch, in_channels, out_channels):
'''Creates a single causal convolution layer.
The layer can change the number of channels.
'''
with tf.name_scope('causal_layer'):
weights_filter = create_variable(
"filter", [self.filter_width, in_channels, out_channels])
return causal_conv(input_batch, weights_filter, 1)
def _create_causal_layer(self, input_batch, in_channels, out_channels):
'''Creates a single causal convolution layer.
The layer can change the number of channels.
'''
with tf.name_scope('causal_layer'):
weights_filter = tf.Variable(
tf.truncated_normal(
[self.filter_width, in_channels, out_channels],
stddev=0.2,
name="filter"))
return causal_conv(input_batch, weights_filter, 1)
def testCausalConv(self):
x1 = np.arange(1, 21, dtype=np.float32)
x = np.append(x1, x1)
x = np.reshape(x, [2, 20, 1])
f = np.reshape(np.array([1, 1], dtype=np.float32), [2, 1, 1])
out = causal_conv(x, f, 4)
with self.test_session() as sess:
result = sess.run(out)
# Causal convolution using numpy
ref = np.convolve(x1, [1, 0, 0, 0, 1])[:-4]
ref = np.append(ref, ref)
ref = np.reshape(ref, [2, 20, 1])
self.assertAllEqual(result, ref)
def _create_dilation_layer(self, input_batch, layer_index, dilation,
in_channels, dilation_channels, skip_channels):
'''Creates a single causal dilated convolution layer.
The layer contains a gated filter that connects to dense output
and to a skip connection:
|-> [gate] -| |-> 1x1 conv -> skip output
| |-> (*) -|
input -|-> [filter] -| |-> 1x1 conv -|
| |-> (+) -> dense output
|------------------------------------|
Where `[gate]` and `[filter]` are causal convolutions with a
non-linear activation at the output.
'''
weights_filter = create_variable(
"filter", [self.filter_width, in_channels, dilation_channels])
weights_gate = create_variable(
"gate", [self.filter_width, in_channels, dilation_channels])
conv_filter = causal_conv(input_batch, weights_filter, dilation)
conv_gate = causal_conv(input_batch, weights_gate, dilation)
if self.use_biases:
biases_filter = create_bias_variable("filter_biases",
[dilation_channels])
biases_gate = create_bias_variable("filter_biases",
[dilation_channels])
conv_filter = tf.add(conv_filter, biases_filter)
conv_gate = tf.add(conv_gate, biases_gate)
out = tf.tanh(conv_filter) * tf.sigmoid(conv_gate)
# The 1x1 conv to produce the dense contribution.
weights_dense = create_variable("dense",
[1, dilation_channels, in_channels])
transformed = tf.nn.conv1d(out,
weights_dense,
stride=1,
padding="SAME",
name="dense")
# The 1x1 conv to produce the skip contribution.
weights_skip = create_variable("skip",
[1, dilation_channels, skip_channels])
skip_contribution = tf.nn.conv1d(out,
weights_skip,
stride=1,
padding="SAME",
name="skip")
if self.use_biases:
biases_dense = create_bias_variable("dense_biases", [in_channels])
transformed = tf.add(transformed, biases_dense)
biases_skip = create_bias_variable("skip_biases", [skip_channels])
skip_contribution = tf.add(skip_contribution, biases_skip)
layer = 'layer{}'.format(layer_index)
tf.histogram_summary(layer + '_filter', weights_filter)
tf.histogram_summary(layer + '_gate', weights_gate)
tf.histogram_summary(layer + '_dense', weights_dense)
tf.histogram_summary(layer + '_skip', weights_skip)
if self.use_biases:
tf.histogram_summary(layer + '_biases_filter', biases_filter)
tf.histogram_summary(layer + '_biases_gate', biases_gate)
tf.histogram_summary(layer + '_biases_dense', biases_dense)
tf.histogram_summary(layer + '_biases_skip', biases_skip)
return skip_contribution, input_batch + transformed
def _create_dilation_layer(self, input_batch, layer_index, dilation,
in_channels, dilation_channels, skip_channels):
'''Creates a single causal dilated convolution layer.'''
weights_filter = tf.Variable(
tf.truncated_normal(
[self.filter_width, in_channels, dilation_channels],
stddev=0.2,
name="filter"))
weights_gate = tf.Variable(
tf.truncated_normal(
[self.filter_width, in_channels, dilation_channels],
stddev=0.2,
name="gate"))
conv_filter = causal_conv(input_batch, weights_filter, dilation)
conv_gate = causal_conv(input_batch, weights_gate, dilation)
if self.use_biases:
biases_filter = tf.Variable(
tf.constant(
0.0, shape=[dilation_channels]),
name="filter_biases")
biases_gate = tf.Variable(
tf.constant(
0.0, shape=[dilation_channels]),
name="gate_biases")
conv_filter = tf.add(conv_filter, biases_filter)
conv_gate = tf.add(conv_gate, biases_gate)
out = tf.tanh(conv_filter) * tf.sigmoid(conv_gate)
weights_dense = tf.Variable(
tf.truncated_normal(
[1, dilation_channels, in_channels], stddev=0.2, name="dense"))
transformed = tf.nn.conv1d(
out, weights_dense, stride=1, padding="SAME", name="dense")
# The 1x1 conv to produce the skip contribution.
weights_skip = tf.Variable(
tf.truncated_normal(
[1, dilation_channels, skip_channels], stddev=0.01),
name="skip")
skip_contribution = tf.nn.conv1d(
out, weights_skip, stride=1, padding="SAME", name="skip")
if self.use_biases:
biases_dense = tf.Variable(
tf.constant(
0.0, shape=[in_channels]), name="dense_biases")
transformed = tf.add(transformed, biases_dense)
biases_skip = tf.Variable(
tf.constant(
0.0, shape=[skip_channels]), name="skip_biases")
skip_contribution = tf.add(skip_contribution, biases_skip)
layer = 'layer{}'.format(layer_index)
tf.histogram_summary(layer + '_filter', weights_filter)
tf.histogram_summary(layer + '_gate', weights_gate)
tf.histogram_summary(layer + '_dense', weights_dense)
tf.histogram_summary(layer + '_skip', weights_skip)
if self.use_biases:
tf.histogram_summary(layer + '_biases_filter', biases_filter)
tf.histogram_summary(layer + '_biases_gate', biases_gate)
tf.histogram_summary(layer + '_biases_dense', biases_dense)
tf.histogram_summary(layer + '_biases_skip', biases_skip)
return skip_contribution, input_batch + transformed
