def wavenet_prior(net, is_training):
from ops import causal_conv, gated_cnn, residual_block
import json
with open('wavenet.json') as file:
wavenet_parameters = json.load(file)
net = causal_conv(net, **wavenet_parameters['preprocess'])
skip_outputs = []
num_layers = len(wavenet_parameters['dilation_rates'])
print('num_layers:', num_layers)
print('net_0:', net.shape)
for i in range(num_layers):
layer_id = 'layer_%d' % (i + 1)
conv_args = wavenet_parameters['residual_stack']
conv_args['dilation_rate'] = wavenet_parameters['dilation_rates'][i]
net = tf.keras.layers.BatchNormalization()(net)
skip, net = residual_block(net, **conv_args)
print('net_%d:' % (i + 1), net.shape,
' dr: %d' % conv_args['dilation_rate'])
skip_outputs.append(skip)
net = tf.keras.layers.Dropout(0.5)(net, training=is_training)
net = sum(skip_outputs)
net = tf.keras.activations.relu(net)
net = causal_conv(net, **wavenet_parameters['postprocess1'])
print('net:', net.shape)
net = causal_conv(net, **wavenet_parameters['postprocess2'])
print('net:', net.shape)
shape = net.shape
net = tf.reshape(net, [-1, shape[1] * shape[2]])
return net
def _create_causal_layer(self, input_batch):
'''Creates a single causal convolution layer.
The layer can change the number of channels.
'''
with tf.name_scope('causal_layer'):
weights_filter = self.variables['causal_layer']['filter']
return causal_conv(input_batch, weights_filter, 1)
def causal_atrous_conv1d(value, filters, rate, padding):
# Using height in 2-D as the 1-D. Adding the batch dimension also.
# Note that for filters using 'SAME' padding, padding zeros are added to the end of the input.
# This means that for causal convolutions, we must shift the output right.
# add zeros to the start and remove the future values from the end.
value_with_batch = tf.expand_dims(value, 0)
# Normally we would use this, but in practice CuDNN does not have implementations for the strided convolutions
# so this only works for CPU.
# value_2d = tf.expand_dims(value_with_batch, 2)
# filters_2d = tf.expand_dims(filters, 1)
# atrous_conv = tf.nn.atrous_conv2d(value_2d, filters_2d, rate, padding)
# # Squeezing out the width and the batch dimensions.
# atr_conv_1d = tf.squeeze(atrous_conv, [0, 2])
# width = tf.shape(value)[0]
# filter_shape = tf.shape(filters)
# filter_width = filter_shape[0]
# filter_width_up = filter_width + (filter_width - 1) * (rate - 1)
# pad_width = filter_width_up - 1
# pad_left = pad_width // 2
# pad_right = pad_width - pad_left
# # We want to shift the result so that acausal values are removed.
# # Any value in the output that makes use of right padding values are acausal.
# # So, we remove pad_right elements from the end, and add as many zeros to the beginning.
# dilation_channels = tf.shape(atr_conv_1d)[1]
# causal = tf.pad(tf.slice(atr_conv_1d, [0, 0], [width - pad_right, dilation_channels]),
# [[pad_right, 0], [0, 0]])
# return causal
# Instead we use this implementation from Igor Babuschkin:
atr_conv_1d_with_batch = ops.causal_conv(value_with_batch, filters, rate)
atr_conv_1d = tf.squeeze(atr_conv_1d_with_batch, [0])
# atr_conv_1d shape is [width, dilation_channels]
return atr_conv_1d
def _create_dilation_layer(self, input_batch, layer_index, dilation,
global_condition_batch, output_width):
'''Creates a single causal dilated convolution layer.
Args:
input_batch: Input to the dilation layer.
layer_index: Integer indicating which layer this is.
dilation: Integer specifying the dilation size.
global_conditioning_batch: Tensor containing the global data upon
which the output is to be conditioned upon. Shape:
[batch size, 1, channels]. The 1 is for the axis
corresponding to time so that the result is broadcast to
all time steps.
The layer contains a gated filter that connects to dense output
and to a skip connection:
|-> [gate] -| |-> 1x1 conv -> skip output
| |-> (*) -|
input -|-> [filter] -| |-> 1x1 conv -|
| |-> (+) -> dense output
|------------------------------------|
Where `[gate]` and `[filter]` are causal convolutions with a
non-linear activation at the output. Biases and global conditioning
are omitted due to the limits of ASCII art.
'''
variables = self.variables['dilated_stack'][layer_index]
weights_filter = variables['filter']
weights_gate = variables['gate']
#weights_filter = tf.Print(weights_filter,["weights_filter:",weights_filter])
conv_filter = causal_conv(input_batch, weights_filter, dilation)
conv_gate = causal_conv(input_batch, weights_gate, dilation)
if global_condition_batch is not None:
weights_gc_filter = variables['gc_filtweights']
conv_filter = conv_filter + tf.nn.conv1d(global_condition_batch,
weights_gc_filter,
stride=1,
padding="SAME",
name="gc_filter")
weights_gc_gate = variables['gc_gateweights']
conv_gate = conv_gate + tf.nn.conv1d(global_condition_batch,
weights_gc_gate,
stride=1,
padding="SAME",
name="gc_gate")
if self.use_biases:
filter_bias = variables['filter_bias']
gate_bias = variables['gate_bias']
conv_filter = tf.add(conv_filter, filter_bias)
conv_gate = tf.add(conv_gate, gate_bias)
out = tf.tanh(conv_filter) * tf.sigmoid(conv_gate)
# The 1x1 conv to produce the residual output
weights_dense = variables['dense']
transformed = tf.nn.conv1d(out,
weights_dense,
stride=1,
padding="SAME",
name="dense")
# The 1x1 conv to produce the skip output
skip_cut = tf.shape(out)[1] - output_width
out_skip = tf.slice(out, [0, skip_cut, 0], [-1, -1, -1])
out_skip = out_skip
weights_skip = variables['skip']
skip_contribution = tf.nn.conv1d(out_skip,
weights_skip,
stride=1,
padding="SAME",
name="skip")
if self.use_biases:
dense_bias = variables['dense_bias']
skip_bias = variables['skip_bias']
transformed = transformed + dense_bias
skip_contribution = skip_contribution + skip_bias
if self.histograms:
layer = 'layer{}'.format(layer_index)
tf.histogram_summary(layer + '_filter', weights_filter)
tf.histogram_summary(layer + '_gate', weights_gate)
tf.histogram_summary(layer + '_dense', weights_dense)
tf.histogram_summary(layer + '_skip', weights_skip)
if self.use_biases:
tf.histogram_summary(layer + '_biases_filter', filter_bias)
tf.histogram_summary(layer + '_biases_gate', gate_bias)
tf.histogram_summary(layer + '_biases_dense', dense_bias)
tf.histogram_summary(layer + '_biases_skip', skip_bias)
input_cut = tf.shape(input_batch)[1] - tf.shape(transformed)[1]
input_batch = tf.slice(input_batch, [0, input_cut, 0], [-1, -1, -1])
self.denseout = input_batch + transformed
self.skip = skip_contribution
return skip_contribution, input_batch + transformed
def residual_block(self,
inputs,
index_stack,
dilation,
nb_filters,
kernel_size,
dropout_rate=0.0,
activation="relu",
is_training=True):
"""Defines the residual block for the WaveNet TCN
Args:
inputs: The previous layer in the model
index_stack: The stack index i.e. which stack in the overall TCN
dilation: The dilation power of 2 we are using for this residual block
nb_filters: The number of convolutional filters to use in this block
kernel_size: The size of the convolutional kernel
activation: The name of the type of activation to use
dropout_rate: Float between 0 and 1. Fraction of the input units to drop.
Returns:
A tuple where the first element is the residual model layer, and the second is the
skip connection
"""
original_x = inputs
with tf.variable_scope("residual_block", reuse=tf.AUTO_REUSE):
with tf.variable_scope("dilated_causal_conv_1"):
# filter_ = tf.get_variable("filter_", shape = [kernel_size, nb_filters, nb_filters], dtype = tf.float32)
filter_shape = [kernel_size, nb_filters, nb_filters]
x = causal_conv(inputs, filter_shape, dilation)
print(x.shape)
with tf.variable_scope("layer_norm_1"):
x = tf.contrib.layers.layer_norm(x)
with tf.variable_scope("activation_1"):
if activation == "norm_relu":
x = tf.nn.relu(x)
x = channel_normalization(x)
elif activation == "wavenet":
x = wave_net_activation(x)
else:
x = tf.nn.relu(x)
with tf.variable_scope("dropout_1"):
x = tf.contrib.layers.dropout(x,
keep_prob=dropout_rate,
noise_shape=[1, 1, nb_filters],
is_training=is_training)
with tf.variable_scope("dilated_causal_conv_2"):
# filter_ = tf.get_variable("filter_", shape = [kernel_size, nb_filters, nb_filters], dtype = tf.float32)
filter_shape = [kernel_size, nb_filters, nb_filters]
x = causal_conv(x, filter_shape, dilation)
with tf.variable_scope("layer_norm_2"):
x = tf.contrib.layers.layer_norm(x)
with tf.variable_scope("activation_2"):
if activation == "norm_relu":
x = tf.nn.relu(x)
x = channel_normalization(x)
elif activation == "wavenet":
x = wave_net_activation(x)
else:
x = tf.nn.relu(x)
with tf.variable_scope("dropout_2"):
x = tf.contrib.layers.dropout(x,
keep_prob=dropout_rate,
noise_shape=[1, 1, nb_filters],
is_training=is_training)
original_x = tf.layers.Conv1D(filters=nb_filters,
kernel_size=1)(original_x)
res_x = tf.add(original_x, x)
return res_x, x