def _create_dilation_layer(self, input_batch, layer_index, dilation, in_channels, dilation_channels): '''Adds a single causal dilated convolution layer.''' weights_filter = tf.Variable( tf.truncated_normal( [self.filter_width, in_channels, dilation_channels], stddev=0.2, name="filter")) weights_gate = tf.Variable( tf.truncated_normal( [self.filter_width, in_channels, dilation_channels], stddev=0.2, name="gate")) conv_filter = causal_conv(input_batch, weights_filter, dilation) conv_gate = causal_conv(input_batch, weights_gate, dilation) out = tf.tanh(conv_filter) * tf.sigmoid(conv_gate) weights_dense = tf.Variable( tf.truncated_normal([1, dilation_channels, in_channels], stddev=0.2, name="dense")) transformed = tf.nn.conv1d(out, weights_dense, stride=1, padding="SAME", name="dense") layer = 'layer{}'.format(layer_index) tf.histogram_summary(layer + '_filter', weights_filter) tf.histogram_summary(layer + '_gate', weights_gate) tf.histogram_summary(layer + '_dense', weights_dense) return transformed, input_batch + transformed
def testNoTimeShift(self): # Input to filter is a time series of values 1..10 x = np.arange(1, 11, dtype=np.float32) # Reshape the input: shape is batch item x duration x channels = 1x10x1 x = np.reshape(x, [1, 10, 1]) # Default shape ordering for conv filter = HWIO for 2d. Since we use # 1d, this just becomes WxIxO where: # W = width AKA number of time steps in time series = 2 # I = input channels = 1 # O = output channels = 1 # Since the filter is size 2, for it to be identity-preserving, one # value is 1.0, the other 0.0 filter = np.reshape(np.array([0.0, 1.0], dtype=np.float32), [2, 1, 1]) # Compute the output out = causal_conv(x, filter, dilation=2) with self.test_session() as sess: result = sess.run(out) # The shapes should be the same. self.assertAllEqual(result.shape, x.shape) # The output time series should be identical to the input series. self.assertAllEqual(result, x)
def _create_causal_layer(self, input_batch, in_channels, out_channels): with tf.name_scope('causal_layer'): weights_filter = tf.Variable( tf.truncated_normal( [self.filter_width, in_channels, out_channels], stddev=0.2, name="filter")) return causal_conv(input_batch, weights_filter, 1)
def _create_causal_layer(self, input_batch, in_channels, out_channels): '''Creates a single causal convolution layer. The layer can change the number of channels. ''' with tf.name_scope('causal_layer'): weights_filter = create_variable( "filter", [self.filter_width, in_channels, out_channels]) return causal_conv(input_batch, weights_filter, 1)
def _create_causal_layer(self, input_batch, in_channels, out_channels): '''Creates a single causal convolution layer. The layer can change the number of channels. ''' with tf.name_scope('causal_layer'): weights_filter = tf.Variable( tf.truncated_normal( [self.filter_width, in_channels, out_channels], stddev=0.2, name="filter")) return causal_conv(input_batch, weights_filter, 1)
def testCausalConv(self): x1 = np.arange(1, 21, dtype=np.float32) x = np.append(x1, x1) x = np.reshape(x, [2, 20, 1]) f = np.reshape(np.array([1, 1], dtype=np.float32), [2, 1, 1]) out = causal_conv(x, f, 4) with self.test_session() as sess: result = sess.run(out) # Causal convolution using numpy ref = np.convolve(x1, [1, 0, 0, 0, 1])[:-4] ref = np.append(ref, ref) ref = np.reshape(ref, [2, 20, 1]) self.assertAllEqual(result, ref)
def _create_dilation_layer(self, input_batch, layer_index, dilation, in_channels, dilation_channels, skip_channels): '''Creates a single causal dilated convolution layer. The layer contains a gated filter that connects to dense output and to a skip connection: |-> [gate] -| |-> 1x1 conv -> skip output | |-> (*) -| input -|-> [filter] -| |-> 1x1 conv -| | |-> (+) -> dense output |------------------------------------| Where `[gate]` and `[filter]` are causal convolutions with a non-linear activation at the output. ''' weights_filter = create_variable( "filter", [self.filter_width, in_channels, dilation_channels]) weights_gate = create_variable( "gate", [self.filter_width, in_channels, dilation_channels]) conv_filter = causal_conv(input_batch, weights_filter, dilation) conv_gate = causal_conv(input_batch, weights_gate, dilation) if self.use_biases: biases_filter = create_bias_variable("filter_biases", [dilation_channels]) biases_gate = create_bias_variable("filter_biases", [dilation_channels]) conv_filter = tf.add(conv_filter, biases_filter) conv_gate = tf.add(conv_gate, biases_gate) out = tf.tanh(conv_filter) * tf.sigmoid(conv_gate) # The 1x1 conv to produce the dense contribution. weights_dense = create_variable("dense", [1, dilation_channels, in_channels]) transformed = tf.nn.conv1d(out, weights_dense, stride=1, padding="SAME", name="dense") # The 1x1 conv to produce the skip contribution. weights_skip = create_variable("skip", [1, dilation_channels, skip_channels]) skip_contribution = tf.nn.conv1d(out, weights_skip, stride=1, padding="SAME", name="skip") if self.use_biases: biases_dense = create_bias_variable("dense_biases", [in_channels]) transformed = tf.add(transformed, biases_dense) biases_skip = create_bias_variable("skip_biases", [skip_channels]) skip_contribution = tf.add(skip_contribution, biases_skip) layer = 'layer{}'.format(layer_index) tf.histogram_summary(layer + '_filter', weights_filter) tf.histogram_summary(layer + '_gate', weights_gate) tf.histogram_summary(layer + '_dense', weights_dense) tf.histogram_summary(layer + '_skip', weights_skip) if self.use_biases: tf.histogram_summary(layer + '_biases_filter', biases_filter) tf.histogram_summary(layer + '_biases_gate', biases_gate) tf.histogram_summary(layer + '_biases_dense', biases_dense) tf.histogram_summary(layer + '_biases_skip', biases_skip) return skip_contribution, input_batch + transformed
def _create_dilation_layer(self, input_batch, layer_index, dilation, in_channels, dilation_channels, skip_channels): '''Creates a single causal dilated convolution layer.''' weights_filter = tf.Variable( tf.truncated_normal( [self.filter_width, in_channels, dilation_channels], stddev=0.2, name="filter")) weights_gate = tf.Variable( tf.truncated_normal( [self.filter_width, in_channels, dilation_channels], stddev=0.2, name="gate")) conv_filter = causal_conv(input_batch, weights_filter, dilation) conv_gate = causal_conv(input_batch, weights_gate, dilation) if self.use_biases: biases_filter = tf.Variable( tf.constant( 0.0, shape=[dilation_channels]), name="filter_biases") biases_gate = tf.Variable( tf.constant( 0.0, shape=[dilation_channels]), name="gate_biases") conv_filter = tf.add(conv_filter, biases_filter) conv_gate = tf.add(conv_gate, biases_gate) out = tf.tanh(conv_filter) * tf.sigmoid(conv_gate) weights_dense = tf.Variable( tf.truncated_normal( [1, dilation_channels, in_channels], stddev=0.2, name="dense")) transformed = tf.nn.conv1d( out, weights_dense, stride=1, padding="SAME", name="dense") # The 1x1 conv to produce the skip contribution. weights_skip = tf.Variable( tf.truncated_normal( [1, dilation_channels, skip_channels], stddev=0.01), name="skip") skip_contribution = tf.nn.conv1d( out, weights_skip, stride=1, padding="SAME", name="skip") if self.use_biases: biases_dense = tf.Variable( tf.constant( 0.0, shape=[in_channels]), name="dense_biases") transformed = tf.add(transformed, biases_dense) biases_skip = tf.Variable( tf.constant( 0.0, shape=[skip_channels]), name="skip_biases") skip_contribution = tf.add(skip_contribution, biases_skip) layer = 'layer{}'.format(layer_index) tf.histogram_summary(layer + '_filter', weights_filter) tf.histogram_summary(layer + '_gate', weights_gate) tf.histogram_summary(layer + '_dense', weights_dense) tf.histogram_summary(layer + '_skip', weights_skip) if self.use_biases: tf.histogram_summary(layer + '_biases_filter', biases_filter) tf.histogram_summary(layer + '_biases_gate', biases_gate) tf.histogram_summary(layer + '_biases_dense', biases_dense) tf.histogram_summary(layer + '_biases_skip', biases_skip) return skip_contribution, input_batch + transformed