def testGhostBNNegativeVirtualBatch(self):
shape = [6, 5, 4, 3]
inp = random_ops.random_uniform(shape, seed=1)
with self.assertRaises(ValueError):
normalization_layers.batch_normalization(
inp, num_virtual_batches=-1)
def testGhostBNVirtualBatch1(self):
shape = [6, 5, 4, 3]
inp = random_ops.random_uniform(shape, seed=1)
out1 = normalization_layers.batch_normalization(inp)
out2 = normalization_layers.batch_normalization(
inp, num_virtual_batches=1)
self.assertListEqual(
out1.shape.as_list(), out2.shape.as_list())
with self.test_session(use_gpu=True) as sess:
sess.run(variables.global_variables_initializer())
x = np.random.random(shape)
y1, y2 = sess.run([out1, out2], feed_dict={inp: x})
self.assertAllClose(y1, y2, atol=1e-5)
def dnn_logit_fn(features, mode):
"""Deep Neural Network logit_fn.
Args:
features: This is the first item returned from the `input_fn`
passed to `train`, `evaluate`, and `predict`. This should be a
single `Tensor` or `dict` of same.
mode: Optional. Specifies if this training, evaluation or prediction. See
`ModeKeys`.
Returns:
A `Tensor` representing the logits, or a list of `Tensor`'s representing
multiple logits in the MultiHead case.
"""
is_training = mode == model_fn.ModeKeys.TRAIN
with variable_scope.variable_scope(
'input_from_feature_columns',
values=tuple(six.itervalues(features)),
partitioner=input_layer_partitioner):
net = feature_column_lib.input_layer(
features=features, feature_columns=feature_columns)
for layer_id, num_hidden_units in enumerate(hidden_units):
with variable_scope.variable_scope(
'hiddenlayer_%d' % layer_id, values=(net,)) as hidden_layer_scope:
net = core_layers.dense(
net,
units=num_hidden_units,
activation=activation_fn,
kernel_initializer=init_ops.glorot_uniform_initializer(),
name=hidden_layer_scope)
if dropout is not None and is_training:
net = core_layers.dropout(net, rate=dropout, training=True)
if batch_norm:
# TODO(hjm): In future, if this becomes popular, we can enable
# customization of the batch normalization params by accepting a
# list of `BatchNormalization` instances as `batch_norm`.
net = normalization.batch_normalization(
net,
# The default momentum 0.99 actually crashes on certain
# problem, so here we use 0.999, which is the default of
# tf.contrib.layers.batch_norm.
momentum=0.999,
training=is_training,
name='batchnorm_%d' % layer_id)
_add_hidden_layer_summary(net, hidden_layer_scope.name)
with variable_scope.variable_scope('logits', values=(net,)) as logits_scope:
logits = core_layers.dense(
net,
units=units,
activation=None,
kernel_initializer=init_ops.glorot_uniform_initializer(),
name=logits_scope)
_add_hidden_layer_summary(logits, logits_scope.name)
return logits
def layer_with_recompute(inputs, is_recomputing=False):
kwarg_values.append(is_recomputing)
out = core_layers.dense(inputs, 2)
out = normalization_layers.batch_normalization(out, training=True)
if is_recomputing:
# Ensure that the updates are not duplicated by popping off the latest
# 2 additions.
update_ops = ops.get_collection_ref(ops.GraphKeys.UPDATE_OPS)
update_ops.pop()
update_ops.pop()
return out
def testGhostBNUnknownBatchSize(self):
np_shape = [10, 5, 4]
tf_shape = [None, 5, 4]
inp = array_ops.placeholder(dtypes.float32, tf_shape)
out = normalization_layers.batch_normalization(
inp, num_virtual_batches=5)
with self.test_session(use_gpu=True) as sess:
sess.run(variables.global_variables_initializer())
x = np.random.random(np_shape)
y = sess.run(out, feed_dict={inp: x})
self.assertListEqual(list(y.shape), np_shape)
def testBatchNormalizeLayerFp16(self):
with self.session() as sess:
with ops.device("/device:IPU:0"):
with variable_scope.variable_scope("", use_resource=True):
a = array_ops.placeholder(np.float16, [4, 64, 64, 4],
name="input_a")
normed = layers_norm.batch_normalization(a, fused=False)
report = ReportJSON(self, sess)
sess.run(variables.global_variables_initializer())
report.reset()
result = sess.run(normed, {a: np.zeros([4, 64, 64, 4])})
self.assertAllClose(result, np.zeros([4, 64, 64, 4]))
report.parse_log()
bl = ['*convert*/Cast*']
report.assert_compute_sets_not_in_blacklist(bl)
report.assert_tensor_input_names("input_a")
def testConvWithBnAndRelu(self):
with ops.device("/device:IPU:0"):
x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2])
with variable_scope.variable_scope("vs", use_resource=True):
y = layers.Conv2D(
2,
1,
use_bias=True,
kernel_initializer=init_ops.ones_initializer())(x)
y = layers_norm.batch_normalization(y, fused=True)
y = nn_ops.relu(y)
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system(True, True, True)
with tu.ipu_session() as sess:
sess.run(variables.global_variables_initializer())
sess.run(report)
sess.run(y, {x: np.zeros([1, 4, 4, 2])})
result = sess.run(report)
self.assertEqual(
len(result),
6) # 2xcompile, 1xupload 1xload, 1xdownload, 1xexecute
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
ok = [
'__seed*', 'host-exchange-local-copy', 'Copy_',
'vs/conv2d/Conv2D/convolution.*/Conv_1x1', 'vs/conv2d/BiasAdd',
'vs/batch_normalization/FusedBatchNorm/batch-norm-inference.*/',
'vs/Relu/custom-call/Nonlinearity'
]
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def test_reference_batch_normalization(self):
"""Check that batch norm from VBN agrees with opensource implementation."""
random_seed.set_random_seed(1234)
batch = random_ops.random_normal([6, 5, 7, 3, 3])
for axis in range(5):
# Get `layers` batchnorm result.
bn_normalized = normalization.batch_normalization(
batch, axis, training=True)
# Get VBN's batch normalization on reference batch.
batch_axis = 0 if axis != 0 else 1 # axis and batch_axis can't same
vbn = virtual_batchnorm.VBN(batch, axis, batch_axis=batch_axis)
vbn_normalized = vbn.reference_batch_normalization()
with self.cached_session(use_gpu=True) as sess:
variables_lib.global_variables_initializer().run()
bn_normalized_np, vbn_normalized_np = sess.run(
[bn_normalized, vbn_normalized])
self.assertAllClose(bn_normalized_np, vbn_normalized_np)
def testBatchNormalizeLayerWithStableStatistics(self):
with self.session() as sess:
with ops.device("/device:IPU:0"):
with variable_scope.variable_scope("", use_resource=True):
a = array_ops.placeholder(np.float32, [4, 64, 64, 4],
name="input_a")
normed = layers_norm.batch_normalization(a, training=True)
ReportJSON(self, sess, use_stable_norm_statistics=True)
sess.run(variables.global_variables_initializer())
# Use a tensor with large mean to test the stability. This blows up with
# the non-stable implementation (NaN output). Use a power-of-two that can
# be represented exactly in float32 to make sure we work with an exact
# mean internally.
input_mean = 2.0**64
inputs = input_mean * np.ones([4, 64, 64, 4])
# y = gamma * (x - mean) / sqrt(variance + epsilon) + beta
# Both (x - mean) and beta_initializer are zero, so this should be zero.
result = sess.run(normed, {a: inputs})
self.assertAllEqual(result, np.zeros([4, 64, 64, 4]))
def test_reference_batch_normalization(self):
"""Check that batch norm from VBN agrees with opensource implementation."""
random_seed.set_random_seed(1234)
batch = random_ops.random_normal([6, 5, 7, 3, 3])
for axis in range(5):
# Get `layers` batchnorm result.
bn_normalized = normalization.batch_normalization(
batch, axis, training=True)
# Get VBN's batch normalization on reference batch.
batch_axis = 0 if axis != 0 else 1 # axis and batch_axis can't same
vbn = virtual_batchnorm.VBN(batch, axis, batch_axis=batch_axis)
vbn_normalized = vbn.reference_batch_normalization()
with self.cached_session(use_gpu=True) as sess:
variables_lib.global_variables_initializer().run()
bn_normalized_np, vbn_normalized_np = sess.run(
[bn_normalized, vbn_normalized])
self.assertAllClose(bn_normalized_np, vbn_normalized_np)
def test_same_as_batchnorm(self):
"""Check that batch norm on set X is the same as ref of X / y on `y`."""
random_seed.set_random_seed(1234)
num_examples = 4
examples = [random_ops.random_normal([5, 7, 3]) for _ in
range(num_examples)]
# Get the result of the opensource batch normalization.
batch_normalized = normalization.batch_normalization(
array_ops.stack(examples), training=True)
for i in range(num_examples):
examples_except_i = array_ops.stack(examples[:i] + examples[i+1:])
# Get the result of VBN's batch normalization.
vbn = virtual_batchnorm.VBN(examples_except_i)
vb_normed = array_ops.squeeze(
vbn(array_ops.expand_dims(examples[i], [0])), [0])
with self.cached_session(use_gpu=True) as sess:
variables_lib.global_variables_initializer().run()
bn_np, vb_np = sess.run([batch_normalized, vb_normed])
self.assertAllClose(bn_np[i, ...], vb_np)
def test_same_as_batchnorm(self):
"""Check that batch norm on set X is the same as ref of X / y on `y`."""
random_seed.set_random_seed(1234)
num_examples = 4
examples = [random_ops.random_normal([5, 7, 3]) for _ in
range(num_examples)]
# Get the result of the opensource batch normalization.
batch_normalized = normalization.batch_normalization(
array_ops.stack(examples), training=True)
for i in range(num_examples):
examples_except_i = array_ops.stack(examples[:i] + examples[i+1:])
# Get the result of VBN's batch normalization.
vbn = virtual_batchnorm.VBN(examples_except_i)
vb_normed = array_ops.squeeze(
vbn(array_ops.expand_dims(examples[i], [0])), [0])
with self.cached_session(use_gpu=True) as sess:
variables_lib.global_variables_initializer().run()
bn_np, vb_np = sess.run([batch_normalized, vb_normed])
self.assertAllClose(bn_np[i, ...], vb_np)
def dnn_logit_fn(inputs, mode):
is_training = mode == ModeKeys.TRAIN
with variable_scope.variable_scope('input_from_feature_columns'):
dnn_inputs = []
for c in column_names:
dnn_inputs.append(inputs[c])
net = array_ops.concat(dnn_inputs, axis=1)
for layer_id, num_hidden_units in enumerate(hidden_units):
with variable_scope.variable_scope(
'hiddenlayer_%d' % layer_id,
values=(net, )) as hidden_layer_scope:
net = core_layers.dense(
net,
units=num_hidden_units,
activation=activation_fn,
kernel_initializer=init_ops.glorot_uniform_initializer(),
name=hidden_layer_scope)
if dropout is not None and is_training:
net = core_layers.dropout(net, rate=dropout, training=True)
if batch_norm:
net = normalization.batch_normalization(
net,
momentum=0.999,
training=is_training,
name='batchnorm_%d' % layer_id)
_add_hidden_layer_summary(net, hidden_layer_scope.name)
with variable_scope.variable_scope('logits',
values=(net, )) as logits_scope:
logits = core_layers.dense(
net,
units=units,
activation=None,
kernel_initializer=init_ops.glorot_uniform_initializer(),
name=logits_scope)
_add_hidden_layer_summary(logits, logits_scope.name)
return logits
def testBatchNormsMatchFwdBwd(self):
with ops.device("/device:IPU:0"):
x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2])
with variable_scope.variable_scope("vs", use_resource=True):
y = convolutional.conv2d(
x,
2,
1,
use_bias=False,
kernel_initializer=init_ops.ones_initializer(),
name='conv1')
y = layers_norm.batch_normalization(y,
fused=True,
training=True)
y = convolutional.conv2d(
y,
2,
1,
use_bias=False,
kernel_initializer=init_ops.ones_initializer(),
name='conv2')
y = layers_norm.batch_normalization(y,
fused=True,
training=True)
y = convolutional.conv2d(
y,
2,
1,
use_bias=False,
kernel_initializer=init_ops.ones_initializer(),
name='conv3')
y = layers_norm.batch_normalization(y,
fused=True,
training=True)
loss = math_ops.reduce_sum(y)
optimizer = gradient_descent.GradientDescentOptimizer(0.1)
train = optimizer.minimize(loss)
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system(True, True, True)
with tu.ipu_session() as sess:
sess.run(variables.global_variables_initializer())
sess.run(report)
sess.run([train, loss], {x: np.zeros([1, 4, 4, 2])})
result = sess.run(report)
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
# One BN for forwards and one BN for grad
# (note that we don't cache gradient application)
ok = [
'__seed*',
'host-exchange-local-copy-',
'Copy_',
'vs/conv1/Conv2D/convolution.*/Conv_1x1',
'vs/batch_normalization/FusedBatchNorm/batch-norm-training.*/',
'Sum/reduce.*/ReduceFinalStage/IntermediateToOutput/Reduce',
'gradients/vs/batch_normalization_2/FusedBatchNorm_grad/FusedBatchNormGrad/batch-norm-grad.*/',
'GradientDescent/update_vs/batch_normalization/',
'GradientDescent/update_vs/batch_normalization_1/',
'GradientDescent/update_vs/batch_normalization_2/',
'gradients/vs/conv3/Conv2D_grad/Conv2DBackpropFilter/fusion.*/Conv_4x4/Convolve',
'gradients/vs/conv3/Conv2D_grad/Conv2DBackpropFilter/fusion.*/AddTo',
]
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testBatchNormsMatchFwdBwdSomeOnShard0SomeOnShard1(self):
with self.session() as sess:
with ops.device("/device:IPU:0"):
x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2])
with variable_scope.variable_scope("vs", use_resource=True):
with ipu.scopes.ipu_shard(0):
y = convolutional.conv2d(
x,
2,
1,
use_bias=False,
kernel_initializer=init_ops.ones_initializer(),
name='conv1')
y = layers_norm.batch_normalization(y,
fused=True,
training=True)
y = convolutional.conv2d(
y,
2,
1,
use_bias=False,
kernel_initializer=init_ops.ones_initializer(),
name='conv2')
y = layers_norm.batch_normalization(y,
fused=True,
training=True)
with ipu.scopes.ipu_shard(1):
y = convolutional.conv2d(
y,
2,
1,
use_bias=False,
kernel_initializer=init_ops.ones_initializer(),
name='conv3')
y = layers_norm.batch_normalization(y,
fused=True,
training=True)
loss = math_ops.reduce_sum(y)
optimizer = gradient_descent.GradientDescentOptimizer(0.1)
train = optimizer.minimize(loss)
report = tu.ReportJSON(self, sess, sharded=True)
tu.move_variable_initialization_to_cpu()
sess.run(variables.global_variables_initializer())
report.reset()
sess.run([train, loss], {x: np.zeros([1, 4, 4, 2])})
report.parse_log()
# Two BN for forwards (on shards 0 and 1) and two BN for grad
# (note that we don't cache gradient application)
# pylint: disable=line-too-long
ok = [
'__seed*',
'*OnTileCopy*',
'Copy_',
'vs/conv1/Conv2D/convolution.*/Conv_1x1',
'vs/conv3/Conv2D/convolution.*/Conv_1x1',
'vs/batch_normalization/FusedBatchNorm*/batch-norm-training.*/',
'vs/batch_normalization_2/FusedBatchNorm*/batch-norm-training.*/',
'Sum/reduce.*/ReduceOnTile/InToIntermediateNoExchange/Reduce',
'Sum/reduce.*/ReduceFinalStage/IntermediateToOutput/Reduce',
'gradients/vs/batch_normalization_2/FusedBatchNorm*_grad/FusedBatchNormGrad*/batch-norm-grad.*/',
'gradients/vs/batch_normalization_1/FusedBatchNorm*_grad/FusedBatchNormGrad*/batch-norm-grad.*/',
'GradientDescent/update_vs/batch_normalization/',
'GradientDescent/update_vs/batch_normalization_1/',
'GradientDescent/update_vs/batch_normalization_2/',
'gradients/vs/conv3/Conv2D_grad/Conv2DBackpropFilter/fusion.*/AddTo',
'gradients/vs/conv3/Conv2D_grad/Conv2DBackpropFilter/fusion.*/Conv_4x4',
'gradients/vs/conv3/Conv2D_grad/Conv2DBackpropFilter/fusion.*/Transpose',
'gradients/vs/conv3/Conv2D_grad/Conv2DBackpropInput/fusion/*Transpose',
'gradients/vs/conv2/Conv2D_grad/Conv2DBackpropInput/fusion.*/*Transpose',
'gradients/vs/conv1/Conv2D_grad/Conv2DBackpropFilter/fusion.*/Conv_4x4',
'gradients/vs/conv1/Conv2D_grad/Conv2DBackpropFilter/fusion.*/AddTo',
'gradients/vs/conv1/Conv2D_grad/Conv2DBackpropFilter/fusion.*/Transpose',
]
# pylint: enable=line-too-long
report.assert_all_compute_sets_and_list(ok)
def build_generator(noise, caption, reuse=False):
with tf.variable_scope("generator") as scope:
if reuse:
tf.get_variable_scope().reuse_variables()
#
t_noise = noise
t_txt = caption
t_txt = dense(inputs=t_txt, units=128, activation=my_leaky_relu)
t_input = tf.concat(values=[t_noise, t_txt], axis=1)
t0 = dense(inputs=t_input, units=128 * 8 * 4 * 4)
t0 = batch_normalization(inputs=t0)
t0 = tf.reshape(tensor=t0, shape=[-1, 4, 4, 128 * 8])
t = conv2d(inputs=t0,
filters=256,
kernel_size=[1, 1],
strides=[1, 1],
padding="valid",
activation=my_leaky_relu)
t = batch_normalization(inputs=t)
t = conv2d(inputs=t,
filters=256,
kernel_size=[3, 3],
strides=[1, 1],
padding="same",
activation=my_leaky_relu)
t = batch_normalization(inputs=t)
t = conv2d(inputs=t,
filters=128 * 8,
kernel_size=[3, 3],
strides=[1, 1],
padding="same")
t = batch_normalization(inputs=t)
t1 = tf.add(t0, t)
t2 = conv2d_transpose(inputs=t1,
filters=128 * 4,
kernel_size=[3, 3],
strides=[2, 2],
padding="same")
t2 = batch_normalization(inputs=t2)
t = conv2d(inputs=t2,
filters=128,
kernel_size=[1, 1],
strides=[1, 1],
padding="valid",
activation=my_leaky_relu)
t = batch_normalization(inputs=t)
t = conv2d(inputs=t,
filters=128,
kernel_size=[3, 3],
strides=[1, 1],
padding="same",
activation=my_leaky_relu)
t = batch_normalization(inputs=t)
t = conv2d(inputs=t,
filters=128 * 4,
kernel_size=[3, 3],
strides=[1, 1],
padding="same")
t = batch_normalization(inputs=t)
t3 = tf.add(t2, t)
t4 = conv2d_transpose(inputs=t3,
filters=128 * 2,
kernel_size=[3, 3],
strides=[2, 2],
padding="same",
activation=my_leaky_relu)
t4 = batch_normalization(inputs=t4)
t5 = conv2d_transpose(inputs=t4,
filters=128,
kernel_size=[3, 3],
strides=[2, 2],
padding="same",
activation=my_leaky_relu)
t5 = batch_normalization(inputs=t5)
t_output = conv2d_transpose(inputs=t5,
filters=3,
kernel_size=[3, 3],
strides=[2, 2],
padding="same",
activation=tf.tanh)
print("\nGenerator Output Shape: {}".format(t_output.shape))
return t_output
def testGhostBNInputOutputShapesMatch(self):
shape = [6, 4, 3]
inp = random_ops.random_uniform(shape, seed=1)
out = normalization_layers.batch_normalization(
inp, num_virtual_batches=2)
self.assertListEqual(out.shape.as_list(), shape)
def build_discriminator(image, caption, reuse=False):
with tf.variable_scope("discriminator") as scope:
if reuse:
tf.get_variable_scope().reuse_variables()
#
t0 = image
t0 = conv2d(inputs=t0,
filters=64,
kernel_size=[4, 4],
strides=[2, 2],
padding="same",
activation=my_leaky_relu)
t1 = conv2d(inputs=t0,
filters=128,
kernel_size=[4, 4],
strides=[2, 2],
padding="same",
activation=my_leaky_relu)
t1 = batch_normalization(inputs=t1)
t2 = conv2d(inputs=t1,
filters=256,
kernel_size=[4, 4],
strides=[2, 2],
padding="same",
activation=my_leaky_relu)
t2 = batch_normalization(inputs=t2)
t3 = conv2d(inputs=t2,
filters=512,
kernel_size=[4, 4],
strides=[2, 2],
padding="same",
activation=None)
t3 = batch_normalization(inputs=t3)
t = conv2d(inputs=t3,
filters=128,
kernel_size=[1, 1],
strides=[1, 1],
padding="valid",
activation=my_leaky_relu)
t = batch_normalization(inputs=t)
t = conv2d(inputs=t,
filters=128,
kernel_size=[3, 3],
strides=[1, 1],
padding="same",
activation=my_leaky_relu)
t = batch_normalization(inputs=t)
t = conv2d(inputs=t,
filters=512,
kernel_size=[3, 3],
strides=[1, 1],
padding="same")
t = batch_normalization(inputs=t)
t4 = tf.add(t3, t)
#
t_txt = dense(inputs=caption, units=128, activation=my_leaky_relu)
t_txt = tf.expand_dims(input=t_txt, axis=1)
t_txt = tf.expand_dims(input=t_txt, axis=1)
t_txt = tf.tile(input=t_txt, multiples=[1, 4, 4, 1])
t_concat = tf.concat(values=[t4, t_txt], axis=3)
t4 = conv2d(inputs=t_concat,
filters=512,
kernel_size=[1, 1],
strides=[1, 1],
padding="valid",
activation=my_leaky_relu)
t4 = batch_normalization(inputs=t4)
t_output = conv2d(inputs=t4,
filters=1,
kernel_size=[4, 4],
strides=[4, 4],
padding="valid")
print("\nDiscriminator Output Shape: {}".format(t_output.shape))
return t_output
def inference(self, x, training=True):
logits = x
with tf.variable_scope("mfcc", self.reuse):
self.reuse = True
kernel = 3
filter_factor = 8
x = batch_normalization(x)
conv1_1 = dilated_conv1d_layer(x,
kernel=kernel,
num_filters=128 * filter_factor,
stride=1,
dilation=1,
name="conv1_1",
padding="SAME")
conv1_2 = dilated_conv1d_layer(x,
kernel=kernel,
num_filters=64 * filter_factor,
stride=1,
dilation=2,
name="conv1_2",
padding="SAME")
conv1_3 = dilated_conv1d_layer(x,
kernel=kernel,
num_filters=32 * filter_factor,
stride=1,
dilation=4,
name="conv1_3",
padding="SAME")
conv1_4 = dilated_conv1d_layer(x,
kernel=kernel,
num_filters=16 * filter_factor,
name="conv1_4",
stride=1,
dilation=8,
padding="SAME")
# conv1_5 = dilated_conv1d_layer(x, kernel=kernel, num_filters=8 * filter_factor, name="conv1_5", stride=1,
# dilation=16,
# padding="SAME")
# conv1_6 = dilated_conv1d_layer(x, kernel=kernel, num_filters=4 * filter_factor, name="conv1_6", stride=1,
# dilation=32,
# padding="SAME")
# conv1_7 = dilated_conv1d_layer(x, kernel=kernel, num_filters=2 * filter_factor, name="conv1_7", stride=1,
# dilation=64,
# padding="SAME")
# conv1_8 = dilated_conv1d_layer(x, kernel=kernel, num_filters=2 * filter_factor, name="conv1_8", stride=1,
# dilation=128,
# padding="SAME")
#
# conv1_merge = tf.concat((conv1_1, conv1_2, conv1_3, conv1_4, conv1_5, conv1_6, conv1_7, conv1_8), axis=2)
conv1_merge = tf.concat((conv1_1, conv1_2, conv1_3, conv1_4),
axis=2)
conv2_1 = dilated_conv1d_layer(conv1_1,
kernel=kernel,
num_filters=1024,
name="conv2_1",
stride=1,
dilation=2,
padding="SAME")
conv3_1 = dilated_conv1d_layer(conv2_1,
kernel=kernel,
num_filters=1024,
name="conv3_1",
stride=1,
dilation=2,
padding="SAME")
conv4_1 = dilated_conv1d_layer(conv3_1,
kernel=kernel,
num_filters=16,
name="conv4_1",
stride=1,
dilation=2,
padding="SAME")
self.layers["x"] = x
self.layers["conv1_1"] = conv1_1
self.layers["conv1_2"] = tf.reshape(
conv1_2,
[-1, x.get_shape()[1].value,
conv1_2.get_shape()[2].value],
name=None)
self.layers["conv1_3"] = tf.reshape(
conv1_3,
[-1, x.get_shape()[1].value,
conv1_3.get_shape()[2].value],
name=None)
self.layers["conv1_4"] = tf.reshape(
conv1_4,
[-1, x.get_shape()[1].value,
conv1_4.get_shape()[2].value],
name=None)
self.layers["conv_merge"] = tf.reshape(
conv1_merge,
[-1,
x.get_shape()[1].value,
conv1_merge.get_shape()[2].value],
name=None)
self.layers["conv3_1"] = conv3_1
if training:
pre_flatten = tf.reshape(
conv4_1,
[-1,
x.get_shape()[1].value,
conv4_1.get_shape()[2].value],
name='pre_flatten')
flatten = flatten1d(pre_flatten, name='flatten')
# fc1 = fully_connected(flatten, units=512, name='fc1')
# fc2 = fully_connected(fc1, units=256, name='fc2')
logits = fully_connected(flatten,
units=26,
name='logits',
activation=False)
return logits