def sg_upconv(tensor, opt):
r"""Applies a up convolution (or convolution transpose).
Args:
tensor: A 4-D `Tensor` (automatically passed by decorator).
opt:
size: A tuple/list of integers of length 2 representing `[kernel height, kernel width]`.
Can be an integer if both values are the same.
If not specified, (4, 4) is set implicitly.
stride: A tuple/list of integers of length 2 or 4 representing stride dimensions.
If the length is 2, i.e., (a, b), the stride is `[1, a, b, 1]`.
If the length is 4, i.e., (a, b, c, d), the stride is `[a, b, c, d]`.
Can be an integer. If the length is a, the stride is `[1, a, a, 1]`.
Default value is [1, 2, 2, 1].
in_dim: A positive `integer`. The size of input dimension.
dim: A positive `integer`. The size of output dimension.
pad: Either `SAME` (Default) or `VALID`.
bias: Boolean. If True, biases are added.
regularizer: A (Tensor -> Tensor or None) function; the result of applying it on a newly created variable
will be added to the collection tf.GraphKeys.REGULARIZATION_LOSSES and can be used for regularization
summary: If True, summaries are added. The default is True.
Returns:
A `Tensor` with the same type as `tensor`.
"""
# default options
opt += tf.sg_opt(size=(4, 4), stride=(1, 2, 2, 1), pad='SAME')
opt.size = opt.size if isinstance(opt.size, (tuple, list)) else [opt.size, opt.size]
opt.stride = opt.stride if isinstance(opt.stride, (tuple, list)) else [1, opt.stride, opt.stride, 1]
opt.stride = [1, opt.stride[0], opt.stride[1], 1] if len(opt.stride) == 2 else opt.stride
# parameter tf.sg_initializer
w = tf.sg_initializer.he_uniform('W', (opt.size[0], opt.size[1], opt.dim, opt.in_dim),
regularizer=opt.regularizer, summary=opt.summary)
b = tf.sg_initializer.constant('b', opt.dim, summary=opt.summary) if opt.bias else 0
# tedious shape handling for conv2d_transpose
shape = tensor.get_shape().as_list()
if opt.pad == "SAME":
out_shape = [tf.shape(tensor)[0], shape[1] * opt.stride[1], shape[2] * opt.stride[2], opt.dim]
else:
out_shape = [tf.shape(tensor)[0], (shape[1] - 1) * opt.stride[1] + opt.size[0],
(shape[2] - 1) * opt.stride[2] + opt.size[1], opt.dim]
# apply convolution
out = tf.nn.conv2d_transpose(tensor, w, output_shape=tf.stack(out_shape),
strides=opt.stride, padding=opt.pad) + b
# reset shape is needed because conv2d_transpose() erase all shape information.
# noinspection PyUnresolvedReferences
out.set_shape([None, out_shape[1], out_shape[2], opt.dim])
return out
def sg_upconv1d(tensor, opt):
r"""Applies 1-D a up convolution (or convolution transpose).
Args:
tensor: A 3-D `Tensor` (automatically passed by decorator).
opt:
size: A positive `integer` representing `[kernel width]`. As a default it is set to 4
stride: A positive `integer` representing stride dimension. As a default it is set to 2
in_dim: A positive `integer`. The size of input dimension.
dim: A positive `integer`. The size of output dimension.
pad: Either `SAME` (Default) or `VALID`.
bias: Boolean. If True, biases are added.
regularizer: A (Tensor -> Tensor or None) function; the result of applying it on a newly created variable
will be added to the collection tf.GraphKeys.REGULARIZATION_LOSSES and can be used for regularization
summary: If True, summaries are added. The default is True.
Returns:
A `Tensor` with the same type as `tensor`.
"""
# default options
opt += tf.sg_opt(size=4, stride=2, pad='SAME')
opt.size = [opt.size, 1]
opt.stride = [1, opt.stride, 1, 1]
# parameter tf.sg_initializer
w = tf.sg_initializer.he_uniform('W', (opt.size[0], opt.size[1], opt.dim, opt.in_dim),
regularizer=opt.regularizer, summary=opt.summary)
b = tf.sg_initializer.constant('b', opt.dim, summary=opt.summary) if opt.bias else 0
# make 4-D tensor
tensor = tensor.sg_expand_dims(axis=2)
# tedious shape handling for conv2d_transpose
shape = tensor.get_shape().as_list()
if opt.pad == "SAME":
out_shape = [tf.shape(tensor)[0], shape[1] * opt.stride[1], shape[2] * opt.stride[2], opt.dim]
else:
out_shape = [tf.shape(tensor)[0], (shape[1] - 1) * opt.stride[1] + opt.size[0],
(shape[2] - 1) * opt.stride[2] + opt.size[1], opt.dim]
# apply convolution
out = tf.nn.conv2d_transpose(tensor, w, output_shape=tf.stack(out_shape),
strides=opt.stride, padding=opt.pad) + b
# reset shape is needed because conv2d_transpose() erase all shape information.
# noinspection PyUnresolvedReferences
out.set_shape([None, out_shape[1], out_shape[2], opt.dim])
# squeeze
out = out.sq_squeeze(axis=2)
return out
def sg_ctc(tensor, opt):
r"""Computes the CTC (Connectionist Temporal Classification) Loss between `tensor` and `target`.
Args:
tensor: A 3-D `float Tensor`.
opt:
target: A `Tensor` with the same length in the first dimension as the `tensor`. Labels. ( Dense tensor )
name: A `string`. A name to display in the tensor board web UI.
Returns:
A 1-D `Tensor` with the same length in the first dimension of the `tensor`.
For example,
```
tensor = [[[2., -1., 3.], [3., 1., -2.]], [[1., -1., 2.], [3., 1., -2.]]]
target = [[2., 1.], [2., 3.]]
tensor.sg_ctc(target=target) => [ 4.45940781 2.43091154]
```
"""
assert opt.target is not None, 'target is mandatory.'
# default sequence length
shape = tf.shape(tensor)
opt += tf.sg_opt(seq_len=tf.ones((shape[0],), dtype=tf.sg_intx) * shape[1], merge=True)
# ctc loss
out = tf.nn.ctc_loss(opt.target.sg_to_sparse(), tensor, opt.seq_len,
ctc_merge_repeated=opt.merge, time_major=False)
out = tf.identity(out, 'ctc')
# add summary
tf.sg_summary_loss(out, name=opt.name)
return out
def sg_upconv(tensor, opt):
# default options
opt += tf.sg_opt(size=(3, 3), stride=(1, 2, 2, 1), pad='SAME')
opt.size = opt.size if isinstance(opt.size,
(tuple, list)) else [opt.size, opt.size]
opt.stride = opt.stride if isinstance(
opt.stride, (tuple, list)) else [1, opt.stride, opt.stride, 1]
opt.stride = [1, opt.stride[0], opt.stride[1], 1] if len(
opt.stride) == 2 else opt.stride
# parameter initialize
w = init.he_uniform('W', (opt.size[0], opt.size[1], opt.dim, opt.in_dim))
if opt.bias:
b = init.constant('b', opt.dim)
# tedious shape handling for conv2d_transpose
shape = tensor.get_shape().as_list()
out_shape = [
tf.shape(tensor)[0], shape[1] * opt.stride[1],
shape[2] * opt.stride[2], opt.dim
]
# apply convolution
out = tf.nn.conv2d_transpose(tensor,
w,
output_shape=tf.pack(out_shape),
strides=opt.stride,
padding=opt.pad) + (b if opt.bias else 0)
# reset shape is needed because conv2d_transpose() erase all shape information.
out.set_shape([None, out_shape[1], out_shape[2], opt.dim])
return out
def sg_upconv(tensor, opt):
r"""Applies a upconvolution (or convolution transpose).
Args:
tensor: A 4-D `Tensor`.
size: A tuple or list of integers of length 2 representing `[kernel height, kernel width]`.
Can be an int if both values are the same.
If not specified, (3, 3) is set implicitly.
The default value is [1, 2, 2, 1].
stride: A tuple or list of integers of length 2 or 4 representing stride dimensions.
If the length is 2, i.e., (a, b), the stride is `[1, a, b, 1]`.
If the length is 4, i.e., (a, b, c, d), the stride is `[a, b, c, d]`.
Can be an int. If the length is an int, i.e., a, the stride is `[1, a, a, 1]`.
in_dim: A positive `integer`. The size of input dimension.
dim: A positive `integer`. The size of output dimension.
pad: Either `SAME` (Default) or `VALID`.
bias: Boolean. If True, biases are added.
Returns:
A `Tensor` with the same type as `tensor`.
"""
# default options
opt += tf.sg_opt(size=(3, 3), stride=(1, 2, 2, 1), pad='SAME')
opt.size = opt.size if isinstance(opt.size,
(tuple, list)) else [opt.size, opt.size]
opt.stride = opt.stride if isinstance(
opt.stride, (tuple, list)) else [1, opt.stride, opt.stride, 1]
opt.stride = [1, opt.stride[0], opt.stride[1], 1] if len(
opt.stride) == 2 else opt.stride
# parameter initialize
w = init.he_uniform('W', (opt.size[0], opt.size[1], opt.dim, opt.in_dim))
if opt.bias:
b = init.constant('b', opt.dim)
# tedious shape handling for conv2d_transpose
shape = tensor.get_shape().as_list()
out_shape = [
tf.shape(tensor)[0], shape[1] * opt.stride[1],
shape[2] * opt.stride[2], opt.dim
]
# apply convolution
out = tf.nn.conv2d_transpose(tensor,
w,
output_shape=tf.pack(out_shape),
strides=opt.stride,
padding=opt.pad) + (b if opt.bias else 0)
# reset shape is needed because conv2d_transpose() erase all shape information.
out.set_shape([None, out_shape[1], out_shape[2], opt.dim])
return out
def sg_upconv1d(tensor, opt):
r"""Applies 1-D a up convolution (or convolution transpose).
Args:
tensor: A 3-D `Tensor` (automatically passed by decorator).
opt:
size: A positive `integer` representing `[kernel width]`. As a default it is set to 4
stride: A positive `integer` representing stride dimension. As a default it is set to 2
in_dim: A positive `integer`. The size of input dimension.
dim: A positive `integer`. The size of output dimension.
pad: Either `SAME` (Default) or `VALID`.
bias: Boolean. If True, biases are added.
Returns:
A `Tensor` with the same type as `tensor`.
"""
# default options
opt += tf.sg_opt(size=4, stride=2, pad='SAME')
opt.size = [opt.size, 1]
opt.stride = [1, opt.stride, 1, 1]
# parameter tf.sg_initializer
w = tf.sg_initializer.he_uniform(
'W', (opt.size[0], opt.size[1], opt.dim, opt.in_dim))
b = tf.sg_initializer.constant('b', opt.dim) if opt.bias else 0
# make 4-D tensor
tensor = tensor.sg_expand_dims(dim=2)
# tedious shape handling for conv2d_transpose
shape = tensor.get_shape().as_list()
out_shape = [
tf.shape(tensor)[0], shape[1] * opt.stride[1],
shape[2] * opt.stride[2], opt.dim
]
# apply convolution
out = tf.nn.conv2d_transpose(tensor,
w,
output_shape=tf.pack(out_shape),
strides=opt.stride,
padding=opt.pad) + b
# reset shape is needed because conv2d_transpose() erase all shape information.
# noinspection PyUnresolvedReferences
out.set_shape([None, out_shape[1], out_shape[2], opt.dim])
# squeeze
out = out.sq_squeeze(dim=2)
return out
def sg_ctc(tensor, opt):
assert opt.target is not None, 'target is mandatory.'
# default sequence length
shape = tf.shape(tensor)
opt += tf.sg_opt(seq_len=tf.ones((shape[0],), dtype=tf.sg_intx) * shape[1])
# ctc loss
out = tf.nn.ctc_loss(tensor, opt.target.sg_to_sparse(), opt.seq_len, time_major=False)
out = tf.identity(out, 'ctc')
# add summary
tf.sg_summary_loss(out)
return out
def sg_to_sparse(tensor, opt):
r"""Converts a dense tensor into a sparse tensor.
See `tf.SparseTensor()` in tensorflow.
Args:
tensor: A `Tensor` with zero-padding (automatically given by chain).
opt:
name: If provided, replace current tensor's name.
Returns:
A `SparseTensor`.
"""
indices = tf.where(tf.not_equal(tensor.sg_float(), 0.))
return tf.SparseTensor(indices=indices,
values=tf.gather_nd(tensor, indices) - 1, # for zero-based index
dense_shape=tf.shape(tensor).sg_cast(dtype=tf.int64))
def sg_ctc(tensor, opt):
r"""Returns softmax cross entropy loss between `tensor` and `target`.
Args:
tensor: A `Tensor`. Logits. Unscaled log probabilities.
target: A `Tensor` with the same length in the first dimension as the `tensor`. Labels. ( Dense tensor )
Returns:
A 1-D `Tensor` with the same shape as `tensor`.
For example,
```
tensor = [[[2, -1, 3], [3, 1, -2]]]
target = [[2, 1]]
tensor.sg_ce(target=target, one_hot=True) => [ 31.32656264 64.13284527]
```
"""
assert opt.target is not None, 'target is mandatory.'
# default sequence length
shape = tf.shape(tensor)
opt += tf.sg_opt(seq_len=tf.ones((shape[0], ), dtype=tf.sg_intx) *
shape[1])
# ctc loss
out = tf.nn.ctc_loss(tensor,
opt.target.sg_to_sparse(),
opt.seq_len,
time_major=False)
out = tf.identity(out, 'ctc')
# add summary
tf.sg_summary_loss(out)
return out
def tower_infer_enc(chars,
scope,
rnn_cell,
dec_cell,
word_emb,
out_reuse_vars=False,
dev='/cpu:0'):
out_rvars = out_reuse_vars
# make embedding matrix for source and target
with tf.device(dev):
with tf.variable_scope('embatch_size', reuse=out_reuse_vars):
# (vocab_size, latent_dim)
emb_char = tf.sg_emb(name='emb_char',
voca_size=Hp.char_vs,
dim=Hp.hd,
dev=dev)
emb_word = tf.sg_emb(name='emb_word',
emb=word_emb,
voca_size=Hp.word_vs,
dim=300,
dev=dev)
chars = tf.cast(chars, tf.int32)
time = tf.constant(0)
inputs = tf.transpose(chars, perm=[1, 0, 2])
input_ta = tensor_array_ops.TensorArray(tf.int32,
size=tf.shape(chars)[1],
dynamic_size=True,
clear_after_read=True)
chars_sent = input_ta.unstack(inputs) #each element is (batch, sentlen)
resp_steps = tf.shape(chars)[1] # number of sentences in paragraph
statm_steps = resp_steps // 2
rnn_state = rnn_cell.zero_state(
Hp.batch_size, tf.float32) #rnn_cell.rnn_state, rnn_cell.rnn_h
maxdecode = 3
# -------------------------------------------- STATEMENT ENCODING -----------------------------------------------
def rnn_cond_stat(time, rnn_state):
return tf.less(time, statm_steps - 1)
def rnn_body_stat(time, rnn_state):
ch = chars_sent.read(time)
ch = tf.reverse_sequence(input=ch,
seq_lengths=[Hp.c_maxlen] * Hp.batch_size,
seq_dim=1)
reuse_vars = out_reuse_vars
# -------------------------- BYTENET ENCODER --------------------------
with tf.variable_scope('encoder'):
# embed table lookup
enc = ch.sg_lookup(emb=emb_char) #(batch, sentlen, latentdim)
# loop dilated conv block
for i in range(Hp.num_blocks):
enc = (enc.sg_res_block(size=5,
rate=1,
name="enc1_%d" % (i),
is_first=True,
reuse_vars=reuse_vars,
dev=dev).sg_res_block(
size=5,
rate=2,
name="enc2_%d" % (i),
reuse_vars=reuse_vars,
dev=dev).sg_res_block(
size=5,
rate=4,
name="enc4_%d" % (i),
reuse_vars=reuse_vars,
dev=dev).sg_res_block(
size=5,
rate=8,
name="enc8_%d" % (i),
reuse_vars=reuse_vars,
dev=dev).sg_res_block(
size=5,
rate=16,
name="enc16_%d" % (i),
reuse_vars=reuse_vars,
dev=dev))
byte_enc = enc
# -------------------------- QCNN + QPOOL ENCODER #1 --------------------------
with tf.variable_scope('quazi'):
#quasi cnn layer ZFO [batch * 3, seqlen, dim2 ]
conv = byte_enc.sg_quasi_conv1d(is_enc=True,
size=4,
name="qconv_1",
dev=dev,
reuse_vars=reuse_vars)
# c = f * c + (1 - f) * z, h = o*c [batch * 4, seqlen, hd]
pool0 = conv.sg_quasi_rnn(is_enc=False,
att=False,
name="qrnn_1",
reuse_vars=reuse_vars,
dev=dev)
qpool_last = pool0[:, -1, :]
# -------------------------- MAXPOOL along time dimension --------------------------
inpt_maxpl = tf.expand_dims(byte_enc,
1) # [batch, 1, seqlen, channels]
maxpool = tf.nn.max_pool(inpt_maxpl, [1, 1, Hp.c_maxlen, 1],
[1, 1, 1, 1], 'VALID')
maxpool = tf.squeeze(maxpool, [1, 2])
# -------------------------- HIGHWAY --------------------------
concat = qpool_last + maxpool
with tf.variable_scope('highway', reuse=reuse_vars):
input_lstm = highway(concat, concat.get_shape()[-1], num_layers=1)
# -------------------------- CONTEXT LSTM --------------------------
input_lstm = tf.nn.dropout(input_lstm, Hp.keep_prob)
with tf.variable_scope('contx_lstm', reuse=reuse_vars):
output, rnn_state = rnn_cell(input_lstm, rnn_state)
return (time + 1, rnn_state)
loop_vars_stat = [time, rnn_state]
time, rnn_state = tf.while_loop\
(rnn_cond_stat, rnn_body_stat, loop_vars_stat, swap_memory=False)
return rnn_state
def sg_to_sparse(tensor, opt):
indices = tf.where(tf.not_equal(tensor.sg_float(), 0.))
return tf.SparseTensor(
indices=indices,
values=tf.gather_nd(tensor, indices) - 1, # for zero-based index
shape=tf.shape(tensor).sg_cast(dtype=tf.int64))
with tf.name_scope('inputLength'):
seq_len = tf.placeholder(tf.int32, [None])
with tf.name_scope('input'):
inputs = tf.placeholder(tf.float32, [None, None, num_mfccs * 2])
targets = tf.sparse_placeholder(tf.int32)
# Stacking rnn cells
with tf.name_scope('cellStack'):
stack = tf.contrib.rnn.MultiRNNCell(
[lstm_cell() for _ in range(num_layers)], state_is_tuple=True)
outputs, _ = tf.nn.dynamic_rnn(stack,
inputs,
seq_len,
dtype=tf.float32)
shape = tf.shape(inputs)
batch_s, TF_max_timesteps = shape[0], shape[1]
with tf.name_scope('outputs'):
outputs = tf.reshape(outputs, [-1, num_hidden])
with tf.name_scope('weights'):
W = tf.Variable(tf.truncated_normal([num_hidden, num_classes],
stddev=0.1),
name='weights')
with tf.name_scope('biases'):
b = tf.get_variable("b",
initializer=tf.constant(0.,
shape=[num_classes]))
with tf.name_scope('logits'):
# CLASSIFY SIZE
classify_size = 2
# mfcc feature of audio
x = data.mfcc
# sequence length except zero-padding
seq_len = tf.not_equal(x.sg_sum(dims=2), 0.).sg_int().sg_sum(dims=1)
# target sentence label
#y = data.label
# target classification label
y = data.label
print("SHAPE", tf.shape(y))
#
# encode graph ( atrous convolution )
#
# residual block
def res_block(tensor, size, rate, dim=num_dim):
# filter convolution
conv_filter = tensor.sg_aconv1d(size=size,
rate=rate,
act='tanh',
bn=True,
dout=0.05)
def tower_loss_manyparams(xx, scope, reu_vars=False):
# make embedding matrix for source and target
reu_vars = reu_vars
with tf.variable_scope('embatch_size', reuse=reu_vars):
# (vocab_size, latent_dim)
emb_x = tf.sg_emb(name='emb_x',
voca_size=Hp.vs,
dim=Hp.hd,
dev=self._dev)
emb_y = tf.sg_emb(name='emb_y',
voca_size=Hp.vs,
dim=Hp.hd,
dev=self._dev)
xx = tf.cast(xx, tf.int32)
time = tf.constant(0)
losses_int = tf.constant(0.0)
inputs = tf.transpose(xx, perm=[1, 0, 2])
input_ta = tensor_array_ops.TensorArray(tf.int32,
size=1,
dynamic_size=True,
clear_after_read=False)
x_sent = input_ta.unstack(inputs) #each element is (batch, sentlen)
n_steps = tf.shape(xx)[1] # number of sentences in paragraph
# generate first an unconditioned sentence
n_input = Hp.hd
subrec1_init = subrec_zero_state(Hp.batch_size, Hp.hd)
subrec2_init = subrec_zero_state(Hp.batch_size, Hp.hd)
with tf.variable_scope("mem", reuse=reu_vars) as scp:
rnn_cell = LSTMCell(in_dim=h, dim=Hp.hd)
crnn_cell = ConvLSTMCell(seqlen=Hp.maxlen,
in_dim=n_input // 2,
dim=Hp.hd // 2)
(rnn_state_init, rnn_h_init) = rnn_cell.zero_state(Hp.batch_size)
# (batch, sentlen, latentdim/2)
(crnn_state_init, crnn_h_init) = crnn_cell.zero_state(Hp.batch_size)
def rnn_cond(time, subrec1, subrec2, rnn_state, rnn_h, crnn_state, crnn_h,
losses):
return tf.less(time, n_steps - 1)
def rnn_body(time, subrec1, subrec2, rnn_state, rnn_h, crnn_state, crnn_h,
losses):
x = x_sent.read(time)
y = x_sent.read(time + 1) # (batch, sentlen) = (16, 200)
# shift target by one step for training source
y_src = tf.concat([tf.zeros((Hp.batch_size, 1), tf.int32), y[:, :-1]],
1)
reuse_vars = time == tf.constant(0) or reu_vars
# -------------------------- BYTENET ENCODER --------------------------
# embed table lookup
enc = x.sg_lookup(emb=emb_x) #(batch, sentlen, latentdim)
# loop dilated conv block
for i in range(num_blocks):
enc = (enc.sg_res_block(
size=5, rate=1, name="enc1_%d" % (i),
reuse_vars=reuse_vars).sg_res_block(
size=5,
rate=2,
name="enc2_%d" % (i),
reuse_vars=reuse_vars).sg_res_block(
size=5,
rate=4,
name="enc4_%d" % (i),
reuse_vars=reuse_vars).sg_res_block(
size=5,
rate=8,
name="enc8_%d" % (i),
reuse_vars=reuse_vars).sg_res_block(
size=5,
rate=16,
name="enc16_%d" % (i),
reuse_vars=reuse_vars))
# -------------------------- QCNN + QPOOL ENCODER with attention #1 --------------------------
#quasi cnn layer ZFO [batch * 3, t, dim2 ]
conv = enc.sg_quasi_conv1d(is_enc=True,
size=3,
name="qconv_1",
reuse_vars=reuse_vars)
#attention layer
# recurrent layer # 1 + final encoder hidden state
subrec1 = tf.tile((subrec1.sg_expand_dims(axis=1)), [1, Hp.maxlen, 1])
concat = conv.sg_concat(target=subrec1,
axis=0) # (batch*4, sentlen, latentdim)
pool = concat.sg_quasi_rnn(is_enc=True,
att=True,
name="qrnn_1",
reuse_vars=reuse_vars)
subrec1 = pool[:Hp.batch_size, -1, :] # last character in sequence
# -------------------------- QCNN + QPOOL ENCODER with attention #2 --------------------------
# quazi cnn ZFO (batch*3, sentlen, latentdim)
conv = pool.sg_quasi_conv1d(is_enc=True,
size=2,
name="qconv_2",
reuse_vars=reuse_vars)
# (batch, sentlen-duplicated, latentdim)
subrec2 = tf.tile((subrec2.sg_expand_dims(axis=1)), [1, Hp.maxlen, 1])
# (batch*4, sentlen, latentdim)
concat = conv.sg_concat(target=subrec2, axis=0)
pool = concat.sg_quasi_rnn(is_enc=True,
att=True,
name="qrnn_2",
reuse_vars=reuse_vars)
subrec2 = pool[:Hp.batch_size, -1, :] # last character in sequence
# -------------------------- ConvLSTM with RESIDUAL connection and MULTIPLICATIVE block --------------------------
#residual block
causal = False # for encoder
crnn_input = (pool[:Hp.batch_size, :, :].sg_bypass_gpus(
name='relu_0', act='relu', bn=(not causal),
ln=causal).sg_conv1d_gpus(name="dimred_0",
size=1,
dev="/cpu:0",
reuse=reuse_vars,
dim=Hp.hd / 2,
act='relu',
bn=(not causal),
ln=causal))
# conv LSTM
with tf.variable_scope("mem/clstm") as scp:
(crnn_state, crnn_h) = crnn_cell(crnn_input, (crnn_state, crnn_h),
size=5,
reuse_vars=reuse_vars)
# dimension recover and residual connection
rnn_input0 = pool[:Hp.batch_size,:,:] + crnn_h\
.sg_conv1d_gpus(name = "diminc_0",size=1,dev="/cpu:0", dim=Hp.hd,reuse=reuse_vars, act='relu', bn=(not causal), ln=causal)
# -------------------------- QCNN + QPOOL ENCODER with attention #3 --------------------------
# pooling for lstm input
# quazi cnn ZFO (batch*3, sentlen, latentdim)
conv = rnn_input0.sg_quasi_conv1d(is_enc=True,
size=2,
name="qconv_3",
reuse_vars=reuse_vars)
pool = conv.sg_quasi_rnn(is_enc=True,
att=False,
name="qrnn_3",
reuse_vars=reuse_vars)
rnn_input = pool[:Hp.batch_size, -1, :] # last character in sequence
# -------------------------- LSTM with RESIDUAL connection and MULTIPLICATIVE block --------------------------
# recurrent block
with tf.variable_scope("mem/lstm") as scp:
(rnn_state, rnn_h) = rnn_cell(rnn_input, (rnn_state, rnn_h))
rnn_h2 = tf.tile(((rnn_h + rnn_input).sg_expand_dims(axis=1)),
[1, Hp.maxlen, 1])
# -------------------------- BYTENET DECODER --------------------------
# CNN decoder
dec = y_src.sg_lookup(emb=emb_y).sg_concat(target=rnn_h2, name="dec")
for i in range(num_blocks):
dec = (dec.sg_res_block(
size=3,
rate=1,
causal=True,
name="dec1_%d" % (i),
reuse_vars=reuse_vars).sg_res_block(
size=3,
rate=2,
causal=True,
name="dec2_%d" % (i),
reuse_vars=reuse_vars).sg_res_block(
size=3,
rate=4,
causal=True,
name="dec4_%d" % (i),
reuse_vars=reuse_vars).sg_res_block(
size=3,
rate=8,
causal=True,
name="dec8_%d" % (i),
reuse_vars=reuse_vars).sg_res_block(
size=3,
rate=16,
causal=True,
name="dec16_%d" % (i),
reuse_vars=reuse_vars))
# final fully convolution layer for softmax
dec = dec.sg_conv1d_gpus(size=1,
dim=Hp.vs,
name="out",
summary=False,
dev=self._dev,
reuse=reuse_vars)
ce_array = dec.sg_ce(target=y, mask=True, name="cross_ent_example")
cross_entropy_mean = tf.reduce_mean(ce_array, name='cross_entropy')
losses = tf.add_n([losses, cross_entropy_mean], name='total_loss')
return (time + 1, subrec1, subrec2, rnn_state, rnn_h, crnn_state,
crnn_h, losses)
