def Net(aa, yt, x):
s=aa.shape[1]
with tf.sg_context(name='NNReg', stride=1, act='leaky_relu', bn=True, reuse=tf.AUTO_REUSE):
yt=tf.expand_dims(yt,2)
v1=tf.expand_dims(x,2).sg_conv(dim=16, size=(1,1), name='gen9',pad="SAME",bn=True)
v2=v1.sg_conv(dim=64, size=(1,1), name='gen1',pad="SAME",bn=True)
v3=v2.sg_conv(dim=128, size=(1,1), name='gen2',pad="SAME",bn=True)
v4=v3.sg_conv(dim=256, size=(1,1), name='gen3',pad="SAME",bn=True)
v5=v4.sg_conv(dim=512, size=(1,1), name='gen4',pad="SAME",bn=True)
v5=tf.tile(tf.expand_dims(tf.reduce_max(v5, axis=1),axis=1),[1,s,1,1])
vv5=v5
v1=yt.sg_conv(dim=16, size=(1,1), name='gen99',pad="SAME",bn=True)
v2=v1.sg_conv(dim=64, size=(1,1), name='gen11',pad="SAME",bn=True)
v3=v2.sg_conv(dim=128, size=(1,1), name='gen22',pad="SAME",bn=True)
v4=v3.sg_conv(dim=256, size=(1,1), name='gen33',pad="SAME",bn=True)
v5=v4.sg_conv(dim=512, size=(1,1), name='gen44',pad="SAME",bn=True)
v5=tf.tile(tf.expand_dims(tf.reduce_max(v5, axis=1),axis=1),[1,s,1,1])
ff=tf.concat([tf.expand_dims(aa,2),v5], axis=-1)
ff=tf.concat([ff,vv5], axis=-1)
f1=ff.sg_conv(dim=256, size=(1,1), name='f1',pad="SAME",bn=True)
f2=f1.sg_conv(dim=128, size=(1,1), name='f2',pad="SAME",bn=True)
f3=f2.sg_conv(dim=2, size=(1,1), name='f3',pad="SAME",bn=False, act="linear")
f3=tf.squeeze(f3,axis=2)
return f3
def sg_quasi_rnn(tensor, opt):
# Split
if opt.att:
H, Z, F, O = tf.split(tensor, 4, axis=0) # (16, 150, 320) for all
else:
Z, F, O = tf.split(tensor, 3, axis=0) # (16, 150, 320) for all
# step func
def step(z, f, o, c):
'''
Runs fo-pooling at each time step
'''
c = f * c + (1 - f) * z
if opt.att: # attention
a = tf.nn.softmax(tf.einsum("ijk,ik->ij", H,
c)) # alpha. (16, 150)
k = (a.sg_expand_dims() * H).sg_sum(
axis=1) # attentional sum. (16, 320)
h = o * (k.sg_dense(act="linear") + \
c.sg_dense(act="linear"))
else:
h = o * c
return h, c # hidden states, (new) cell memories
# Do rnn loop
c, hs = 0, []
timesteps = tensor.get_shape().as_list()[1]
for t in range(timesteps):
z = Z[:, t, :] # (16, 320)
f = F[:, t, :] # (16, 320)
o = O[:, t, :] # (16, 320)
# apply step function
h, c = step(z, f, o, c) # (16, 320), (16, 320)
# save result
hs.append(h.sg_expand_dims(axis=1))
# Concat to return
H = tf.concat(hs, 1) # (16, 150, 320)
seqlen = tf.to_int32(
tf.reduce_sum(tf.sign(tf.abs(tf.reduce_sum(H, axis=-1))),
1)) # (16,) float32
h = tf.reverse_sequence(input=H, seq_lengths=seqlen,
seq_dim=1)[:, 0, :] # last hidden state vector
if opt.is_enc:
H_z = tf.tile((h.sg_dense(act="linear").sg_expand_dims(axis=1)),
[1, timesteps, 1])
H_f = tf.tile((h.sg_dense(act="linear").sg_expand_dims(axis=1)),
[1, timesteps, 1])
H_o = tf.tile((h.sg_dense(act="linear").sg_expand_dims(axis=1)),
[1, timesteps, 1])
concatenated = tf.concat([H, H_z, H_f, H_o], 0) # (16*4, 150, 320)
return concatenated
else:
return H # (16, 150, 320)
def sg_quasi_rnn(tensor, opt):
# Split
if opt.att:
H, Z, F, O = tf.split(axis=0, num_or_size_splits=4,
value=tensor) # (16, 150, 320) for all
else:
Z, F, O = tf.split(axis=0, num_or_size_splits=3,
value=tensor) # (16, 150, 320) for all
# step func
def step(z, f, o, c):
'''
Runs fo-pooling at each time step
'''
c = f * c + (1 - f) * z
if opt.att: # attention
a = tf.nn.softmax(tf.einsum("ijk,ik->ij", H,
c)) # alpha. (16, 150)
k = (a.sg_expand_dims() * H).sg_sum(
dims=1) # attentional sum. (16, 150)
h = o * (k.sg_dense(act="linear") + c.sg_dense(act="linear"))
else:
h = o * c
return h, c # hidden states, (new) cell memories
# Do rnn loop
c, hs = 0, []
timesteps = tensor.get_shape().as_list()[1]
for t in range(timesteps):
z = Z[:, t, :] # (16, 320)
f = F[:, t, :] # (16, 320)
o = O[:, t, :] # (16, 320)
# apply step function
h, c = step(z, f, o, c) # (16, 320), (16, 320)
# save result
hs.append(h.sg_expand_dims(dim=1))
# Concat to return
H = tf.concat(axis=1, values=hs) # (16, 150, 320)
if opt.is_enc:
H_z = tf.tile((h.sg_dense(act="linear").sg_expand_dims(dim=1)),
[1, timesteps, 1])
H_f = tf.tile((h.sg_dense(act="linear").sg_expand_dims(dim=1)),
[1, timesteps, 1])
H_o = tf.tile((h.sg_dense(act="linear").sg_expand_dims(dim=1)),
[1, timesteps, 1])
concatenated = tf.concat(axis=0, values=[H, H_z, H_f,
H_o]) # (16*4, 150, 320)
return concatenated
else:
return H # (16, 150, 320)
def sg_quasi_conv1d(tensor, opt):
opt += tf.sg_opt(is_enc=False)
# Split into H and H_zfo
H = tensor[:Hp.bs]
H_z = tensor[Hp.bs:2 * Hp.bs]
H_f = tensor[2 * Hp.bs:3 * Hp.bs]
H_o = tensor[3 * Hp.bs:]
if opt.is_enc:
H_z, H_f, H_o = 0, 0, 0
# Convolution and merging
with tf.sg_context(act="linear",
causal=(not opt.is_enc),
bn=opt.is_enc,
ln=(not opt.is_enc)):
Z = H.sg_aconv1d() + H_z # (16, 300, 320)
F = H.sg_aconv1d() + H_f # (16, 300, 320)
O = H.sg_aconv1d() + H_o # (16, 300, 320)
# Activation
Z = Z.sg_bypass(act="tanh") # (16, 300, 320)
F = F.sg_bypass(act="sigmoid") # (16, 300, 320)
O = O.sg_bypass(act="sigmoid") # (16, 300, 320)
# Masking
M = tf.sign(tf.abs(H))[:, :, :1] # (16, 300, 1) float32. 0 or 1
Z *= M # broadcasting
F *= M # broadcasting
O *= M # broadcasting
# Concat
ZFO = tf.concat(axis=0, values=[Z, F, O])
return ZFO # (16*3, 150, 320)
def sg_emb(**kwargs):
r"""Returns a look-up table for embedding.
kwargs:
name: A name for the layer.
emb: A 2-D array (optional).
If None, the resulting tensor should have the shape of
`[vocabulary size, embedding dimension size]`.
Note that its first row is filled with 0's associated with padding.
in_dim: A positive `integer`. The size of input dimension.
dim: A positive `integer`. The size of output dimension.
voca_size: A positive integer. The size of vocabulary.
Returns:
A 2-D `Tensor` of float32.
"""
opt = tf.sg_opt(kwargs)
assert opt.name is not None, 'name is mandatory.'
if opt.emb is None:
# initialize embedding matrix
assert opt.voca_size is not None, 'voca_size is mandatory.'
assert opt.dim is not None, 'dim is mandatory.'
w = tf.sg_initializer.he_uniform(opt.name,
(opt.voca_size - 1, opt.dim))
else:
# use given embedding matrix
w = tf.sg_initializer.external(opt.name, value=opt.emb)
# 1st row should be zero and not be updated by backprop because of zero padding.
emb = tf.concat(0, [tf.zeros((1, opt.dim), dtype=tf.sg_floatx), w])
return emb
def sg_inverse_periodic_shuffle(tensor, opt):
# default factor
opt += tf.sg_opt(factor=2)
# get current shape
batch, row, col, channel = tensor.get_shape().as_list()
# get target shape and channel num
channel_factor = opt.factor * opt.factor
# intermediate shape for shuffling
shape_1 = [
batch, row / opt.factor, col / opt.factor,
channel_factor // opt.factor, channel_factor // opt.factor
]
shape_2 = [batch, row / opt.factor, col / opt.factor, channel_factor]
# reshape and transpose for periodic shuffling for each channel
out = []
for i in range(channel):
out.append(tensor[:, :, :, i].sg_expand_dims().sg_reshape(
shape=shape_1).sg_transpose(perm=(0, 1, 3, 2,
4)).sg_reshape(shape=shape_2))
# final output
out = tf.concat(3, out)
return tf.identity(out, name=opt.name)
def sg_emb(**kwargs):
r"""Returns an embedding layer or a look-up table.
Args:
name: A name for the layer (required).
emb: A 2-D array. Has the shape of `[vocabulary size -1, embedding dimension size]`.
Note that the first row is filled with 0's because they correspond to padding.
in_dim: A positive `integer`. The size of input dimension.
dim: A positive `integer`. The size of output dimension.
voca_size: A positive int32.
Returns:
A 2-D tensor.
"""
opt = tf.sg_opt(kwargs)
assert opt.name is not None, 'name is mandatory.'
import sg_initializer as init
if opt.emb is None:
# initialize embedding matrix
assert opt.voca_size is not None, 'voca_size is mandatory.'
assert opt.dim is not None, 'dim is mandatory.'
w = init.he_uniform(opt.name, (opt.voca_size - 1, opt.dim))
else:
# use given embedding matrix
w = init.external(opt.name, value=opt.emb)
# 1st row should be zero and not be updated by backprop because of zero padding.
emb = tf.concat(0, [tf.zeros((1, opt.dim), dtype=tf.sg_floatx), w])
return emb
def rnn_classify(x, num_classes, is_test=False):
with tf.sg_context(name='rnn_classify'):
fw_cell = tf.nn.rnn_cell.MultiRNNCell(
[lstm_cell(is_test) for _ in range(num_blocks)],
state_is_tuple=True)
bw_cell = tf.nn.rnn_cell.MultiRNNCell(
[lstm_cell(is_test) for _ in range(num_blocks)],
state_is_tuple=True)
words_used_in_sent = tf.sign(
tf.reduce_max(tf.abs(x), reduction_indices=2))
length = tf.cast(
tf.reduce_sum(words_used_in_sent, reduction_indices=1), tf.int32)
outputs, _ = tf.nn.bidirectional_dynamic_rnn(fw_cell,
bw_cell,
x,
dtype=tf.float32,
sequence_length=length)
output = tf.concat(outputs, 2).sg_reshape(shape=[-1, 2 * latent_dim])
prediction = output.sg_dense(dim=num_classes, name='dense')
res = tf.reshape(prediction,
[x.get_shape().as_list()[0], -1, num_classes])
return res
def sg_periodic_shuffle(tensor, opt):
# default factor
opt += tf.sg_opt(factor=2)
# get current shape
batch, row, col, channel = tensor.get_shape().as_list()
# get target channel num
channel_target = channel / (opt.factor * opt.factor)
channel_factor = channel / channel_target
# intermediate shape for shuffling
shape_1 = [
batch, row, col, channel_factor / opt.factor,
channel_factor / opt.factor
]
shape_2 = [batch, row * opt.factor, col * opt.factor, 1]
# reshape and transpose for periodic shuffling for each channel
out = []
for i in range(channel_target):
out.append(
(tensor[:, :, :, i * channel_factor:(i + 1) *
channel_factor]).sg_reshape(shape=shape_1).sg_transpose(
perm=(0, 1, 3, 2, 4)).sg_reshape(shape=shape_2))
# final output
out = tf.concat(3, out)
return tf.identity(out, name=opt.name)
def sg_rnn(tensor, opt):
r"""Applies a simple rnn.
Args:
tensor: A 3-D `Tensor`.
in_dim: A positive `integer`. The size of input dimension.
dim: A positive `integer`. The size of output dimension.
bias: Boolean. If True, biases are added.
ln: Boolean. If True, layer normalization is applied.
init_state: A 2-D `Tensor`. If None, the initial state is set to zeros.
last_only: Boolean. If True, the outputs in the last time step are returned.
Returns:
A `Tensor`. If last_only is False, the output tensor has shape [batch size, time steps, dim].
If last_only is True, the shape will be [batch size, dim].
"""
# layer normalization
ln = lambda v: _ln_rnn(v, gamma, beta) if opt.ln else v
# step function
def step(h, x):
# simple rnn
### Replace tensor[:, i, :] with x. bryan ###
y = ln(
tf.matmul(tensor[:, i, :], w) + tf.matmul(h, u) +
(b if opt.bias else 0))
return y
# parameter initialize
w = init.orthogonal('W', (opt.in_dim, opt.dim))
u = init.identity('U', opt.dim)
if opt.bias:
b = init.constant('b', opt.dim)
# layer normalization parameters
if opt.ln:
# offset, scale parameter
beta = init.constant('beta', opt.dim)
gamma = init.constant('gamma', opt.dim, value=1)
# initial state
init_h = opt.init_state if opt.init_state is not None \
else tf.zeros((tensor.get_shape().as_list()[0], opt.dim), dtype=tf.sg_floatx)
# do rnn loop
h, out = init_h, []
for i in range(tensor.get_shape().as_list()[1]):
# apply step func
h = step(h, tensor[:, i, :])
# save result
out.append(h.sg_expand_dims(dim=1))
# merge tensor
if opt.last_only:
out = out[-1].sg_squeeze(dim=1)
else:
out = tf.concat(1, out)
return out
def sg_densenet_layer(x, opt):
r"""Applies basic architecture of densenet layer.
Note that the fc layers in the original architecture
will be replaced with fully convolutional layers.
For convenience, We still call them fc layers, though.
Args:
x: A `Tensor`.
opt:
dim: An integer. Dimension for this resnet layer
num: Number of times to repeat
act: String. 'relu' (default). the activation function name
trans: Boolean. If True(default), transition layer will be applied.
reuse: Boolean(Optional). If True, all variables will be loaded from previous network.
name: String. (optional) Used as convolution layer prefix
Returns:
A `Tensor`.
"""
assert opt.dim is not None, 'dim is mandatory.'
assert opt.num is not None, 'num is mandatory.'
# default stride
opt += tf.sg_opt(stride=1, act='relu', trans=True)
# format convolutional layer name
def cname(index):
return opt.name if opt.name is None else opt.name + '_%d' % index
# dense layer
with tf.sg_context(bias=False, reuse=opt.reuse):
out = x
for i in range(opt.num):
# dense block
out_new = (out.sg_bypass(act=opt.act,
bn=True,
name=cname(3 * i + 1)).sg_conv(
dim=opt.dim // 4,
size=1,
act=opt.act,
bn=True,
name=cname(3 * i + 2)).sg_conv(
dim=opt.dim,
size=3,
name=cname(3 * i + 3)))
out = tf.concat([out_new, out], 3)
# transition layer
if opt.trans:
out = (out.sg_bypass(act=opt.act, bn=True,
name=cname(3 * i + 4)).sg_conv(
size=1,
name=cname(3 * i + 5)).sg_pool(avg=True))
return out
def q_process(t1, t2):
'''
Processes each training sample so that it fits in the queue.
'''
# Lstrip zeros
zeros = tf.equal(t1, tf.zeros_like(t1)).sg_int().sg_sum()
t1 = t1[zeros:]
t2 = t2[zeros:]
# zero-PrePadding
t1 = tf.concat([tf.zeros([Hyperparams.seqlen-1], tf.int32), t1], 0)# 49 zero-prepadding
t2 = tf.concat([tf.zeros([Hyperparams.seqlen-1], tf.int32), t2], 0)# 49 zero-prepadding
# radom crop
stacked = tf.stack((t1, t2))
cropped = tf.random_crop(stacked, [2, Hyperparams.seqlen])
t1, t2 = cropped[0], cropped[1]
t2 = t2[-1]
return t1, t2
def rnn_classify(x, num_classes, is_test=False):
with tf.sg_context(name='rnn_classify'):
fw_cell = tf.nn.rnn_cell.MultiRNNCell([lstm_cell(is_test) for _ in range(num_blocks)], state_is_tuple=True)
bw_cell = tf.nn.rnn_cell.MultiRNNCell([lstm_cell(is_test) for _ in range(num_blocks)], state_is_tuple=True)
words_used_in_sent = tf.sign(tf.reduce_max(tf.abs(x), reduction_indices=2))
length = tf.cast(tf.reduce_sum(words_used_in_sent, reduction_indices=1), tf.int32)
outputs, _ = tf.nn.bidirectional_dynamic_rnn(fw_cell, bw_cell, x, dtype=tf.float32, sequence_length=length)
output = tf.concat(outputs, 2).sg_reshape(shape=[-1, 2 * latent_dim])
prediction = output.sg_dense(dim=num_classes, name='dense')
res = tf.reshape(prediction, [x.get_shape().as_list()[0], -1, num_classes])
return res
def sg_rnn(tensor, opt):
# layer normalization
ln = lambda v: _ln_rnn(v, gamma, beta) if opt.ln else v
# step function
def step(h, x):
# simple rnn
y = ln(
tf.matmul(tensor[:, i, :], w) + tf.matmul(h, u) +
(b if opt.bias else 0))
return y
# parameter initialize
w = init.orthogonal('W', (opt.in_dim, opt.dim))
u = init.identity('U', opt.dim)
if opt.bias:
b = init.constant('b', opt.dim)
# layer normalization parameters
if opt.ln:
# offset, scale parameter
beta = init.constant('beta', opt.dim)
gamma = init.constant('gamma', opt.dim, value=1)
# initial state
init_h = opt.init_state if opt.init_state is not None \
else tf.zeros((tensor.get_shape().as_list()[0], opt.dim), dtype=tf.sg_floatx)
# do rnn loop
h, out = init_h, []
for i in range(tensor.get_shape().as_list()[1]):
# apply step func
h = step(h, tensor[:, i, :])
# save result
out.append(h.sg_expand_dims(dim=1))
# merge tensor
if opt.last_only:
out = out[-1].sg_squeeze(dim=1)
else:
out = tf.concat(1, out)
return out
def sg_emb(**kwargs):
opt = tf.sg_opt(kwargs)
assert opt.name is not None, 'name is mandatory.'
import sg_initializer as init
if opt.emb is None:
# initialize embedding matrix
assert opt.voca_size is not None, 'voca_size is mandatory.'
assert opt.dim is not None, 'dim is mandatory.'
w = init.he_uniform(opt.name, (opt.voca_size - 1, opt.dim))
else:
# use given embedding matrix
w = init.external(opt.name, value=opt.emb)
# 1st row should be zero and not be updated by backprop because of zero padding.
emb = tf.concat(0, [tf.zeros((1, opt.dim), dtype=tf.sg_floatx), w])
return emb
def sg_periodic_shuffle(tensor, opt):
r""" Periodic shuffle transformation for SubPixel CNN.
(see [Shi et al. 2016](http://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/Shi_Real-Time_Single_Image_CVPR_2016_paper.pdf)
Args:
tensor: A tensor (automatically given by chain).
opt:
factor: factor to multiply shape by. Default is 2.
name : If provided, it replaces current tensor's name.
Returns:
A tensor
"""
# default factor
opt += tf.sg_opt(factor=2)
# get current shape
batch, row, col, channel = tensor.get_shape().as_list()
# get target channel num
channel_target = channel // (opt.factor * opt.factor)
channel_factor = channel // channel_target
# intermediate shape for shuffling
shape_1 = [
batch, row, col, channel_factor // opt.factor,
channel_factor // opt.factor
]
shape_2 = [batch, row * opt.factor, col * opt.factor, 1]
# reshape and transpose for periodic shuffling for each channel
out = []
for i in range(channel_target):
out.append(
(tensor[:, :, :, i * channel_factor:(i + 1) *
channel_factor]).sg_reshape(shape=shape_1).sg_transpose(
perm=(0, 1, 3, 2, 4)).sg_reshape(shape=shape_2))
# final output
out = tf.concat(axis=3, values=out)
return tf.identity(out, name=opt.name)
def sg_concat(tensor, opt):
r"""Concatenates tensors along a axis.
See `tf.concat()` in tensorflow.
Args:
tensor: A `Tensor` (automatically given by chain).
opt:
target: A `Tensor`. Must have the same rank as `tensor`, and
all dimensions except `opt.dim` must be equal.
axis : Target axis. Default is the last one.
name: If provided, replace current tensor's name.
Returns:
A `Tensor`.
"""
assert opt.target is not None, 'target is mandatory.'
opt += tf.sg_opt(axis=tensor.get_shape().ndims-1)
target = opt.target if isinstance(opt.target, (tuple, list)) else [opt.target]
return tf.concat([tensor] + target, opt.axis, name=opt.name)
def sg_quasi_conv1d(tensor, opt):
'''
Args:
tensor: A 3-D tensor of either [batch size, time steps, embedding size] for original
X or [batch size * 4, time steps, embedding size] for the others.
'''
opt += tf.sg_opt(is_enc=False)
# Split into H and H_zfo
H = tensor[:Hp.batch_size]
H_z = tensor[Hp.batch_size:2 * Hp.batch_size]
H_f = tensor[2 * Hp.batch_size:3 * Hp.batch_size]
H_o = tensor[3 * Hp.batch_size:]
if opt.is_enc:
H_z, H_f, H_o = 0, 0, 0
# Convolution and merging
with tf.sg_context(size=opt.size, act="linear", causal=(not opt.is_enc)):
Z = H.sg_aconv1d() + H_z # (16, 150, 320)
F = H.sg_aconv1d() + H_f # (16, 150, 320)
O = H.sg_aconv1d() + H_o # (16, 150, 320)
# Activation
with tf.sg_context(ln=True):
Z = Z.sg_bypass(act="tanh") # (16, 150, 320)
F = F.sg_bypass(act="sigmoid") # (16, 150, 320)
O = O.sg_bypass(act="sigmoid") # (16, 150, 320)
# Masking
M = tf.sign(tf.abs(tf.reduce_sum(
H, axis=-1, keep_dims=True))) # (16, 150, 1) float32. 0 or 1
Z *= M # broadcasting
F *= M # broadcasting
O *= M # broadcasting
# Concat
ZFO = tf.concat([Z, F, O], 0)
return ZFO # (16*3, 150, 320)
def sg_quasi_conv1d(tensor, opt):
opt += tf.sg_opt(is_enc=False, causal=True)
# Split into H and H_zfo
H = tensor[:Hp.batch_size]
H_z = tensor[Hp.batch_size:2 * Hp.batch_size]
H_f = tensor[2 * Hp.batch_size:3 * Hp.batch_size]
H_o = tensor[3 * Hp.batch_size:]
if opt.is_enc:
H_z, H_f, H_o = 0, 0, 0
# Convolution and merging
with tf.sg_context(size=opt.size,
act="linear",
causal=opt.causal and (not opt.is_enc),
dev=opt.dev,
reuse=opt.reuse_vars):
Z = H.sg_aconv1d_gpus(name="aconvz_" +
opt.name) + H_z # (b, seqlen, hd)
F = H.sg_aconv1d_gpus(name="aconvf_" +
opt.name) + H_f # (b, seqlen, hd)
O = H.sg_aconv1d_gpus(name="aconvo_" +
opt.name) + H_o # (b, seqlen, hd)
# Activation
with tf.sg_context(dev=opt.dev, reuse=opt.reuse_vars):
Z = Z.sg_bypass_gpus(act="tanh",
name="tanhz_" + opt.name) # (b, seqlen, hd)
F = F.sg_bypass_gpus(act="sigmoid",
name="sigmf_" + opt.name) # (b, seqlen, hd)
O = O.sg_bypass_gpus(act="sigmoid",
name="sigmo_" + opt.name) # (b, seqlen, hd)
ZFO = tf.concat([Z, F, O], 0)
return ZFO # (batch*3, seqlen, hiddim)
# inputs
#
# MNIST input tensor ( with QueueRunner )
data = tf.sg_data.Mnist(batch_size=batch_size)
# input images and label
x = data.train.image
y = data.train.label
# labels for discriminator
y_real = tf.ones(batch_size)
y_fake = tf.zeros(batch_size)
# discriminator labels ( half 1s, half 0s )
y_disc = tf.concat(0, [y, y * 0])
# categorical latent variable
z_cat = tf.multinomial(
tf.ones((batch_size, cat_dim), dtype=tf.sg_floatx) / cat_dim,
1).sg_squeeze().sg_int()
# continuous latent variable
z_con = tf.random_normal((batch_size, con_dim))
# random latent variable dimension
z_rand = tf.random_normal((batch_size, rand_dim))
# latent variable
z = tf.concat(1, [z_cat.sg_one_hot(depth=cat_dim), z_con, z_rand])
#
# Computational graph
#
# hyper parameters # batch_size = 16 # batch size # # inputs # # ComTrans parallel corpus input tensor ( with QueueRunner ) data = ComTrans(batch_size=batch_size) # source, target sentence x, y = data.source, data.target # shift target for training source y_in = tf.concat([tf.zeros((batch_size, 1), tf.sg_intx), y[:, :-1]], axis=1) # vocabulary size voca_size = data.voca_size # make embedding matrix for source and target emb_x = tf.sg_emb(name='emb_x', voca_size=voca_size, dim=latent_dim) emb_y = tf.sg_emb(name='emb_y', voca_size=voca_size, dim=latent_dim) # latent from embed table z_x = x.sg_lookup(emb=emb_x) z_y = y_in.sg_lookup(emb=emb_y) # encode graph ( atrous convolution ) enc = encode(z_x) # concat merge target source
# inputs
#
# MNIST input tensor ( with QueueRunner )
data = tf.sg_data.Mnist(batch_size=batch_size)
# input images and label
x = data.train.image
y = data.train.label
# labels for discriminator
y_real = tf.ones(batch_size)
y_fake = tf.zeros(batch_size)
# discriminator labels ( half 1s, half 0s )
y_disc = tf.concat([y, y * 0], 0)
# categorical latent variable
z_cat = tf.multinomial(
tf.ones((batch_size, cat_dim), dtype=tf.sg_floatx) / cat_dim,
1).sg_squeeze().sg_int()
# continuous latent variable
z_con = tf.random_normal((batch_size, con_dim))
# random latent variable dimension
z_rand = tf.random_normal((batch_size, rand_dim))
# latent variable
z = tf.concat(axis=1, values=[z_cat.sg_one_hot(depth=cat_dim), z_con, z_rand])
#
# Computational graph
#
# MNIST input tensor ( with QueueRunner )
data = tf.sg_data.Mnist(batch_size=batch_size)
# input images
x = data.train.image
# corrupted image
x_small = tf.image.resize_bicubic(x, (14, 14))
x_nearest = tf.image.resize_images(x_small, (28, 28),
tf.image.ResizeMethod.NEAREST_NEIGHBOR)
# generator labels ( all ones )
y = tf.ones(batch_size, dtype=tf.sg_floatx)
# discriminator labels ( half 1s, half 0s )
y_disc = tf.concat([y, y * 0], 0)
#
# create generator
#
# I've used ESPCN scheme
# http://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/Shi_Real-Time_Single_Image_CVPR_2016_paper.pdf
#
# generator network
with tf.sg_context(name='generator', act='relu', bn=True):
gen = (x_small.sg_conv(dim=32).sg_conv().sg_conv(
dim=4, act='sigmoid', bn=False).sg_periodic_shuffle(factor=2))
# add image summary
tf.sg_summary_image(gen)
num_cont = 2 # continuous variable number num_dim = 30 # total latent dimension ( category + continuous + noise ) # # inputs # # input tensor ( with QueueRunner ) data = TimeSeriesData(batch_size=batch_size) x = data.X # generator labels ( all ones ) y = tf.ones(batch_size, dtype=tf.sg_floatx) # discriminator labels ( half 1s, half 0s ) y_disc = tf.concat(0, [y, y * 0]) # # create generator # # random class number z_cat = tf.multinomial(tf.ones((batch_size, num_category), dtype=tf.sg_floatx) / num_category, 1).sg_squeeze() # random seed = random categorical variable + random uniform z = z_cat.sg_one_hot(depth=num_category).sg_concat(target=tf.random_uniform((batch_size, num_dim-num_category))) # random continuous variable z_cont = z[:, num_category:num_category+num_cont]
x = data.train.image
# labels for discriminator
y_real = tf.ones(batch_size)
y_fake = tf.zeros(batch_size)
# categorical latent variable
z_cat = tf.multinomial(
tf.ones((batch_size, cat_dim), dtype=tf.sg_floatx) / cat_dim,
1).sg_squeeze().sg_int()
# continuous latent variable
z_con = tf.random_uniform((batch_size, con_dim))
# random latent variable dimension
z_rand = tf.random_uniform((batch_size, rand_dim))
# latent variable
z = tf.concat([z_cat.sg_one_hot(depth=cat_dim), z_con, z_rand], 1)
#
# Computational graph
#
# generator
gen = generator(z)
# add image summary
tf.sg_summary_image(x, name='real')
tf.sg_summary_image(gen, name='fake')
# discriminator
disc_real, _, _ = discriminator(x)
disc_fake, cat_fake, con_fake = discriminator(gen)
# MNIST input tensor ( with QueueRunner )
data = tf.sg_data.Mnist(batch_size=batch_size)
# input images
x = data.train.image
# corrupted image
x_small = tf.image.resize_bicubic(x, (14, 14))
x_nearest = tf.image.resize_images(x_small, (28, 28), tf.image.ResizeMethod.NEAREST_NEIGHBOR)
# generator labels ( all ones )
y = tf.ones(batch_size, dtype=tf.sg_floatx)
# discriminator labels ( half 1s, half 0s )
y_disc = tf.concat(0, [y, y * 0])
#
# create generator
#
# I've used ESPCN scheme
# http://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/Shi_Real-Time_Single_Image_CVPR_2016_paper.pdf
#
# generator network
with tf.sg_context(name='generator', act='relu', bn=True):
gen = (x_small
.sg_conv(dim=32)
.sg_conv()
.sg_conv(dim=4, act='sigmoid', bn=False)
.sg_periodic_shuffle(factor=2))
def sg_concat(tensor, opt):
assert opt.target is not None, 'target is mandatory.'
opt += tf.sg_opt(dim=tensor.get_shape().ndims - 1)
target = opt.target if isinstance(opt.target,
(tuple, list)) else [opt.target]
return tf.concat(opt.dim, [tensor] + target, name=opt.name)
# convert to tensor queue
self.train.image, self.train.label = \
_data_to_tensor([x.astype('float32'), label.astype('int32')], batch_size, name='train')
# calc total batch count
self.train.num_batch = label.shape[0] // batch_size
# MNIST input tensor ( with QueueRunner )
data = Mnist(batch_size=batch_size)
# input images
x = data.train.image
# generator labels ( all ones )
y = tf.ones(batch_size, dtype=tf.sg_floatx)
# discriminator labels ( half 1s, half 0s )
y_disc = tf.concat([y, y * 0], 0)
#
# create generator
#
# random class number
z_cat = tf.multinomial(
tf.ones((batch_size, num_category), dtype=tf.sg_floatx) / num_category,
1).sg_squeeze().sg_int()
# random seed = random categorical variable + random uniform
z = z_cat.sg_one_hot(depth=num_category).sg_concat(
target=tf.random_uniform((batch_size, num_dim - num_category)))
# random continuous variable
z_cont = z[:, num_category:num_category + num_cont]
# category label
label = tf.concat([data.train.label, z_cat], 0)
# generator network
num_cont = 2 # continuous variable number
num_dim = 30 # total latent dimension ( category + continuous + noise )
#
# inputs
#
# input tensor ( with QueueRunner )
data = TimeSeriesData(batch_size=batch_size)
x = data.X
# generator labels ( all ones )
y = tf.ones(batch_size, dtype=tf.sg_floatx)
# discriminator labels ( half 1s, half 0s )
y_disc = tf.concat(0, [y, y * 0])
#
# create generator
#
# random class number
z_cat = tf.multinomial(
tf.ones((batch_size, num_category), dtype=tf.sg_floatx) / num_category,
1).sg_squeeze()
# random seed = random categorical variable + random uniform
z = z_cat.sg_one_hot(depth=num_category).sg_concat(
target=tf.random_uniform((batch_size, num_dim - num_category)))
# random continuous variable
def sg_lstm(tensor, opt):
r"""Applies an LSTM.
Args:
tensor: A 3-D `Tensor`.
in_dim: A positive `integer`. The size of input dimension.
dim: A positive `integer`. The size of output dimension.
bias: Boolean. If True, biases are added.
ln: Boolean. If True, layer normalization is applied.
init_state: A 2-D `Tensor`. If None, the initial state is set to zeros.
last_only: Boolean. If True, the outputs in the last time step are returned.
Returns:
A `Tensor`. If last_only is False, the output tensor has shape [batch size, time steps, dim].
If last_only is True, the shape will be [batch size, dim].
"""
# layer normalization
ln = lambda v: _ln_rnn(v, gamma, beta) if opt.ln else v
# step func
def step(h, c, x):
# forget gate
f = tf.sigmoid(
ln(
tf.matmul(x, w_f) + tf.matmul(h, u_f) +
(b_f if opt.bias else 0)))
# input gate
i = tf.sigmoid(
ln(
tf.matmul(x, w_i) + tf.matmul(h, u_i) +
(b_i if opt.bias else 0)))
# new cell value
cc = tf.tanh(
ln(
tf.matmul(x, w_c) + tf.matmul(h, u_c) +
(b_c if opt.bias else 0)))
# out gate
o = tf.sigmoid(
ln(
tf.matmul(x, w_o) + tf.matmul(h, u_o) +
(b_o if opt.bias else 0)))
# cell update
cell = f * c + i * cc
# final output
y = o * tf.tanh(cell)
return y, cell
# parameter initialize
w_i = init.orthogonal('W_i', (opt.in_dim, opt.dim))
u_i = init.identity('U_i', opt.dim)
w_f = init.orthogonal('W_f', (opt.in_dim, opt.dim))
u_f = init.identity('U_f', opt.dim)
w_o = init.orthogonal('W_o', (opt.in_dim, opt.dim))
u_o = init.identity('U_o', opt.dim)
w_c = init.orthogonal('W_c', (opt.in_dim, opt.dim))
u_c = init.identity('U_c', opt.dim)
if opt.bias:
b_i = init.constant('b_i', opt.dim)
b_f = init.constant('b_f', opt.dim)
b_o = init.constant('b_o', opt.dim, value=1)
b_c = init.constant('b_c', opt.dim)
# layer normalization parameters
if opt.ln:
# offset, scale parameter
beta = init.constant('beta', opt.dim)
gamma = init.constant('gamma', opt.dim, value=1)
# initial state
init_h = opt.init_state if opt.init_state is not None \
else tf.zeros((tensor.get_shape().as_list()[0], opt.dim), dtype=tf.sg_floatx)
# do rnn loop
h, c, out = init_h, init_h, []
for i in range(tensor.get_shape().as_list()[1]):
# apply step function
h, c = step(h, c, tensor[:, i, :])
# save result
out.append(h.sg_expand_dims(dim=1))
# merge tensor
if opt.last_only:
out = out[-1].sg_squeeze(dim=1)
else:
out = tf.concat(1, out)
return out
def sg_gru(tensor, opt):
r"""Applies a GRU.
Args:
tensor: A 3-D `Tensor`.
in_dim: A positive `integer`. The size of input dimension.
dim: A positive `integer`. The size of output dimension.
bias: Boolean. If True, biases are added.
ln: Boolean. If True, layer normalization is applied.
init_state: A 2-D `Tensor`. If None, the initial state is set to zeros.
last_only: Boolean. If True, the outputs in the last time step are returned.
Returns:
A `Tensor`. If last_only is False, the output tensor has shape [batch size, time steps, dim].
If last_only is True, the shape will be [batch size, dim].
"""
# layer normalization
ln = lambda v: _ln_rnn(v, gamma, beta) if opt.ln else v
# step func
def step(h, x):
# update gate
z = tf.sigmoid(
ln(
tf.matmul(x, w_z) + tf.matmul(h, u_z) +
(b_z if opt.bias else 0)))
# reset gate
r = tf.sigmoid(
ln(
tf.matmul(x, w_r) + tf.matmul(h, u_r) +
(b_r if opt.bias else 0)))
# h_hat
hh = tf.tanh(
ln(
tf.matmul(x, w_h) + tf.matmul(r * h, u_h) +
(b_h if opt.bias else 0)))
# final output
y = (1. - z) * h + z * hh
return y
# parameter initialize
w_z = init.orthogonal('W_z', (opt.in_dim, opt.dim))
u_z = init.identity('U_z', opt.dim)
w_r = init.orthogonal('W_r', (opt.in_dim, opt.dim))
u_r = init.identity('U_r', opt.dim)
w_h = init.orthogonal('W_h', (opt.in_dim, opt.dim))
u_h = init.identity('U_h', opt.dim)
if opt.bias:
b_z = init.constant('b_z', opt.dim)
b_r = init.constant('b_r', opt.dim)
b_h = init.constant('b_h', opt.dim)
# layer normalization parameters
if opt.ln:
# offset, scale parameter
beta = init.constant('beta', opt.dim)
gamma = init.constant('gamma', opt.dim, value=1)
# initial state
init_h = opt.init_state if opt.init_state is not None \
else tf.zeros((tensor.get_shape().as_list()[0], opt.dim), dtype=tf.sg_floatx)
# do rnn loop
h, out = init_h, []
for i in range(tensor.get_shape().as_list()[1]):
# apply step function
h = step(h, tensor[:, i, :])
# save result
out.append(h.sg_expand_dims(dim=1))
# merge tensor
if opt.last_only:
out = out[-1].sg_squeeze(dim=1)
else:
out = tf.concat(1, out)
return out
def sg_quasi_rnn(tensor, opt):
# Split
if opt.att:
H, Z, F, O = tf.split(axis=0, num_or_size_splits=4,
value=tensor) # (b, seqlen, hd) for all
else:
Z, F, O = tf.split(axis=0, num_or_size_splits=3,
value=tensor) # (b, seqlen, hd) for all
# step func
def step(z, f, o, c):
'''
Runs fo-pooling at each time step
'''
c = f * c + (1 - f) * z
if opt.att: # attention
a = tf.nn.softmax(tf.einsum("ijk,ik->ij", H,
c)) # alpha. (b, seqlen)
k = (a.sg_expand_dims() * H).sg_sum(
axis=1) # attentional sum. (b, seqlen)
h = o * (k.sg_dense_gpus(act="linear",name = "k%d_%s"%(t,opt.name),dev = opt.dev,reuse=opt.reuse_vars)\
+ c.sg_dense_gpus(act="linear",name = "c%d_%s"%(t,opt.name),dev = opt.dev,reuse=opt.reuse_vars))
else:
h = o * c
return h, c # hidden states, (new) cell memories
# Do rnn loop
c, hs = 0, []
timesteps = tensor.get_shape().as_list()[1]
for t in range(timesteps):
z = Z[:, t, :] # (b, hd)
f = F[:, t, :] # (b, hd)
o = O[:, t, :] # (b, hd)
# apply step function
h, c = step(z, f, o, c) # (b, hd), (b, hd)
# save result
hs.append(h.sg_expand_dims(axis=1))
# Concat to return
H = tf.concat(hs, 1) # (b, seqlen, hd)
if opt.is_enc:
H_z = tf.tile(
(h.sg_dense_gpus(act="linear",
name="z_%s" % (opt.name),
dev=opt.dev,
reuse=opt.reuse_vars).sg_expand_dims(axis=1)),
[1, timesteps, 1])
H_f = tf.tile(
(h.sg_dense_gpus(act="linear",
name="f_%s" % (opt.name),
dev=opt.dev,
reuse=opt.reuse_vars).sg_expand_dims(axis=1)),
[1, timesteps, 1])
H_o = tf.tile(
(h.sg_dense_gpus(act="linear",
name="o_%s" % (opt.name),
dev=opt.dev,
reuse=opt.reuse_vars).sg_expand_dims(axis=1)),
[1, timesteps, 1])
concatenated = tf.concat(axis=0, values=[H, H_z, H_f,
H_o]) # (b*4, seqlen, hd)
return concatenated
else:
return H # (b, seqlen, hd)
# # input tensor ( with QueueRunner ) data = SpectrumData(batch_size=batch_size) x = data.X # labels for discriminator y_real = tf.ones(batch_size) y_fake = tf.zeros(batch_size) # continuous latent variable z_con = tf.random_uniform((batch_size, con_dim)) # random latent variable dimension z_rand = tf.random_uniform((batch_size, rand_dim)) # latent variable z = tf.concat([z_con, z_rand], 1) # # Computational graph # # generator gen = generator(z) # discriminator disc_real, _ = discriminator(x) disc_fake, con_fake = discriminator(gen) # # loss #
