def sg_conv1d(tensor, opt):
r"""Applies a 1-D convolution.
Args:
tensor: A `Tensor`.
size: An `integer` representing `[kernel width]`.
If not specified, 2 is set implicitly.
stride: An `integer`. The number of entries by which
the filter is moved right at each step.
in_dim: An `integer`. The size of input dimension.
dim: An `integer`. The size of output dimension.
pad: Either `SAME` (Default) or `VALID`.
bias: Boolean. Whether to add biases to the filters.
Returns:
A `Tensor` with the same type as `tensor`.
"""
# default options
opt += tf.sg_opt(size=2, stride=1, pad='SAME')
# parameter initialize
w = init.he_uniform('W', (opt.size, opt.in_dim, opt.dim))
if opt.bias:
b = init.constant('b', opt.dim)
# apply convolution
out = tf.nn.conv1d(tensor, w, stride=opt.stride,
padding=opt.pad) + (b if opt.bias else 0)
return out
def sg_emb(**kwargs):
r"""Returns an embedding layer or a look-up table.
Args:
name: A name for the layer (required).
emb: A 2-D array. Has the shape of `[vocabulary size -1, embedding dimension size]`.
Note that the first row is filled with 0's because they correspond to padding.
in_dim: A positive `integer`. The size of input dimension.
dim: A positive `integer`. The size of output dimension.
voca_size: A positive int32.
Returns:
A 2-D tensor.
"""
opt = tf.sg_opt(kwargs)
assert opt.name is not None, 'name is mandatory.'
import sg_initializer as init
if opt.emb is None:
# initialize embedding matrix
assert opt.voca_size is not None, 'voca_size is mandatory.'
assert opt.dim is not None, 'dim is mandatory.'
w = init.he_uniform(opt.name, (opt.voca_size - 1, opt.dim))
else:
# use given embedding matrix
w = init.external(opt.name, value=opt.emb)
# 1st row should be zero and not be updated by backprop because of zero padding.
emb = tf.concat(0, [tf.zeros((1, opt.dim), dtype=tf.sg_floatx), w])
return emb
def sg_aconv(tensor, opt):
r"""Applies a 2-D atrous (or dilated) convolution.
Args:
tensor: A 4-D `Tensor`.
size: A tuple or list of integers of length 2 representing `[kernel height, kernel width]`.
Can be an int if both values are the same.
If not specified, (3, 3) is set automatically.
rate: A positive int32. The stride with which we sample input values across
the `height` and `width` dimensions. Default is 2.
in_dim: An `integer`. The size of input dimension.
dim: An `integer`. The size of output dimension.
pad: Either `SAME` (Default) or `VALID`.
bias: Boolean. Whether to add biases to the filters.
Returns:
A `Tensor` with the same type as `tensor`.
"""
# default options
opt += tf.sg_opt(size=(3, 3), rate=2, pad='SAME')
opt.size = opt.size if isinstance(opt.size,
(tuple, list)) else [opt.size, opt.size]
# parameter initialize
w = init.he_uniform('W', (opt.size[0], opt.size[1], opt.in_dim, opt.dim))
if opt.bias:
b = init.constant('b', opt.dim)
# apply convolution
out = tf.nn.atrous_conv2d(tensor, w, rate=opt.rate,
padding=opt.pad) + (b if opt.bias else 0)
return out
def sg_aconv1d(tensor, opt):
# default options
opt += tf.sg_opt(size=(2 if opt.causal else 3), rate=1, pad='SAME')
# parameter initialize
w = init.he_uniform('W', (1, opt.size, opt.in_dim, opt.dim))
if opt.bias:
b = init.constant('b', opt.dim)
if opt.causal:
# pre-padding for causality
if opt.pad == 'SAME':
pad_len = (opt.size - 1) * opt.rate # padding size
x = tf.pad(tensor,
[[0, 0], [pad_len, 0], [0, 0]]).sg_expand_dims(dim=1)
else:
x = tensor.sg_expand_dims(dim=1)
# apply 2d convolution
out = tf.nn.atrous_conv2d(x, w, rate=opt.rate,
padding='VALID') + (b if opt.bias else 0)
else:
# apply 2d convolution
out = tf.nn.atrous_conv2d(
tensor.sg_expand_dims(dim=1), w, rate=opt.rate,
padding=opt.pad) + (b if opt.bias else 0)
# reduce dimension
out = out.sg_squeeze(dim=1)
return out
def sg_upconv(tensor, opt):
# default options
opt += tf.sg_opt(size=(3, 3), stride=(1, 2, 2, 1), pad='SAME')
opt.size = opt.size if isinstance(opt.size,
(tuple, list)) else [opt.size, opt.size]
opt.stride = opt.stride if isinstance(
opt.stride, (tuple, list)) else [1, opt.stride, opt.stride, 1]
opt.stride = [1, opt.stride[0], opt.stride[1], 1] if len(
opt.stride) == 2 else opt.stride
# parameter initialize
w = init.he_uniform('W', (opt.size[0], opt.size[1], opt.dim, opt.in_dim))
if opt.bias:
b = init.constant('b', opt.dim)
# tedious shape handling for conv2d_transpose
shape = tensor.get_shape().as_list()
out_shape = [
tf.shape(tensor)[0], shape[1] * opt.stride[1],
shape[2] * opt.stride[2], opt.dim
]
# apply convolution
out = tf.nn.conv2d_transpose(tensor,
w,
output_shape=tf.pack(out_shape),
strides=opt.stride,
padding=opt.pad) + (b if opt.bias else 0)
# reset shape is needed because conv2d_transpose() erase all shape information.
out.set_shape([None, out_shape[1], out_shape[2], opt.dim])
return out
def sg_dense(tensor, opt):
# parameter initialize
w = init.he_uniform('W', (opt.in_dim, opt.dim))
if opt.bias:
b = init.constant('b', opt.dim)
# apply transform
out = tf.matmul(tensor, w) + (b if opt.bias else 0)
return out
def sg_upconv(tensor, opt):
r"""Applies a upconvolution (or convolution transpose).
Args:
tensor: A 4-D `Tensor`.
size: A tuple or list of integers of length 2 representing `[kernel height, kernel width]`.
Can be an int if both values are the same.
If not specified, (3, 3) is set implicitly.
The default value is [1, 2, 2, 1].
stride: A tuple or list of integers of length 2 or 4 representing stride dimensions.
If the length is 2, i.e., (a, b), the stride is `[1, a, b, 1]`.
If the length is 4, i.e., (a, b, c, d), the stride is `[a, b, c, d]`.
Can be an int. If the length is an int, i.e., a, the stride is `[1, a, a, 1]`.
in_dim: A positive `integer`. The size of input dimension.
dim: A positive `integer`. The size of output dimension.
pad: Either `SAME` (Default) or `VALID`.
bias: Boolean. If True, biases are added.
Returns:
A `Tensor` with the same type as `tensor`.
"""
# default options
opt += tf.sg_opt(size=(3, 3), stride=(1, 2, 2, 1), pad='SAME')
opt.size = opt.size if isinstance(opt.size,
(tuple, list)) else [opt.size, opt.size]
opt.stride = opt.stride if isinstance(
opt.stride, (tuple, list)) else [1, opt.stride, opt.stride, 1]
opt.stride = [1, opt.stride[0], opt.stride[1], 1] if len(
opt.stride) == 2 else opt.stride
# parameter initialize
w = init.he_uniform('W', (opt.size[0], opt.size[1], opt.dim, opt.in_dim))
if opt.bias:
b = init.constant('b', opt.dim)
# tedious shape handling for conv2d_transpose
shape = tensor.get_shape().as_list()
out_shape = [
tf.shape(tensor)[0], shape[1] * opt.stride[1],
shape[2] * opt.stride[2], opt.dim
]
# apply convolution
out = tf.nn.conv2d_transpose(tensor,
w,
output_shape=tf.pack(out_shape),
strides=opt.stride,
padding=opt.pad) + (b if opt.bias else 0)
# reset shape is needed because conv2d_transpose() erase all shape information.
out.set_shape([None, out_shape[1], out_shape[2], opt.dim])
return out
def sg_aconv(tensor, opt):
# default options
opt += tf.sg_opt(size=(3, 3), rate=2, pad='VALID')
opt.size = opt.size if isinstance(opt.size, (tuple, list)) else [opt.size, opt.size]
# parameter initialize
w = init.he_uniform('W', (opt.size[0], opt.size[1], opt.in_dim, opt.dim))
if opt.bias:
b = init.constant('b', opt.dim)
# apply convolution
out = tf.nn.atrous_conv2d(tensor, w, rate=opt.rate, padding=opt.pad) + (b if opt.bias else 0)
return out
def sg_conv1d(tensor, opt):
# default options
opt += tf.sg_opt(size=2, stride=1, pad='SAME')
# parameter initialize
w = init.he_uniform('W', (opt.size, opt.in_dim, opt.dim))
if opt.bias:
b = init.constant('b', opt.dim)
# apply convolution
out = tf.nn.conv1d(tensor, w, stride=opt.stride,
padding=opt.pad) + (b if opt.bias else 0)
return out
def sg_conv(tensor, opt):
# default options
opt += tf.sg_opt(size=(3, 3), stride=(1, 1, 1, 1), pad='SAME')
opt.size = opt.size if isinstance(opt.size, (tuple, list)) else [opt.size, opt.size]
opt.stride = opt.stride if isinstance(opt.stride, (tuple, list)) else [1, opt.stride, opt.stride, 1]
opt.stride = [1, opt.stride[0], opt.stride[1], 1] if len(opt.stride) == 2 else opt.stride
# parameter initialize
w = init.he_uniform('W', (opt.size[0], opt.size[1], opt.in_dim, opt.dim))
if opt.bias:
b = init.constant('b', opt.dim)
# apply convolution
out = tf.nn.conv2d(tensor, w, strides=opt.stride, padding=opt.pad) + (b if opt.bias else 0)
return out
def sg_aconv1d(tensor, opt):
r"""Applies 1-D atrous (or dilated) convolution.
Args:
tensor: A 3-D `Tensor`.
causal: Boolean. If True, zeros are padded before the time axis such that
each activation unit doesn't have receptive neurons beyond the equivalent time step.
size: An `integer` representing `[kernel width]`. As a default it is set to 2
if causal is True, 3 otherwise.
rate: A positive int32. The stride with which we sample input values across
the `height` and `width` dimensions. Default is 1.
in_dim: An `integer`. The size of input dimension.
dim: An `integer`. The size of output dimension.
pad: Either `SAME` (Default) or `VALID`.
bias: Boolean. Whether to add biases to the filters.
Returns:
A `Tensor` with the same type as `tensor`.
"""
# default options
opt += tf.sg_opt(size=(2 if opt.causal else 3), rate=1, pad='SAME')
# parameter initialize
w = init.he_uniform('W', (1, opt.size, opt.in_dim, opt.dim))
if opt.bias:
b = init.constant('b', opt.dim)
if opt.causal:
# pre-padding for causality
if opt.pad == 'SAME':
pad_len = (opt.size - 1) * opt.rate # padding size
x = tf.pad(tensor,
[[0, 0], [pad_len, 0], [0, 0]]).sg_expand_dims(dim=1)
else:
x = tensor.sg_expand_dims(dim=1)
# apply 2d convolution
out = tf.nn.atrous_conv2d(x, w, rate=opt.rate,
padding='VALID') + (b if opt.bias else 0)
else:
# apply 2d convolution
out = tf.nn.atrous_conv2d(
tensor.sg_expand_dims(dim=1), w, rate=opt.rate,
padding=opt.pad) + (b if opt.bias else 0)
# reduce dimension
out = out.sg_squeeze(dim=1)
return out
def sg_emb(**kwargs):
opt = tf.sg_opt(kwargs)
assert opt.name is not None, 'name is mandatory.'
import sg_initializer as init
if opt.emb is None:
# initialize embedding matrix
assert opt.voca_size is not None, 'voca_size is mandatory.'
assert opt.dim is not None, 'dim is mandatory.'
w = init.he_uniform(opt.name, (opt.voca_size - 1, opt.dim))
else:
# use given embedding matrix
w = init.external(opt.name, value=opt.emb)
# 1st row should be zero and not be updated by backprop because of zero padding.
emb = tf.concat(0, [tf.zeros((1, opt.dim), dtype=tf.sg_floatx), w])
return emb
def sg_dense(tensor, opt):
r"""Applies a full connection.
Args:
tensor: A 2-D `Tensor`.
in_dim: An `integer`. The size of input dimension.
dim: An `integer`. The size of output dimension.
bias: Boolean. If True, biases are added.
Returns:
A `Tensor` with the same type as `tensor`.
"""
# parameter initialize
w = init.he_uniform('W', (opt.in_dim, opt.dim))
if opt.bias:
b = init.constant('b', opt.dim)
# apply transform
out = tf.matmul(tensor, w) + (b if opt.bias else 0)
return out
def sg_conv(tensor, opt):
r"""Applies a 2-D convolution.
Args:
tensor: A 4-D `Tensor`.
size: A tuple or list of integers of length 2 representing `[kernel height, kernel width]`.
Can be an int if both values are the same.
If not specified, (3, 3) is set implicitly.
stride: A tuple or list of integers of length 2 or 4 representing stride dimensions.
If the length is 2, i.e., (a, b), the stride is `[1, a, b, 1]`.
If the length is 4, i.e., (a, b, c, d), the stride is `[a, b, c, d]`.
Can be an int. If the length is an int, i.e., a, the stride is `[1, a, a, 1]`.
The default value is [1, 1, 1, 1].
in_dim: An `integer`. The size of input dimension.
dim: An `integer`. The size of output dimension.
pad: Either `SAME` (Default) or `VALID`.
bias: Boolean. If True, biases are added.
Returns:
A `Tensor` with the same type as `tensor`.
"""
# default options
opt += tf.sg_opt(size=(3, 3), stride=(1, 1, 1, 1), pad='SAME')
opt.size = opt.size if isinstance(opt.size,
(tuple, list)) else [opt.size, opt.size]
opt.stride = opt.stride if isinstance(
opt.stride, (tuple, list)) else [1, opt.stride, opt.stride, 1]
opt.stride = [1, opt.stride[0], opt.stride[1], 1] if len(
opt.stride) == 2 else opt.stride
# parameter initialize
w = init.he_uniform('W', (opt.size[0], opt.size[1], opt.in_dim, opt.dim))
if opt.bias:
b = init.constant('b', opt.dim)
# apply convolution
out = tf.nn.conv2d(tensor, w, strides=opt.stride,
padding=opt.pad) + (b if opt.bias else 0)
return out
