def __call__(self, x_t, state, size, scope=None, reuse_vars=False):
(prev_c, prev_h) = state
scope = scope or tf.get_variable_scope()
print("____reuse_______", reuse_vars)
with tf.variable_scope(scope, reuse=True):
w_ic = tf.get_variable("w_ic")
w_fc = tf.get_variable("w_fc")
w_oc = tf.get_variable("w_oc")
with tf.sg_context(dev=self._dev, reuse=reuse_vars):
i = x_t.sg_conv1d_gpus(name = "ix_",size=size)+\
prev_h.sg_conv1d_gpus(name = "ih_",size=size)+\
prev_c*w_ic
f = x_t.sg_aconv1d_gpus(name = "fx_",size=size)+\
prev_h.sg_aconv1d_gpus(name = "fh_",size=size)+\
prev_c*w_fc
c = x_t.sg_conv1d_gpus(name = "cx_",size=size)+\
prev_h.sg_conv1d_gpus(name = "ch_",size=size)
o = x_t.sg_conv1d_gpus(name = "ox_",size=size)+\
prev_h.sg_conv1d_gpus(name = "oh_",size=size)+\
prev_c*w_oc
new_c = prev_c * tf.sigmoid(f) + tf.sigmoid(i) * self._activation(c)
new_h = self._activation(new_c) * tf.sigmoid(o)
return (new_c, new_h)
def linear(input_, output_size, scope=None):
'''
Linear map: output[k] = sum_i(Matrix[k, i] * args[i] ) + Bias[k]
Args:
args: a tensor or a list of 2D, batch x n, Tensors.
output_size: int, second dimension of W[i].
scope: VariableScope for the created subgraph; defaults to "Linear".
Returns:
A 2D Tensor with shape [batch x output_size] equal to
sum_i(args[i] * W[i]), where W[i]s are newly created matrices.
Raises:
ValueError: if some of the arguments has unspecified or wrong shape.
'''
shape = input_.get_shape().as_list()
if len(shape) != 2:
raise ValueError("Linear is expecting 2D arguments: %s" % str(shape))
if not shape[1]:
raise ValueError("Linear expects shape[1] of arguments: %s" %
str(shape))
input_size = shape[1]
# Now the computation.
with tf.variable_scope(scope or "SimpleLinear"):
matrix = tf.get_variable("Matrix", [output_size, input_size],
dtype=input_.dtype)
bias_term = tf.get_variable("Bias", [output_size], dtype=input_.dtype)
return tf.matmul(input_, tf.transpose(matrix)) + bias_term
def __init__(self,
in_dim,
dim,
forget_bias=1.0,
activation=tf.tanh,
ln=True,
bias=True,
dtype=tf.float32,
dev='/cpu:0',
batch_size=3):
self._in_dim = in_dim
self._dim = dim
self._forget_bias = forget_bias
self._activation = activation
self._ln = False
self._bias = bias
self._dev = dev
self._size = self._in_dim * self._dim
self._initializer = tf.contrib.layers.xavier_initializer(
) #tf.random_normal_initializer()
self._dtype = dtype
with tf.device(self._dev):
with tf.variable_scope("lstm") as scp:
#self.rnn_state = tf.get_variable("rnn_c",(batch_size, self._dim), dtype=tf.sg_floatx,initializer=tf.constant_initializer(0.0),trainable=False)
#self.rnn_h = tf.get_variable("rnn_h",(batch_size, self._dim), dtype=tf.sg_floatx,initializer=tf.constant_initializer(0.0),trainable=False)
self.rnn_state, self.rnn_h = tf.zeros(
(batch_size, self._dim), dtype=tf.sg_floatx), tf.zeros(
(batch_size, self._dim), dtype=tf.sg_floatx)
w_i2h = tf.get_variable(
'w_i2h', (self._in_dim, 4 * self._dim),
dtype=tf.float32,
initializer=tf.contrib.layers.xavier_initializer(),
trainable=True)
w_h2h = tf.get_variable(
'w_h2h', (self._dim, 4 * self._dim),
dtype=tf.float32,
initializer=tf.contrib.layers.xavier_initializer(),
trainable=True)
w_b = tf.get_variable(
'w_b', (1, 4 * self._dim),
dtype=tf.float32,
initializer=tf.contrib.layers.xavier_initializer(),
trainable=True) if self._bias == True else 0.0
if self._ln:
with tf.variable_scope("ln_rnn"):
beta = tf.get_variable(
'beta',
self._dim,
dtype=tf.sg_floatx,
initializer=tf.constant_initializer(0.0),
trainable=True)
gamma = tf.get_variable(
'gamma',
self._dim,
dtype=tf.sg_floatx,
initializer=tf.constant_initializer(1.0),
trainable=True)
def _linear(self, arys):
scope = tf.get_variable_scope()
with tf.variable_scope(scope, reuse=True):
w_i2h = tf.get_variable("w_i2h")
w_h2h = tf.get_variable("w_h2h")
w_b = tf.get_variable("w_b") if self._bias == True else 0
i2h = tf.matmul(arys[0], w_i2h)
h2h = tf.matmul(arys[1], w_h2h)
out = i2h + h2h + w_b
return out
def make_states(self, batch_size):
seqlen = self._seqlen
self.crnn_state = tf.get_variable(
"crnn_c", (batch_size, seqlen, self._dim),
dtype=tf.sg_floatx,
initializer=tf.constant_initializer(0.0),
trainable=False)
self.crnn_h = tf.get_variable("crnn_h",
(batch_size, seqlen, self._dim),
dtype=tf.sg_floatx,
initializer=tf.constant_initializer(0.0),
trainable=False)
def constant(name, shape, value=0, dtype=tf.sg_floatx, summary=True, regularizer=None, trainable=True):
r"""Creates a tensor variable of which initial values are `value` and shape is `shape`.
Args:
name: The name of new variable.
shape: A tuple/list of integers or an integer.
If shape is an integer, it is converted to a list.
value: A Python scalar. All elements of the initialized variable
will be set to this value. Default is 0.
dtype: The data type. Only floating point types are supported. Default is float32.
summary: If True, add this constant to tensor board summary.
regularizer: A (Tensor -> Tensor or None) function; the result of applying it on a newly created variable
will be added to the collection tf.GraphKeys.REGULARIZATION_LOSSES and can be used for regularization
trainable: If True, add this constant to trainable collection. Default is True.
Returns:
A `Variable`.
"""
shape = shape if isinstance(shape, (tuple, list)) else [shape]
x = tf.get_variable(name, shape, dtype=dtype,
initializer=tf.constant_initializer(value),
regularizer=regularizer, trainable=trainable)
# add summary
if summary:
tf.sg_summary_param(x)
return x
def orthogonal(name, shape, scale=1.1, dtype=tf.sg_floatx, summary=True):
r"""Creates a tensor variable of which initial values are of
an orthogonal ndarray.
See [Saxe et al. 2014.](http://arxiv.org/pdf/1312.6120.pdf)
Args:
name: The name of new variable.
shape: A tuple/list of integers.
scale: A Python scalar.
dtype: Either float32 or float64.
summary: If True, add this constant to tensor board summary.
Returns:
A `Variable`.
"""
flat_shape = (shape[0], np.prod(shape[1:]))
a = np.random.normal(0.0, 1.0, flat_shape)
u, _, v = np.linalg.svd(a, full_matrices=False)
# pick the one with the correct shape
q = u if u.shape == flat_shape else v
q = q.reshape(shape)
# create variable
x = tf.get_variable(name,
initializer=tf.constant(scale *
q[:shape[0], :shape[1]],
dtype=dtype))
# add summary
if not tf.get_variable_scope().reuse and summary:
tf.sg_summary_param(x)
return x
def constant(name, shape, value=0, dtype=tf.sg_floatx, summary=True):
r"""Creates a tensor variable of which initial values are `value` and shape is `shape`.
Args:
name: The name of new variable.
shape: A tuple/list of integers or an integer.
If shape is an integer, it is converted to a list.
value: A Python scalar. All elements of the initialized variable
will be set to this value. Default is 0.
dtype: The data type. Only floating point types are supported. Default is float32.
summary: If True, add this constant to tensor board summary.
Returns:
A `Variable`.
"""
shape = shape if isinstance(shape, (tuple, list)) else [shape]
x = tf.get_variable(name,
shape,
dtype=dtype,
initializer=tf.constant_initializer(value))
# add summary
if not tf.get_variable_scope().reuse and summary:
tf.sg_summary_param(x)
return x
def identity(name, dim, scale=1, dtype=tf.sg_floatx):
x = tf.get_variable(name,
initializer=tf.constant(np.eye(dim) * scale, dtype=dtype))
# add summary
if not tf.get_variable_scope().reuse:
tf.sg_summary_param(x)
return x
def external(name, value, dtype=tf.sg_floatx, summary=True, regularizer=None, trainable=True):
r"""Creates a tensor variable of which initial values are `value`.
For example,
```
external("external", [3,3,1,2])
=> [3. 3. 1. 2.]
```
Args:
name: The name of new variable.
value: A constant value (or list) of output type `dtype`.
dtype: The type of the elements of the resulting tensor.
summary: If True, add this constant to tensor board summary.
regularizer: A (Tensor -> Tensor or None) function; the result of applying it on a newly created variable
will be added to the collection tf.GraphKeys.REGULARIZATION_LOSSES and can be used for regularization
trainable: If True, add this constant to trainable collection. Default is True.
Returns:
A `Variable`. Has the same contents as `value` of `dtype`.
"""
# create variable
x = tf.get_variable(name,
initializer=tf.constant(value, dtype=dtype),
regularizer=regularizer, trainable=trainable)
# add summary
if summary:
tf.sg_summary_param(x)
return x
def uniform(name, shape, scale=0.05, dtype=tf.sg_floatx, summary=True, regularizer=None, trainable=True):
r"""Creates a tensor variable of which initial values are
random numbers based on uniform distribution.
Note that the default value of `scale` (=0.05) is different from
the min/max values (=0.0, 1.0) of tf.random_uniform_initializer.
Args:
name: The name of the new variable.
shape: A tuple/list of integers or an integer.
If shape is an integer, it's converted to a list.
scale: A Python scalar. All initial values should be in range `[-scale, scale)`. Default is .05.
dtype: The data type. Only floating point types are supported. Default is float32.
summary: If True, add this constant to tensor board summary.
regularizer: A (Tensor -> Tensor or None) function; the result of applying it on a newly created variable
will be added to the collection tf.GraphKeys.REGULARIZATION_LOSSES and can be used for regularization
trainable: If True, add this constant to trainable collection. Default is True.
Returns:
A `Variable`.
"""
shape = shape if isinstance(shape, (tuple, list)) else [shape]
x = tf.get_variable(name, shape, dtype=dtype,
initializer=tf.random_uniform_initializer(minval=-scale, maxval=scale),
regularizer=regularizer, trainable=trainable)
# add summary
if summary:
tf.sg_summary_param(x)
return x
def orthogonal(name, shape, scale=1.1, dtype=tf.sg_floatx):
r"""Returns a random orthogonal initializer.
See Saxe et al. 2014 `http://arxiv.org/pdf/1312.6120.pdf`
Args:
name: A string. The name of the new or existing variable.
shape: A list or tuple of integers.
scale: A Python scalr.
dtype = A float32 or float64.
Returns:
A `Tensor` variable.
"""
flat_shape = (shape[0], np.prod(shape[1:]))
a = np.random.normal(0.0, 1.0, flat_shape)
u, _, v = np.linalg.svd(a, full_matrices=False)
# pick the one with the correct shape
q = u if u.shape == flat_shape else v
q = q.reshape(shape)
# create variable
x = tf.get_variable(name,
initializer=tf.constant(scale *
q[:shape[0], :shape[1]],
dtype=dtype))
# add summary
if not tf.get_variable_scope().reuse:
tf.sg_summary_param(x)
return x
def orthogonal(name, shape, scale=1.1, dtype=tf.sg_floatx, summary=True, regularizer=None, trainable=True):
r"""Creates a tensor variable of which initial values are of
an orthogonal ndarray.
See [Saxe et al. 2014.](http://arxiv.org/pdf/1312.6120.pdf)
Args:
name: The name of new variable.
shape: A tuple/list of integers.
scale: A Python scalar.
dtype: Either float32 or float64.
summary: If True, add this constant to tensor board summary.
regularizer: A (Tensor -> Tensor or None) function; the result of applying it on a newly created variable
will be added to the collection tf.GraphKeys.REGULARIZATION_LOSSES and can be used for regularization
trainable: If True, add this constant to trainable collection. Default is True.
Returns:
A `Variable`.
"""
flat_shape = (shape[0], np.prod(shape[1:]))
a = np.random.normal(0.0, 1.0, flat_shape)
u, _, v = np.linalg.svd(a, full_matrices=False)
# pick the one with the correct shape
q = u if u.shape == flat_shape else v
q = q.reshape(shape)
# create variable
x = tf.get_variable(name,
initializer=tf.constant(scale * q[:shape[0], :shape[1]], dtype=dtype),
regularizer=regularizer, trainable=trainable)
# add summary
if summary:
tf.sg_summary_param(x)
return x
def external(name, value, dtype=tf.sg_floatx, summary=True):
r"""Creates a tensor variable of which initial values are `value`.
For example,
```
external("external", [3,3,1,2])
=> [3. 3. 1. 2.]
```
Args:
name: The name of new variable.
value: A constant value (or list) of output type `dtype`.
dtype: The type of the elements of the resulting tensor.
summary: If True, add this constant to tensor board summary.
Returns:
A `Variable`. Has the same contents as `value` of `dtype`.
"""
# create variable
x = tf.get_variable(name, initializer=tf.constant(value, dtype=dtype))
# add summary
if not tf.get_variable_scope().reuse and summary:
tf.sg_summary_param(x)
return x
def identity(name, dim, scale=1, dtype=tf.sg_floatx, summary=True):
r"""Creates a tensor variable of which initial values are of
an identity matrix.
Note that the default value of `scale` (=0.05) is different from
the min/max values (=0.0, 1.0) of tf.random_uniform_initializer.
For example,
```
identity("identity", 3, 2) =>
[[2. 0. 0.]
[0. 2. 0.]
[0. 0. 2.]]
```
Args:
name: The name of new variable.
dim: An int. The size of the first and second dimension of the output tensor.
scale: A Python scalar. The value on the diagonal.
dtype: The type of the elements of the resulting tensor.
summary: If True, add this constant to tensor board summary.
Returns:
A 2-D `Variable`.
"""
x = tf.get_variable(name,
initializer=tf.constant(np.eye(dim) * scale,
dtype=dtype))
# add summary
if not tf.get_variable_scope().reuse and summary:
tf.sg_summary_param(x)
return x
def uniform(name, shape, scale=0.05, dtype=tf.sg_floatx, summary=True):
r"""Creates a tensor variable of which initial values are
random numbers based on uniform distribution.
Note that the default value of `scale` (=0.05) is different from
the min/max values (=0.0, 1.0) of tf.random_uniform_initializer.
Args:
name: The name of the new variable.
shape: A tuple/list of integers or an integer.
If shape is an integer, it's converted to a list.
scale: A Python scalar. All initial values should be in range `[-scale, scale)`. Default is .05.
dtype: The data type. Only floating point types are supported. Default is float32.
summary: If True, add this constant to tensor board summary.
Returns:
A `Variable`.
"""
shape = shape if isinstance(shape, (tuple, list)) else [shape]
x = tf.get_variable(name,
shape,
dtype=dtype,
initializer=tf.random_uniform_initializer(
minval=-scale, maxval=scale))
# add summary
if not tf.get_variable_scope().reuse and summary:
tf.sg_summary_param(x)
return x
def __init__(self,
seqlen,
in_dim,
dim,
forget_bias=1.0,
activation=tf.tanh,
ln=True,
bias=True,
dtype=tf.float32,
dev='/cpu:0',
batch_size=3):
self._in_dim = in_dim
self._dim = dim
self._forget_bias = forget_bias
self._activation = activation
self._ln = ln
self._dev = dev
self._seqlen = seqlen
self._bias = bias
self._size = int(self._in_dim * self._dim)
self._initializer = tf.contrib.layers.xavier_initializer(
) #tf.random_normal_initializer()
self._dtype = dtype
with tf.device(self._dev):
with tf.variable_scope("clstm") as scp:
#self.crnn_state = tf.get_variable("crnn_c",(batch_size, seqlen, self._dim), dtype=tf.sg_floatx,initializer=tf.constant_initializer(0.0),trainable=False)
#self.crnn_h = tf.get_variable("crnn_h",(batch_size, seqlen, self._dim), dtype=tf.sg_floatx,initializer=tf.constant_initializer(0.0),trainable=False)
w_ic = tf.get_variable(
'w_ic', (self._seqlen, self._dim),
dtype=tf.float32,
initializer=tf.contrib.layers.xavier_initializer(),
trainable=True)
w_fc = tf.get_variable(
'w_fc', (self._seqlen, self._dim),
dtype=tf.float32,
initializer=tf.contrib.layers.xavier_initializer(),
trainable=True)
w_oc = tf.get_variable(
'w_oc', (self._seqlen, self._dim),
dtype=tf.float32,
initializer=tf.contrib.layers.xavier_initializer(),
trainable=True)
self.make_states(batch_size)
def uniform(name, shape, scale=0.05, dtype=tf.sg_floatx):
shape = shape if isinstance(shape, (tuple, list)) else [shape]
x = tf.get_variable(name, shape, dtype=dtype,
initializer=tf.random_uniform_initializer(minval=-scale, maxval=scale))
# add summary
if not tf.get_variable_scope().reuse:
tf.sg_summary_param(x)
return x
def external(name, value, dtype=tf.sg_floatx):
# create variable
x = tf.get_variable(name,
initializer=tf.constant(value, dtype=dtype))
# add summary
if not tf.get_variable_scope().reuse:
tf.sg_summary_param(x)
return x
def constant(name, shape, value=0, dtype=tf.sg_floatx):
shape = shape if isinstance(shape, (tuple, list)) else [shape]
x = tf.get_variable(name, shape, dtype=dtype,
initializer=tf.constant_initializer(value))
# add summary
if not tf.get_variable_scope().reuse:
tf.sg_summary_param(x)
return x
def __call__(self, tensor, state, scope=None):
(prev_c, prev_h) = state
# i = input_gate, c = new cell value for update, f = forget_gate, o = output_gate
lstm_matrix = self._linear([tensor, prev_h])
i, c, f, o = tf.split(value=lstm_matrix, num_or_size_splits=4, axis=1)
if self._ln:
with tf.variable_scope("ln_rnn", reuse=True):
beta = tf.get_variable('beta')
gamma = tf.get_variable('gamma')
ln = lambda v: _ln_rnn(v, gamma, beta) if self._ln else v
# do rnn loop
new_c = prev_c * tf.sigmoid(ln(f)) + tf.sigmoid(
ln(i)) * self._activation(ln(c))
new_h = self._activation(new_c) * tf.sigmoid(ln(o))
return (new_c, new_h)
def embed(inputs, vocab_size, embed_size, variable_scope):
'''
inputs = tf.expand_dims(tf.range(5), 0) => (1, 5)
_embed(inputs, 5, 10) => (1, 5, 10)
'''
with tf.variable_scope(variable_scope):
lookup_table = tf.get_variable('lookup_table',
dtype=tf.float32,
shape=[vocab_size, embed_size],
initializer=tf.truncated_normal_initializer())
return tf.nn.embedding_lookup(lookup_table, inputs)
def constant(name, shape, value=0, dtype=tf.sg_floatx):
r"""Returns an initializer of `shape` with all elements set to a scalar `value`.
"""
shape = shape if isinstance(shape, (tuple, list)) else [shape]
x = tf.get_variable(name,
shape,
dtype=dtype,
initializer=tf.constant_initializer(value))
# add summary
if not tf.get_variable_scope().reuse:
tf.sg_summary_param(x)
return x
def init_custom_embeddings(name, embeddings_matrix, summary=True, trainable=False):
"""
Initializes the embedding vector with custom preloaded embeddings
"""
embedding = np.array(embeddings_matrix)
emb = tf.get_variable(name, shape=embedding.shape, initializer=tf.constant_initializer(embedding),
trainable=trainable)
if summary:
tf.sg_summary_param(emb)
return emb
def init_custom_embeddings(name, embeddings_matrix, summary=True, trainable=False):
"""
Initializes the embedding vector with custom preloaded embeddings
"""
embedding = np.array(embeddings_matrix)
emb = tf.get_variable(name, shape=embedding.shape, initializer=tf.constant_initializer(embedding),
trainable=trainable)
if summary:
tf.sg_summary_param(emb)
return emb
def uniform(name, shape, scale=0.05, dtype=tf.sg_floatx):
r"""Returns an initializer of random numbers based on uniform distribution.
Note that the default value of `scale` (=0.05) is different from
the min/max values (=0.0, 1.0) of tf.random_uniform_initializer.
"""
shape = shape if isinstance(shape, (tuple, list)) else [shape]
x = tf.get_variable(name,
shape,
dtype=dtype,
initializer=tf.random_uniform_initializer(
minval=-scale, maxval=scale))
# add summary
if not tf.get_variable_scope().reuse:
tf.sg_summary_param(x)
return x
def orthogonal(name, shape, scale=1.1, dtype=tf.sg_floatx):
# Sax et aE. ( http://arxiv.org/pdf/1312.6120.pdf )
flat_shape = (shape[0], np.prod(shape[1:]))
a = np.random.normal(0.0, 1.0, flat_shape)
u, _, v = np.linalg.svd(a, full_matrices=False)
# pick the one with the correct shape
q = u if u.shape == flat_shape else v
q = q.reshape(shape)
# create variable
x = tf.get_variable(name,
initializer=tf.constant(scale * q[:shape[0], :shape[1]], dtype=dtype))
# add summary
if not tf.get_variable_scope().reuse:
tf.sg_summary_param(x)
return x
def external(name, value, dtype=tf.sg_floatx):
r"""Returns an initializer of `value`.
Args:
name: A string. The name of the new or existing variable.
value: A constant value (or array) of output type `dtype`.
dtype: The type of the elements of the resulting tensor. (optional)
Returns:
A `Tensor` variable.
"""
# create variable
x = tf.get_variable(name, initializer=tf.constant(value, dtype=dtype))
# add summary
if not tf.get_variable_scope().reuse:
tf.sg_summary_param(x)
return x
def identity(name,
dim,
scale=1,
dtype=tf.sg_floatx,
summary=True,
regularizer=None,
trainable=True):
r"""Creates a tensor variable of which initial values are of
an identity matrix.
Note that the default value of `scale` (=0.05) is different from
the min/max values (=0.0, 1.0) of tf.random_uniform_initializer.
For example,
```
identity("identity", 3, 2) =>
[[2. 0. 0.]
[0. 2. 0.]
[0. 0. 2.]]
```
Args:
name: The name of new variable.
dim: An int. The size of the first and second dimension of the output tensor.
scale: A Python scalar. The value on the diagonal.
dtype: The type of the elements of the resulting tensor.
summary: If True, add this constant to tensor board summary.
regularizer: A (Tensor -> Tensor or None) function; the result of applying it on a newly created variable
will be added to the collection tf.GraphKeys.REGULARIZATION_LOSSES and can be used for regularization
trainable: If True, add this constant to trainable collection. Default is True.
Returns:
A 2-D `Variable`.
"""
x = tf.get_variable(name,
initializer=tf.constant(np.eye(dim) * scale,
dtype=dtype),
regularizer=regularizer,
trainable=trainable)
# add summary
if summary:
tf.sg_summary_param(x)
return x
def identity(name, dim, scale=1, dtype=tf.sg_floatx):
r"""Returns an initializer of a 2-D identity tensor.
Args:
name: A string. The name of the new or existing variable.
dim: An int. The size of the first and second dimension of the output tensor
scale: An int (optional). The value on the diagonal.
shape: Shape of the new or existing variable.
dtype: A tensor datatype.
Returns:
A 2-D tensor variable with the value of `scale` on the diagoanl and zeros elsewhere.
"""
x = tf.get_variable(name,
initializer=tf.constant(np.eye(dim) * scale,
dtype=dtype))
# add summary
if not tf.get_variable_scope().reuse:
tf.sg_summary_param(x)
return x
def __init__(self,
x,
y,
num_batch,
vocab_size,
emb_dim,
hidden_dim,
max_ep=240,
infer_shape=(1, 1),
mode="train"):
self.num_batch = num_batch
self.emb_dim = emb_dim
self.hidden_dim = hidden_dim
self.vocab_size = vocab_size
self.max_len_infer = 512
self.max_ep = max_ep
# reuse = len([t for t in tf.global_variables() if t.name.startswith('gen')]) > 0
reuse = (mode == 'infer')
if mode == "train":
self.x = x
self.y = y
elif mode == "infer":
self.x = tf.placeholder(tf.int32, shape=infer_shape)
self.y = tf.placeholder(tf.int32, shape=infer_shape)
with tf.variable_scope("gen_embs", reuse=reuse):
self.emb_x = tf.get_variable("emb_x",
[self.vocab_size, self.emb_dim])
self.emb_y = tf.get_variable("emb_y",
[self.vocab_size, self.emb_dim])
self.X = tf.nn.embedding_lookup(self.emb_x, self.x)
self.Y = tf.nn.embedding_lookup(self.emb_y, self.y)
with tf.sg_context(name='gen', reuse=reuse):
# self.emb_x = tf.Variable(tf.random_uniform([self.vocab_size, self.emb_dim], 0.0, 1.0), name="emb_x")
# self.emb_y = tf.Variable(tf.random_uniform([self.vocab_size, self.emb_dim], 0.0, 1.0), name="emb_y")
# self.emb_x = tf.sg_emb(name='emb_x', voca_size=self.vocab_size, dim=self.emb_dim) # (68,16)
# self.emb_y = tf.sg_emb(name='emb_y', voca_size=self.vocab_size, dim=self.emb_dim) # (68,16)
# self.X = self.x.sg_lookup(emb=self.emb_x) # (8,63,16)
# self.Y = self.y.sg_lookup(emb=self.emb_y) # (8,63,16)
if mode == "train":
self.lstm_layer = self.X.sg_lstm(in_dim=self.emb_dim,
dim=self.vocab_size,
name="lstm") # (8, 63, 68)
self.test = self.lstm_layer.sg_softmax(name="testtt")
print "mazum??"
print self.test
elif mode == "infer":
self.lstm_layer = self.X.sg_lstm(in_dim=self.emb_dim,
dim=self.vocab_size,
last_only=True,
name="lstm")
self.log_prob = tf.log(self.lstm_layer)
# next_token: select by distribution probability, preds: select by argmax
self.multinormed = tf.multinomial(self.log_prob, 1)
self.next_token = tf.cast(
tf.reshape(tf.multinomial(self.log_prob, 1),
[1, infer_shape[0]]), tf.int32)
self.preds = self.lstm_layer.sg_argmax()
if mode == "train":
self.loss = self.lstm_layer.sg_ce(target=self.y)
self.istarget = tf.not_equal(self.y, 0).sg_float()
self.reduced_loss = (self.loss.sg_sum()) / (
self.istarget.sg_sum() + 0.0000001)
tf.sg_summary_loss(self.reduced_loss, "reduced_loss")
