def __call__(self, x_t, state, size, scope=None, reuse_vars=False): (prev_c, prev_h) = state scope = scope or tf.get_variable_scope() print("____reuse_______", reuse_vars) with tf.variable_scope(scope, reuse=True): w_ic = tf.get_variable("w_ic") w_fc = tf.get_variable("w_fc") w_oc = tf.get_variable("w_oc") with tf.sg_context(dev=self._dev, reuse=reuse_vars): i = x_t.sg_conv1d_gpus(name = "ix_",size=size)+\ prev_h.sg_conv1d_gpus(name = "ih_",size=size)+\ prev_c*w_ic f = x_t.sg_aconv1d_gpus(name = "fx_",size=size)+\ prev_h.sg_aconv1d_gpus(name = "fh_",size=size)+\ prev_c*w_fc c = x_t.sg_conv1d_gpus(name = "cx_",size=size)+\ prev_h.sg_conv1d_gpus(name = "ch_",size=size) o = x_t.sg_conv1d_gpus(name = "ox_",size=size)+\ prev_h.sg_conv1d_gpus(name = "oh_",size=size)+\ prev_c*w_oc new_c = prev_c * tf.sigmoid(f) + tf.sigmoid(i) * self._activation(c) new_h = self._activation(new_c) * tf.sigmoid(o) return (new_c, new_h)
def linear(input_, output_size, scope=None): ''' Linear map: output[k] = sum_i(Matrix[k, i] * args[i] ) + Bias[k] Args: args: a tensor or a list of 2D, batch x n, Tensors. output_size: int, second dimension of W[i]. scope: VariableScope for the created subgraph; defaults to "Linear". Returns: A 2D Tensor with shape [batch x output_size] equal to sum_i(args[i] * W[i]), where W[i]s are newly created matrices. Raises: ValueError: if some of the arguments has unspecified or wrong shape. ''' shape = input_.get_shape().as_list() if len(shape) != 2: raise ValueError("Linear is expecting 2D arguments: %s" % str(shape)) if not shape[1]: raise ValueError("Linear expects shape[1] of arguments: %s" % str(shape)) input_size = shape[1] # Now the computation. with tf.variable_scope(scope or "SimpleLinear"): matrix = tf.get_variable("Matrix", [output_size, input_size], dtype=input_.dtype) bias_term = tf.get_variable("Bias", [output_size], dtype=input_.dtype) return tf.matmul(input_, tf.transpose(matrix)) + bias_term
def __init__(self, in_dim, dim, forget_bias=1.0, activation=tf.tanh, ln=True, bias=True, dtype=tf.float32, dev='/cpu:0', batch_size=3): self._in_dim = in_dim self._dim = dim self._forget_bias = forget_bias self._activation = activation self._ln = False self._bias = bias self._dev = dev self._size = self._in_dim * self._dim self._initializer = tf.contrib.layers.xavier_initializer( ) #tf.random_normal_initializer() self._dtype = dtype with tf.device(self._dev): with tf.variable_scope("lstm") as scp: #self.rnn_state = tf.get_variable("rnn_c",(batch_size, self._dim), dtype=tf.sg_floatx,initializer=tf.constant_initializer(0.0),trainable=False) #self.rnn_h = tf.get_variable("rnn_h",(batch_size, self._dim), dtype=tf.sg_floatx,initializer=tf.constant_initializer(0.0),trainable=False) self.rnn_state, self.rnn_h = tf.zeros( (batch_size, self._dim), dtype=tf.sg_floatx), tf.zeros( (batch_size, self._dim), dtype=tf.sg_floatx) w_i2h = tf.get_variable( 'w_i2h', (self._in_dim, 4 * self._dim), dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer(), trainable=True) w_h2h = tf.get_variable( 'w_h2h', (self._dim, 4 * self._dim), dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer(), trainable=True) w_b = tf.get_variable( 'w_b', (1, 4 * self._dim), dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer(), trainable=True) if self._bias == True else 0.0 if self._ln: with tf.variable_scope("ln_rnn"): beta = tf.get_variable( 'beta', self._dim, dtype=tf.sg_floatx, initializer=tf.constant_initializer(0.0), trainable=True) gamma = tf.get_variable( 'gamma', self._dim, dtype=tf.sg_floatx, initializer=tf.constant_initializer(1.0), trainable=True)
def _linear(self, arys): scope = tf.get_variable_scope() with tf.variable_scope(scope, reuse=True): w_i2h = tf.get_variable("w_i2h") w_h2h = tf.get_variable("w_h2h") w_b = tf.get_variable("w_b") if self._bias == True else 0 i2h = tf.matmul(arys[0], w_i2h) h2h = tf.matmul(arys[1], w_h2h) out = i2h + h2h + w_b return out
def make_states(self, batch_size): seqlen = self._seqlen self.crnn_state = tf.get_variable( "crnn_c", (batch_size, seqlen, self._dim), dtype=tf.sg_floatx, initializer=tf.constant_initializer(0.0), trainable=False) self.crnn_h = tf.get_variable("crnn_h", (batch_size, seqlen, self._dim), dtype=tf.sg_floatx, initializer=tf.constant_initializer(0.0), trainable=False)
def constant(name, shape, value=0, dtype=tf.sg_floatx, summary=True, regularizer=None, trainable=True): r"""Creates a tensor variable of which initial values are `value` and shape is `shape`. Args: name: The name of new variable. shape: A tuple/list of integers or an integer. If shape is an integer, it is converted to a list. value: A Python scalar. All elements of the initialized variable will be set to this value. Default is 0. dtype: The data type. Only floating point types are supported. Default is float32. summary: If True, add this constant to tensor board summary. regularizer: A (Tensor -> Tensor or None) function; the result of applying it on a newly created variable will be added to the collection tf.GraphKeys.REGULARIZATION_LOSSES and can be used for regularization trainable: If True, add this constant to trainable collection. Default is True. Returns: A `Variable`. """ shape = shape if isinstance(shape, (tuple, list)) else [shape] x = tf.get_variable(name, shape, dtype=dtype, initializer=tf.constant_initializer(value), regularizer=regularizer, trainable=trainable) # add summary if summary: tf.sg_summary_param(x) return x
def orthogonal(name, shape, scale=1.1, dtype=tf.sg_floatx, summary=True): r"""Creates a tensor variable of which initial values are of an orthogonal ndarray. See [Saxe et al. 2014.](http://arxiv.org/pdf/1312.6120.pdf) Args: name: The name of new variable. shape: A tuple/list of integers. scale: A Python scalar. dtype: Either float32 or float64. summary: If True, add this constant to tensor board summary. Returns: A `Variable`. """ flat_shape = (shape[0], np.prod(shape[1:])) a = np.random.normal(0.0, 1.0, flat_shape) u, _, v = np.linalg.svd(a, full_matrices=False) # pick the one with the correct shape q = u if u.shape == flat_shape else v q = q.reshape(shape) # create variable x = tf.get_variable(name, initializer=tf.constant(scale * q[:shape[0], :shape[1]], dtype=dtype)) # add summary if not tf.get_variable_scope().reuse and summary: tf.sg_summary_param(x) return x
def constant(name, shape, value=0, dtype=tf.sg_floatx, summary=True): r"""Creates a tensor variable of which initial values are `value` and shape is `shape`. Args: name: The name of new variable. shape: A tuple/list of integers or an integer. If shape is an integer, it is converted to a list. value: A Python scalar. All elements of the initialized variable will be set to this value. Default is 0. dtype: The data type. Only floating point types are supported. Default is float32. summary: If True, add this constant to tensor board summary. Returns: A `Variable`. """ shape = shape if isinstance(shape, (tuple, list)) else [shape] x = tf.get_variable(name, shape, dtype=dtype, initializer=tf.constant_initializer(value)) # add summary if not tf.get_variable_scope().reuse and summary: tf.sg_summary_param(x) return x
def identity(name, dim, scale=1, dtype=tf.sg_floatx): x = tf.get_variable(name, initializer=tf.constant(np.eye(dim) * scale, dtype=dtype)) # add summary if not tf.get_variable_scope().reuse: tf.sg_summary_param(x) return x
def external(name, value, dtype=tf.sg_floatx, summary=True, regularizer=None, trainable=True): r"""Creates a tensor variable of which initial values are `value`. For example, ``` external("external", [3,3,1,2]) => [3. 3. 1. 2.] ``` Args: name: The name of new variable. value: A constant value (or list) of output type `dtype`. dtype: The type of the elements of the resulting tensor. summary: If True, add this constant to tensor board summary. regularizer: A (Tensor -> Tensor or None) function; the result of applying it on a newly created variable will be added to the collection tf.GraphKeys.REGULARIZATION_LOSSES and can be used for regularization trainable: If True, add this constant to trainable collection. Default is True. Returns: A `Variable`. Has the same contents as `value` of `dtype`. """ # create variable x = tf.get_variable(name, initializer=tf.constant(value, dtype=dtype), regularizer=regularizer, trainable=trainable) # add summary if summary: tf.sg_summary_param(x) return x
def uniform(name, shape, scale=0.05, dtype=tf.sg_floatx, summary=True, regularizer=None, trainable=True): r"""Creates a tensor variable of which initial values are random numbers based on uniform distribution. Note that the default value of `scale` (=0.05) is different from the min/max values (=0.0, 1.0) of tf.random_uniform_initializer. Args: name: The name of the new variable. shape: A tuple/list of integers or an integer. If shape is an integer, it's converted to a list. scale: A Python scalar. All initial values should be in range `[-scale, scale)`. Default is .05. dtype: The data type. Only floating point types are supported. Default is float32. summary: If True, add this constant to tensor board summary. regularizer: A (Tensor -> Tensor or None) function; the result of applying it on a newly created variable will be added to the collection tf.GraphKeys.REGULARIZATION_LOSSES and can be used for regularization trainable: If True, add this constant to trainable collection. Default is True. Returns: A `Variable`. """ shape = shape if isinstance(shape, (tuple, list)) else [shape] x = tf.get_variable(name, shape, dtype=dtype, initializer=tf.random_uniform_initializer(minval=-scale, maxval=scale), regularizer=regularizer, trainable=trainable) # add summary if summary: tf.sg_summary_param(x) return x
def orthogonal(name, shape, scale=1.1, dtype=tf.sg_floatx): r"""Returns a random orthogonal initializer. See Saxe et al. 2014 `http://arxiv.org/pdf/1312.6120.pdf` Args: name: A string. The name of the new or existing variable. shape: A list or tuple of integers. scale: A Python scalr. dtype = A float32 or float64. Returns: A `Tensor` variable. """ flat_shape = (shape[0], np.prod(shape[1:])) a = np.random.normal(0.0, 1.0, flat_shape) u, _, v = np.linalg.svd(a, full_matrices=False) # pick the one with the correct shape q = u if u.shape == flat_shape else v q = q.reshape(shape) # create variable x = tf.get_variable(name, initializer=tf.constant(scale * q[:shape[0], :shape[1]], dtype=dtype)) # add summary if not tf.get_variable_scope().reuse: tf.sg_summary_param(x) return x
def orthogonal(name, shape, scale=1.1, dtype=tf.sg_floatx, summary=True, regularizer=None, trainable=True): r"""Creates a tensor variable of which initial values are of an orthogonal ndarray. See [Saxe et al. 2014.](http://arxiv.org/pdf/1312.6120.pdf) Args: name: The name of new variable. shape: A tuple/list of integers. scale: A Python scalar. dtype: Either float32 or float64. summary: If True, add this constant to tensor board summary. regularizer: A (Tensor -> Tensor or None) function; the result of applying it on a newly created variable will be added to the collection tf.GraphKeys.REGULARIZATION_LOSSES and can be used for regularization trainable: If True, add this constant to trainable collection. Default is True. Returns: A `Variable`. """ flat_shape = (shape[0], np.prod(shape[1:])) a = np.random.normal(0.0, 1.0, flat_shape) u, _, v = np.linalg.svd(a, full_matrices=False) # pick the one with the correct shape q = u if u.shape == flat_shape else v q = q.reshape(shape) # create variable x = tf.get_variable(name, initializer=tf.constant(scale * q[:shape[0], :shape[1]], dtype=dtype), regularizer=regularizer, trainable=trainable) # add summary if summary: tf.sg_summary_param(x) return x
def external(name, value, dtype=tf.sg_floatx, summary=True): r"""Creates a tensor variable of which initial values are `value`. For example, ``` external("external", [3,3,1,2]) => [3. 3. 1. 2.] ``` Args: name: The name of new variable. value: A constant value (or list) of output type `dtype`. dtype: The type of the elements of the resulting tensor. summary: If True, add this constant to tensor board summary. Returns: A `Variable`. Has the same contents as `value` of `dtype`. """ # create variable x = tf.get_variable(name, initializer=tf.constant(value, dtype=dtype)) # add summary if not tf.get_variable_scope().reuse and summary: tf.sg_summary_param(x) return x
def identity(name, dim, scale=1, dtype=tf.sg_floatx, summary=True): r"""Creates a tensor variable of which initial values are of an identity matrix. Note that the default value of `scale` (=0.05) is different from the min/max values (=0.0, 1.0) of tf.random_uniform_initializer. For example, ``` identity("identity", 3, 2) => [[2. 0. 0.] [0. 2. 0.] [0. 0. 2.]] ``` Args: name: The name of new variable. dim: An int. The size of the first and second dimension of the output tensor. scale: A Python scalar. The value on the diagonal. dtype: The type of the elements of the resulting tensor. summary: If True, add this constant to tensor board summary. Returns: A 2-D `Variable`. """ x = tf.get_variable(name, initializer=tf.constant(np.eye(dim) * scale, dtype=dtype)) # add summary if not tf.get_variable_scope().reuse and summary: tf.sg_summary_param(x) return x
def uniform(name, shape, scale=0.05, dtype=tf.sg_floatx, summary=True): r"""Creates a tensor variable of which initial values are random numbers based on uniform distribution. Note that the default value of `scale` (=0.05) is different from the min/max values (=0.0, 1.0) of tf.random_uniform_initializer. Args: name: The name of the new variable. shape: A tuple/list of integers or an integer. If shape is an integer, it's converted to a list. scale: A Python scalar. All initial values should be in range `[-scale, scale)`. Default is .05. dtype: The data type. Only floating point types are supported. Default is float32. summary: If True, add this constant to tensor board summary. Returns: A `Variable`. """ shape = shape if isinstance(shape, (tuple, list)) else [shape] x = tf.get_variable(name, shape, dtype=dtype, initializer=tf.random_uniform_initializer( minval=-scale, maxval=scale)) # add summary if not tf.get_variable_scope().reuse and summary: tf.sg_summary_param(x) return x
def __init__(self, seqlen, in_dim, dim, forget_bias=1.0, activation=tf.tanh, ln=True, bias=True, dtype=tf.float32, dev='/cpu:0', batch_size=3): self._in_dim = in_dim self._dim = dim self._forget_bias = forget_bias self._activation = activation self._ln = ln self._dev = dev self._seqlen = seqlen self._bias = bias self._size = int(self._in_dim * self._dim) self._initializer = tf.contrib.layers.xavier_initializer( ) #tf.random_normal_initializer() self._dtype = dtype with tf.device(self._dev): with tf.variable_scope("clstm") as scp: #self.crnn_state = tf.get_variable("crnn_c",(batch_size, seqlen, self._dim), dtype=tf.sg_floatx,initializer=tf.constant_initializer(0.0),trainable=False) #self.crnn_h = tf.get_variable("crnn_h",(batch_size, seqlen, self._dim), dtype=tf.sg_floatx,initializer=tf.constant_initializer(0.0),trainable=False) w_ic = tf.get_variable( 'w_ic', (self._seqlen, self._dim), dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer(), trainable=True) w_fc = tf.get_variable( 'w_fc', (self._seqlen, self._dim), dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer(), trainable=True) w_oc = tf.get_variable( 'w_oc', (self._seqlen, self._dim), dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer(), trainable=True) self.make_states(batch_size)
def uniform(name, shape, scale=0.05, dtype=tf.sg_floatx): shape = shape if isinstance(shape, (tuple, list)) else [shape] x = tf.get_variable(name, shape, dtype=dtype, initializer=tf.random_uniform_initializer(minval=-scale, maxval=scale)) # add summary if not tf.get_variable_scope().reuse: tf.sg_summary_param(x) return x
def external(name, value, dtype=tf.sg_floatx): # create variable x = tf.get_variable(name, initializer=tf.constant(value, dtype=dtype)) # add summary if not tf.get_variable_scope().reuse: tf.sg_summary_param(x) return x
def constant(name, shape, value=0, dtype=tf.sg_floatx): shape = shape if isinstance(shape, (tuple, list)) else [shape] x = tf.get_variable(name, shape, dtype=dtype, initializer=tf.constant_initializer(value)) # add summary if not tf.get_variable_scope().reuse: tf.sg_summary_param(x) return x
def __call__(self, tensor, state, scope=None): (prev_c, prev_h) = state # i = input_gate, c = new cell value for update, f = forget_gate, o = output_gate lstm_matrix = self._linear([tensor, prev_h]) i, c, f, o = tf.split(value=lstm_matrix, num_or_size_splits=4, axis=1) if self._ln: with tf.variable_scope("ln_rnn", reuse=True): beta = tf.get_variable('beta') gamma = tf.get_variable('gamma') ln = lambda v: _ln_rnn(v, gamma, beta) if self._ln else v # do rnn loop new_c = prev_c * tf.sigmoid(ln(f)) + tf.sigmoid( ln(i)) * self._activation(ln(c)) new_h = self._activation(new_c) * tf.sigmoid(ln(o)) return (new_c, new_h)
def embed(inputs, vocab_size, embed_size, variable_scope): ''' inputs = tf.expand_dims(tf.range(5), 0) => (1, 5) _embed(inputs, 5, 10) => (1, 5, 10) ''' with tf.variable_scope(variable_scope): lookup_table = tf.get_variable('lookup_table', dtype=tf.float32, shape=[vocab_size, embed_size], initializer=tf.truncated_normal_initializer()) return tf.nn.embedding_lookup(lookup_table, inputs)
def constant(name, shape, value=0, dtype=tf.sg_floatx): r"""Returns an initializer of `shape` with all elements set to a scalar `value`. """ shape = shape if isinstance(shape, (tuple, list)) else [shape] x = tf.get_variable(name, shape, dtype=dtype, initializer=tf.constant_initializer(value)) # add summary if not tf.get_variable_scope().reuse: tf.sg_summary_param(x) return x
def init_custom_embeddings(name, embeddings_matrix, summary=True, trainable=False): """ Initializes the embedding vector with custom preloaded embeddings """ embedding = np.array(embeddings_matrix) emb = tf.get_variable(name, shape=embedding.shape, initializer=tf.constant_initializer(embedding), trainable=trainable) if summary: tf.sg_summary_param(emb) return emb
def uniform(name, shape, scale=0.05, dtype=tf.sg_floatx): r"""Returns an initializer of random numbers based on uniform distribution. Note that the default value of `scale` (=0.05) is different from the min/max values (=0.0, 1.0) of tf.random_uniform_initializer. """ shape = shape if isinstance(shape, (tuple, list)) else [shape] x = tf.get_variable(name, shape, dtype=dtype, initializer=tf.random_uniform_initializer( minval=-scale, maxval=scale)) # add summary if not tf.get_variable_scope().reuse: tf.sg_summary_param(x) return x
def orthogonal(name, shape, scale=1.1, dtype=tf.sg_floatx): # Sax et aE. ( http://arxiv.org/pdf/1312.6120.pdf ) flat_shape = (shape[0], np.prod(shape[1:])) a = np.random.normal(0.0, 1.0, flat_shape) u, _, v = np.linalg.svd(a, full_matrices=False) # pick the one with the correct shape q = u if u.shape == flat_shape else v q = q.reshape(shape) # create variable x = tf.get_variable(name, initializer=tf.constant(scale * q[:shape[0], :shape[1]], dtype=dtype)) # add summary if not tf.get_variable_scope().reuse: tf.sg_summary_param(x) return x
def external(name, value, dtype=tf.sg_floatx): r"""Returns an initializer of `value`. Args: name: A string. The name of the new or existing variable. value: A constant value (or array) of output type `dtype`. dtype: The type of the elements of the resulting tensor. (optional) Returns: A `Tensor` variable. """ # create variable x = tf.get_variable(name, initializer=tf.constant(value, dtype=dtype)) # add summary if not tf.get_variable_scope().reuse: tf.sg_summary_param(x) return x
def identity(name, dim, scale=1, dtype=tf.sg_floatx, summary=True, regularizer=None, trainable=True): r"""Creates a tensor variable of which initial values are of an identity matrix. Note that the default value of `scale` (=0.05) is different from the min/max values (=0.0, 1.0) of tf.random_uniform_initializer. For example, ``` identity("identity", 3, 2) => [[2. 0. 0.] [0. 2. 0.] [0. 0. 2.]] ``` Args: name: The name of new variable. dim: An int. The size of the first and second dimension of the output tensor. scale: A Python scalar. The value on the diagonal. dtype: The type of the elements of the resulting tensor. summary: If True, add this constant to tensor board summary. regularizer: A (Tensor -> Tensor or None) function; the result of applying it on a newly created variable will be added to the collection tf.GraphKeys.REGULARIZATION_LOSSES and can be used for regularization trainable: If True, add this constant to trainable collection. Default is True. Returns: A 2-D `Variable`. """ x = tf.get_variable(name, initializer=tf.constant(np.eye(dim) * scale, dtype=dtype), regularizer=regularizer, trainable=trainable) # add summary if summary: tf.sg_summary_param(x) return x
def identity(name, dim, scale=1, dtype=tf.sg_floatx): r"""Returns an initializer of a 2-D identity tensor. Args: name: A string. The name of the new or existing variable. dim: An int. The size of the first and second dimension of the output tensor scale: An int (optional). The value on the diagonal. shape: Shape of the new or existing variable. dtype: A tensor datatype. Returns: A 2-D tensor variable with the value of `scale` on the diagoanl and zeros elsewhere. """ x = tf.get_variable(name, initializer=tf.constant(np.eye(dim) * scale, dtype=dtype)) # add summary if not tf.get_variable_scope().reuse: tf.sg_summary_param(x) return x
def __init__(self, x, y, num_batch, vocab_size, emb_dim, hidden_dim, max_ep=240, infer_shape=(1, 1), mode="train"): self.num_batch = num_batch self.emb_dim = emb_dim self.hidden_dim = hidden_dim self.vocab_size = vocab_size self.max_len_infer = 512 self.max_ep = max_ep # reuse = len([t for t in tf.global_variables() if t.name.startswith('gen')]) > 0 reuse = (mode == 'infer') if mode == "train": self.x = x self.y = y elif mode == "infer": self.x = tf.placeholder(tf.int32, shape=infer_shape) self.y = tf.placeholder(tf.int32, shape=infer_shape) with tf.variable_scope("gen_embs", reuse=reuse): self.emb_x = tf.get_variable("emb_x", [self.vocab_size, self.emb_dim]) self.emb_y = tf.get_variable("emb_y", [self.vocab_size, self.emb_dim]) self.X = tf.nn.embedding_lookup(self.emb_x, self.x) self.Y = tf.nn.embedding_lookup(self.emb_y, self.y) with tf.sg_context(name='gen', reuse=reuse): # self.emb_x = tf.Variable(tf.random_uniform([self.vocab_size, self.emb_dim], 0.0, 1.0), name="emb_x") # self.emb_y = tf.Variable(tf.random_uniform([self.vocab_size, self.emb_dim], 0.0, 1.0), name="emb_y") # self.emb_x = tf.sg_emb(name='emb_x', voca_size=self.vocab_size, dim=self.emb_dim) # (68,16) # self.emb_y = tf.sg_emb(name='emb_y', voca_size=self.vocab_size, dim=self.emb_dim) # (68,16) # self.X = self.x.sg_lookup(emb=self.emb_x) # (8,63,16) # self.Y = self.y.sg_lookup(emb=self.emb_y) # (8,63,16) if mode == "train": self.lstm_layer = self.X.sg_lstm(in_dim=self.emb_dim, dim=self.vocab_size, name="lstm") # (8, 63, 68) self.test = self.lstm_layer.sg_softmax(name="testtt") print "mazum??" print self.test elif mode == "infer": self.lstm_layer = self.X.sg_lstm(in_dim=self.emb_dim, dim=self.vocab_size, last_only=True, name="lstm") self.log_prob = tf.log(self.lstm_layer) # next_token: select by distribution probability, preds: select by argmax self.multinormed = tf.multinomial(self.log_prob, 1) self.next_token = tf.cast( tf.reshape(tf.multinomial(self.log_prob, 1), [1, infer_shape[0]]), tf.int32) self.preds = self.lstm_layer.sg_argmax() if mode == "train": self.loss = self.lstm_layer.sg_ce(target=self.y) self.istarget = tf.not_equal(self.y, 0).sg_float() self.reduced_loss = (self.loss.sg_sum()) / ( self.istarget.sg_sum() + 0.0000001) tf.sg_summary_loss(self.reduced_loss, "reduced_loss")