def __call__(self, inputs, state, timestep=0, scope=None): with tf.device("/gpu:" + str(self._gpu_for_layer)): """Long short-term memory cell (LSTM).""" with tf.variable_scope(scope or type(self).__name__): # "BasicLSTMCell" # Parameters of gates are concatenated into one multiply for efficiency. c, h = state concat = linear([inputs, h], self._num_units * 4, False, 0.0) concat = layers.layer_norm(concat, reuse=None) # i = input_gate, j = new_input, f = forget_gate, o = output_gate i, j, f, o = tf.split(concat, 4, 1) if self.use_recurrent_dropout and self.is_training: input_contribution = tf.nn.dropout( tf.tanh(j), self.recurrent_dropout_factor) else: input_contribution = tf.tanh(j) new_c = c * tf.sigmoid(f + self._forget_bias) + tf.sigmoid( i) * input_contribution with tf.variable_scope('new_h_output'): new_h = tf.tanh(layers.layer_norm( new_c, reuse=None)) * tf.sigmoid(o) new_state = rnn_cell_impl.LSTMStateTuple(new_c, new_h) # return new_h, tf.concat([new_h, new_c], 1) #purposely reversed return new_h, new_state
def layer_norm(in_tensor, data_format='NHWC'): if data_format == 'NCHW': in_tensor = tf.transpose(in_tensor, (0, 2, 3, 1)) out_tensor = layer_norm(in_tensor) if data_format == 'NCHW': out_tensor = tf.transpose(out_tensor, (0, 3, 1, 2)) return out_tensor
def _norm(self, inp, scope): with vs.variable_scope(scope) as scope: shape = inp.get_shape()[-1:] gamma_init = init_ops.constant_initializer(self._g) beta_init = init_ops.constant_initializer(self._b) gamma = vs.get_variable("gamma", shape=shape, initializer=gamma_init) # pylint: disable=unused-variable beta = vs.get_variable("beta", shape=shape, initializer=beta_init) # pylint: disable=unused-variable normalized = layers.layer_norm(inp, reuse=True, scope=scope) return normalized
def _norm(self, inp, scope): shape = inp.get_shape()[-1:] gamma_init = init_ops.constant_initializer(self._g) beta_init = init_ops.constant_initializer(self._b) with vs.variable_scope(scope): # Initialize beta and gamma for use by layer_norm. vs.get_variable("gamma", shape=shape, initializer=gamma_init) vs.get_variable("beta", shape=shape, initializer=beta_init) normalized = layers.layer_norm(inp, reuse=True, scope=scope) return normalized
def layer(x, name=None): with variable_scope.variable_scope(name, default_name="layer"): x = layers.layer_norm(x) x = convolutional.conv1d( x, 10, 1, use_bias=False, kernel_initializer=init_ops.constant_initializer(42.42)) x = nn_ops.relu(x) return x
def _norm(self, inp, scope="layer_norm"): """ TODO """ shape = inp.get_shape()[-1:] gamma_init = init_ops.constant_initializer(1) beta_init = init_ops.constant_initializer(1) with tf.variable_scope(scope): tf.get_variable("gamma", shape=shape, initializer=gamma_init) tf.get_variable("beta", shape=shape, initializer=beta_init) normalized = layers.layer_norm(inp, reuse=True, scope=scope) return normalized
def _norm(self, inp, scope="layer_norm"): """ """ shape = inp.get_shape()[-1:] gamma_init = init_ops.constant_initializer(self._g) beta_init = init_ops.constant_initializer(self._b) with vs.variable_scope(scope, reuse=tf.AUTO_REUSE): vs.get_variable("gamma", shape=shape, initializer=gamma_init) vs.get_variable("beta", shape=shape, initializer=beta_init) normalized = layers.layer_norm(inp, reuse=True, scope=scope) return normalized
def _norm(self, inp, scope="layer_norm"): """ Performs layer normalization on the hidden state. inp = the input to be normalized Returns inp normalized by learned parameters gamma and beta """ shape = inp.get_shape()[-1:] gamma_init = init_ops.constant_initializer(self._g) beta_init = init_ops.constant_initializer(self._b) with vs.variable_scope(scope): vs.get_variable("gamma", shape=shape, initializer=gamma_init) vs.get_variable("beta", shape=shape, initializer=beta_init) normalized = layers.layer_norm(inp, reuse=True, scope=scope) return normalized
def _norm(self, inp, scope="layer_norm"): """ Performs layer normalization on the hidden state. inp = the input to be normalized scope = name for the variable scope, just leave as default Returns inp normalized by learned parameters gamma and beta """ shape = inp.get_shape()[-1:] gamma_init = init_ops.constant_initializer(1) beta_init = init_ops.constant_initializer(1) with tf.variable_scope(scope): tf.get_variable("gamma", shape=shape, initializer=gamma_init) tf.get_variable("beta", shape=shape, initializer=beta_init) normalized = layers.layer_norm(inp, reuse=True, scope=scope) return normalized
def _norm(self, inp, scope): return layers.layer_norm(inp, reuse=True, scope=scope)