def __call__(self, inputs, state, scope=None):
with tf.device("/gpu:"+str(self._gpu_for_layer)):
"""JZS3, mutant 2 with n units cells."""
with tf.variable_scope(scope or type(self).__name__): # "JZS1Cell"
with tf.variable_scope("Zinput"): # Reset gate and update gate.
# We start with bias of 1.0 to not reset and not update.
'''equation 1'''
z = tf.sigmoid(lfe.enhanced_linear([inputs, tf.tanh(state)],
self._num_units, True, 1.0, weight_initializer = self._weight_initializer))
'''equation 2'''
with tf.variable_scope("Rinput"):
r = tf.sigmoid(lfe.enhanced_linear([inputs, state],
self._num_units, True, 1.0, weight_initializer = self._weight_initializer))
'''equation 3'''
with tf.variable_scope("Candidate"):
component_0 = linear.linear([state*r,inputs],
self._num_units, True)
component_2 = (tf.tanh(component_0))*z
component_3 = state*(1 - z)
h_t = component_2 + component_3
return h_t, h_t #there is only one hidden state output to keep track of.
def __call__(self, inputs, state, scope=None):
with tf.device("/gpu:"+str(self._gpu_for_layer)):
"""JZS1, mutant 1 with n units cells."""
with tf.variable_scope(scope or type(self).__name__): # "JZS1Cell"
with tf.variable_scope("Zinput"): # Reset gate and update gate.
# We start with bias of 1.0 to not reset and not update.
'''equation 1 z = sigm(WxzXt+Bz), x_t is inputs'''
z = tf.sigmoid(lfe.enhanced_linear([inputs],
self._num_units, True, 1.0, weight_initializer = self._weight_initializer))
with tf.variable_scope("Rinput"):
'''equation 2 r = sigm(WxrXt+Whrht+Br), h_t is the previous state'''
r = tf.sigmoid(lfe.enhanced_linear([inputs,state],
self._num_units, True, 1.0, weight_initializer = self._weight_initializer))
'''equation 3'''
with tf.variable_scope("Candidate"):
component_0 = linear.linear([r*state],
self._num_units, True)
component_1 = tf.tanh(tf.tanh(inputs) + component_0)
component_2 = component_1*z
component_3 = state*(1 - z)
h_t = component_2 + component_3
return h_t, h_t #there is only one hidden state output to keep track of.
def __call__(self, inputs, state, scope=None):
with tf.device("/gpu:"+str(self._gpu_for_layer)):
"""Long short-term memory cell (LSTM)."""
with tf.variable_scope(scope or type(self).__name__): # "BasicLSTMCell"
# Parameters of gates are concatenated into one multiply for efficiency.
c, h = tf.split(1, 2, state)
concat = lfe.enhanced_linear([inputs, h], 4 * self._num_units, True, weight_initializer = self._weight_initializer)
# i = input_gate, j = new_input, f = forget_gate, o = output_gate
i, j, f, o = tf.split(1, 4, concat)
new_c = c * tf.sigmoid(f + self._forget_bias) + tf.sigmoid(i) * tf.tanh(j)
new_h = tf.tanh(new_c) * tf.sigmoid(o)
return new_h, tf.concat(1, [new_c, new_h])
'''important, the second part is the hidden state!, thus a lstm with n cells had a hidden state of dimenson 2n'''
def __call__(self, inputs, state,scope=None):
with tf.device("/gpu:"+str(self._gpu_for_layer)):
"""Gated recurrent unit (GRU) with nunits cells."""
with tf.variable_scope(scope or type(self).__name__): # "GRUCell"
with tf.variable_scope("Gates"): # Reset gate and update gate.
# We start with bias of 1.0 to not reset and not udpate.
r, u = tf.split(1, 2, lfe.enhanced_linear([inputs, state],
2 * self._num_units, True, 1.0, weight_initializer = self._weight_initializer))
r, u = tf.sigmoid(r), tf.sigmoid(u)
with tf.variable_scope("Candidate"): #you need a different one because you're doing a new linear
#notice they have the activation/non-linear step right here!
c = tf.tanh(linear.linear([inputs, r * state], self._num_units, True))
new_h = u * state + (1 - u) * c
return new_h, new_h
'''nick, notice that for the gru, the output and the hidden state are literally the same thing'''