def __call__(self, inputs, state, scope=None):
"""Run the cell and output projection on inputs, starting from state."""
output, res_state = self._cell(inputs, state)
# Default scope: "OutputProjectionWrapper"
with tf.variable_scope(scope or type(self).__name__):
projected = linear.linear(output, self._output_size, True)
return projected, res_state
def __call__(self, inputs, state, scope=None):
with tf.device("/gpu:"+str(self._gpu_for_layer)):
"""JZS3, mutant 2 with n units cells."""
with tf.variable_scope(scope or type(self).__name__): # "JZS1Cell"
with tf.variable_scope("Zinput"): # Reset gate and update gate.
# We start with bias of 1.0 to not reset and not update.
'''equation 1'''
z = tf.sigmoid(lfe.enhanced_linear([inputs, tf.tanh(state)],
self._num_units, True, 1.0, weight_initializer = self._weight_initializer))
'''equation 2'''
with tf.variable_scope("Rinput"):
r = tf.sigmoid(lfe.enhanced_linear([inputs, state],
self._num_units, True, 1.0, weight_initializer = self._weight_initializer))
'''equation 3'''
with tf.variable_scope("Candidate"):
component_0 = linear.linear([state*r,inputs],
self._num_units, True)
component_2 = (tf.tanh(component_0))*z
component_3 = state*(1 - z)
h_t = component_2 + component_3
return h_t, h_t #there is only one hidden state output to keep track of.
def __call__(self, inputs, state, scope=None):
with tf.device("/gpu:"+str(self._gpu_for_layer)):
"""JZS1, mutant 1 with n units cells."""
with tf.variable_scope(scope or type(self).__name__): # "JZS1Cell"
with tf.variable_scope("Zinput"): # Reset gate and update gate.
# We start with bias of 1.0 to not reset and not update.
'''equation 1 z = sigm(WxzXt+Bz), x_t is inputs'''
z = tf.sigmoid(lfe.enhanced_linear([inputs],
self._num_units, True, 1.0, weight_initializer = self._weight_initializer))
with tf.variable_scope("Rinput"):
'''equation 2 r = sigm(WxrXt+Whrht+Br), h_t is the previous state'''
r = tf.sigmoid(lfe.enhanced_linear([inputs,state],
self._num_units, True, 1.0, weight_initializer = self._weight_initializer))
'''equation 3'''
with tf.variable_scope("Candidate"):
component_0 = linear.linear([r*state],
self._num_units, True)
component_1 = tf.tanh(tf.tanh(inputs) + component_0)
component_2 = component_1*z
component_3 = state*(1 - z)
h_t = component_2 + component_3
return h_t, h_t #there is only one hidden state output to keep track of.
def __call__(self, inputs, state,scope=None):
with tf.device("/gpu:"+str(self._gpu_for_layer)):
"""Gated recurrent unit (GRU) with nunits cells."""
with tf.variable_scope(scope or type(self).__name__): # "GRUCell"
with tf.variable_scope("Gates"): # Reset gate and update gate.
# We start with bias of 1.0 to not reset and not udpate.
r, u = tf.split(1, 2, lfe.enhanced_linear([inputs, state],
2 * self._num_units, True, 1.0, weight_initializer = self._weight_initializer))
r, u = tf.sigmoid(r), tf.sigmoid(u)
with tf.variable_scope("Candidate"): #you need a different one because you're doing a new linear
#notice they have the activation/non-linear step right here!
c = tf.tanh(linear.linear([inputs, r * state], self._num_units, True))
new_h = u * state + (1 - u) * c
return new_h, new_h
'''nick, notice that for the gru, the output and the hidden state are literally the same thing'''
def __call__(self, inputs, state, scope=None):
"""Run the input projection and then the cell."""
# Default scope: "InputProjectionWrapper"
with tf.variable_scope(scope or type(self).__name__):
projected = linear.linear(inputs, self._cell.input_size, True)
return self._cell(projected, state)
def __call__(self, inputs, state, scope=None):
with tf.device("/gpu:"+str(self._gpu_for_layer)):
print('testing')
with tf.variable_scope(scope or type(self).__name__): # "UnitaryRNNCell"
with tf.variable_scope("UnitaryGates"): # Reset gate and update gate.
'''just for sake of consistency, we'll keep some var names the same as authors'''
n_hidden = self._num_units
h_prev = state
'''development nick version here'''
step1 = unitary_linear.times_diag_tf(h_prev, n_hidden) #this will create a diagonal tensor with given diagonal values
#work on times_reflection next
modulus = T.sqrt(lin_output_re ** 2 + lin_output_im ** 2)
rescale = T.maximum(modulus + hidden_bias.dimshuffle('x',0), 0.) / (modulus + 1e-5)
nonlin_output_re = lin_output_re * rescale
nonlin_output_im = lin_output_im * rescale
h_t = tf.concat(1, [nonlin_output_re,
nonlin_output_im])
#keep in mind that you can use tf.complex to convert two numbers into a complex number -- this works for tensors!
return h_t, h_t #check if h_t is the same as the output?????
'''list of complex number functions in tf
1. tf.complex -- makes complex number
2. complex_abs -- finds the absolute value of the tensor
3. tf.conj -- makes conjugate
4. tf.imag -- returns imaginary part -- go back and forth between complex and imag
5. tf.real -- returns real part'''
#keep in mind that identity matricies are a form of diagonal matricies, but they just have ones.
'''----------------------------end of unitary rnn cell--------------------------'''
# We start with bias of 1.0 to not reset and not update.
'''First, we will start with the hidden linear transform
W = D3R2F-1D2PermR1FD1
Keep in mind that originally the equation would be W = VDV*, but it leads to too much computation/memory o(n^2)'''
step1 = times_diag(h_prev, n_hidden, theta[0,:])
step2 = step1
# step2 = do_fft(step1, n_hidden)
step3 = times_reflection(step2, n_hidden, reflection[0,:])
step4 = vec_permutation(step3, n_hidden, index_permute)
step5 = times_diag(step4, n_hidden, theta[1,:])
step6 = step5
# step6 = do_ifft(step5, n_hidden)
step7 = times_reflection(step6, n_hidden, reflection[1,:])
step8 = times_diag(step7, n_hidden, theta[2,:])
step9 = scale_diag(step8, n_hidden, scale)
hidden_lin_output = step9
z = tf.sigmoid(linear.linear([inputs],
self._num_units, True, 1.0))
'''equation 2 r = sigm(WxrXt+Whrht+Br), h_t is the previous state'''
r = tf.sigmoid((linear.linear([inputs,state],
self._num_units, True, 1.0)))
'''equation 3'''
with tf.variable_scope("Candidate"):
component_0 = linear.linear([r*state],
self._num_units, True)
component_1 = tf.tanh(tf.tanh(inputs) + component_0)
component_2 = component_1*z
component_3 = state*(1 - z)
h_t = component_2 + component_3
h_t = tf.concat(concat_dim = 1, value =[nonlin_output_re, nonlin_output_im]) #I know here you need to concatenate the real and imaginary parts
return h_t, h_t #there is only one hidden state output to keep track of.
