def __call__(self, inputs, state, scope=None): """Run the cell and output projection on inputs, starting from state.""" output, res_state = self._cell(inputs, state) # Default scope: "OutputProjectionWrapper" with tf.variable_scope(scope or type(self).__name__): projected = linear.linear(output, self._output_size, True) return projected, res_state
def __call__(self, inputs, state, scope=None): with tf.device("/gpu:"+str(self._gpu_for_layer)): """JZS3, mutant 2 with n units cells.""" with tf.variable_scope(scope or type(self).__name__): # "JZS1Cell" with tf.variable_scope("Zinput"): # Reset gate and update gate. # We start with bias of 1.0 to not reset and not update. '''equation 1''' z = tf.sigmoid(lfe.enhanced_linear([inputs, tf.tanh(state)], self._num_units, True, 1.0, weight_initializer = self._weight_initializer)) '''equation 2''' with tf.variable_scope("Rinput"): r = tf.sigmoid(lfe.enhanced_linear([inputs, state], self._num_units, True, 1.0, weight_initializer = self._weight_initializer)) '''equation 3''' with tf.variable_scope("Candidate"): component_0 = linear.linear([state*r,inputs], self._num_units, True) component_2 = (tf.tanh(component_0))*z component_3 = state*(1 - z) h_t = component_2 + component_3 return h_t, h_t #there is only one hidden state output to keep track of.
def __call__(self, inputs, state, scope=None): with tf.device("/gpu:"+str(self._gpu_for_layer)): """JZS1, mutant 1 with n units cells.""" with tf.variable_scope(scope or type(self).__name__): # "JZS1Cell" with tf.variable_scope("Zinput"): # Reset gate and update gate. # We start with bias of 1.0 to not reset and not update. '''equation 1 z = sigm(WxzXt+Bz), x_t is inputs''' z = tf.sigmoid(lfe.enhanced_linear([inputs], self._num_units, True, 1.0, weight_initializer = self._weight_initializer)) with tf.variable_scope("Rinput"): '''equation 2 r = sigm(WxrXt+Whrht+Br), h_t is the previous state''' r = tf.sigmoid(lfe.enhanced_linear([inputs,state], self._num_units, True, 1.0, weight_initializer = self._weight_initializer)) '''equation 3''' with tf.variable_scope("Candidate"): component_0 = linear.linear([r*state], self._num_units, True) component_1 = tf.tanh(tf.tanh(inputs) + component_0) component_2 = component_1*z component_3 = state*(1 - z) h_t = component_2 + component_3 return h_t, h_t #there is only one hidden state output to keep track of.
def __call__(self, inputs, state,scope=None): with tf.device("/gpu:"+str(self._gpu_for_layer)): """Gated recurrent unit (GRU) with nunits cells.""" with tf.variable_scope(scope or type(self).__name__): # "GRUCell" with tf.variable_scope("Gates"): # Reset gate and update gate. # We start with bias of 1.0 to not reset and not udpate. r, u = tf.split(1, 2, lfe.enhanced_linear([inputs, state], 2 * self._num_units, True, 1.0, weight_initializer = self._weight_initializer)) r, u = tf.sigmoid(r), tf.sigmoid(u) with tf.variable_scope("Candidate"): #you need a different one because you're doing a new linear #notice they have the activation/non-linear step right here! c = tf.tanh(linear.linear([inputs, r * state], self._num_units, True)) new_h = u * state + (1 - u) * c return new_h, new_h '''nick, notice that for the gru, the output and the hidden state are literally the same thing'''
def __call__(self, inputs, state, scope=None): """Run the input projection and then the cell.""" # Default scope: "InputProjectionWrapper" with tf.variable_scope(scope or type(self).__name__): projected = linear.linear(inputs, self._cell.input_size, True) return self._cell(projected, state)
def __call__(self, inputs, state, scope=None): with tf.device("/gpu:"+str(self._gpu_for_layer)): print('testing') with tf.variable_scope(scope or type(self).__name__): # "UnitaryRNNCell" with tf.variable_scope("UnitaryGates"): # Reset gate and update gate. '''just for sake of consistency, we'll keep some var names the same as authors''' n_hidden = self._num_units h_prev = state '''development nick version here''' step1 = unitary_linear.times_diag_tf(h_prev, n_hidden) #this will create a diagonal tensor with given diagonal values #work on times_reflection next modulus = T.sqrt(lin_output_re ** 2 + lin_output_im ** 2) rescale = T.maximum(modulus + hidden_bias.dimshuffle('x',0), 0.) / (modulus + 1e-5) nonlin_output_re = lin_output_re * rescale nonlin_output_im = lin_output_im * rescale h_t = tf.concat(1, [nonlin_output_re, nonlin_output_im]) #keep in mind that you can use tf.complex to convert two numbers into a complex number -- this works for tensors! return h_t, h_t #check if h_t is the same as the output????? '''list of complex number functions in tf 1. tf.complex -- makes complex number 2. complex_abs -- finds the absolute value of the tensor 3. tf.conj -- makes conjugate 4. tf.imag -- returns imaginary part -- go back and forth between complex and imag 5. tf.real -- returns real part''' #keep in mind that identity matricies are a form of diagonal matricies, but they just have ones. '''----------------------------end of unitary rnn cell--------------------------''' # We start with bias of 1.0 to not reset and not update. '''First, we will start with the hidden linear transform W = D3R2F-1D2PermR1FD1 Keep in mind that originally the equation would be W = VDV*, but it leads to too much computation/memory o(n^2)''' step1 = times_diag(h_prev, n_hidden, theta[0,:]) step2 = step1 # step2 = do_fft(step1, n_hidden) step3 = times_reflection(step2, n_hidden, reflection[0,:]) step4 = vec_permutation(step3, n_hidden, index_permute) step5 = times_diag(step4, n_hidden, theta[1,:]) step6 = step5 # step6 = do_ifft(step5, n_hidden) step7 = times_reflection(step6, n_hidden, reflection[1,:]) step8 = times_diag(step7, n_hidden, theta[2,:]) step9 = scale_diag(step8, n_hidden, scale) hidden_lin_output = step9 z = tf.sigmoid(linear.linear([inputs], self._num_units, True, 1.0)) '''equation 2 r = sigm(WxrXt+Whrht+Br), h_t is the previous state''' r = tf.sigmoid((linear.linear([inputs,state], self._num_units, True, 1.0))) '''equation 3''' with tf.variable_scope("Candidate"): component_0 = linear.linear([r*state], self._num_units, True) component_1 = tf.tanh(tf.tanh(inputs) + component_0) component_2 = component_1*z component_3 = state*(1 - z) h_t = component_2 + component_3 h_t = tf.concat(concat_dim = 1, value =[nonlin_output_re, nonlin_output_im]) #I know here you need to concatenate the real and imaginary parts return h_t, h_t #there is only one hidden state output to keep track of.