class Decoderlstm: def __init__(self): self.layers = hps.DecoderRNN_layers self.size = hps.DecoderRNN_size self.lstm_list = [LSTMCell(self.size) for _ in range(self.layers)] self.lstm_cell = RNN(self.lstm_list, return_state=True, return_sequences=True) def build(self, input_shape): self.lstm_cell.build(input_shape) self.trainable_weights = self.lstm_cell._trainable_weights self.weights = self.lstm_cell.weights def compute_output_shape(self, inputs): return self.lstm_cell.compute_output_shape(inputs) def get_initial_state(self, inputs): return self.lstm_cell.get_initial_state(inputs) def __call__(self, inputs, initial_state): return self.lstm_cell(inputs, initial_state=initial_state)
class LMU(Layer): """ A layer of trainable low-dimensional delay systems. Each unit buffers its encoded input by internally representing a low-dimensional (i.e., compressed) version of the input window. Nonlinear decodings of this representation provide computations across the window, such as its derivative, energy, median value, etc (*). Note that decoders can span across all of the units. By default the window lengths are trained via backpropagation, as well as the encoding and decoding weights. Optionally, the state-space matrices that implement the low-dimensional delay system can be trained as well, but these are shared across all of the units in the layer. Based on the occurrence of the recurrent connections, this layer will choose different implementations of evaluating the delay system. If any recurrent connections are enabled, evaluation will occur sequentially with a Keras RNN layer using the ``LMUCell`` cell class. If all recurrent connections are disabled, evaluation of the delay system will be computed as the convolution of the input sequence with the impulse response of the LMU cell, using the ``LMUCellFFT`` cell class. (*) Voelker and Eliasmith (2018). Improving spiking dynamical networks: Accurate delays, higher-order synapses, and time cells. Neural Computation, 30(3): 569-609. (*) Voelker and Eliasmith. "Methods and systems for implementing dynamic neural networks." U.S. Patent Application No. 15/243,223. Filing date: 2016-08-22. """ def __init__( self, units, order, theta, # relative to dt=1 method="zoh", realizer=Identity(), # TODO: Deprecate? factory=LegendreDelay, # TODO: Deprecate? memory_to_memory=True, hidden_to_memory=True, hidden_to_hidden=True, trainable_input_encoders=True, trainable_hidden_encoders=True, trainable_memory_encoders=True, trainable_input_kernel=True, trainable_hidden_kernel=True, trainable_memory_kernel=True, trainable_A=False, trainable_B=False, input_encoders_initializer="lecun_uniform", hidden_encoders_initializer="lecun_uniform", memory_encoders_initializer=Constant(0), # 'lecun_uniform', input_kernel_initializer="glorot_normal", hidden_kernel_initializer="glorot_normal", memory_kernel_initializer="glorot_normal", hidden_activation="tanh", return_sequences=False, **kwargs): # Note: Setting memory_to_memory, hidden_to_memory, and hidden_to_hidden to # False doesn't actually remove the connections, but only initializes the # weights to be zero and non-trainable (when using the LMUCell). # This behaviour may change pending a future API decision. self.units = units self.order = order self.theta = theta self.method = method self.realizer = realizer self.factory = factory self.memory_to_memory = memory_to_memory self.hidden_to_memory = hidden_to_memory self.hidden_to_hidden = hidden_to_hidden self.trainable_input_encoders = trainable_input_encoders self.trainable_hidden_encoders = (trainable_hidden_encoders if hidden_to_memory else False) self.trainable_memory_encoders = (trainable_memory_encoders if memory_to_memory else False) self.trainable_input_kernel = trainable_input_kernel self.trainable_hidden_kernel = (trainable_hidden_kernel if hidden_to_hidden else False) self.trainable_memory_kernel = trainable_memory_kernel self.trainable_A = trainable_A self.trainable_B = trainable_B self.input_encoders_initializer = input_encoders_initializer self.hidden_encoders_initializer = (hidden_encoders_initializer if hidden_to_memory else Constant(0)) self.memory_encoders_initializer = (memory_encoders_initializer if memory_to_memory else Constant(0)) self.input_kernel_initializer = input_kernel_initializer self.hidden_kernel_initializer = (hidden_kernel_initializer if hidden_to_hidden else Constant(0)) self.memory_kernel_initializer = memory_kernel_initializer self.hidden_activation = hidden_activation self.return_sequences = return_sequences super().__init__(**kwargs) if self.fft_check(): self.lmu_layer = LMUCellFFT( units=self.units, order=self.order, theta=self.theta, trainable_input_encoders=self.trainable_input_encoders, trainable_input_kernel=self.trainable_input_kernel, trainable_memory_kernel=self.trainable_memory_kernel, input_encoders_initializer=self.input_encoders_initializer, input_kernel_initializer=self.input_kernel_initializer, memory_kernel_initializer=self.memory_kernel_initializer, hidden_activation=self.hidden_activation, return_sequences=self.return_sequences, ) else: self.lmu_layer = RNN( LMUCell( units=self.units, order=self.order, theta=self.theta, method=self.method, realizer=self.realizer, factory=self.factory, trainable_input_encoders=self.trainable_input_encoders, trainable_hidden_encoders=self.trainable_hidden_encoders, trainable_memory_encoders=self.trainable_memory_encoders, trainable_input_kernel=self.trainable_input_kernel, trainable_hidden_kernel=self.trainable_hidden_kernel, trainable_memory_kernel=self.trainable_memory_kernel, trainable_A=self.trainable_A, trainable_B=self.trainable_B, input_encoders_initializer=self.input_encoders_initializer, hidden_encoders_initializer=self. hidden_encoders_initializer, memory_encoders_initializer=self. memory_encoders_initializer, input_kernel_initializer=self.input_kernel_initializer, hidden_kernel_initializer=self.hidden_kernel_initializer, memory_kernel_initializer=self.memory_kernel_initializer, hidden_activation=self.hidden_activation, ), return_sequences=self.return_sequences, ) def call(self, inputs): """ Calls the layer with inputs. """ return self.lmu_layer.call(inputs) def build(self, input_shape): """ Initializes network parameters. """ self.lmu_layer.build(input_shape) self.built = True def fft_check(self): """ Checks if recurrent connections are enabled to automatically switch to FFT. """ # Note: Only the flags are checked here. The alternative would be to check the # weight initializers and trainiable flag settings, however it is cumbersome # to check against all initializers forms that initialize the recurrent weights # to 0. # # These flags used below exist in other LMUCell implementations, and will be # brought forward in a future API decisions. return not (self.memory_to_memory or self.hidden_to_memory or self.hidden_to_hidden) def get_config(self): """ Overrides the tensorflow get_config function. """ config = super().get_config() config.update( dict( units=self.units, order=self.order, theta=self.theta, method=self.method, factory=self.factory, memory_to_memory=self.memory_to_memory, hidden_to_memory=self.hidden_to_memory, hidden_to_hidden=self.hidden_to_hidden, trainable_input_encoders=self.trainable_input_encoders, trainable_hidden_encoders=self.trainable_hidden_encoders, trainable_memory_encoders=self.trainable_memory_encoders, trainable_input_kernel=self.trainable_input_kernel, trainable_hidden_kernel=self.trainable_hidden_kernel, trainable_memory_kernel=self.trainable_memory_kernel, trainable_A=self.trainable_A, trainable_B=self.trainable_B, input_encorders_initializer=self.input_encoders_initializer, hidden_encoders_initializer=self.hidden_encoders_initializer, memory_encoders_initializer=self.memory_encoders_initializer, input_kernel_initializer=self.input_kernel_initializer, hidden_kernel_initializer=self.hidden_kernel_initializer, memory_kernel_initializer=self.memory_kernel_initializer, hidden_activation=self.hidden_activation, return_sequences=self.return_sequences, )) return config