def __init__(self, size, initialHistory=gpu.zeros((0, 0)), baseLayerClass=SigmoidLayer, connectionClass=FullConnection): """ A recurrent layer extends the activation layer by adding a full recurrent connection from the output of the layer to its input, delayed by a timestep. """ # Properly inherit the AbstractLayer AbstractLayer.__init__(self) # Extract the layerSize from the provided activation layer self.baseLayer = baseLayerClass(size) self.layerSize = size self.historyLayer = HistoryLayer(self.layerSize, initialHistory) # A recurrent layer has an input port, history port and output port self.input = self.baseLayer.input self.output = self.baseLayer.output # Make two connections - the recurrent connection to the history and a connection from # the history to the activationLayer self.recurrentConnection = connectionClass(self.output, self.historyLayer.input) self.historyConnection = IdentityConnection(self.historyLayer.output, self.input) # Keep track of how many timesteps there were, and the initial history incase of reset self.timestep = 0
def __init__(self, size, initialHistory = gpu.zeros((0,0)), baseLayerClass = SigmoidLayer, connectionClass = FullConnection): """ A recurrent layer extends the activation layer by adding a full recurrent connection from the output of the layer to its input, delayed by a timestep. """ # Properly inherit the AbstractLayer AbstractLayer.__init__(self) # Extract the layerSize from the provided activation layer self.baseLayer = baseLayerClass(size) self.layerSize = size self.historyLayer = HistoryLayer(self.layerSize, initialHistory) # A recurrent layer has an input port, history port and output port self.input = self.baseLayer.input self.output = self.baseLayer.output # Make two connections - the recurrent connection to the history and a connection from # the history to the activationLayer self.recurrentConnection = connectionClass(self.output, self.historyLayer.input) self.historyConnection = IdentityConnection(self.historyLayer.output, self.input) # Keep track of how many timesteps there were, and the initial history incase of reset self.timestep = 0
class RecurrentLayer(AbstractLayer): """ A layer which implements a delay in time """ def __init__(self, size, initialHistory = gpu.zeros((0,0)), baseLayerClass = SigmoidLayer, connectionClass = FullConnection): """ A recurrent layer extends the activation layer by adding a full recurrent connection from the output of the layer to its input, delayed by a timestep. """ # Properly inherit the AbstractLayer AbstractLayer.__init__(self) # Extract the layerSize from the provided activation layer self.baseLayer = baseLayerClass(size) self.layerSize = size self.historyLayer = HistoryLayer(self.layerSize, initialHistory) # A recurrent layer has an input port, history port and output port self.input = self.baseLayer.input self.output = self.baseLayer.output # Make two connections - the recurrent connection to the history and a connection from # the history to the activationLayer self.recurrentConnection = connectionClass(self.output, self.historyLayer.input) self.historyConnection = IdentityConnection(self.historyLayer.output, self.input) # Keep track of how many timesteps there were, and the initial history incase of reset self.timestep = 0 def forward(self): """ Perform a forward step - set the output to the current history """ # Nothing much to do, simply call forward on the activation layer and recurrent connection self.historyLayer.forward() self.historyConnection.forward() self.baseLayer.forward() self.recurrentConnection.forward() def backward(self): """ Perform the backprop step on the activation layer and recurrent connection """ self.recurrentConnection.backward() self.baseLayer.backward() self.historyConnection.backward() self.historyLayer.backward() def step(self): """ Step forward in time. Propagate the history input to the history output """ self.timestep += 1 self.historyLayer.step() def reset(self): """ Set the history to the original initial history and set timestep to zero """ self.timestep = 0 self.historyLayer.reset() def getRecurrentConnection(self): """ Provide the recurrent connection """ return self.recurrentConnection def setInitialHistory(self, history): """ Setup the initial history of this layer """ self.historyLayer.initialHistory = history self.historyLayer.output.value = gpu.garray(np.copy(history.as_numpy_array())) def setHistoryDelta(self, delta): """ Set the delta on the history layer to the provided value """ self.historyLayer.setDelta(delta) # We need to propagate the delta backward to the appropriate ports self.backward() def zeroInitialHistoryBatch(self, batchSize): """ Set the initial history to zeros for the provided batch size """ zero_history = gpu.zeros((batchSize, self.layerSize)) self.setInitialHistory(zero_history) def zeroHistoryDeltaBatch(self, batchSize): """ Set the initial history delta to zeros for the provided batch size """ zero_delta = gpu.zeros((batchSize, self.layerSize)) self.setHistoryDelta(zero_delta) def backstep(self): """ Step backward in time. Propagate the input delta to the history """ self.timestep -= 1 self.historyLayer.backstep()
class RecurrentLayer(AbstractLayer): """ A layer which implements a delay in time """ def __init__(self, size, initialHistory=gpu.zeros((0, 0)), baseLayerClass=SigmoidLayer, connectionClass=FullConnection): """ A recurrent layer extends the activation layer by adding a full recurrent connection from the output of the layer to its input, delayed by a timestep. """ # Properly inherit the AbstractLayer AbstractLayer.__init__(self) # Extract the layerSize from the provided activation layer self.baseLayer = baseLayerClass(size) self.layerSize = size self.historyLayer = HistoryLayer(self.layerSize, initialHistory) # A recurrent layer has an input port, history port and output port self.input = self.baseLayer.input self.output = self.baseLayer.output # Make two connections - the recurrent connection to the history and a connection from # the history to the activationLayer self.recurrentConnection = connectionClass(self.output, self.historyLayer.input) self.historyConnection = IdentityConnection(self.historyLayer.output, self.input) # Keep track of how many timesteps there were, and the initial history incase of reset self.timestep = 0 def forward(self): """ Perform a forward step - set the output to the current history """ # Nothing much to do, simply call forward on the activation layer and recurrent connection self.historyLayer.forward() self.historyConnection.forward() self.baseLayer.forward() self.recurrentConnection.forward() def backward(self): """ Perform the backprop step on the activation layer and recurrent connection """ self.recurrentConnection.backward() self.baseLayer.backward() self.historyConnection.backward() self.historyLayer.backward() def step(self): """ Step forward in time. Propagate the history input to the history output """ self.timestep += 1 self.historyLayer.step() def reset(self): """ Set the history to the original initial history and set timestep to zero """ self.timestep = 0 self.historyLayer.reset() def getRecurrentConnection(self): """ Provide the recurrent connection """ return self.recurrentConnection def setInitialHistory(self, history): """ Setup the initial history of this layer """ self.historyLayer.initialHistory = history self.historyLayer.output.value = gpu.garray( np.copy(history.as_numpy_array())) def setHistoryDelta(self, delta): """ Set the delta on the history layer to the provided value """ self.historyLayer.setDelta(delta) # We need to propagate the delta backward to the appropriate ports self.backward() def zeroInitialHistoryBatch(self, batchSize): """ Set the initial history to zeros for the provided batch size """ zero_history = gpu.zeros((batchSize, self.layerSize)) self.setInitialHistory(zero_history) def zeroHistoryDeltaBatch(self, batchSize): """ Set the initial history delta to zeros for the provided batch size """ zero_delta = gpu.zeros((batchSize, self.layerSize)) self.setHistoryDelta(zero_delta) def backstep(self): """ Step backward in time. Propagate the input delta to the history """ self.timestep -= 1 self.historyLayer.backstep()