def __init__(self,indim,outdim,hiddim=6): self.network=RecurrentNetwork() ##CREATE MODULES self._in_layer = LinearLayer(indim+outdim) self._hid_layer = LSTMLayer(hiddim,peepholes=False) self._out_layer = LinearLayer(outdim) self._bias = BiasUnit() ##ADD MODULES self.network.addInputModule(self._in_layer) self.network.addModule(self._hid_layer) self.network.addModule(self._bias) self.network.addOutputModule(self._out_layer) self._last_hidden_layer = None self._first_hidden_layer = None ###CREATE CONNECTIONS self._hid_to_out_connection = FullConnection(self._hid_layer , self._out_layer) self._in_to_hid_connection = FullConnection(self._in_layer , self._hid_layer) self._out_to_hid_connection=FullConnection(self._out_layer,self._hid_layer) ##ADD CONNECTIONS self.network.addConnection(self._hid_to_out_connection) self.network.addConnection(self._in_to_hid_connection) self.network.addConnection(FullConnection(self._bias, self._hid_layer)) self.network.addRecurrentConnection(self._out_to_hid_connection) self.network.sortModules() self.backprojectionFactor = 1
def __init__(self, indim, outdim, hiddim=6): Module.__init__(self, indim, outdim) self._network = RecurrentNetwork() self._in_layer = LinearLayer(indim + outdim) self._hid_layer = LSTMLayer(hiddim) self._out_layer = LinearLayer(outdim) self._bias = BiasUnit() self._network.addInputModule(self._in_layer) self._network.addModule(self._hid_layer) self._network.addModule(self._bias) self._network.addOutputModule(self._out_layer) self._hid_to_out_connection = FullConnection(self._hid_layer , self._out_layer) self._in_to_hid_connection = FullConnection(self._in_layer , self._hid_layer) self._network.addConnection(self._hid_to_out_connection) self._network.addConnection(self._in_to_hid_connection) self._network.addConnection(FullConnection(self._bias, self._hid_layer)) self._network.sortModules() self.offset = self._network.offset self.backprojectionFactor = 1.0 return
def __init__(self, outdim, hiddim=15): """ Create an EvolinoNetwork with for sequences of dimension outdim and hiddim dimension of the RNN Layer.""" indim = 0 Module.__init__(self, indim, outdim) self._network = RecurrentNetwork() self._in_layer = LinearLayer(indim + outdim) self._hid_layer = LSTMLayer(hiddim) self._out_layer = LinearLayer(outdim) self._bias = BiasUnit() self._network.addInputModule(self._in_layer) self._network.addModule(self._hid_layer) self._network.addModule(self._bias) self._network.addOutputModule(self._out_layer) self._in_to_hid_connection = FullConnection(self._in_layer, self._hid_layer) self._bias_to_hid_connection = FullConnection(self._bias, self._hid_layer) self._hid_to_out_connection = FullConnection(self._hid_layer, self._out_layer) self._network.addConnection(self._in_to_hid_connection) self._network.addConnection(self._bias_to_hid_connection) self._network.addConnection(self._hid_to_out_connection) self._recurrent_connection = FullConnection(self._hid_layer, self._hid_layer) self._network.addRecurrentConnection(self._recurrent_connection) self._network.sortModules() self._network.reset() self.offset = self._network.offset self.backprojectionFactor = 0.01
class EvolinoNetwork(Module): """ Model class to be trained by the EvolinoTrainer.""" def __init__(self, outdim, hiddim=15): """ Create an EvolinoNetwork with for sequences of dimension outdim and hiddim dimension of the RNN Layer.""" indim = 0 Module.__init__(self, indim, outdim) self._network = RecurrentNetwork() self._in_layer = LinearLayer(indim + outdim) self._hid_layer = LSTMLayer(hiddim) self._out_layer = LinearLayer(outdim) self._bias = BiasUnit() self._network.addInputModule(self._in_layer) self._network.addModule(self._hid_layer) self._network.addModule(self._bias) self._network.addOutputModule(self._out_layer) self._in_to_hid_connection = FullConnection(self._in_layer, self._hid_layer) self._bias_to_hid_connection = FullConnection(self._bias, self._hid_layer) self._hid_to_out_connection = FullConnection(self._hid_layer, self._out_layer) self._network.addConnection(self._in_to_hid_connection) self._network.addConnection(self._bias_to_hid_connection) self._network.addConnection(self._hid_to_out_connection) self._recurrent_connection = FullConnection(self._hid_layer, self._hid_layer) self._network.addRecurrentConnection(self._recurrent_connection) self._network.sortModules() self._network.reset() self.offset = self._network.offset self.backprojectionFactor = 0.01 def reset(self): """ Resets the underlying network """ self._network.reset() def washout(self, sequence): """ Force the network to process the sequence instead of the backprojection values. Used for adjusting the RNN's state. Returns the outputs of the RNN that are needed for linear regression.""" assert len(sequence) != 0 assert self.outdim == len(sequence[0]) raw_outputs = [] for val in sequence: backprojection = self._getLastOutput() backprojection *= self.backprojectionFactor self._activateNetwork(backprojection) raw_out = self._getRawOutput() raw_outputs.append(raw_out) self._setLastOutput(val) return array(raw_outputs) def _activateNetwork(self, input): """ Run the activate method of the underlying network.""" assert len(input) == self._network.indim output = array(self._network.activate(input)) self.offset = self._network.offset return output def activate(self, input): raise NotImplementedError( '.activate() is not supported, use .extrapolate()') def extrapolate(self, sequence, length): """ Extrapolate 'sequence' for 'length' steps and return the extrapolated sequence as array. Extrapolating is realized by reseting the network, then washing it out with the supplied sequence, and then generating a sequence.""" self.reset() self.washout(sequence) return self.generate(length) def generate(self, length): """ Generate a sequence of specified length. Use .reset() and .washout() before.""" generated_sequence = [] #empty(length) for _ in xrange(length): backprojection = self._getLastOutput() backprojection *= self.backprojectionFactor out = self._activateNetwork(backprojection) generated_sequence.append(out) return array(generated_sequence) def _getLastOutput(self): """Return the current output of the linear output layer.""" if self.offset == 0: return zeros(self.outdim) else: return self._out_layer.outputbuffer[self.offset - 1] def _setLastOutput(self, output): """Force the current output of the linear output layer to 'output'.""" self._out_layer.outputbuffer[self.offset - 1][:] = output # # Genome related # def _validateGenomeLayer(self, layer): """Validate the type and state of a layer.""" assert isinstance(layer, LSTMLayer) assert not layer.peepholes def getGenome(self): """Return the RNN's Genome.""" return self._getGenomeOfLayer(self._hid_layer) def setGenome(self, weights): """Set the RNN's Genome.""" weights = deepcopy(weights) self._setGenomeOfLayer(self._hid_layer, weights) def _getGenomeOfLayer(self, layer): """Return the genome of a single layer.""" self._validateGenomeLayer(layer) connections = self._getInputConnectionsOfLayer(layer) layer_weights = [] # iterate cells of layer for cell_idx in range(layer.outdim): # todo: the evolino paper uses a different order of weights for the genotype of a lstm cell cell_weights = [] # iterate weight types (ingate, forgetgate, cell and outgate) for t in range(4): # iterate connections for c in connections: # iterate sources of connection for i in range(c.indim): idx = i + cell_idx * c.indim + t * layer.outdim * c.indim cell_weights.append(c.params[idx]) layer_weights.append(cell_weights) return layer_weights def _setGenomeOfLayer(self, layer, weights): """Set the genome of a single layer.""" self._validateGenomeLayer(layer) connections = self._getInputConnectionsOfLayer(layer) # iterate cells of layer for cell_idx in range(layer.outdim): # todo: the evolino paper uses a different order of weights for the genotype of a lstm cell cell_weights = weights[cell_idx] # iterate weight types (ingate, forgetgate, cell and outgate) for t in range(4): # iterate connections for c in connections: # iterate sources of connection for i in range(c.indim): idx = i + cell_idx * c.indim + t * layer.outdim * c.indim c.params[idx] = cell_weights.pop(0) # # Linear Regression related # def setOutputWeightMatrix(self, W): """Set the weight matrix of the linear output layer.""" c = self._hid_to_out_connection c.params[:] = W.flatten() def getOutputWeightMatrix(self): """Return the weight matrix of the linear output layer.""" c = self._hid_to_out_connection p = c.params return reshape(p, (c.outdim, c.indim)) def _getRawOutput(self): """Return the current output of the RNN. This is needed for linear regression, which calculates the weight matrix of the linear output layer.""" return copy(self._hid_layer.outputbuffer[self.offset - 1]) # # Topology Helper # def _getInputConnectionsOfLayer(self, layer): """Return a list of all input connections for the layer.""" connections = [] all_cons = list(self._network.recurrentConns) all_cons += sum(self._network.connections.values(), []) for c in all_cons: if c.outmod is layer: if not isinstance(c, FullConnection): raise NotImplementedError( "Only FullConnections are supported") connections.append(c) return connections
class EvolinoNetwork(Module): """ Model class to be trained by the EvolinoTrainer.""" def __init__(self, outdim, hiddim=15): """ Create an EvolinoNetwork with for sequences of dimension outdim and hiddim dimension of the RNN Layer.""" indim = 0 Module.__init__(self, indim, outdim) self._network = RecurrentNetwork() self._in_layer = LinearLayer(indim + outdim) self._hid_layer = LSTMLayer(hiddim) self._out_layer = LinearLayer(outdim) self._bias = BiasUnit() self._network.addInputModule(self._in_layer) self._network.addModule(self._hid_layer) self._network.addModule(self._bias) self._network.addOutputModule(self._out_layer) self._in_to_hid_connection = FullConnection(self._in_layer, self._hid_layer) self._bias_to_hid_connection = FullConnection(self._bias, self._hid_layer) self._hid_to_out_connection = FullConnection(self._hid_layer, self._out_layer) self._network.addConnection(self._in_to_hid_connection) self._network.addConnection(self._bias_to_hid_connection) self._network.addConnection(self._hid_to_out_connection) self._recurrent_connection = FullConnection(self._hid_layer, self._hid_layer) self._network.addRecurrentConnection(self._recurrent_connection) self._network.sortModules() self._network.reset() self.offset = self._network.offset self.backprojectionFactor = 0.01 def reset(self): """ Resets the underlying network """ self._network.reset() def washout(self, sequence): """ Force the network to process the sequence instead of the backprojection values. Used for adjusting the RNN's state. Returns the outputs of the RNN that are needed for linear regression.""" assert len(sequence) != 0 assert self.outdim == len(sequence[0]) raw_outputs = [] for val in sequence: backprojection = self._getLastOutput() backprojection *= self.backprojectionFactor self._activateNetwork(backprojection) raw_out = self._getRawOutput() raw_outputs.append(raw_out) self._setLastOutput(val) return array(raw_outputs) def _activateNetwork(self, input): """ Run the activate method of the underlying network.""" assert len(input) == self._network.indim output = array(self._network.activate(input)) self.offset = self._network.offset return output def activate(self, input): raise NotImplementedError( '.activate() is not supported, use .extrapolate()') def extrapolate(self, sequence, length): """ Extrapolate 'sequence' for 'length' steps and return the extrapolated sequence as array. Extrapolating is realized by reseting the network, then washing it out with the supplied sequence, and then generating a sequence.""" self.reset() self.washout(sequence) return self.generate(length) def generate(self, length): """ Generate a sequence of specified length. Use .reset() and .washout() before.""" generated_sequence = [] #empty(length) for _ in range(length): backprojection = self._getLastOutput() backprojection *= self.backprojectionFactor out = self._activateNetwork(backprojection) generated_sequence.append(out) return array(generated_sequence) def _getLastOutput(self): """Return the current output of the linear output layer.""" if self.offset == 0: return zeros(self.outdim) else: return self._out_layer.outputbuffer[self.offset - 1] def _setLastOutput(self, output): """Force the current output of the linear output layer to 'output'.""" self._out_layer.outputbuffer[self.offset - 1][:] = output # # Genome related # def _validateGenomeLayer(self, layer): """Validate the type and state of a layer.""" assert isinstance(layer, LSTMLayer) assert not layer.peepholes def getGenome(self): """Return the RNN's Genome.""" return self._getGenomeOfLayer(self._hid_layer) def setGenome(self, weights): """Set the RNN's Genome.""" weights = deepcopy(weights) self._setGenomeOfLayer(self._hid_layer, weights) def _getGenomeOfLayer(self, layer): """Return the genome of a single layer.""" self._validateGenomeLayer(layer) connections = self._getInputConnectionsOfLayer(layer) layer_weights = [] # iterate cells of layer for cell_idx in range(layer.outdim): # todo: the evolino paper uses a different order of weights for the genotype of a lstm cell cell_weights = [] # iterate weight types (ingate, forgetgate, cell and outgate) for t in range(4): # iterate connections for c in connections: # iterate sources of connection for i in range(c.indim): idx = i + cell_idx * c.indim + t * layer.outdim * c.indim cell_weights.append(c.params[idx]) layer_weights.append(cell_weights) return layer_weights def _setGenomeOfLayer(self, layer, weights): """Set the genome of a single layer.""" self._validateGenomeLayer(layer) connections = self._getInputConnectionsOfLayer(layer) # iterate cells of layer for cell_idx in range(layer.outdim): # todo: the evolino paper uses a different order of weights for the genotype of a lstm cell cell_weights = weights[cell_idx] # iterate weight types (ingate, forgetgate, cell and outgate) for t in range(4): # iterate connections for c in connections: # iterate sources of connection for i in range(c.indim): idx = i + cell_idx * c.indim + t * layer.outdim * c.indim c.params[idx] = cell_weights.pop(0) # # Linear Regression related # def setOutputWeightMatrix(self, W): """Set the weight matrix of the linear output layer.""" c = self._hid_to_out_connection c.params[:] = W.flatten() def getOutputWeightMatrix(self): """Return the weight matrix of the linear output layer.""" c = self._hid_to_out_connection p = c.params return reshape(p, (c.outdim, c.indim)) def _getRawOutput(self): """Return the current output of the RNN. This is needed for linear regression, which calculates the weight matrix of the linear output layer.""" return copy(self._hid_layer.outputbuffer[self.offset - 1]) # # Topology Helper # def _getInputConnectionsOfLayer(self, layer): """Return a list of all input connections for the layer.""" connections = [] all_cons = list(self._network.recurrentConns) all_cons += sum(list(self._network.connections.values()), []) for c in all_cons: if c.outmod is layer: if not isinstance(c, FullConnection): raise NotImplementedError( "Only FullConnections are supported") connections.append(c) return connections
print(identifier, net.activate((0, 0)), net.activate((0, 1)), net.activate((1, 0)), net.activate((1, 1))) ds = SupervisedDataSet(2, 1) ds.addSample((0, 0), (0, )) ds.addSample((0, 1), (1, )) ds.addSample((1, 0), (1, )) ds.addSample((1, 1), (0, )) for input, target in ds: print(input, target) #define net net = RecurrentNetwork() net.addInputModule(LinearLayer(2, name="il")) net.addModule(SigmoidLayer(4, name="h1")) net.addModule(SigmoidLayer(4, name="h2")) net.addOutputModule(LinearLayer(1, name="ol")) c1 = FullConnection(net["il"], net["h1"]) c2 = FullConnection(net["h1"], net["h2"]) c3 = FullConnection(net["h2"], net["ol"]) cr1 = FullConnection(net["h1"], net["h1"]) net.addConnection(c1) net.addConnection(c2) net.addConnection(c3) net.addRecurrentConnection(cr1) net.sortModules() print(net)
def buildSimpleLSTMNetwork(peepholes=False): N = RecurrentNetwork('simpleLstmNet') i = LinearLayer(1, name='i') h = LSTMLayer(1, peepholes=peepholes, name='lstm') o = LinearLayer(1, name='o') b = BiasUnit('bias') N.addModule(b) N.addOutputModule(o) N.addInputModule(i) N.addModule(h) N.addConnection(FullConnection(i, h, name='f1')) N.addConnection(FullConnection(b, h, name='f2')) N.addRecurrentConnection(FullConnection(h, h, name='r1')) N.addConnection(FullConnection(h, o, name='r1')) N.sortModules() return N
def mk_nn(mx_lag): #Construct the Neural Network #n = FeedForwardNetwork() n = RecurrentNetwork() n.addInputModule(LinearLayer(mx_lag, name='in')) n.addModule(BiasUnit('bias')) n.addModule(LSTMLayer(5, name='hidden1')) n.addModule(LSTMLayer(5, name='hidden2')) #n.addModule(TanhLayer(10, name='hidden2')) #n.addModule(TanhLayer(10, name='hidden3')) n.addOutputModule(LinearLayer(1, name='out')) #add connections n.addConnection(FullConnection(n['in'], n['hidden1'], name='c1')) n.addConnection(FullConnection(n['hidden1'], n['hidden2'], name='c2')) n.addConnection(FullConnection(n['bias'], n['hidden1'], name='c3')) #n.addConnection(FullConnection(n['hidden2'], n['hidden3'], name='c5')) #n.addRecurrentConnection(FullConnection(n['hidden1'], n['hidden1'])) n.addConnection(FullConnection(n['hidden2'], n['out'], name='c4')) #n.addConnection(FullConnection(n['hidden1'], n['out'], name='c2')) n.sortModules() return n
def buildSimpleMDLSTMNetwork(peepholes = False): N = RecurrentNetwork('simpleMDLstmNet') i = LinearLayer(1, name = 'i') dim = 1 h = MDLSTMLayer(dim, peepholes = peepholes, name = 'MDlstm') o = LinearLayer(1, name = 'o') b = BiasUnit('bias') N.addModule(b) N.addOutputModule(o) N.addInputModule(i) N.addModule(h) N.addConnection(FullConnection(i, h, outSliceTo = 4*dim, name = 'f1')) N.addConnection(FullConnection(b, h, outSliceTo = 4*dim, name = 'f2')) N.addRecurrentConnection(FullConnection(h, h, inSliceTo = dim, outSliceTo = 4*dim, name = 'r1')) N.addRecurrentConnection(IdentityConnection(h, h, inSliceFrom = dim, outSliceFrom = 4*dim, name = 'rstate')) N.addConnection(FullConnection(h, o, inSliceTo = dim, name = 'f3')) N.sortModules() return N
def _randomizePhase(self, c): theta0 = np.random.randn(self.outdim * self.indim) theta0 = np.exp(1j * 2 * np.pi * theta0) return c * reshape(theta0, (self.outdim, self.indim)) def _forwardImplementation(self, inbuf, outbuf): outbuf += inbuf def _backwardImplementation(self, outerr, inerr, inbuf): #CHECKME: not setting derivatives -- this means the multiplicative weight is never updated! inerr += 0 if __name__ == "__main__": # from pybrain.tests import runModuleTestSuite # import pygfnn.tests.unittests.structure.connections.test_gfnn_connections as test # runModuleTestSuite(test) from pybrain.structure.networks.recurrent import RecurrentNetwork from pybrain import LinearLayer, FullConnection, FullNotSelfConnection, IdentityConnection from pygfnn import GFNNLayer, RealIdentityConnection, RealMeanFieldConnection N = RecurrentNetwork('simpleGFNN') i = LinearLayer(1, name = 'i') h = GFNNLayer(200, name = 'gfnn') o = LinearLayer(200, name = 'o') N.addOutputModule(o) N.addInputModule(i) N.addModule(h) N.addConnection(GFNNExtConnection(i, h, name = 'f1')) N.addRecurrentConnection(GFNNIntConnection(h, h, name = 'r1')) N.addConnection(RealIdentityConnection(h, o, name = 'i1')) N.sortModules()
def buildSimpleLSTMNetwork(peepholes=False): N = RecurrentNetwork("simpleLstmNet") i = LinearLayer(1, name="i") h = LSTMLayer(1, peepholes=peepholes, name="lstm") o = LinearLayer(1, name="o") b = BiasUnit("bias") N.addModule(b) N.addOutputModule(o) N.addInputModule(i) N.addModule(h) N.addConnection(FullConnection(i, h, name="f1")) N.addConnection(FullConnection(b, h, name="f2")) N.addRecurrentConnection(FullConnection(h, h, name="r1")) N.addConnection(FullConnection(h, o, name="r1")) N.sortModules() return N
def buildLSTMNetwork(self): # create network and modules net = RecurrentNetwork() inp = LinearLayer(self.n_input, name="Input") h1 = LSTMLayer(3, name='LSTM') h2 = SigmoidLayer(10, name='sigm') outp = LinearLayer(self.numActions, name='output') # add modules net.addOutputModule(outp) net.addInputModule(inp) net.addModule(h1) net.addModule(h2) # create connections from input net.addConnection(FullConnection(inp, h1, name="input_LSTM")) net.addConnection(FullConnection(inp, h2, name="input_sigm")) # create connections from LSTM net.addConnection(FullConnection(h1, h2, name="LSTM_sigm")) # add whichever recurrent connections net.addRecurrentConnection(FullConnection(h1, h1, name='LSTM_rec')) net.addRecurrentConnection(FullConnection(h2, h1, name='sigm_LSTM_rec')) # create connections to output net.addConnection(FullConnection(h1, outp, name="LSTM_outp")) net.addConnection(FullConnection(h2, outp, name="sigm_outp")) # finish up net.sortModules() net.randomize() self.printModules(net) self.e = [0 for param in range(len(net.params))] # for each action, need to accumulate the gradient self.accumulated_gradients = [[0 for param in range(len(net.params))] for i in range(self.numActions)] return net
def buildSimpleMDLSTMNetwork(peepholes=False): N = RecurrentNetwork('simpleMDLstmNet') i = LinearLayer(1, name='i') dim = 1 h = MDLSTMLayer(dim, peepholes=peepholes, name='MDlstm') o = LinearLayer(1, name='o') b = BiasUnit('bias') N.addModule(b) N.addOutputModule(o) N.addInputModule(i) N.addModule(h) N.addConnection(FullConnection(i, h, outSliceTo=4 * dim, name='f1')) N.addConnection(FullConnection(b, h, outSliceTo=4 * dim, name='f2')) N.addRecurrentConnection( FullConnection(h, h, inSliceTo=dim, outSliceTo=4 * dim, name='r1')) N.addRecurrentConnection( IdentityConnection(h, h, inSliceFrom=dim, outSliceFrom=4 * dim, name='rstate')) N.addConnection(FullConnection(h, o, inSliceTo=dim, name='f3')) N.sortModules() return N
class FinancialNetwork(object): def __init__(self,indim,outdim,hiddim=6): self.network=RecurrentNetwork() ##CREATE MODULES self._in_layer = LinearLayer(indim+outdim) self._hid_layer = LSTMLayer(hiddim,peepholes=False) self._out_layer = LinearLayer(outdim) self._bias = BiasUnit() ##ADD MODULES self.network.addInputModule(self._in_layer) self.network.addModule(self._hid_layer) self.network.addModule(self._bias) self.network.addOutputModule(self._out_layer) self._last_hidden_layer = None self._first_hidden_layer = None ###CREATE CONNECTIONS self._hid_to_out_connection = FullConnection(self._hid_layer , self._out_layer) self._in_to_hid_connection = FullConnection(self._in_layer , self._hid_layer) self._out_to_hid_connection=FullConnection(self._out_layer,self._hid_layer) ##ADD CONNECTIONS self.network.addConnection(self._hid_to_out_connection) self.network.addConnection(self._in_to_hid_connection) self.network.addConnection(FullConnection(self._bias, self._hid_layer)) self.network.addRecurrentConnection(self._out_to_hid_connection) self.network.sortModules() self.backprojectionFactor = 1 def getGenome(self): weights = [] for layer in self.getHiddenLayers(): if isinstance(layer, LSTMLayer): # if layer is not self._recurrence_layer: weights += self._getGenomeOfLayer(layer) return weights def setGenome(self, weights): """ Sets the Genome of the network. See class description for more details. """ weights = deepcopy(weights) for layer in self.getHiddenLayers(): if isinstance(layer, LSTMLayer): # if layer is not self._recurrence_layer: self._setGenomeOfLayer(layer, weights) def _setGenomeOfLayer(self, layer, weights): dim = layer.outdim connections = self._getInputConnectionsOfLayer(layer) for cell_idx in range(dim): cell_weights = weights.pop(0) for c in connections: params = c.params params[cell_idx + 0 * dim] = cell_weights.pop(0) params[cell_idx + 1 * dim] = cell_weights.pop(0) params[cell_idx + 2 * dim] = cell_weights.pop(0) params[cell_idx + 3 * dim] = cell_weights.pop(0) assert not len(cell_weights) def _getGenomeOfLayer(self, layer): dim = layer.outdim layer_weights = [] connections = self._getInputConnectionsOfLayer(layer) for cell_idx in range(dim): # todo: the evolino paper uses a different order of weights for the genotype of a lstm cell cell_weights = [] for c in connections: cell_weights += [ c.params[ cell_idx + 0 * dim ], c.params[ cell_idx + 1 * dim ], c.params[ cell_idx + 2 * dim ], c.params[ cell_idx + 3 * dim ] ] layer_weights.append(cell_weights) return layer_weights def _getInputConnectionsOfLayer(self, layer): """ Returns a list of all input connections for the layer. """ connections = [] for c in sum(list(self.network.connections.values()), []): if c.outmod is layer: if not isinstance(c, FullConnection): raise NotImplementedError("At the time there is only support for FullConnection") connections.append(c) return connections def getHiddenLayers(self): """ Returns a list of all hidden layers. """ layers = [] network = self.network for m in network.modules: if m not in network.inmodules and m not in network.outmodules: layers.append(m) return layers def _getLastOutput(self): if self.network.offset == 0: return zeros(self.network.outdim) else: return self._out_layer.outputbuffer[self.network.offset - 1] def _setLastOutput(self, output): self._out_layer.outputbuffer[self.network.offset - 1][:] = output def washout(self,input): self.network.offset=0 lstmvalues=[] for val in input: backprojection=self._getLastOutput() backprojection*=self.backprojectionFactor input=append(val,backprojection) output=self.network.activate(input) self._setLastOutput(output) lstmvalues.append(self._hid_layer.outputbuffer[self.network.offset - 1]) return lstmvalues def reset(self): self.network.reset() def getOutputWeightMatrix(self): c=self._hid_to_out_connection W=c.params return reshape(W, (c.outdim, c.indim)) def setOutputWeightMatrix(self,W): c=self._hid_to_out_connection p=c.params p[:]=W.flatten() def activate(self,input): outputs=[] for val in input: backprojection=self._getLastOutput() backprojection*=self.backprojectionFactor inputtrain=append(val,backprojection) output=self.network.activate(inputtrain) self._setLastOutput(output) outputs.append(self._out_layer.outputbuffer[self.network.offset - 1]) return outputs
def buildSimpleLSTMNetwork(peepholes = False): N = RecurrentNetwork('simpleLstmNet') i = LinearLayer(1, name = 'i') h = LSTMLayer(1, peepholes = peepholes, name = 'lstm') o = LinearLayer(1, name = 'o') b = BiasUnit('bias') N.addModule(b) N.addOutputModule(o) N.addInputModule(i) N.addModule(h) N.addConnection(FullConnection(i, h, name = 'f1')) N.addConnection(FullConnection(b, h, name = 'f2')) N.addRecurrentConnection(FullConnection(h, h, name = 'r1')) N.addConnection(FullConnection(h, o, name = 'r1')) N.sortModules() return N
class EvolinoNetwork(Module): def __init__(self, indim, outdim, hiddim=6): Module.__init__(self, indim, outdim) self._network = RecurrentNetwork() self._in_layer = LinearLayer(indim + outdim) self._hid_layer = LSTMLayer(hiddim) self._out_layer = LinearLayer(outdim) self._bias = BiasUnit() self._network.addInputModule(self._in_layer) self._network.addModule(self._hid_layer) self._network.addModule(self._bias) self._network.addOutputModule(self._out_layer) self._hid_to_out_connection = FullConnection(self._hid_layer , self._out_layer) self._in_to_hid_connection = FullConnection(self._in_layer , self._hid_layer) self._network.addConnection(self._hid_to_out_connection) self._network.addConnection(self._in_to_hid_connection) self._network.addConnection(FullConnection(self._bias, self._hid_layer)) self._network.sortModules() self.offset = self._network.offset self.backprojectionFactor = 1.0 return def reset(self): self._network.reset() return def washout(self, inputs, targets, first_idx=None, last_idx=None): assert self.indim == len(inputs[0]) assert self.outdim == len(targets[0]) assert len(inputs) == len(targets) if first_idx is None: first_idx = 0 if last_idx is None: last_idx = len(targets) - 1 raw_outputs = [] for i in xrange(first_idx, last_idx + 1): backprojection = self._getLastOutput() backprojection *= self.backprojectionFactor full_inp = self._createFullInput(inputs[i], backprojection) self._activateNetwork(full_inp) raw_out = self._getRawOutput() raw_outputs.append(np.array(raw_out)) self._setLastOutput(targets[i]) return np.array(raw_outputs) def extrapolate(self, inputs, targets, length): """ Extrapolate 'sequence' for 'length' steps and return the extrapolated sequence as array. Extrapolating is realized by reseting the network, then washing it out with the supplied sequence, and then generating a sequence. """ testing_idx = len(targets) self.reset() self.washout(inputs[:testing_idx], targets) outputs = self.generate(inputs[testing_idx:], length) return outputs def generate(self, inputs, length): generated_sequence = [] for i in xrange(length): output = self.activate(inputs[i]) generated_sequence.append(output) return np.array(generated_sequence) def _activateNetwork(self, input): assert len(input) == self._network.indim output = self._network.activate(input) self.offset = self._network.offset return output def activate(self, input): assert len(input) == self.indim backprojection = self._getLastOutput() backprojection *= self.backprojectionFactor full_inp = self._createFullInput(input, backprojection) out = self._activateNetwork(full_inp) self._setLastOutput(out) return out def calculateOutput(self, dataset, washout_ratio): # iterate through all sequences collected_input = None collected_output = None collected_target = None for i in range(dataset.getNumSequences()): seq = dataset.getSequence(i) input = seq[0] target = seq[1] washout_steps = int(len(input) * washout_ratio) washout_input = input[:washout_steps ] washout_target = target[:washout_steps ] calculation_target = target[washout_steps:] # reset self.reset() # washout self.washout(washout_input, washout_target) # collect calculation data outputs = [] inputs = [] for inp in input[washout_steps:]: out = self.activate(inp) inputs.append(inp) outputs.append(out) # collect output and targets if collected_input is not None: collected_input = np.append(collected_input, inputs, axis=0) else: collected_input = np.array(inputs) if collected_output is not None: collected_output = np.append(collected_output, outputs, axis=0) else: collected_output = np.array(outputs) if collected_target is not None: collected_target = np.append(collected_target, calculation_target, axis=0) else: collected_target = calculation_target return collected_input, collected_output, collected_target def _createFullInput(self, input, output): if self.indim > 0: return np.append(input, output) return np.array(output) def _getLastOutput(self): if self.offset == 0: return np.zeros(self.outdim) #return self._out_layer.outputbuffer[self.offset - 1] return self._network.outputbuffer[self.offset - 1] def _setLastOutput(self, output): #self._out_layer.outputbuffer[self.offset - 1][:] = output self._network.outputbuffer[self.offset - 1][:] = output return # ======================================================== Genome related === def _validateGenomeLayer(self, layer): """ Validates the type and state of a layer """ assert isinstance(layer, LSTMLayer) assert not layer.peepholes return def getGenome(self): """ Returns the Genome of the network. See class description for more details. """ return self._getGenomeOfLayer(self._hid_layer) def setGenome(self, weights): """ Sets the Genome of the network. See class description for more details. """ weights = deepcopy(weights) self._setGenomeOfLayer(self._hid_layer, weights) return def _getGenomeOfLayer(self, layer): """ Returns the genome of a single layer. """ self._validateGenomeLayer(layer) dim = layer.outdim layer_weights = [] connections = self._getInputConnectionsOfLayer(layer) for cell_idx in range(dim): # todo: the evolino paper uses a different order of weights for the genotype of a lstm cell cell_weights = [] for c in connections: cell_weights += [ c.params[ cell_idx + 0 * dim ], c.params[ cell_idx + 1 * dim ], c.params[ cell_idx + 2 * dim ], c.params[ cell_idx + 3 * dim ] ] layer_weights.append(cell_weights) return layer_weights def _setGenomeOfLayer(self, layer, weights): """ Sets the genome of a single layer. """ self._validateGenomeLayer(layer) dim = layer.outdim connections = self._getInputConnectionsOfLayer(layer) for cell_idx in range(dim): cell_weights = weights.pop(0) for c in connections: params = c.params params[cell_idx + 0 * dim] = cell_weights.pop(0) params[cell_idx + 1 * dim] = cell_weights.pop(0) params[cell_idx + 2 * dim] = cell_weights.pop(0) params[cell_idx + 3 * dim] = cell_weights.pop(0) assert not len(cell_weights) return # ============================================ Linear Regression related === def setOutputWeightMatrix(self, W): """ Sets the weight matrix of the output layer's input connection. """ c = self._hid_to_out_connection c.params[:] = W.flatten() return def getOutputWeightMatrix(self): """ Sets the weight matrix of the output layer's input connection. """ c = self._hid_to_out_connection #p = c.getParameters() p = c.params[:] return np.reshape(p, (c.outdim, c.indim)) def _getRawOutput(self): """ Returns the current output of the last hidden layer. This is needed for linear regression, which calculates the weight matrix W of the full connection between this layer and the output layer. """ return copy(self._hid_layer.outputbuffer[self.offset - 1]) # ====================================================== Topology Helper === def _getInputConnectionsOfLayer(self, layer): """ Returns a list of all input connections for the layer. """ connections = [] for c in sum(self._network.connections.values(), []): if c.outmod is layer: if not isinstance(c, FullConnection): raise NotImplementedError("At the time there is only support for FullConnection") connections.append(c) return connections