def setInputScaling(self, newInputScaling): inputScaling = B.ones(self.n_input) * self._inputScaling self._expandedInputScaling = B.vstack( (B.array(1.0), inputScaling.reshape(-1, 1))).flatten() self._WInput = self._WInput * (self._expandedInputScaling / self._inputScaling) self._inputScaling = newInputScaling
def propagateOneStep(self, inputData, outputData, step, transientTime=0, verbose=0, steps="auto", learn=False): if (verbose > 0): bar = progressbar.ProgressBar(max_value=inputLength, redirect_stdout=True, poll_interval=0.0001) bar.update(0) x = self._x #updates states u, x = self.update(inputData=inputData, x=x) self._x = x self._X = B.vstack((B.array(self._outputBias), self._outputInputScaling*u, x)) # calculate output #estimatedData = self.out_activation(B.dot(self._WOut, self._X).T) #y_step = estimatedData Y = B.dot(self._WOut, self._X) if(learn): #learning rate rate = 0.1 #calculate target activation y_target = self.out_inverse_activation(outputData).T #solve for Wout by using expected_output and states wout = B.dot(y_target.reshape(self.n_output,1), B.pinv(self._X)) #.reshape(self.n_output,1) self._WOut = rate*wout + (1-rate)*self._WOut if (verbose > 0): bar.update(t) if (verbose > 0): bar.finish() return self._X, Y
def autocorrelation(self, x): n = x.shape[0] variance = B.var(x) x = x - B.mean(x) r = B.correlate(x, x, mode="full")[-n:] assert B.allclose( r, B.array([(x[:n - k] * x[-(n - k):]).sum() for k in range(n)])) result = r / (variance * (B.arange(n, 0, -1))) return result
def update(self, inputData, outputData=None, x=None): if x is None: x = self._x if self._WFeedback is None: # reshape the data u = inputData.reshape(self.n_input, 1) # update the states transmission = self.calculateLinearNetworkTransmissions(u, x) x *= 1.0 - self._leakingRate x += self._leakingRate * self._activation( transmission + (np.random.rand() - 0.5) * self._noiseLevel) return u else: # the input is allowed to be "empty" (size=0) if self.n_input != 0: # reshape the data u = inputData.reshape(self.n_input, 1) outputData = outputData.reshape(self.n_output, 1) # update the states transmission = self.calculateLinearNetworkTransmissions(u, x) x *= 1.0 - self._leakingRate x += self._leakingRate * self._activation(transmission + B.dot( self._WFeedback, B.vstack((B.array(self._outputBias), outputData)), ) + (np.random.rand() - 0.5) * self._noiseLevel) return u else: # reshape the data outputData = outputData.reshape(self.n_output, 1) # update the states transmission = B.dot(self._W, x) x *= 1.0 - self._leakingRate x += self._leakingRate * self._activation(transmission + B.dot( self._WFeedback, B.vstack((B.array(self._outputBias), outputData)), ) + (np.random.rand() - 0.5) * self._noiseLevel) return B.empty((0, 1))
def predict( self, inputData, continuation=True, initialData=None, update_processor=lambda x: x, verbose=0, ): inputData = B.array(inputData) # let some input run through the ESN to initialize its states from a new starting value if not continuation: self._x = B.zeros(self._x.shape) if initialData is not None: if self._WFeedback is None: for t in range(initialData.shape[0]): super(PredictionESN, self).update(initialData[t]) else: if type(initialData) is tuple: initialDataInput, initialDataOutput = initialData if len(initialDataInput) != len(initialDataOutput): raise ValueError( "Length of the inputData and the outputData of the initialData tuple do not match." ) else: raise ValueError( "initialData has to be a tuple consisting out of the input and the output data." ) super(PredictionESN, self).update(initialDataInput[t], initialDataOutput[t]) X = self.propagate(inputData, verbose=verbose) if self._WFeedback is not None: X, _ = X # calculate the prediction using the trained model if self._solver in [ "sklearn_auto", "sklearn_lsqr", "sklearn_sag", "sklearn_svd", "sklearn_svr", ]: Y = self._ridgeSolver.predict(X.T).reshape((self.n_output, -1)) else: Y = B.dot(self._WOut, X) # apply the output activation function Y = update_processor(self.out_activation(Y)) # return the result return Y.T
def predict_loop(self, inputData, outputData1, outputData2, continuation=True, initialData=None, update_processor=lambda x: x, verbose=0): inputData = B.array(inputData) #let some input run through the ESN to initialize its states from a new starting value if (not continuation): self._esn1._x = B.zeros(self._esn1._x.shape) self._esn2._x = B.zeros(self._esn2._x.shape) total_length = inputData.shape[0] aggregated_y1 = B.empty((total_length, self._n_output1)) aggregated_y2 = B.empty((total_length, self._n_output2)) # X1, y1 = self._esn1.propagate(inputData=inputData, outputData=None, transientTime=self._transientTime, verbose=verbose-1) # Y1 = B.dot(self._esn1._WOut, X1) # Y1 = update_processor(self.out_activation(Y1)).T y2 = self.out_activation(B.dot(self._esn2._WOut, self._esn2._X).T) for i in range(total_length): inputDataWithFeedback = B.zeros((self.n_input + self._n_output2)) inputDataWithFeedback[:self.n_input] = inputData[i, :] inputDataWithFeedback[self.n_input:] = y2 #update models X1, _ = self._esn1.propagateOneStep( inputData=inputDataWithFeedback, outputData=None, step=i, transientTime=self._transientTime, verbose=verbose - 1, learn=False) y1 = B.dot(self._esn1._WOut, X1) aggregated_y1[i, :] = update_processor(self.out_activation(y1)).T #output from 1st layer and correct output #input2 = outputData1[i] - y1.reshape(self._n_output1) # input2 = B.vstack((y1.reshape(self._n_output1), outputData1[i])) # x2, y2 = self._esn2.propagateOneStep(inputData=input2, outputData=None, step=i, transientTime=self._transientTime, verbose=verbose-1, learn=False) # Y_target2[i,:] = y2.reshape(self._n_output2) training_error1 = B.sqrt(B.mean((aggregated_y1 - outputData1)**2)) print("training errors") print(training_error1) return aggregated_y1, aggregated_y2
def __init__(self, n_input, n_reservoir, n_output, spectralRadius=1.0, noiseLevel=0.01, inputScaling=None, leakingRate=1.0, feedbackScaling = 1.0, reservoirDensity=0.2, randomSeed=None, out_activation=lambda x: x, out_inverse_activation=lambda x: x, weightGeneration='naive', bias=1.0, outputBias=1.0, outputInputScaling=1.0, feedback=False, inputDensity=1.0, activation = B.tanh, activationDerivation=lambda x: 1.0/B.cosh(x)**2): self.n_input = n_input self.n_reservoir = n_reservoir self.n_output = n_output self._spectralRadius = spectralRadius self._noiseLevel = noiseLevel self._reservoirDensity = reservoirDensity self._leakingRate = leakingRate self._feedbackScaling = feedbackScaling self.inputDensity = inputDensity self._activation = activation self._activationDerivation = activationDerivation self._inputScaling = inputScaling #this seems to be initialized somewhere (mystery) #self._x = np.zeros(( 1+ n_reservoir, 1)) if self._inputScaling is None: self._inputScaling = 1.0 if np.isscalar(self._inputScaling): self._inputScaling = B.ones(n_input) * self._inputScaling else: if len(self._inputScaling) != self.n_input: raise ValueError("Dimension of inputScaling ({0}) does not match the input data dimension ({1})".format(len(self._inputScaling), n_input)) self._inputScaling = inputScaling self._expandedInputScaling = B.vstack((B.array(1.0), self._inputScaling.reshape(-1,1))).flatten() self.out_activation = out_activation self.out_inverse_activation = out_inverse_activation if randomSeed is not None: rnd.seed(randomSeed) self._bias = bias self._outputBias = outputBias self._outputInputScaling = outputInputScaling self._createReservoir(weightGeneration, feedback)
def predict(self, inputData, outputData1, outputData2, continuation=True, initialData=None, update_processor=lambda x: x, verbose=0): inputData = B.array(inputData) #let some input run through the ESN to initialize its states from a new starting value if (not continuation): self._esn1._x = B.zeros(self._esn1._x.shape) self._esn2._x = B.zeros(self._esn2._x.shape) total_length = inputData.shape[0] aggregated_y1 = B.empty((total_length, self._n_output1)) aggregated_y2 = B.empty((total_length, self._n_output2)) #put pixels and switch data together inputDataWithFeedback = B.zeros( (total_length, self.n_input + self._n_output2)) inputDataWithFeedback[:, :self.n_input] = inputData inputDataWithFeedback[:, self.n_input:] = outputData2 X1, _ = self._esn1.propagate(inputData=inputDataWithFeedback, outputData=None, transientTime=self._transientTime, verbose=verbose - 1) aggregated_y1 = B.dot(self._esn1._WOut, X1) aggregated_y1 = update_processor(self.out_activation(aggregated_y1)).T training_error1 = B.sqrt(B.mean((aggregated_y1 - outputData1)**2)) print("predict errors") print(training_error1) return aggregated_y1, aggregated_y2
def fit_loop(self, inputData, outputData1, outputData2, transientTime="AutoReduce", transientTimeCalculationEpsilon=1e-3, transientTimeCalculationLength=20, verbose=0): #check the input data if self.n_input != 0: if len(inputData.shape) == 3 and len(outputData1.shape) > 1: #multiple time series are used with a shape (timeseries, time, dimension) -> (timeseries, time, dimension) if inputData.shape[0] != outputData1.shape[0]: raise ValueError( "Amount of input and output datasets is not equal - {0} != {1}" .format(inputData.shape[0], outputData1.shape[0])) if inputData.shape[1] != outputData1.shape[1]: raise ValueError( "Amount of input and output time steps is not equal - {0} != {1}" .format(inputData.shape[1], outputData1.shape[1])) else: if inputData.shape[0] != outputData1.shape[0]: raise ValueError( "Amount of input and output time steps is not equal - {0} != {1}" .format(inputData.shape[0], outputData1.shape[0])) else: if inputData is not None: raise ValueError( "n_input has been set to zero. Therefore, the given inputData will not be used." ) inputData = B.array(inputData) outputData1 = B.array(outputData1) outputData2 = B.array(outputData2) self._transientTime = transientTime self._esn1.resetState() self._esn2.resetState() total_length = inputData.shape[0] print("total_length ", total_length) if (verbose > 0): bar = progressbar.ProgressBar(max_value=total_length, redirect_stdout=True, poll_interval=0.0001) bar.update(0) #should be named aggregated_y aggregated_y1 = B.empty((outputData1.shape)) aggregated_y2 = B.empty((outputData2.shape)) # X1, _ = self._esn1.propagate(inputData=inputData, outputData=outputData1, transientTime=transientTime, verbose=verbose-1) # self._esn1._X = X1 # Y_target = self.out_inverse_activation(outputData1).T # self._esn1._WOut = B.dot(Y_target, B.pinv(X1)) # y1 = self.out_activation((B.dot(self._esn1._WOut, X1)).T) # training_error1 = B.sqrt(B.mean((y1.reshape((total_length, self._n_output1))- outputData1)**2)) # training_error2 = 0 y2 = self.out_activation(B.dot(self._esn2._WOut, self._esn2._X).T) for i in range((total_length)): #input: input + output from layer 2 inputDataWithFeedback = B.zeros((self.n_input + self._n_output2)) inputDataWithFeedback[:self.n_input] = inputData[i, :] inputDataWithFeedback[self.n_input:] = y2 #update models X1, untrained_y1 = self._esn1.propagateOneStep( inputData=inputDataWithFeedback, outputData=outputData1[i], step=i, transientTime=transientTime, verbose=verbose - 1, learn=True) #self._esn1._X = X1 #y after model update (ideally we would use y before model update) #y1 = self.out_activation((B.dot(self._esn1._WOut, X1)).T) aggregated_y1[i, :] = untrained_y1.reshape(self._n_output1) #output from 1st layer and correct output # input2 = B.vstack((y1.reshape(self._n_output1), outputData1[i])) # x2, y2 = self._esn2.propagateOneStep(inputData=input2, outputData=outputData2[i], step=i, transientTime=transientTime, verbose=verbose-1, learn=True) # Y_target2[i,:] = y2.reshape(self._n_output2) if (verbose > 0): bar.update(i) if (verbose > 0): bar.finish() self._training_res = aggregated_y1 training_error1 = B.sqrt(B.mean((aggregated_y1 - outputData1)**2)) training_error2 = B.sqrt(B.mean((aggregated_y2 - outputData2)**2)) print("training errors") print(training_error1) print(training_error2) return training_error1, training_error2
def fit(self, inputData, outputData1, outputData2, transientTime="AutoReduce", transientTimeCalculationEpsilon=1e-3, transientTimeCalculationLength=20, verbose=0): #check the input data if self.n_input != 0: if len(inputData.shape) == 3 and len(outputData1.shape) > 1: #multiple time series are used with a shape (timeseries, time, dimension) -> (timeseries, time, dimension) if inputData.shape[0] != outputData1.shape[0]: raise ValueError( "Amount of input and output datasets is not equal - {0} != {1}" .format(inputData.shape[0], outputData1.shape[0])) if inputData.shape[1] != outputData1.shape[1]: raise ValueError( "Amount of input and output time steps is not equal - {0} != {1}" .format(inputData.shape[1], outputData1.shape[1])) else: if inputData.shape[0] != outputData1.shape[0]: raise ValueError( "Amount of input and output time steps is not equal - {0} != {1}" .format(inputData.shape[0], outputData1.shape[0])) else: if inputData is not None: raise ValueError( "n_input has been set to zero. Therefore, the given inputData will not be used." ) inputData = B.array(inputData) outputData1 = B.array(outputData1) outputData2 = B.array(outputData2) self._transientTime = transientTime self._esn1.resetState() self._esn2.resetState() total_length = inputData.shape[0] print("total_length ", total_length) aggregated_y1 = B.empty((outputData1.shape)) aggregated_y2 = B.empty((outputData2.shape)) #put pixels and switch data together inputDataWithFeedback = B.zeros( (total_length, self.n_input + self._n_output2)) inputDataWithFeedback[:, :self.n_input] = inputData inputDataWithFeedback[:, self.n_input:] = outputData2 X1, _ = self._esn1.propagate(inputData=inputDataWithFeedback, outputData=outputData1, transientTime=transientTime, verbose=verbose - 1) self._esn1._X = X1 Y_target = self.out_inverse_activation(outputData1).T self._esn1._WOut = B.dot(Y_target, B.pinv(X1)) aggregated_y1 = self.out_activation((B.dot(self._esn1._WOut, X1)).T) training_error1 = B.sqrt( B.mean((aggregated_y1.reshape( (total_length, self._n_output1)) - outputData1)**2)) training_error2 = 0 training_error1 = B.sqrt(B.mean((aggregated_y1 - outputData1)**2)) training_error2 = B.sqrt(B.mean((aggregated_y2 - outputData2)**2)) return training_error1, training_error2
def calculateLinearNetworkTransmissions(self, u, x=None): if x is None: x = self._x return B.dot(self._WInput, B.vstack((B.array(self._bias), u))) + B.dot(self._W, x)
def propagate(self, inputData, outputData=None, transientTime=0, verbose=0, x=None, steps="auto", feedbackData=None): if x is None: x = self._x inputLength = steps if inputData is None: if outputData is not None: inputLength = len(outputData) else: inputLength = len(inputData) if inputLength == "auto": raise ValueError("inputData and outputData are both None. Therefore, steps must not be `auto`.") # define states' matrix X = B.zeros((1 + self.n_input + self.n_reservoir, inputLength - transientTime)) if (verbose > 0): bar = progressbar.ProgressBar(max_value=inputLength, redirect_stdout=True, poll_interval=0.0001) bar.update(0) if self._WFeedback is None: #do not distinguish between whether inputData is None or not, as the feedback has been disabled #therefore, the input has to be anything but None for t in range(inputLength): u, x = self.update(inputData[t], x=x) if (t >= transientTime): #add valueset to the states' matrix X[:,t-transientTime] = B.vstack((B.array(self._outputBias), self._outputInputScaling*u, x))[:,0] Y = B.dot(self._WOut, X) if (verbose > 0): bar.update(t) else: if outputData is None: Y = B.empty((inputLength-transientTime, self.n_output)) if feedbackData is None: feedbackData = B.zeros((1, self.n_output)) if inputData is None: for t in range(inputLength): self.update(None, feedbackData, x=x) if (t >= transientTime): #add valueset to the states' matrix X[:,t-transientTime] = B.vstack((B.array(self._outputBias), x))[:,0] if outputData is None: #calculate the prediction using the trained model if (self._solver in ["sklearn_auto", "sklearn_lsqr", "sklearn_sag", "sklearn_svd"]): feedbackData = self._ridgeSolver.predict(B.vstack((B.array(self._outputBias), self._x)).T) else: feedbackData = B.dot(self._WOut, B.vstack((B.array(self._outputBias), self._x))) if t >= transientTime: Y[t-transientTime, :] = feedbackData else: feedbackData = outputData[t] if (verbose > 0): bar.update(t) else: for t in range(inputLength): u = self.update(inputData[t], feedbackData, x=x) if (t >= transientTime): #add valueset to the states' matrix X[:,t-transientTime] = B.vstack((B.array(self._outputBias), self._outputInputScaling*u, x))[:,0] if outputData is None: #calculate the prediction using the trained model if (self._solver in ["sklearn_auto", "sklearn_lsqr", "sklearn_sag", "sklearn_svd"]): feedbackData = self._ridgeSolver.predict(B.vstack((B.array(self._outputBias), self._outputInputScaling*u, self._x)).T) else: feedbackData = B.dot(self._WOut, B.vstack((B.array(self._outputBias), self._outputInputScaling*u, self._x))) Y[t, :] = feedbackData.ravel() else: feedbackData = outputData[t] if (verbose > 0): bar.update(t) if (verbose > 0): bar.finish() return X, Y
def _createReservoir(self, weightGeneration, feedback=False, verbose=False): # naive generation of the matrix W by using random weights if weightGeneration == "naive": # random weight matrix from -0.5 to 0.5 self._W = B.array(B.rand(self.n_reservoir, self.n_reservoir) - 0.5) # set sparseness% to zero mask = B.rand(self.n_reservoir, self.n_reservoir) > self._reservoirDensity self._W[mask] = 0.0 _W_eigenvalues = B.abs(B.eigenval(self._W)[0]) self._W *= self._spectralRadius / B.max(_W_eigenvalues) # generation using the SORM technique (see http://ftp.math.uni-rostock.de/pub/preprint/2012/pre12_01.pdf) elif weightGeneration == "SORM": self._W = B.identity(self.n_reservoir) number_nonzero_elements = (self._reservoirDensity * self.n_reservoir * self.n_reservoir) i = 0 while B.count_nonzero(self._W) < number_nonzero_elements: i += 1 Q = self.create_random_rotation_matrix() self._W = Q.dot(self._W) self._W *= self._spectralRadius # generation using the proposed method of Yildiz elif weightGeneration == "advanced": # two create W we must follow some steps: # at first, create a W = |W| # make it sparse # then scale its spectral radius to rho(W) = 1 (according to Yildiz with x(n+1) = (1-a)*x(n)+a*f(...)) # then change randomly the signs of the matrix # random weight matrix from 0 to 0.5 self._W = B.array(B.rand(self.n_reservoir, self.n_reservoir) / 2) # set sparseness% to zero mask = B.rand(self.n_reservoir, self.n_reservoir) > self._reservoirDensity self._W[mask] = 0.0 _W_eigenvalue = B.max(B.abs(B.eigvals(self._W))) self._W *= self._spectralRadius / _W_eigenvalue if verbose: M = self._leakingRate * self._W + ( 1 - self._leakingRate) * B.identity(n=self._W.shape[0]) M_eigenvalue = B.max(B.abs(B.eigenval(M)[0])) print("eff. spectral radius: {0}".format(M_eigenvalue)) # change random signs random_signs = B.power(-1, B.randint(1, 3, (self.n_reservoir, ))) self._W = B.multiply(self._W, random_signs) elif weightGeneration == "custom": pass else: raise ValueError( "The weightGeneration property must be one of the following values: naive, advanced, SORM, custom" ) # check of the user is really using one of the internal methods, or wants to create W by his own if weightGeneration != "custom": self._createInputMatrix() # create the optional feedback matrix if feedback: self._WFeedback = B.rand(self.n_reservoir, 1 + self.n_output) - 0.5 self._WFeedback *= self._feedbackScaling else: self._WFeedback = None
def __init__( self, n_input, n_reservoir, n_output, spectralRadius=1.0, noiseLevel=0.01, inputScaling=None, leakingRate=1.0, feedbackScaling=1.0, reservoirDensity=0.2, randomSeed=None, out_activation=lambda x: x, out_inverse_activation=lambda x: x, weightGeneration="naive", bias=1.0, outputBias=1.0, outputInputScaling=1.0, feedback=False, inputDensity=1.0, activation=B.tanh, activationDerivative=lambda x: 1.0 / B.cosh(x)**2, ): """Implementation of a ESN. Args: n_input : Dimensionality of the input. n_reservoir : Number of units in the reservoir. n_output : Dimensionality of the output. spectralRadius : Spectral radius of the reservoir's connection/weight matrix. noiseLevel : Magnitude of noise that is added to the input while fitting to prevent overfitting. inputScaling : Scaling factor of the input. leakingRate : Convex combination factor between 0 and 1 that weights current and new state value. feedbackScaling : Rescaling factor of the output-to-input feedback in the update process. reservoirDensity : Percentage of non-zero weight connections in the reservoir. randomSeed : Seed for random processes, e.g. weight initialization. out_activation : Final activation function (i.e. activation function of the output). out_inverse_activation : Inverse of the final activation function weightGeneration : Algorithm to generate weight matrices. Choices: naive, SORM, advanced, custom bias : Size of the bias added for the internal update process. outputBias : Size of the bias added for the final linear regression of the output. outputInputScaling : Rescaling factor for the input of the ESN for the regression. feedback : Include output-input feedback in the ESN. inputDensity : Percentage of non-zero weights in the input-to-reservoir weight matrix. activation : (Non-linear) Activation function. activationDerivative : Derivative of the activation function. """ self.n_input = n_input self.n_reservoir = n_reservoir self.n_output = n_output self._spectralRadius = spectralRadius self._noiseLevel = noiseLevel self._reservoirDensity = reservoirDensity self._leakingRate = leakingRate self._feedbackScaling = feedbackScaling self.inputDensity = inputDensity self._activation = activation self._activationDerivative = activationDerivative self._inputScaling = inputScaling if self._inputScaling is None: self._inputScaling = 1.0 if np.isscalar(self._inputScaling): self._inputScaling = B.ones(n_input) * self._inputScaling else: if len(self._inputScaling) != self.n_input: raise ValueError( "Dimension of inputScaling ({0}) does not match the input data dimension ({1})" .format(len(self._inputScaling), n_input)) self._inputScaling = inputScaling self._expandedInputScaling = B.vstack( (B.array(1.0), self._inputScaling.reshape(-1, 1))).flatten() self.out_activation = out_activation self.out_inverse_activation = out_inverse_activation if randomSeed is not None: B.seed(randomSeed) np.random.seed(randomSeed) self._bias = bias self._outputBias = outputBias self._outputInputScaling = outputInputScaling self._createReservoir(weightGeneration, feedback)
def generate(self, n, inputData=None, initialOutputData=None, continuation=True, initialData=None, update_processor=lambda x: x, verbose=0): #initialOutputData is the output of the last step BEFORE the generation shall start, e.g. the last step of the training's output #check the input data #if (self.n_input != self.n_output): # raise ValueError("n_input does not equal n_output. The generation mode uses its own output as its input. Therefore, n_input has to be equal to n_output - please adjust these numbers!") if inputData is not None: inputData = B.array(inputData) if initialOutputData is not None: initialOutputData = B.array(initialOutputData) if initialData is not None: initialData = B.array(initialData) if initialOutputData is None and initialData is None: raise ValueError( "Either intitialOutputData or initialData must be different from None, as the network needs an initial output value" ) if initialOutputData is None and initialData is not None: initialOutputData = initialData[1][-1] if inputData is not None: inputData = B.array(inputData) if initialData is not None: initialData = B.array(initialData) #let some input run through the ESN to initialize its states from a new starting value if not continuation: self._x = B.zeros(self._x.shape) if initialData is not None: if type(initialData) is tuple: initialDataInput, initialDataOutput = initialData if initialDataInput is not None and len( initialDataInput) != len(initialDataOutput): raise ValueError( "Length of the inputData and the outputData of the initialData tuple do not match." ) else: raise ValueError( "initialData has to be a tuple consisting out of the input and the output data." ) for t in range(initialDataInput.shape[0]): super(PredictionESN, self).update(initialDataInput[t], initialDataOutput[t]) if self.n_input != 0: if inputData is None: raise ValueError("inputData must not be None.") elif len(inputData) < n: raise ValueError("Length of inputData has to be >= n.") _, Y = self.propagate(inputData, None, verbose=verbose, steps=n, previousOutputData=initialOutputData) Y = update_processor(Y) #return the result return Y.T
def fit(self, inputData, outputData, transientTime="AutoReduce", transientTimeCalculationEpsilon=1e-3, transientTimeCalculationLength=20, verbose=0): #check the input data if self.n_input != 0: if len(inputData.shape) == 3 and len(outputData.shape) > 1: #multiple time series are used with a shape (timeseries, time, dimension) -> (timeseries, time, dimension) if inputData.shape[0] != outputData.shape[0]: raise ValueError( "Amount of input and output datasets is not equal - {0} != {1}" .format(inputData.shape[0], outputData.shape[0])) if inputData.shape[1] != outputData.shape[1]: raise ValueError( "Amount of input and output time steps is not equal - {0} != {1}" .format(inputData.shape[1], outputData.shape[1])) else: if inputData.shape[0] != outputData.shape[0]: raise ValueError( "Amount of input and output time steps is not equal - {0} != {1}" .format(inputData.shape[0], outputData.shape[0])) else: if inputData is not None: raise ValueError( "n_input has been set to zero. Therefore, the given inputData will not be used." ) if inputData is not None: inputData = B.array(inputData) if outputData is not None: outputData = B.array(outputData) #reshape the input/output data to have the shape (timeseries, time, dimension) if len(outputData.shape) <= 2: outputData = outputData.reshape((1, -1, self.n_output)) if inputData is not None: if len(inputData.shape) <= 2: inputData = inputData.reshape((1, -1, self.n_input)) self.resetState() # Automatic transient time calculations if transientTime == "Auto": transientTime = self.calculateTransientTime( inputData[0], outputData[0], transientTimeCalculationEpsilon, transientTimeCalculationLength) if transientTime == "AutoReduce": if (inputData is None and outputData.shape[2] == 1) or inputData.shape[2] == 1: transientTime = self.calculateTransientTime( inputData[0], outputData[0], transientTimeCalculationEpsilon, transientTimeCalculationLength) transientTime = self.reduceTransientTime( inputData[0], outputData[0], transientTime) else: print( "Transient time reduction is supported only for 1 dimensional input." ) if inputData is not None: partialLength = (inputData.shape[1] - transientTime) totalLength = inputData.shape[0] * partialLength timeseriesCount = inputData.shape[0] elif outputData is not None: partialLength = (outputData.shape[1] - transientTime) totalLength = outputData.shape[0] * partialLength timeseriesCount = outputData.shape[0] else: raise ValueError("Either input or output data must not to be None") self._X = B.empty((1 + self.n_input + self.n_reservoir, totalLength)) if (verbose > 0): bar = progressbar.ProgressBar(max_value=totalLength, redirect_stdout=True, poll_interval=0.0001) bar.update(0) for i in range(timeseriesCount): if inputData is not None: xx, yy = self.propagate(inputData[i], outputData[i], transientTime, verbose - 1) self._X[:, i * partialLength:(i + 1) * partialLength] = xx else: xx, yy = self.propagate(None, outputData[i], transientTime, verbose - 1) self._X[:, i * partialLength:(i + 1) * partialLength] = xx if (verbose > 0): bar.update(i) if (verbose > 0): bar.finish() #define the target values Y_target = B.empty((outputData.shape[2], totalLength)) for i in range(timeseriesCount): Y_target[:, i * partialLength:(i + 1) * partialLength] = self.out_inverse_activation( outputData[i]).T[:, transientTime:] if (self._solver == "pinv"): self._WOut = B.dot(Y_target, B.pinv(self._X)) #calculate the training prediction now train_prediction = self.out_activation((B.dot(self._WOut, self._X)).T) # elif (self._solver == "lsqr"): # X_T = self._X.T # self._WOut = B.dot(B.dot(Y_target, X_T),B.inv(B.dot(self._X,X_T) + self._regressionParameters[0]*B.identity(1+self.n_input+self.n_reservoir))) # """ # #alternative representation of the equation # Xt = X.T # A = np.dot(X, Y_target.T) # B = np.linalg.inv(np.dot(X, Xt) + regression_parameter*np.identity(1+self.n_input+self.n_reservoir)) # self._WOut = np.dot(B, A) # self._WOut = self._WOut.T # """ # #calculate the training prediction now # train_prediction = self.out_activation(B.dot(self._WOut, self._X).T) elif (self._solver in [ "sklearn_auto", "sklearn_lsqr", "sklearn_sag", "sklearn_svd" ]): mode = self._solver[8:] params = self._regressionParameters params["solver"] = mode self._ridgeSolver = Ridge(**params) self._ridgeSolver.fit(self._X.T, Y_target.T) #calculate the training prediction now train_prediction = self.out_activation( self._ridgeSolver.predict(self._X.T)) elif (self._solver in ["sklearn_svr", "sklearn_svc"]): self._ridgeSolver = SVR(**self._regressionParameters) self._ridgeSolver.fit(self._X.T, Y_target.T.ravel()) #calculate the training prediction now train_prediction = self.out_activation( self._ridgeSolver.predict(self._X.T)) #calculate the training error now #flatten the outputData outputData = outputData[:, transientTime:, :].reshape(totalLength, -1) training_error = B.sqrt(B.mean((train_prediction - outputData)**2)) return training_error
def _createReservoir(self, weightGeneration, feedback=False, verbose=False): #naive generation of the matrix W by using random weights if weightGeneration == 'naive': #random weight matrix from -0.5 to 0.5 self._W = B.array(rnd.rand(self.n_reservoir, self.n_reservoir) - 0.5) #set sparseness% to zero mask = rnd.rand(self.n_reservoir, self.n_reservoir) > self._reservoirDensity self._W[mask] = 0.0 _W_eigenvalues = B.abs(B.eigenval(self._W)[0]) self._W *= self._spectralRadius / B.max(_W_eigenvalues) #generation using the SORM technique (see http://ftp.math.uni-rostock.de/pub/preprint/2012/pre12_01.pdf) elif weightGeneration == "SORM": self._W = B.identity(self.n_reservoir) number_nonzero_elements = self._reservoirDensity * self.n_reservoir * self.n_reservoir i = 0 while np.count_nonzero(self._W) < number_nonzero_elements: i += 1 Q = self.create_random_rotation_matrix() self._W = Q.dot(self._W) self._W *= self._spectralRadius #generation using the proposed method of Yildiz elif weightGeneration == 'advanced': #two create W we must follow some steps: #at first, create a W = |W| #make it sparse #then scale its spectral radius to rho(W) = 1 (according to Yildiz with x(n+1) = (1-a)*x(n)+a*f(...)) #then change randomly the signs of the matrix #random weight matrix from 0 to 0.5 self._W = B.array(rnd.rand(self.n_reservoir, self.n_reservoir) / 2) #set sparseness% to zero mask = B.rand(self.n_reservoir, self.n_reservoir) > self._reservoirDensity self._W[mask] = 0.0 from scipy.sparse.linalg.eigen.arpack.arpack import ArpackNoConvergence #just calculate the largest EV - hopefully this is the right code to do so... try: #this is just a good approximation, so this code might fail _W_eigenvalue = B.max(np.abs(sp.sparse.linalg.eigs(self._W, k=1)[0])) except ArpackNoConvergence: #this is the safe fall back method to calculate the EV _W_eigenvalue = B.max(B.abs(sp.linalg.eigvals(self._W))) #_W_eigenvalue = B.max(B.abs(np.linalg.eig(self._W)[0])) self._W *= self._spectralRadius / _W_eigenvalue if verbose: M = self._leakingRate*self._W + (1 - self._leakingRate)*np.identity(n=self._W.shape[0]) M_eigenvalue = B.max(B.abs(B.eigenval(M)[0]))#np.max(np.abs(sp.sparse.linalg.eigs(M, k=1)[0])) print("eff. spectral radius: {0}".format(M_eigenvalue)) #change random signs random_signs = B.power(-1, rnd.random_integers(self.n_reservoir, self.n_reservoir)) self._W = B.multiply(self._W, random_signs) elif weightGeneration == 'custom': pass else: raise ValueError("The weightGeneration property must be one of the following values: naive, advanced, SORM, custom") #check of the user is really using one of the internal methods, or wants to create W by his own if (weightGeneration != 'custom'): self._createInputMatrix() #create the optional feedback matrix if feedback: self._WFeedback = B.rand(self.n_reservoir, 1 + self.n_output) - 0.5 self._WFeedback *= self._feedbackScaling else: self._WFeedback = None