def __init__(self, n_input, n_reservoir, n_output, spectralRadius=1.0, noiseLevel=0.0, inputScaling=None, leakingRate=1.0, feedbackScaling=1.0, reservoirDensity=0.2, randomSeed=None, out_activation=lambda x: x, out_inverse_activation=lambda x: x, weightGeneration='naive', bias=1.0, outputBias=1.0, feedback=False, outputInputScaling=1.0, inputDensity=1.0, solver='pinv', regressionParameters={}, activation=B.tanh, activationDerivation=lambda x: 1.0 / B.cosh(x)**2): super(PredictionESN, self).__init__(n_input=n_input, n_reservoir=n_reservoir, n_output=n_output, spectralRadius=spectralRadius, noiseLevel=noiseLevel, inputScaling=inputScaling, leakingRate=leakingRate, feedbackScaling=feedbackScaling, reservoirDensity=reservoirDensity, randomSeed=randomSeed, feedback=feedback, out_activation=out_activation, out_inverse_activation=out_inverse_activation, weightGeneration=weightGeneration, bias=bias, outputBias=outputBias, outputInputScaling=outputInputScaling, inputDensity=inputDensity, activation=activation, activationDerivation=activationDerivation) self._solver = solver self._regressionParameters = regressionParameters self._x = B.zeros((self.n_reservoir, 1)) #self._WOut = np.random.rand(n_output, n_reservoir+1)#bias term self._WOut = np.random.rand(n_output, 1 + n_input + n_reservoir) #bias term #this seems to have been defined somewhere already self._X = B.zeros((1 + n_input + n_reservoir, 1)) """
def predict_loop(self, inputData, outputData1, outputData2, continuation=True, initialData=None, update_processor=lambda x: x, verbose=0): inputData = B.array(inputData) #let some input run through the ESN to initialize its states from a new starting value if (not continuation): self._esn1._x = B.zeros(self._esn1._x.shape) self._esn2._x = B.zeros(self._esn2._x.shape) total_length = inputData.shape[0] aggregated_y1 = B.empty((total_length, self._n_output1)) aggregated_y2 = B.empty((total_length, self._n_output2)) # X1, y1 = self._esn1.propagate(inputData=inputData, outputData=None, transientTime=self._transientTime, verbose=verbose-1) # Y1 = B.dot(self._esn1._WOut, X1) # Y1 = update_processor(self.out_activation(Y1)).T y2 = self.out_activation(B.dot(self._esn2._WOut, self._esn2._X).T) for i in range(total_length): inputDataWithFeedback = B.zeros((self.n_input + self._n_output2)) inputDataWithFeedback[:self.n_input] = inputData[i, :] inputDataWithFeedback[self.n_input:] = y2 #update models X1, _ = self._esn1.propagateOneStep( inputData=inputDataWithFeedback, outputData=None, step=i, transientTime=self._transientTime, verbose=verbose - 1, learn=False) y1 = B.dot(self._esn1._WOut, X1) aggregated_y1[i, :] = update_processor(self.out_activation(y1)).T #output from 1st layer and correct output #input2 = outputData1[i] - y1.reshape(self._n_output1) # input2 = B.vstack((y1.reshape(self._n_output1), outputData1[i])) # x2, y2 = self._esn2.propagateOneStep(inputData=input2, outputData=None, step=i, transientTime=self._transientTime, verbose=verbose-1, learn=False) # Y_target2[i,:] = y2.reshape(self._n_output2) training_error1 = B.sqrt(B.mean((aggregated_y1 - outputData1)**2)) print("training errors") print(training_error1) return aggregated_y1, aggregated_y2
def predict( self, inputData, update_processor=lambda x: x, transientTime=0, verbose=0 ): if len(inputData.shape) == 1: inputData = inputData[None, :] predictionLength = inputData.shape[1] Y = B.empty((inputData.shape[0], self.n_output)) if verbose > 0: bar = progressbar.ProgressBar( max_value=inputData.shape[0], redirect_stdout=True, poll_interval=0.0001 ) bar.update(0) for n in range(inputData.shape[0]): # reset the state self._x = B.zeros(self._x.shape) X = self.propagate(inputData[n], transientTime) # calculate the prediction using the trained model if self._solver in [ "sklearn_auto", "sklearn_lsqr", "sklearn_sag", "sklearn_svd", "sklearn_svr", ]: y = self._ridgeSolver.predict(X.T).reshape((self.n_output, -1)) else: y = B.dot(self._W_out, X) Y[n] = np.mean(y, 1) if verbose > 0: bar.update(n) if verbose > 0: bar.finish() # return the result result = B.zeros(Y.shape) for i, n in enumerate(B.argmax(Y, 1)): result[i, n] = 1.0 return result
def SWD(series, intervall): differences = B.zeros(series.shape[0] - 2 * intervall) reference_series = series[:intervall] for i in range(intervall, series.shape[0] - intervall): differences[i - intervall] = B.sum( B.abs(reference_series - series[i:i + intervall])) return B.argmin(differences) + intervall, differences
def predict( self, inputData, continuation=True, initialData=None, update_processor=lambda x: x, verbose=0, ): inputData = B.array(inputData) # let some input run through the ESN to initialize its states from a new starting value if not continuation: self._x = B.zeros(self._x.shape) if initialData is not None: if self._WFeedback is None: for t in range(initialData.shape[0]): super(PredictionESN, self).update(initialData[t]) else: if type(initialData) is tuple: initialDataInput, initialDataOutput = initialData if len(initialDataInput) != len(initialDataOutput): raise ValueError( "Length of the inputData and the outputData of the initialData tuple do not match." ) else: raise ValueError( "initialData has to be a tuple consisting out of the input and the output data." ) super(PredictionESN, self).update(initialDataInput[t], initialDataOutput[t]) X = self.propagate(inputData, verbose=verbose) if self._WFeedback is not None: X, _ = X # calculate the prediction using the trained model if self._solver in [ "sklearn_auto", "sklearn_lsqr", "sklearn_sag", "sklearn_svd", "sklearn_svr", ]: Y = self._ridgeSolver.predict(X.T).reshape((self.n_output, -1)) else: Y = B.dot(self._WOut, X) # apply the output activation function Y = update_processor(self.out_activation(Y)) # return the result return Y.T
def predict(self, inputData, outputData1, outputData2, continuation=True, initialData=None, update_processor=lambda x: x, verbose=0): inputData = B.array(inputData) #let some input run through the ESN to initialize its states from a new starting value if (not continuation): self._esn1._x = B.zeros(self._esn1._x.shape) self._esn2._x = B.zeros(self._esn2._x.shape) total_length = inputData.shape[0] aggregated_y1 = B.empty((total_length, self._n_output1)) aggregated_y2 = B.empty((total_length, self._n_output2)) #put pixels and switch data together inputDataWithFeedback = B.zeros( (total_length, self.n_input + self._n_output2)) inputDataWithFeedback[:, :self.n_input] = inputData inputDataWithFeedback[:, self.n_input:] = outputData2 X1, _ = self._esn1.propagate(inputData=inputDataWithFeedback, outputData=None, transientTime=self._transientTime, verbose=verbose - 1) aggregated_y1 = B.dot(self._esn1._WOut, X1) aggregated_y1 = update_processor(self.out_activation(aggregated_y1)).T training_error1 = B.sqrt(B.mean((aggregated_y1 - outputData1)**2)) print("predict errors") print(training_error1) return aggregated_y1, aggregated_y2
def getStateAtGivenPoint(inputs, outputs, targetTime): # inputs: input of reserovoir # outputs: output of reservoir # targetTime: time at which the state is wanted # propagates the inputs/outputs till given point in time and returns the state of the reservoir at this point x = B.zeros((self.n_reservoir, 1)) length = inputs.shape[0] if inputs is not None else outputs.shape[0] length = min(length, targetTime) for t in range(length): u = inputs[t].reshape(-1, 1) if inputs is not None else None o = outputs[t].reshape(-1, 1) if outputs is not None else None self.update(u, o, x) return x
def calculate_esn_mi_input_scaling(input_data, output_data): if len(input_data) != len(output_data): raise ValueError( "input_data and output_data do not have the same length - {0} vs. {1}" .format(len(input_data), len(output_data))) # Scott's rule to calculate nbins std_output = B.std(output_data) nbins = int( B.ceil(2.0 / (3.5 * std_output / B.power(len(input_data), 1.0 / 3.0)))) mi = B.zeros(input_data.shape[1]) for i in range(len(mi)): mi[i] = calculate_mutualinformation(input_data[:, i], output_data, nbins) scaling = mi / B.max(mi) return scaling
def __init__(self, inputShape, n_reservoir, filterSize=1, stride=1, borderMode="mirror", nWorkers="auto", spectralRadius=1.0, noiseLevel=0.0, inputScaling=None, leakingRate=1.0, reservoirDensity=0.2, randomSeed=None, averageOutputWeights=True, out_activation=lambda x: x, out_inverse_activation=lambda x: x, weightGeneration='naive', bias=1.0, outputBias=1.0, outputInputScaling=1.0, inputDensity=1.0, solver='pinv', regressionParameters={}, activation=B.tanh, activationDerivation=lambda x: 1.0 / B.cosh(x) ** 2): self._averageOutputWeights = averageOutputWeights if averageOutputWeights and solver != "lsqr": raise ValueError( "`averageOutputWeights` can only be set to `True` when `solver` is set to `lsqr` (Ridge Regression)") self._borderMode = borderMode if not borderMode in ["mirror", "padding", "edge", "wrap"]: raise ValueError( "`borderMode` must be set to one of the following values: `mirror`, `padding`, `edge` or `wrap`.") self._regressionParameters = regressionParameters self._solver = solver n_inputDimensions = len(inputShape) if filterSize % 2 == 0: raise ValueError("filterSize has to be an odd number (1, 3, 5, ...).") self._filterSize = filterSize self._filterWidth = int(np.floor(filterSize / 2)) self._stride = stride self._n_input = int(np.power(np.ceil(filterSize / stride), n_inputDimensions)) self.n_inputDimensions = n_inputDimensions self.inputShape = inputShape if not self._averageOutputWeights: self._WOuts = B.empty((np.prod(inputShape), 1, self._n_input + n_reservoir + 1)) self._WOut = None else: self._WOuts = None self._WOut = B.zeros((1, self._n_input + n_reservoir + 1)) self._xs = B.empty((np.prod(inputShape), n_reservoir, 1)) if nWorkers == "auto": self._nWorkers = np.max((cpu_count() - 1, 1)) else: self._nWorkers = nWorkers manager = Manager() self.sharedNamespace = manager.Namespace() if hasattr(self, "fitWorkerID") == False or self.parallelWorkerIDs is None: self.parallelWorkerIDs = manager.Queue() for i in range(self._nWorkers): self.parallelWorkerIDs.put((i)) super(SpatioTemporalESN, self).__init__(n_input=self._n_input, n_reservoir=n_reservoir, n_output=1, spectralRadius=spectralRadius, noiseLevel=noiseLevel, inputScaling=inputScaling, leakingRate=leakingRate, reservoirDensity=reservoirDensity, randomSeed=randomSeed, out_activation=out_activation, out_inverse_activation=out_inverse_activation, weightGeneration=weightGeneration, bias=bias, outputBias=outputBias, outputInputScaling=outputInputScaling, inputDensity=inputDensity, activation=activation, activationDerivation=activationDerivation) """
def generate(self, n, inputData=None, initialOutputData=None, continuation=True, initialData=None, update_processor=lambda x: x, verbose=0): #initialOutputData is the output of the last step BEFORE the generation shall start, e.g. the last step of the training's output #check the input data #if (self.n_input != self.n_output): # raise ValueError("n_input does not equal n_output. The generation mode uses its own output as its input. Therefore, n_input has to be equal to n_output - please adjust these numbers!") if inputData is not None: inputData = B.array(inputData) if initialOutputData is not None: initialOutputData = B.array(initialOutputData) if initialData is not None: initialData = B.array(initialData) if initialOutputData is None and initialData is None: raise ValueError( "Either intitialOutputData or initialData must be different from None, as the network needs an initial output value" ) if initialOutputData is None and initialData is not None: initialOutputData = initialData[1][-1] if inputData is not None: inputData = B.array(inputData) if initialData is not None: initialData = B.array(initialData) #let some input run through the ESN to initialize its states from a new starting value if not continuation: self._x = B.zeros(self._x.shape) if initialData is not None: if type(initialData) is tuple: initialDataInput, initialDataOutput = initialData if initialDataInput is not None and len( initialDataInput) != len(initialDataOutput): raise ValueError( "Length of the inputData and the outputData of the initialData tuple do not match." ) else: raise ValueError( "initialData has to be a tuple consisting out of the input and the output data." ) for t in range(initialDataInput.shape[0]): super(PredictionESN, self).update(initialDataInput[t], initialDataOutput[t]) if self.n_input != 0: if inputData is None: raise ValueError("inputData must not be None.") elif len(inputData) < n: raise ValueError("Length of inputData has to be >= n.") _, Y = self.propagate(inputData, None, verbose=verbose, steps=n, previousOutputData=initialOutputData) Y = update_processor(Y) #return the result return Y.T
def fit( self, inputData, outputData, transientTime="AutoReduce", transientTimeCalculationEpsilon=1e-3, transientTimeCalculationLength=20, verbose=0, ): # check the input data if inputData.shape[0] != outputData.shape[0]: raise ValueError( "Amount of input and output datasets is not equal - {0} != {1}" .format(inputData.shape[0], outputData.shape[0])) nSequences = inputData.shape[0] trainingLength = inputData.shape[1] self._x = B.zeros((self.n_reservoir, 1)) # Automatic transient time calculations if transientTime == "Auto": transientTime = self.calculateTransientTime( inputData, outputData, transientTimeCalculationEpsilon, transientTimeCalculationLength, ) if transientTime == "AutoReduce": if (inputData is None and outputData.shape[1] == 1) or inputData.shape[1] == 1: transientTime = self.calculateTransientTime( inputData, outputData, transientTimeCalculationEpsilon, transientTimeCalculationLength, ) transientTime = self.reduceTransientTime( inputData, outputData, transientTime) else: print( "Transient time reduction is supported only for 1 dimensional input." ) self._X = B.zeros(( 1 + self.n_input + self.n_reservoir, nSequences * (trainingLength - transientTime), )) Y_target = B.zeros( (self.n_output, (trainingLength - transientTime) * nSequences)) if verbose > 0: bar = progressbar.ProgressBar(max_value=len(inputData), redirect_stdout=True, poll_interval=0.0001) bar.update(0) for n in range(len(inputData)): self._x = B.zeros((self.n_reservoir, 1)) self._X[:, n * (trainingLength - transientTime):(n + 1) * (trainingLength - transientTime), ] = self.propagate( inputData[n], transientTime=transientTime, verbose=0) # set the target values Y_target[:, n * (trainingLength - transientTime):(n + 1) * (trainingLength - transientTime), ] = np.tile( self.out_inverse_activation(outputData[n]), trainingLength - transientTime, ).T if verbose > 0: bar.update(n) if verbose > 0: bar.finish() if self._solver == "pinv": self._WOut = B.dot(Y_target, B.pinv(self._X)) # calculate the training prediction now train_prediction = self.out_activation((B.dot(self._WOut, self._X)).T) elif self._solver == "lsqr": X_T = self._X.T self._WOut = B.dot( B.dot(Y_target, X_T), B.inv( B.dot(self._X, X_T) + self._regressionParameters[0] * B.identity(1 + self.n_input + self.n_reservoir)), ) """ #alternative represantation of the equation Xt = X.T A = np.dot(X, Y_target.T) B = np.linalg.inv(np.dot(X, Xt) + regression_parameter*np.identity(1+self.n_input+self.n_reservoir)) self._WOut = np.dot(B, A) self._WOut = self._WOut.T """ # calculate the training prediction now train_prediction = self.out_activation( B.dot(self._WOut, self._X).T) elif self._solver in [ "sklearn_auto", "sklearn_lsqr", "sklearn_sag", "sklearn_svd", ]: mode = self._solver[8:] params = self._regressionParameters params["solver"] = mode self._ridgeSolver = Ridge(**params) self._ridgeSolver.fit(self._X.T, Y_target.T) # calculate the training prediction now train_prediction = self.out_activation( self._ridgeSolver.predict(self._X.T)) elif self._solver in ["sklearn_svr", "sklearn_svc"]: self._ridgeSolver = SVR(**self._regressionParameters) self._ridgeSolver.fit(self._X.T, Y_target.T.flatten()) # calculate the training prediction now train_prediction = self.out_activation( self._ridgeSolver.predict(self._X.T)) train_prediction = np.mean(train_prediction, 0) # calculate the training error now training_error = B.sqrt(B.mean((train_prediction - outputData.T)**2)) return training_error
def fit_loop(self, inputData, outputData1, outputData2, transientTime="AutoReduce", transientTimeCalculationEpsilon=1e-3, transientTimeCalculationLength=20, verbose=0): #check the input data if self.n_input != 0: if len(inputData.shape) == 3 and len(outputData1.shape) > 1: #multiple time series are used with a shape (timeseries, time, dimension) -> (timeseries, time, dimension) if inputData.shape[0] != outputData1.shape[0]: raise ValueError( "Amount of input and output datasets is not equal - {0} != {1}" .format(inputData.shape[0], outputData1.shape[0])) if inputData.shape[1] != outputData1.shape[1]: raise ValueError( "Amount of input and output time steps is not equal - {0} != {1}" .format(inputData.shape[1], outputData1.shape[1])) else: if inputData.shape[0] != outputData1.shape[0]: raise ValueError( "Amount of input and output time steps is not equal - {0} != {1}" .format(inputData.shape[0], outputData1.shape[0])) else: if inputData is not None: raise ValueError( "n_input has been set to zero. Therefore, the given inputData will not be used." ) inputData = B.array(inputData) outputData1 = B.array(outputData1) outputData2 = B.array(outputData2) self._transientTime = transientTime self._esn1.resetState() self._esn2.resetState() total_length = inputData.shape[0] print("total_length ", total_length) if (verbose > 0): bar = progressbar.ProgressBar(max_value=total_length, redirect_stdout=True, poll_interval=0.0001) bar.update(0) #should be named aggregated_y aggregated_y1 = B.empty((outputData1.shape)) aggregated_y2 = B.empty((outputData2.shape)) # X1, _ = self._esn1.propagate(inputData=inputData, outputData=outputData1, transientTime=transientTime, verbose=verbose-1) # self._esn1._X = X1 # Y_target = self.out_inverse_activation(outputData1).T # self._esn1._WOut = B.dot(Y_target, B.pinv(X1)) # y1 = self.out_activation((B.dot(self._esn1._WOut, X1)).T) # training_error1 = B.sqrt(B.mean((y1.reshape((total_length, self._n_output1))- outputData1)**2)) # training_error2 = 0 y2 = self.out_activation(B.dot(self._esn2._WOut, self._esn2._X).T) for i in range((total_length)): #input: input + output from layer 2 inputDataWithFeedback = B.zeros((self.n_input + self._n_output2)) inputDataWithFeedback[:self.n_input] = inputData[i, :] inputDataWithFeedback[self.n_input:] = y2 #update models X1, untrained_y1 = self._esn1.propagateOneStep( inputData=inputDataWithFeedback, outputData=outputData1[i], step=i, transientTime=transientTime, verbose=verbose - 1, learn=True) #self._esn1._X = X1 #y after model update (ideally we would use y before model update) #y1 = self.out_activation((B.dot(self._esn1._WOut, X1)).T) aggregated_y1[i, :] = untrained_y1.reshape(self._n_output1) #output from 1st layer and correct output # input2 = B.vstack((y1.reshape(self._n_output1), outputData1[i])) # x2, y2 = self._esn2.propagateOneStep(inputData=input2, outputData=outputData2[i], step=i, transientTime=transientTime, verbose=verbose-1, learn=True) # Y_target2[i,:] = y2.reshape(self._n_output2) if (verbose > 0): bar.update(i) if (verbose > 0): bar.finish() self._training_res = aggregated_y1 training_error1 = B.sqrt(B.mean((aggregated_y1 - outputData1)**2)) training_error2 = B.sqrt(B.mean((aggregated_y2 - outputData2)**2)) print("training errors") print(training_error1) print(training_error2) return training_error1, training_error2
def fit(self, inputData, outputData1, outputData2, transientTime="AutoReduce", transientTimeCalculationEpsilon=1e-3, transientTimeCalculationLength=20, verbose=0): #check the input data if self.n_input != 0: if len(inputData.shape) == 3 and len(outputData1.shape) > 1: #multiple time series are used with a shape (timeseries, time, dimension) -> (timeseries, time, dimension) if inputData.shape[0] != outputData1.shape[0]: raise ValueError( "Amount of input and output datasets is not equal - {0} != {1}" .format(inputData.shape[0], outputData1.shape[0])) if inputData.shape[1] != outputData1.shape[1]: raise ValueError( "Amount of input and output time steps is not equal - {0} != {1}" .format(inputData.shape[1], outputData1.shape[1])) else: if inputData.shape[0] != outputData1.shape[0]: raise ValueError( "Amount of input and output time steps is not equal - {0} != {1}" .format(inputData.shape[0], outputData1.shape[0])) else: if inputData is not None: raise ValueError( "n_input has been set to zero. Therefore, the given inputData will not be used." ) inputData = B.array(inputData) outputData1 = B.array(outputData1) outputData2 = B.array(outputData2) self._transientTime = transientTime self._esn1.resetState() self._esn2.resetState() total_length = inputData.shape[0] print("total_length ", total_length) aggregated_y1 = B.empty((outputData1.shape)) aggregated_y2 = B.empty((outputData2.shape)) #put pixels and switch data together inputDataWithFeedback = B.zeros( (total_length, self.n_input + self._n_output2)) inputDataWithFeedback[:, :self.n_input] = inputData inputDataWithFeedback[:, self.n_input:] = outputData2 X1, _ = self._esn1.propagate(inputData=inputDataWithFeedback, outputData=outputData1, transientTime=transientTime, verbose=verbose - 1) self._esn1._X = X1 Y_target = self.out_inverse_activation(outputData1).T self._esn1._WOut = B.dot(Y_target, B.pinv(X1)) aggregated_y1 = self.out_activation((B.dot(self._esn1._WOut, X1)).T) training_error1 = B.sqrt( B.mean((aggregated_y1.reshape( (total_length, self._n_output1)) - outputData1)**2)) training_error2 = 0 training_error1 = B.sqrt(B.mean((aggregated_y1 - outputData1)**2)) training_error2 = B.sqrt(B.mean((aggregated_y2 - outputData2)**2)) return training_error1, training_error2
def propagate(self, inputData, outputData=None, transientTime=0, verbose=0, x=None, steps="auto", feedbackData=None): if x is None: x = self._x inputLength = steps if inputData is None: if outputData is not None: inputLength = len(outputData) else: inputLength = len(inputData) if inputLength == "auto": raise ValueError("inputData and outputData are both None. Therefore, steps must not be `auto`.") # define states' matrix X = B.zeros((1 + self.n_input + self.n_reservoir, inputLength - transientTime)) if (verbose > 0): bar = progressbar.ProgressBar(max_value=inputLength, redirect_stdout=True, poll_interval=0.0001) bar.update(0) if self._WFeedback is None: #do not distinguish between whether inputData is None or not, as the feedback has been disabled #therefore, the input has to be anything but None for t in range(inputLength): u, x = self.update(inputData[t], x=x) if (t >= transientTime): #add valueset to the states' matrix X[:,t-transientTime] = B.vstack((B.array(self._outputBias), self._outputInputScaling*u, x))[:,0] Y = B.dot(self._WOut, X) if (verbose > 0): bar.update(t) else: if outputData is None: Y = B.empty((inputLength-transientTime, self.n_output)) if feedbackData is None: feedbackData = B.zeros((1, self.n_output)) if inputData is None: for t in range(inputLength): self.update(None, feedbackData, x=x) if (t >= transientTime): #add valueset to the states' matrix X[:,t-transientTime] = B.vstack((B.array(self._outputBias), x))[:,0] if outputData is None: #calculate the prediction using the trained model if (self._solver in ["sklearn_auto", "sklearn_lsqr", "sklearn_sag", "sklearn_svd"]): feedbackData = self._ridgeSolver.predict(B.vstack((B.array(self._outputBias), self._x)).T) else: feedbackData = B.dot(self._WOut, B.vstack((B.array(self._outputBias), self._x))) if t >= transientTime: Y[t-transientTime, :] = feedbackData else: feedbackData = outputData[t] if (verbose > 0): bar.update(t) else: for t in range(inputLength): u = self.update(inputData[t], feedbackData, x=x) if (t >= transientTime): #add valueset to the states' matrix X[:,t-transientTime] = B.vstack((B.array(self._outputBias), self._outputInputScaling*u, x))[:,0] if outputData is None: #calculate the prediction using the trained model if (self._solver in ["sklearn_auto", "sklearn_lsqr", "sklearn_sag", "sklearn_svd"]): feedbackData = self._ridgeSolver.predict(B.vstack((B.array(self._outputBias), self._outputInputScaling*u, self._x)).T) else: feedbackData = B.dot(self._WOut, B.vstack((B.array(self._outputBias), self._outputInputScaling*u, self._x))) Y[t, :] = feedbackData.ravel() else: feedbackData = outputData[t] if (verbose > 0): bar.update(t) if (verbose > 0): bar.finish() return X, Y
def __init__( self, n_input, n_reservoir, n_output, spectralRadius=1.0, noiseLevel=0.0, inputScaling=None, leakingRate=1.0, feedbackScaling=1.0, reservoirDensity=0.2, randomSeed=None, out_activation=lambda x: x, out_inverse_activation=lambda x: x, weightGeneration="naive", bias=1.0, outputBias=1.0, feedback=False, outputInputScaling=1.0, inputDensity=1.0, solver="pinv", regressionParameters={}, activation=B.tanh, activationDerivative=lambda x: 1.0 / B.cosh(x)**2, ): """ ESN that predicts (steps of) a time series based on a time series. Args: n_input : Dimensionality of the input. n_reservoir : Number of units in the reservoir. n_output : Dimensionality of the output. spectralRadius : Spectral radius of the reservoir's connection/weight matrix. noiseLevel : Magnitude of noise that is added to the input while fitting to prevent overfitting. inputScaling : Scaling factor of the input. leakingRate : Convex combination factor between 0 and 1 that weights current and new state value. feedbackScaling : Rescaling factor of the output-to-input feedback in the update process. reservoirDensity : Percentage of non-zero weight connections in the reservoir. randomSeed : Seed for random processes, e.g. weight initialization. out_activation : Final activation function (i.e. activation function of the output). out_inverse_activation : Inverse of the final activation function weightGeneration : Algorithm to generate weight matrices. Choices: naive, SORM, advanced, custom bias : Size of the bias added for the internal update process. outputBias : Size of the bias added for the final linear regression of the output. feedback : Include output-input feedback in the ESN. outputInputScaling : Rescaling factor for the input of the ESN for the regression. inputDensity : Percentage of non-zero weights in the input-to-reservoir weight matrix. solver : Algorithm to find output matrix. Choices: pinv, lsqr. regressionParameters : Arguments to the solving algorithm. For LSQR this controls the L2 regularization. activation : (Non-linear) Activation function. activationDerivative : Derivative of the activation function. """ super(PredictionESN, self).__init__( n_input=n_input, n_reservoir=n_reservoir, n_output=n_output, spectralRadius=spectralRadius, noiseLevel=noiseLevel, inputScaling=inputScaling, leakingRate=leakingRate, feedbackScaling=feedbackScaling, reservoirDensity=reservoirDensity, randomSeed=randomSeed, feedback=feedback, out_activation=out_activation, out_inverse_activation=out_inverse_activation, weightGeneration=weightGeneration, bias=bias, outputBias=outputBias, outputInputScaling=outputInputScaling, inputDensity=inputDensity, activation=activation, activationDerivative=activationDerivative, ) self._solver = solver self._regressionParameters = regressionParameters self._x = B.zeros((self.n_reservoir, 1)) """
def resetState(self, index=None): if index is None: self._x = B.zeros((self._nWorkers, self.n_reservoir, 1)) else: self._x[index] = B.zeros((self.n_reservoir, 1))
def predict(self, inputData, transientTime=0, update_processor=lambda x: x, verbose=0): rank = len(inputData.shape) - 1 if rank != self.n_inputDimensions: raise ValueError( "The `inputData` does not have a suitable shape. It has to have {0} spatial dimensions and 1 temporal dimension.".format( self.n_inputDimensions)) manager = Manager() predictQueue = manager.Queue() # workaround as predict does not support batches atm # add dummy dimension to let embedInputData work properly (is optimized to work for batches) inputData = inputData.reshape(1, *inputData.shape) modifiedInputData = self._embedInputData(inputData) modifiedInputData = modifiedInputData[0] inputData = inputData[0] self.transientTime = transientTime self.sharedNamespace.transientTime = transientTime predictionOutput = B.zeros(np.insert(self.inputShape, 0, inputData.shape[0] - transientTime)) jobs = np.stack(np.meshgrid(*[np.arange(x) + self._filterWidth for x in inputData.shape[1:]]), axis=rank).reshape(-1, rank).tolist() nJobs = len(jobs) self.resetState() iterator = PredictionArrayIterator(modifiedInputData, jobs, self._filterWidth, self._stride, self) pool = Pool(processes=self._nWorkers, initializer=SpatioTemporalESN._init_predictProcess, initargs=[predictQueue, self]) pool.map_async(self._predictProcess, iterator, chunksize=200)#, chunksize=1) def _processPoolWorkerResults(): nJobsDone = 0 if verbose > 0: bar = progressbar.ProgressBar(max_value=nJobs, redirect_stdout=True, poll_interval=0.0001) bar.update(0) while nJobsDone < nJobs: data = predictQueue.get() # result of predicting indices, prediction, state = data id = self._uniqueIDFromIndices(indices) self._xs[id] = state # update the values predictionOutput[tuple([Ellipsis] + indices)] = prediction nJobsDone += 1 if verbose > 0: bar.update(nJobsDone) if verbose > 1: print(nJobsDone) if verbose > 0: bar.finish() _processPoolWorkerResults() pool.close() return predictionOutput
def __init__( self, inputShape, n_reservoir, filterSize=1, stride=1, borderMode="mirror", nWorkers="auto", spectralRadius=1.0, noiseLevel=0.0, inputScaling=None, leakingRate=1.0, reservoirDensity=0.2, randomSeed=None, averageOutputWeights=True, out_activation=lambda x: x, out_inverse_activation=lambda x: x, weightGeneration="naive", bias=1.0, outputBias=1.0, outputInputScaling=1.0, inputDensity=1.0, solver="pinv", regressionParameters={}, activation=B.tanh, activationDerivative=lambda x: 1.0 / B.cosh(x)**2, chunkSize=16, ): """ ESN that predicts (steps of) a spatio-temporal time series based on a time series. Args: inputShape : Shape of the input w/o the time axis, e.g. (W, H) for a 2D input. n_reservoir : Number of units in the reservoir. filterSize : Size of patches used to predict a single output element. stride : Stride between different patches. borderMode : How to handle border values. Choices: mirror, padding, edge, wrap. nWorkers : Number of CPU threads executed in parallel to solve the problem. spectralRadius : Spectral radius of the reservoir's connection/weight matrix. noiseLevel : Magnitude of noise that is added to the input while fitting to prevent overfitting. inputScaling : Scaling factor of the input. leakingRate : Convex combination factor between 0 and 1 that weights current and new state value. reservoirDensity : Percentage of non-zero weight connections in the reservoir. randomSeed : Seed for random processes, e.g. weight initialization. averageOutputWeights : Average output matrices after fitting across all pixels or use a distinct matrix per pixel. The former assumes homogeneity of the problem across all pixels. out_activation : Final activation function (i.e. activation function of the output). out_inverse_activation : Inverse of the final activation function weightGeneration : Algorithm to generate weight matrices. Choices: naive, SORM, advanced, custom bias : Size of the bias added for the internal update process. outputBias : Size of the bias added for the final linear regression of the output. outputInputScaling : Rescaling factor for the input of the ESN for the regression. inputDensity : Percentage of non-zero weights in the input-to-reservoir weight matrix. solver : Algorithm to find output matrix. Choices: pinv, lsqr. regressionParameters : Arguments to the solving algorithm. For LSQR this controls the L2 regularization. activation : (Non-linear) Activation function. activationDerivative : Derivative of the activation function. chunkSize : Internal parameter for the multi-threading. For long time series this should be reduced to avoid OOM errors/getting stuck and to reduce memory consumption. """ self._averageOutputWeights = averageOutputWeights if averageOutputWeights and solver != "lsqr": raise ValueError( "`averageOutputWeights` can only be set to `True` when `solver` is set to `lsqr` (Ridge Regression)" ) self._borderMode = borderMode if not borderMode in ["mirror", "padding", "edge", "wrap"]: raise ValueError( "`borderMode` must be set to one of the following values: `mirror`, `padding`, `edge` or `wrap`." ) self._regressionParameters = regressionParameters self._solver = solver n_inputDimensions = len(inputShape) if filterSize % 2 == 0: raise ValueError( "filterSize has to be an odd number (1, 3, 5, ...).") self._filterSize = filterSize self._filterWidth = int(np.floor(filterSize / 2)) self._stride = stride self._n_input = int( np.power(np.ceil(filterSize / stride), n_inputDimensions)) self.n_inputDimensions = n_inputDimensions self.inputShape = inputShape if not self._averageOutputWeights: self._WOuts = B.empty( (np.prod(inputShape), 1, self._n_input + n_reservoir + 1)) self._WOut = None else: self._WOuts = None self._WOut = B.zeros((1, self._n_input + n_reservoir + 1)) self._xs = B.empty((np.prod(inputShape), n_reservoir, 1)) if nWorkers == "auto": self._nWorkers = np.max((cpu_count() - 1, 1)) else: self._nWorkers = nWorkers manager = Manager() self.sharedNamespace = manager.Namespace() if hasattr(self, "fitWorkerID") == False or self.parallelWorkerIDs is None: self.parallelWorkerIDs = manager.Queue() for i in range(self._nWorkers): self.parallelWorkerIDs.put((i)) self._chunkSize = chunkSize super(SpatioTemporalESN, self).__init__( n_input=self._n_input, n_reservoir=n_reservoir, n_output=1, spectralRadius=spectralRadius, noiseLevel=noiseLevel, inputScaling=inputScaling, leakingRate=leakingRate, reservoirDensity=reservoirDensity, randomSeed=randomSeed, out_activation=out_activation, out_inverse_activation=out_inverse_activation, weightGeneration=weightGeneration, bias=bias, outputBias=outputBias, outputInputScaling=outputInputScaling, inputDensity=inputDensity, activation=activation, activationDerivative=activationDerivative, ) """