def predict_loop(self, inputData, outputData1, outputData2, continuation=True, initialData=None, update_processor=lambda x: x, verbose=0): inputData = B.array(inputData) #let some input run through the ESN to initialize its states from a new starting value if (not continuation): self._esn1._x = B.zeros(self._esn1._x.shape) self._esn2._x = B.zeros(self._esn2._x.shape) total_length = inputData.shape[0] aggregated_y1 = B.empty((total_length, self._n_output1)) aggregated_y2 = B.empty((total_length, self._n_output2)) # X1, y1 = self._esn1.propagate(inputData=inputData, outputData=None, transientTime=self._transientTime, verbose=verbose-1) # Y1 = B.dot(self._esn1._WOut, X1) # Y1 = update_processor(self.out_activation(Y1)).T y2 = self.out_activation(B.dot(self._esn2._WOut, self._esn2._X).T) for i in range(total_length): inputDataWithFeedback = B.zeros((self.n_input + self._n_output2)) inputDataWithFeedback[:self.n_input] = inputData[i, :] inputDataWithFeedback[self.n_input:] = y2 #update models X1, _ = self._esn1.propagateOneStep( inputData=inputDataWithFeedback, outputData=None, step=i, transientTime=self._transientTime, verbose=verbose - 1, learn=False) y1 = B.dot(self._esn1._WOut, X1) aggregated_y1[i, :] = update_processor(self.out_activation(y1)).T #output from 1st layer and correct output #input2 = outputData1[i] - y1.reshape(self._n_output1) # input2 = B.vstack((y1.reshape(self._n_output1), outputData1[i])) # x2, y2 = self._esn2.propagateOneStep(inputData=input2, outputData=None, step=i, transientTime=self._transientTime, verbose=verbose-1, learn=False) # Y_target2[i,:] = y2.reshape(self._n_output2) training_error1 = B.sqrt(B.mean((aggregated_y1 - outputData1)**2)) print("training errors") print(training_error1) return aggregated_y1, aggregated_y2
def _fitProcess(self, data): try: inData, outData, indices, state = data transientTime = self.sharedNamespace.transientTime partialLength = self.sharedNamespace.partialLength totalLength = self.sharedNamespace.totalLength timeseriesCount = self.sharedNamespace.timeseriesCount workerID = self.parallelWorkerIDs.get() self._x[workerID] = state # propagate X = B.empty((1 + self.n_input + self.n_reservoir, totalLength)) for i in range(timeseriesCount): X[:, i * partialLength:(i + 1) * partialLength] = self.propagate(inData[i], transientTime=transientTime, x=self._x[workerID], verbose=0) # define the target values Y_target = B.empty((1, totalLength)) for i in range(timeseriesCount): Y_target[:, i * partialLength:(i + 1) * partialLength] = self.out_inverse_activation(outData[i]).T[:, transientTime:] # now fit WOut = None if self._solver == "pinv": WOut = B.dot(Y_target, B.pinv(X)) elif self._solver == "lsqr": X_T = X.T WOut = B.dot(B.dot(Y_target, X_T), B.inv( B.dot(X, X_T) + self._regressionParameters[0] * B.identity(1 + self.n_input + self.n_reservoir))) # calculate the training prediction now # trainingPrediction = self.out_activation(B.dot(WOut, X).T) # store the state and the output matrix of the worker SpatioTemporalESN._fitProcess.fitQueue.put( ([x - self._filterWidth for x in indices], self._x[workerID].copy(), WOut.copy())) self.parallelWorkerIDs.put(workerID) except Exception as ex: print(ex) import traceback traceback.print_exc() SpatioTemporalESN._fitProcess.fitQueue.put(([x - self._filterWidth for x in indices], None, None)) self.parallelWorkerIDs.put(workerID)
def calculateTransientTime(self, inputs, outputs, epsilon, proximityLength = None): # inputs: input of reserovoir # outputs: output of reservoir # epsilon: given constant # proximity length: number of steps for which all states have to be epsilon close to declare convergance # initializes two initial states as far as possible from each other in [-1,1] regime and tests when they converge-> this is transient time length = inputs.shape[0] if inputs is not None else outputs.shape[0] if proximityLength is None: proximityLength = int(length * 0.1) if proximityLength < 3: proximityLength = 3 initial_x = B.empty((2, self.n_reservoir, 1)) initial_x[0] = - B.ones((self.n_reservoir, 1)) initial_x[1] = B.ones((self.n_reservoir, 1)) countedConsecutiveSteps = 0 length = inputs.shape[0] if inputs is not None else outputs.shape[0] for t in range(length): if B.max(B.ptp(initial_x, axis=0)) < epsilon: if countedConsecutiveSteps >= proximityLength: return t - proximityLength else: countedConsecutiveSteps += 1 else: countedConsecutiveSteps = 0 u = inputs[t].reshape(-1, 1) if inputs is not None else None o = outputs[t].reshape(-1, 1) if outputs is not None else None for i in range(initial_x.shape[0]): self.update(u, o, initial_x[i]) #transient time could not be determined raise ValueError("Transient time could not be determined - maybe the proximityLength is too big.")
def getEquilibriumState(inputs, outputs, epsilon = 1e-3): # inputs: input of reserovoir # outputs: output of reservoir # epsilon: given constant # returns the equilibrium state when esn is fed with the first state of input x = B.empty((2, self.n_reservoir, 1)) while not B.max(B.ptp(x, axis=0)) < epsilon: x[0] = x[1] u = inputs[0].reshape(-1, 1) if inputs is not None else None o = outputs[0].reshape(-1, 1) if outputs is not None else None self.update(u, o, x[1]) return x[1]
def predict(self, inputData, outputData1, outputData2, continuation=True, initialData=None, update_processor=lambda x: x, verbose=0): inputData = B.array(inputData) #let some input run through the ESN to initialize its states from a new starting value if (not continuation): self._esn1._x = B.zeros(self._esn1._x.shape) self._esn2._x = B.zeros(self._esn2._x.shape) total_length = inputData.shape[0] aggregated_y1 = B.empty((total_length, self._n_output1)) aggregated_y2 = B.empty((total_length, self._n_output2)) #put pixels and switch data together inputDataWithFeedback = B.zeros( (total_length, self.n_input + self._n_output2)) inputDataWithFeedback[:, :self.n_input] = inputData inputDataWithFeedback[:, self.n_input:] = outputData2 X1, _ = self._esn1.propagate(inputData=inputDataWithFeedback, outputData=None, transientTime=self._transientTime, verbose=verbose - 1) aggregated_y1 = B.dot(self._esn1._WOut, X1) aggregated_y1 = update_processor(self.out_activation(aggregated_y1)).T training_error1 = B.sqrt(B.mean((aggregated_y1 - outputData1)**2)) print("predict errors") print(training_error1) return aggregated_y1, aggregated_y2
def predict( self, inputData, update_processor=lambda x: x, transientTime=0, verbose=0 ): if len(inputData.shape) == 1: inputData = inputData[None, :] predictionLength = inputData.shape[1] Y = B.empty((inputData.shape[0], self.n_output)) if verbose > 0: bar = progressbar.ProgressBar( max_value=inputData.shape[0], redirect_stdout=True, poll_interval=0.0001 ) bar.update(0) for n in range(inputData.shape[0]): # reset the state self._x = B.zeros(self._x.shape) X = self.propagate(inputData[n], transientTime) # calculate the prediction using the trained model if self._solver in [ "sklearn_auto", "sklearn_lsqr", "sklearn_sag", "sklearn_svd", "sklearn_svr", ]: y = self._ridgeSolver.predict(X.T).reshape((self.n_output, -1)) else: y = B.dot(self._W_out, X) Y[n] = np.mean(y, 1) if verbose > 0: bar.update(n) if verbose > 0: bar.finish() # return the result result = B.zeros(Y.shape) for i, n in enumerate(B.argmax(Y, 1)): result[i, n] = 1.0 return result
def update(self, inputData, outputData=None, x=None): if x is None: x = self._x if self._WFeedback is None: # reshape the data u = inputData.reshape(self.n_input, 1) # update the states transmission = self.calculateLinearNetworkTransmissions(u, x) x *= 1.0 - self._leakingRate x += self._leakingRate * self._activation( transmission + (np.random.rand() - 0.5) * self._noiseLevel) return u else: # the input is allowed to be "empty" (size=0) if self.n_input != 0: # reshape the data u = inputData.reshape(self.n_input, 1) outputData = outputData.reshape(self.n_output, 1) # update the states transmission = self.calculateLinearNetworkTransmissions(u, x) x *= 1.0 - self._leakingRate x += self._leakingRate * self._activation(transmission + B.dot( self._WFeedback, B.vstack((B.array(self._outputBias), outputData)), ) + (np.random.rand() - 0.5) * self._noiseLevel) return u else: # reshape the data outputData = outputData.reshape(self.n_output, 1) # update the states transmission = B.dot(self._W, x) x *= 1.0 - self._leakingRate x += self._leakingRate * self._activation(transmission + B.dot( self._WFeedback, B.vstack((B.array(self._outputBias), outputData)), ) + (np.random.rand() - 0.5) * self._noiseLevel) return B.empty((0, 1))
def __init__(self, inputShape, n_reservoir, filterSize=1, stride=1, borderMode="mirror", nWorkers="auto", spectralRadius=1.0, noiseLevel=0.0, inputScaling=None, leakingRate=1.0, reservoirDensity=0.2, randomSeed=None, averageOutputWeights=True, out_activation=lambda x: x, out_inverse_activation=lambda x: x, weightGeneration='naive', bias=1.0, outputBias=1.0, outputInputScaling=1.0, inputDensity=1.0, solver='pinv', regressionParameters={}, activation=B.tanh, activationDerivation=lambda x: 1.0 / B.cosh(x) ** 2): self._averageOutputWeights = averageOutputWeights if averageOutputWeights and solver != "lsqr": raise ValueError( "`averageOutputWeights` can only be set to `True` when `solver` is set to `lsqr` (Ridge Regression)") self._borderMode = borderMode if not borderMode in ["mirror", "padding", "edge", "wrap"]: raise ValueError( "`borderMode` must be set to one of the following values: `mirror`, `padding`, `edge` or `wrap`.") self._regressionParameters = regressionParameters self._solver = solver n_inputDimensions = len(inputShape) if filterSize % 2 == 0: raise ValueError("filterSize has to be an odd number (1, 3, 5, ...).") self._filterSize = filterSize self._filterWidth = int(np.floor(filterSize / 2)) self._stride = stride self._n_input = int(np.power(np.ceil(filterSize / stride), n_inputDimensions)) self.n_inputDimensions = n_inputDimensions self.inputShape = inputShape if not self._averageOutputWeights: self._WOuts = B.empty((np.prod(inputShape), 1, self._n_input + n_reservoir + 1)) self._WOut = None else: self._WOuts = None self._WOut = B.zeros((1, self._n_input + n_reservoir + 1)) self._xs = B.empty((np.prod(inputShape), n_reservoir, 1)) if nWorkers == "auto": self._nWorkers = np.max((cpu_count() - 1, 1)) else: self._nWorkers = nWorkers manager = Manager() self.sharedNamespace = manager.Namespace() if hasattr(self, "fitWorkerID") == False or self.parallelWorkerIDs is None: self.parallelWorkerIDs = manager.Queue() for i in range(self._nWorkers): self.parallelWorkerIDs.put((i)) super(SpatioTemporalESN, self).__init__(n_input=self._n_input, n_reservoir=n_reservoir, n_output=1, spectralRadius=spectralRadius, noiseLevel=noiseLevel, inputScaling=inputScaling, leakingRate=leakingRate, reservoirDensity=reservoirDensity, randomSeed=randomSeed, out_activation=out_activation, out_inverse_activation=out_inverse_activation, weightGeneration=weightGeneration, bias=bias, outputBias=outputBias, outputInputScaling=outputInputScaling, inputDensity=inputDensity, activation=activation, activationDerivation=activationDerivation) """
def fit(self, inputData, outputData, transientTime="AutoReduce", transientTimeCalculationEpsilon=1e-3, transientTimeCalculationLength=20, verbose=0): #check the input data if self.n_input != 0: if len(inputData.shape) == 3 and len(outputData.shape) > 1: #multiple time series are used with a shape (timeseries, time, dimension) -> (timeseries, time, dimension) if inputData.shape[0] != outputData.shape[0]: raise ValueError( "Amount of input and output datasets is not equal - {0} != {1}" .format(inputData.shape[0], outputData.shape[0])) if inputData.shape[1] != outputData.shape[1]: raise ValueError( "Amount of input and output time steps is not equal - {0} != {1}" .format(inputData.shape[1], outputData.shape[1])) else: if inputData.shape[0] != outputData.shape[0]: raise ValueError( "Amount of input and output time steps is not equal - {0} != {1}" .format(inputData.shape[0], outputData.shape[0])) else: if inputData is not None: raise ValueError( "n_input has been set to zero. Therefore, the given inputData will not be used." ) if inputData is not None: inputData = B.array(inputData) if outputData is not None: outputData = B.array(outputData) #reshape the input/output data to have the shape (timeseries, time, dimension) if len(outputData.shape) <= 2: outputData = outputData.reshape((1, -1, self.n_output)) if inputData is not None: if len(inputData.shape) <= 2: inputData = inputData.reshape((1, -1, self.n_input)) self.resetState() # Automatic transient time calculations if transientTime == "Auto": transientTime = self.calculateTransientTime( inputData[0], outputData[0], transientTimeCalculationEpsilon, transientTimeCalculationLength) if transientTime == "AutoReduce": if (inputData is None and outputData.shape[2] == 1) or inputData.shape[2] == 1: transientTime = self.calculateTransientTime( inputData[0], outputData[0], transientTimeCalculationEpsilon, transientTimeCalculationLength) transientTime = self.reduceTransientTime( inputData[0], outputData[0], transientTime) else: print( "Transient time reduction is supported only for 1 dimensional input." ) if inputData is not None: partialLength = (inputData.shape[1] - transientTime) totalLength = inputData.shape[0] * partialLength timeseriesCount = inputData.shape[0] elif outputData is not None: partialLength = (outputData.shape[1] - transientTime) totalLength = outputData.shape[0] * partialLength timeseriesCount = outputData.shape[0] else: raise ValueError("Either input or output data must not to be None") self._X = B.empty((1 + self.n_input + self.n_reservoir, totalLength)) if (verbose > 0): bar = progressbar.ProgressBar(max_value=totalLength, redirect_stdout=True, poll_interval=0.0001) bar.update(0) for i in range(timeseriesCount): if inputData is not None: xx, yy = self.propagate(inputData[i], outputData[i], transientTime, verbose - 1) self._X[:, i * partialLength:(i + 1) * partialLength] = xx else: xx, yy = self.propagate(None, outputData[i], transientTime, verbose - 1) self._X[:, i * partialLength:(i + 1) * partialLength] = xx if (verbose > 0): bar.update(i) if (verbose > 0): bar.finish() #define the target values Y_target = B.empty((outputData.shape[2], totalLength)) for i in range(timeseriesCount): Y_target[:, i * partialLength:(i + 1) * partialLength] = self.out_inverse_activation( outputData[i]).T[:, transientTime:] if (self._solver == "pinv"): self._WOut = B.dot(Y_target, B.pinv(self._X)) #calculate the training prediction now train_prediction = self.out_activation((B.dot(self._WOut, self._X)).T) # elif (self._solver == "lsqr"): # X_T = self._X.T # self._WOut = B.dot(B.dot(Y_target, X_T),B.inv(B.dot(self._X,X_T) + self._regressionParameters[0]*B.identity(1+self.n_input+self.n_reservoir))) # """ # #alternative representation of the equation # Xt = X.T # A = np.dot(X, Y_target.T) # B = np.linalg.inv(np.dot(X, Xt) + regression_parameter*np.identity(1+self.n_input+self.n_reservoir)) # self._WOut = np.dot(B, A) # self._WOut = self._WOut.T # """ # #calculate the training prediction now # train_prediction = self.out_activation(B.dot(self._WOut, self._X).T) elif (self._solver in [ "sklearn_auto", "sklearn_lsqr", "sklearn_sag", "sklearn_svd" ]): mode = self._solver[8:] params = self._regressionParameters params["solver"] = mode self._ridgeSolver = Ridge(**params) self._ridgeSolver.fit(self._X.T, Y_target.T) #calculate the training prediction now train_prediction = self.out_activation( self._ridgeSolver.predict(self._X.T)) elif (self._solver in ["sklearn_svr", "sklearn_svc"]): self._ridgeSolver = SVR(**self._regressionParameters) self._ridgeSolver.fit(self._X.T, Y_target.T.ravel()) #calculate the training prediction now train_prediction = self.out_activation( self._ridgeSolver.predict(self._X.T)) #calculate the training error now #flatten the outputData outputData = outputData[:, transientTime:, :].reshape(totalLength, -1) training_error = B.sqrt(B.mean((train_prediction - outputData)**2)) return training_error
def fit_loop(self, inputData, outputData1, outputData2, transientTime="AutoReduce", transientTimeCalculationEpsilon=1e-3, transientTimeCalculationLength=20, verbose=0): #check the input data if self.n_input != 0: if len(inputData.shape) == 3 and len(outputData1.shape) > 1: #multiple time series are used with a shape (timeseries, time, dimension) -> (timeseries, time, dimension) if inputData.shape[0] != outputData1.shape[0]: raise ValueError( "Amount of input and output datasets is not equal - {0} != {1}" .format(inputData.shape[0], outputData1.shape[0])) if inputData.shape[1] != outputData1.shape[1]: raise ValueError( "Amount of input and output time steps is not equal - {0} != {1}" .format(inputData.shape[1], outputData1.shape[1])) else: if inputData.shape[0] != outputData1.shape[0]: raise ValueError( "Amount of input and output time steps is not equal - {0} != {1}" .format(inputData.shape[0], outputData1.shape[0])) else: if inputData is not None: raise ValueError( "n_input has been set to zero. Therefore, the given inputData will not be used." ) inputData = B.array(inputData) outputData1 = B.array(outputData1) outputData2 = B.array(outputData2) self._transientTime = transientTime self._esn1.resetState() self._esn2.resetState() total_length = inputData.shape[0] print("total_length ", total_length) if (verbose > 0): bar = progressbar.ProgressBar(max_value=total_length, redirect_stdout=True, poll_interval=0.0001) bar.update(0) #should be named aggregated_y aggregated_y1 = B.empty((outputData1.shape)) aggregated_y2 = B.empty((outputData2.shape)) # X1, _ = self._esn1.propagate(inputData=inputData, outputData=outputData1, transientTime=transientTime, verbose=verbose-1) # self._esn1._X = X1 # Y_target = self.out_inverse_activation(outputData1).T # self._esn1._WOut = B.dot(Y_target, B.pinv(X1)) # y1 = self.out_activation((B.dot(self._esn1._WOut, X1)).T) # training_error1 = B.sqrt(B.mean((y1.reshape((total_length, self._n_output1))- outputData1)**2)) # training_error2 = 0 y2 = self.out_activation(B.dot(self._esn2._WOut, self._esn2._X).T) for i in range((total_length)): #input: input + output from layer 2 inputDataWithFeedback = B.zeros((self.n_input + self._n_output2)) inputDataWithFeedback[:self.n_input] = inputData[i, :] inputDataWithFeedback[self.n_input:] = y2 #update models X1, untrained_y1 = self._esn1.propagateOneStep( inputData=inputDataWithFeedback, outputData=outputData1[i], step=i, transientTime=transientTime, verbose=verbose - 1, learn=True) #self._esn1._X = X1 #y after model update (ideally we would use y before model update) #y1 = self.out_activation((B.dot(self._esn1._WOut, X1)).T) aggregated_y1[i, :] = untrained_y1.reshape(self._n_output1) #output from 1st layer and correct output # input2 = B.vstack((y1.reshape(self._n_output1), outputData1[i])) # x2, y2 = self._esn2.propagateOneStep(inputData=input2, outputData=outputData2[i], step=i, transientTime=transientTime, verbose=verbose-1, learn=True) # Y_target2[i,:] = y2.reshape(self._n_output2) if (verbose > 0): bar.update(i) if (verbose > 0): bar.finish() self._training_res = aggregated_y1 training_error1 = B.sqrt(B.mean((aggregated_y1 - outputData1)**2)) training_error2 = B.sqrt(B.mean((aggregated_y2 - outputData2)**2)) print("training errors") print(training_error1) print(training_error2) return training_error1, training_error2
def __init__(self, n_input, n_output1, n_output2, n_reservoir, spectralRadius=1.0, noiseLevel=0.0, inputScaling=None, leakingRate=1.0, feedbackScaling=1.0, reservoirDensity=0.2, randomSeed=None, out_activation=lambda x: x, out_inverse_activation=lambda x: x, weightGeneration='naive', bias=1.0, outputBias=1.0, outputInputScaling=1.0, inputDensity=1.0, solver='pinv', regressionParameters={}, activation=B.tanh, activationDerivation=lambda x: 1.0 / B.cosh(x)**2): #probably not needed super(StackedESN, self).__init__(n_input=n_input, n_reservoir=n_reservoir, n_output=n_output1, spectralRadius=spectralRadius, noiseLevel=noiseLevel, inputScaling=inputScaling, leakingRate=leakingRate, feedbackScaling=feedbackScaling, reservoirDensity=reservoirDensity, randomSeed=randomSeed, feedback=False, out_activation=out_activation, out_inverse_activation=out_inverse_activation, weightGeneration=weightGeneration, bias=bias, outputBias=outputBias, outputInputScaling=outputInputScaling, inputDensity=inputDensity, activation=activation, activationDerivation=activationDerivation) #layers self._esn1 = PredictionESN( n_input=(n_input + n_output2), n_reservoir=n_reservoir, n_output=n_output1, spectralRadius=spectralRadius, noiseLevel=noiseLevel, inputScaling=inputScaling, leakingRate=leakingRate, feedbackScaling=feedbackScaling, reservoirDensity=reservoirDensity, randomSeed=randomSeed, feedback=False, out_activation=out_activation, out_inverse_activation=out_inverse_activation, weightGeneration=weightGeneration, bias=bias, outputBias=outputBias, outputInputScaling=outputInputScaling, inputDensity=inputDensity, activation=activation, activationDerivation=activationDerivation) #esn_2 takes coordinates in (estimated and actual) self._esn2 = PredictionESN( n_input=n_output1 * 2, n_reservoir=n_reservoir, n_output=n_output2, spectralRadius=spectralRadius, noiseLevel=noiseLevel, inputScaling=inputScaling, leakingRate=leakingRate, feedbackScaling=feedbackScaling, reservoirDensity=reservoirDensity, randomSeed=randomSeed, feedback=False, out_activation=out_activation, out_inverse_activation=out_inverse_activation, weightGeneration=weightGeneration, bias=bias, outputBias=outputBias, outputInputScaling=outputInputScaling, inputDensity=inputDensity, activation=activation, activationDerivation=activationDerivation) self._solver = solver self._regressionParameters = regressionParameters self._transientTime = 0 self._n_output1 = n_output1 self._n_output2 = n_output2 self._training_res = B.empty((2, 2)) """
def fit(self, inputData, outputData1, outputData2, transientTime="AutoReduce", transientTimeCalculationEpsilon=1e-3, transientTimeCalculationLength=20, verbose=0): #check the input data if self.n_input != 0: if len(inputData.shape) == 3 and len(outputData1.shape) > 1: #multiple time series are used with a shape (timeseries, time, dimension) -> (timeseries, time, dimension) if inputData.shape[0] != outputData1.shape[0]: raise ValueError( "Amount of input and output datasets is not equal - {0} != {1}" .format(inputData.shape[0], outputData1.shape[0])) if inputData.shape[1] != outputData1.shape[1]: raise ValueError( "Amount of input and output time steps is not equal - {0} != {1}" .format(inputData.shape[1], outputData1.shape[1])) else: if inputData.shape[0] != outputData1.shape[0]: raise ValueError( "Amount of input and output time steps is not equal - {0} != {1}" .format(inputData.shape[0], outputData1.shape[0])) else: if inputData is not None: raise ValueError( "n_input has been set to zero. Therefore, the given inputData will not be used." ) inputData = B.array(inputData) outputData1 = B.array(outputData1) outputData2 = B.array(outputData2) self._transientTime = transientTime self._esn1.resetState() self._esn2.resetState() total_length = inputData.shape[0] print("total_length ", total_length) aggregated_y1 = B.empty((outputData1.shape)) aggregated_y2 = B.empty((outputData2.shape)) #put pixels and switch data together inputDataWithFeedback = B.zeros( (total_length, self.n_input + self._n_output2)) inputDataWithFeedback[:, :self.n_input] = inputData inputDataWithFeedback[:, self.n_input:] = outputData2 X1, _ = self._esn1.propagate(inputData=inputDataWithFeedback, outputData=outputData1, transientTime=transientTime, verbose=verbose - 1) self._esn1._X = X1 Y_target = self.out_inverse_activation(outputData1).T self._esn1._WOut = B.dot(Y_target, B.pinv(X1)) aggregated_y1 = self.out_activation((B.dot(self._esn1._WOut, X1)).T) training_error1 = B.sqrt( B.mean((aggregated_y1.reshape( (total_length, self._n_output1)) - outputData1)**2)) training_error2 = 0 training_error1 = B.sqrt(B.mean((aggregated_y1 - outputData1)**2)) training_error2 = B.sqrt(B.mean((aggregated_y2 - outputData2)**2)) return training_error1, training_error2
def reduceTransientTime(self, inputs, outputs, initialTransientTime, epsilon = 1e-3, proximityLength = 50): # inputs: input of reserovoir # outputs: output of reservoir # epsilon: given constant # proximity length: number of steps for which all states have to be epsilon close to declare convergance # initialTransientTime: transient time with calculateTransientTime() method estimated # finds initial state with lower transient time and sets internal state to this state # returns the new transient time by calculating the convergence time of initial states found with SWD and Equilibrium method def getEquilibriumState(inputs, outputs, epsilon = 1e-3): # inputs: input of reserovoir # outputs: output of reservoir # epsilon: given constant # returns the equilibrium state when esn is fed with the first state of input x = B.empty((2, self.n_reservoir, 1)) while not B.max(B.ptp(x, axis=0)) < epsilon: x[0] = x[1] u = inputs[0].reshape(-1, 1) if inputs is not None else None o = outputs[0].reshape(-1, 1) if outputs is not None else None self.update(u, o, x[1]) return x[1] def getStateAtGivenPoint(inputs, outputs, targetTime): # inputs: input of reserovoir # outputs: output of reservoir # targetTime: time at which the state is wanted # propagates the inputs/outputs till given point in time and returns the state of the reservoir at this point x = B.zeros((self.n_reservoir, 1)) length = inputs.shape[0] if inputs is not None else outputs.shape[0] length = min(length, targetTime) for t in range(length): u = inputs[t].reshape(-1, 1) if inputs is not None else None o = outputs[t].reshape(-1, 1) if outputs is not None else None self.update(u, o, x) return x length = inputs.shape[0] if inputs is not None else outputs.shape[0] if proximityLength is None: proximityLength = int(length * 0.1) if proximityLength < 3: proximityLength = 3 x = B.empty((2, self.n_reservoir, 1)) equilibriumState = getEquilibriumState(inputs, outputs) if inputs is None: swdPoint, _ = hp.SWD(outputs, int(initialTransientTime*0.8)) else: swdPoint, _ = hp.SWD(inputs, int(initialTransientTime * 0.8)) swdState = getStateAtGivenPoint(inputs, outputs, swdPoint) x[0] = equilibriumState x[1] = swdState transientTime = 0 countedConsecutiveSteps = 0 for t in range(length): if B.max(B.ptp(x, axis=0)) < epsilon: countedConsecutiveSteps += 1 if countedConsecutiveSteps > proximityLength: transientTime = t - proximityLength break else: countedConsecutiveSteps = 0 u = inputs[t].reshape(-1, 1) if inputs is not None else None o = outputs[t].reshape(-1, 1) if outputs is not None else None for i in range(x.shape[0]): self.update(u, o, x[i]) self._x = x[0] return transientTime
def propagate(self, inputData, outputData=None, transientTime=0, verbose=0, x=None, steps="auto", feedbackData=None): if x is None: x = self._x inputLength = steps if inputData is None: if outputData is not None: inputLength = len(outputData) else: inputLength = len(inputData) if inputLength == "auto": raise ValueError("inputData and outputData are both None. Therefore, steps must not be `auto`.") # define states' matrix X = B.zeros((1 + self.n_input + self.n_reservoir, inputLength - transientTime)) if (verbose > 0): bar = progressbar.ProgressBar(max_value=inputLength, redirect_stdout=True, poll_interval=0.0001) bar.update(0) if self._WFeedback is None: #do not distinguish between whether inputData is None or not, as the feedback has been disabled #therefore, the input has to be anything but None for t in range(inputLength): u, x = self.update(inputData[t], x=x) if (t >= transientTime): #add valueset to the states' matrix X[:,t-transientTime] = B.vstack((B.array(self._outputBias), self._outputInputScaling*u, x))[:,0] Y = B.dot(self._WOut, X) if (verbose > 0): bar.update(t) else: if outputData is None: Y = B.empty((inputLength-transientTime, self.n_output)) if feedbackData is None: feedbackData = B.zeros((1, self.n_output)) if inputData is None: for t in range(inputLength): self.update(None, feedbackData, x=x) if (t >= transientTime): #add valueset to the states' matrix X[:,t-transientTime] = B.vstack((B.array(self._outputBias), x))[:,0] if outputData is None: #calculate the prediction using the trained model if (self._solver in ["sklearn_auto", "sklearn_lsqr", "sklearn_sag", "sklearn_svd"]): feedbackData = self._ridgeSolver.predict(B.vstack((B.array(self._outputBias), self._x)).T) else: feedbackData = B.dot(self._WOut, B.vstack((B.array(self._outputBias), self._x))) if t >= transientTime: Y[t-transientTime, :] = feedbackData else: feedbackData = outputData[t] if (verbose > 0): bar.update(t) else: for t in range(inputLength): u = self.update(inputData[t], feedbackData, x=x) if (t >= transientTime): #add valueset to the states' matrix X[:,t-transientTime] = B.vstack((B.array(self._outputBias), self._outputInputScaling*u, x))[:,0] if outputData is None: #calculate the prediction using the trained model if (self._solver in ["sklearn_auto", "sklearn_lsqr", "sklearn_sag", "sklearn_svd"]): feedbackData = self._ridgeSolver.predict(B.vstack((B.array(self._outputBias), self._outputInputScaling*u, self._x)).T) else: feedbackData = B.dot(self._WOut, B.vstack((B.array(self._outputBias), self._outputInputScaling*u, self._x))) Y[t, :] = feedbackData.ravel() else: feedbackData = outputData[t] if (verbose > 0): bar.update(t) if (verbose > 0): bar.finish() return X, Y
def __init__( self, inputShape, n_reservoir, filterSize=1, stride=1, borderMode="mirror", nWorkers="auto", spectralRadius=1.0, noiseLevel=0.0, inputScaling=None, leakingRate=1.0, reservoirDensity=0.2, randomSeed=None, averageOutputWeights=True, out_activation=lambda x: x, out_inverse_activation=lambda x: x, weightGeneration="naive", bias=1.0, outputBias=1.0, outputInputScaling=1.0, inputDensity=1.0, solver="pinv", regressionParameters={}, activation=B.tanh, activationDerivative=lambda x: 1.0 / B.cosh(x)**2, chunkSize=16, ): """ ESN that predicts (steps of) a spatio-temporal time series based on a time series. Args: inputShape : Shape of the input w/o the time axis, e.g. (W, H) for a 2D input. n_reservoir : Number of units in the reservoir. filterSize : Size of patches used to predict a single output element. stride : Stride between different patches. borderMode : How to handle border values. Choices: mirror, padding, edge, wrap. nWorkers : Number of CPU threads executed in parallel to solve the problem. spectralRadius : Spectral radius of the reservoir's connection/weight matrix. noiseLevel : Magnitude of noise that is added to the input while fitting to prevent overfitting. inputScaling : Scaling factor of the input. leakingRate : Convex combination factor between 0 and 1 that weights current and new state value. reservoirDensity : Percentage of non-zero weight connections in the reservoir. randomSeed : Seed for random processes, e.g. weight initialization. averageOutputWeights : Average output matrices after fitting across all pixels or use a distinct matrix per pixel. The former assumes homogeneity of the problem across all pixels. out_activation : Final activation function (i.e. activation function of the output). out_inverse_activation : Inverse of the final activation function weightGeneration : Algorithm to generate weight matrices. Choices: naive, SORM, advanced, custom bias : Size of the bias added for the internal update process. outputBias : Size of the bias added for the final linear regression of the output. outputInputScaling : Rescaling factor for the input of the ESN for the regression. inputDensity : Percentage of non-zero weights in the input-to-reservoir weight matrix. solver : Algorithm to find output matrix. Choices: pinv, lsqr. regressionParameters : Arguments to the solving algorithm. For LSQR this controls the L2 regularization. activation : (Non-linear) Activation function. activationDerivative : Derivative of the activation function. chunkSize : Internal parameter for the multi-threading. For long time series this should be reduced to avoid OOM errors/getting stuck and to reduce memory consumption. """ self._averageOutputWeights = averageOutputWeights if averageOutputWeights and solver != "lsqr": raise ValueError( "`averageOutputWeights` can only be set to `True` when `solver` is set to `lsqr` (Ridge Regression)" ) self._borderMode = borderMode if not borderMode in ["mirror", "padding", "edge", "wrap"]: raise ValueError( "`borderMode` must be set to one of the following values: `mirror`, `padding`, `edge` or `wrap`." ) self._regressionParameters = regressionParameters self._solver = solver n_inputDimensions = len(inputShape) if filterSize % 2 == 0: raise ValueError( "filterSize has to be an odd number (1, 3, 5, ...).") self._filterSize = filterSize self._filterWidth = int(np.floor(filterSize / 2)) self._stride = stride self._n_input = int( np.power(np.ceil(filterSize / stride), n_inputDimensions)) self.n_inputDimensions = n_inputDimensions self.inputShape = inputShape if not self._averageOutputWeights: self._WOuts = B.empty( (np.prod(inputShape), 1, self._n_input + n_reservoir + 1)) self._WOut = None else: self._WOuts = None self._WOut = B.zeros((1, self._n_input + n_reservoir + 1)) self._xs = B.empty((np.prod(inputShape), n_reservoir, 1)) if nWorkers == "auto": self._nWorkers = np.max((cpu_count() - 1, 1)) else: self._nWorkers = nWorkers manager = Manager() self.sharedNamespace = manager.Namespace() if hasattr(self, "fitWorkerID") == False or self.parallelWorkerIDs is None: self.parallelWorkerIDs = manager.Queue() for i in range(self._nWorkers): self.parallelWorkerIDs.put((i)) self._chunkSize = chunkSize super(SpatioTemporalESN, self).__init__( n_input=self._n_input, n_reservoir=n_reservoir, n_output=1, spectralRadius=spectralRadius, noiseLevel=noiseLevel, inputScaling=inputScaling, leakingRate=leakingRate, reservoirDensity=reservoirDensity, randomSeed=randomSeed, out_activation=out_activation, out_inverse_activation=out_inverse_activation, weightGeneration=weightGeneration, bias=bias, outputBias=outputBias, outputInputScaling=outputInputScaling, inputDensity=inputDensity, activation=activation, activationDerivative=activationDerivative, ) """