def __init__(self,
n_input,
n_reservoir,
n_output,
spectralRadius=1.0,
noiseLevel=0.0,
inputScaling=None,
leakingRate=1.0,
feedbackScaling=1.0,
reservoirDensity=0.2,
randomSeed=None,
out_activation=lambda x: x,
out_inverse_activation=lambda x: x,
weightGeneration='naive',
bias=1.0,
outputBias=1.0,
feedback=False,
outputInputScaling=1.0,
inputDensity=1.0,
solver='pinv',
regressionParameters={},
activation=B.tanh,
activationDerivation=lambda x: 1.0 / B.cosh(x)**2):
super(PredictionESN,
self).__init__(n_input=n_input,
n_reservoir=n_reservoir,
n_output=n_output,
spectralRadius=spectralRadius,
noiseLevel=noiseLevel,
inputScaling=inputScaling,
leakingRate=leakingRate,
feedbackScaling=feedbackScaling,
reservoirDensity=reservoirDensity,
randomSeed=randomSeed,
feedback=feedback,
out_activation=out_activation,
out_inverse_activation=out_inverse_activation,
weightGeneration=weightGeneration,
bias=bias,
outputBias=outputBias,
outputInputScaling=outputInputScaling,
inputDensity=inputDensity,
activation=activation,
activationDerivation=activationDerivation)
self._solver = solver
self._regressionParameters = regressionParameters
self._x = B.zeros((self.n_reservoir, 1))
#self._WOut = np.random.rand(n_output, n_reservoir+1)#bias term
self._WOut = np.random.rand(n_output,
1 + n_input + n_reservoir) #bias term
#this seems to have been defined somewhere already
self._X = B.zeros((1 + n_input + n_reservoir, 1))
"""
def predict_loop(self,
inputData,
outputData1,
outputData2,
continuation=True,
initialData=None,
update_processor=lambda x: x,
verbose=0):
inputData = B.array(inputData)
#let some input run through the ESN to initialize its states from a new starting value
if (not continuation):
self._esn1._x = B.zeros(self._esn1._x.shape)
self._esn2._x = B.zeros(self._esn2._x.shape)
total_length = inputData.shape[0]
aggregated_y1 = B.empty((total_length, self._n_output1))
aggregated_y2 = B.empty((total_length, self._n_output2))
# X1, y1 = self._esn1.propagate(inputData=inputData, outputData=None, transientTime=self._transientTime, verbose=verbose-1)
# Y1 = B.dot(self._esn1._WOut, X1)
# Y1 = update_processor(self.out_activation(Y1)).T
y2 = self.out_activation(B.dot(self._esn2._WOut, self._esn2._X).T)
for i in range(total_length):
inputDataWithFeedback = B.zeros((self.n_input + self._n_output2))
inputDataWithFeedback[:self.n_input] = inputData[i, :]
inputDataWithFeedback[self.n_input:] = y2
#update models
X1, _ = self._esn1.propagateOneStep(
inputData=inputDataWithFeedback,
outputData=None,
step=i,
transientTime=self._transientTime,
verbose=verbose - 1,
learn=False)
y1 = B.dot(self._esn1._WOut, X1)
aggregated_y1[i, :] = update_processor(self.out_activation(y1)).T
#output from 1st layer and correct output
#input2 = outputData1[i] - y1.reshape(self._n_output1)
# input2 = B.vstack((y1.reshape(self._n_output1), outputData1[i]))
# x2, y2 = self._esn2.propagateOneStep(inputData=input2, outputData=None, step=i, transientTime=self._transientTime, verbose=verbose-1, learn=False)
# Y_target2[i,:] = y2.reshape(self._n_output2)
training_error1 = B.sqrt(B.mean((aggregated_y1 - outputData1)**2))
print("training errors")
print(training_error1)
return aggregated_y1, aggregated_y2
def predict(
self, inputData, update_processor=lambda x: x, transientTime=0, verbose=0
):
if len(inputData.shape) == 1:
inputData = inputData[None, :]
predictionLength = inputData.shape[1]
Y = B.empty((inputData.shape[0], self.n_output))
if verbose > 0:
bar = progressbar.ProgressBar(
max_value=inputData.shape[0], redirect_stdout=True, poll_interval=0.0001
)
bar.update(0)
for n in range(inputData.shape[0]):
# reset the state
self._x = B.zeros(self._x.shape)
X = self.propagate(inputData[n], transientTime)
# calculate the prediction using the trained model
if self._solver in [
"sklearn_auto",
"sklearn_lsqr",
"sklearn_sag",
"sklearn_svd",
"sklearn_svr",
]:
y = self._ridgeSolver.predict(X.T).reshape((self.n_output, -1))
else:
y = B.dot(self._W_out, X)
Y[n] = np.mean(y, 1)
if verbose > 0:
bar.update(n)
if verbose > 0:
bar.finish()
# return the result
result = B.zeros(Y.shape)
for i, n in enumerate(B.argmax(Y, 1)):
result[i, n] = 1.0
return result
def SWD(series, intervall):
differences = B.zeros(series.shape[0] - 2 * intervall)
reference_series = series[:intervall]
for i in range(intervall, series.shape[0] - intervall):
differences[i - intervall] = B.sum(
B.abs(reference_series - series[i:i + intervall]))
return B.argmin(differences) + intervall, differences
def predict(
self,
inputData,
continuation=True,
initialData=None,
update_processor=lambda x: x,
verbose=0,
):
inputData = B.array(inputData)
# let some input run through the ESN to initialize its states from a new starting value
if not continuation:
self._x = B.zeros(self._x.shape)
if initialData is not None:
if self._WFeedback is None:
for t in range(initialData.shape[0]):
super(PredictionESN, self).update(initialData[t])
else:
if type(initialData) is tuple:
initialDataInput, initialDataOutput = initialData
if len(initialDataInput) != len(initialDataOutput):
raise ValueError(
"Length of the inputData and the outputData of the initialData tuple do not match."
)
else:
raise ValueError(
"initialData has to be a tuple consisting out of the input and the output data."
)
super(PredictionESN, self).update(initialDataInput[t],
initialDataOutput[t])
X = self.propagate(inputData, verbose=verbose)
if self._WFeedback is not None:
X, _ = X
# calculate the prediction using the trained model
if self._solver in [
"sklearn_auto",
"sklearn_lsqr",
"sklearn_sag",
"sklearn_svd",
"sklearn_svr",
]:
Y = self._ridgeSolver.predict(X.T).reshape((self.n_output, -1))
else:
Y = B.dot(self._WOut, X)
# apply the output activation function
Y = update_processor(self.out_activation(Y))
# return the result
return Y.T
def predict(self,
inputData,
outputData1,
outputData2,
continuation=True,
initialData=None,
update_processor=lambda x: x,
verbose=0):
inputData = B.array(inputData)
#let some input run through the ESN to initialize its states from a new starting value
if (not continuation):
self._esn1._x = B.zeros(self._esn1._x.shape)
self._esn2._x = B.zeros(self._esn2._x.shape)
total_length = inputData.shape[0]
aggregated_y1 = B.empty((total_length, self._n_output1))
aggregated_y2 = B.empty((total_length, self._n_output2))
#put pixels and switch data together
inputDataWithFeedback = B.zeros(
(total_length, self.n_input + self._n_output2))
inputDataWithFeedback[:, :self.n_input] = inputData
inputDataWithFeedback[:, self.n_input:] = outputData2
X1, _ = self._esn1.propagate(inputData=inputDataWithFeedback,
outputData=None,
transientTime=self._transientTime,
verbose=verbose - 1)
aggregated_y1 = B.dot(self._esn1._WOut, X1)
aggregated_y1 = update_processor(self.out_activation(aggregated_y1)).T
training_error1 = B.sqrt(B.mean((aggregated_y1 - outputData1)**2))
print("predict errors")
print(training_error1)
return aggregated_y1, aggregated_y2
def getStateAtGivenPoint(inputs, outputs, targetTime):
# inputs: input of reserovoir
# outputs: output of reservoir
# targetTime: time at which the state is wanted
# propagates the inputs/outputs till given point in time and returns the state of the reservoir at this point
x = B.zeros((self.n_reservoir, 1))
length = inputs.shape[0] if inputs is not None else outputs.shape[0]
length = min(length, targetTime)
for t in range(length):
u = inputs[t].reshape(-1, 1) if inputs is not None else None
o = outputs[t].reshape(-1, 1) if outputs is not None else None
self.update(u, o, x)
return x
def calculate_esn_mi_input_scaling(input_data, output_data):
if len(input_data) != len(output_data):
raise ValueError(
"input_data and output_data do not have the same length - {0} vs. {1}"
.format(len(input_data), len(output_data)))
# Scott's rule to calculate nbins
std_output = B.std(output_data)
nbins = int(
B.ceil(2.0 / (3.5 * std_output / B.power(len(input_data), 1.0 / 3.0))))
mi = B.zeros(input_data.shape[1])
for i in range(len(mi)):
mi[i] = calculate_mutualinformation(input_data[:, i], output_data,
nbins)
scaling = mi / B.max(mi)
return scaling
def __init__(self, inputShape, n_reservoir,
filterSize=1, stride=1, borderMode="mirror", nWorkers="auto",
spectralRadius=1.0, noiseLevel=0.0, inputScaling=None,
leakingRate=1.0, reservoirDensity=0.2, randomSeed=None, averageOutputWeights=True,
out_activation=lambda x: x, out_inverse_activation=lambda x: x,
weightGeneration='naive', bias=1.0, outputBias=1.0,
outputInputScaling=1.0, inputDensity=1.0, solver='pinv', regressionParameters={}, activation=B.tanh,
activationDerivation=lambda x: 1.0 / B.cosh(x) ** 2):
self._averageOutputWeights = averageOutputWeights
if averageOutputWeights and solver != "lsqr":
raise ValueError(
"`averageOutputWeights` can only be set to `True` when `solver` is set to `lsqr` (Ridge Regression)")
self._borderMode = borderMode
if not borderMode in ["mirror", "padding", "edge", "wrap"]:
raise ValueError(
"`borderMode` must be set to one of the following values: `mirror`, `padding`, `edge` or `wrap`.")
self._regressionParameters = regressionParameters
self._solver = solver
n_inputDimensions = len(inputShape)
if filterSize % 2 == 0:
raise ValueError("filterSize has to be an odd number (1, 3, 5, ...).")
self._filterSize = filterSize
self._filterWidth = int(np.floor(filterSize / 2))
self._stride = stride
self._n_input = int(np.power(np.ceil(filterSize / stride), n_inputDimensions))
self.n_inputDimensions = n_inputDimensions
self.inputShape = inputShape
if not self._averageOutputWeights:
self._WOuts = B.empty((np.prod(inputShape), 1, self._n_input + n_reservoir + 1))
self._WOut = None
else:
self._WOuts = None
self._WOut = B.zeros((1, self._n_input + n_reservoir + 1))
self._xs = B.empty((np.prod(inputShape), n_reservoir, 1))
if nWorkers == "auto":
self._nWorkers = np.max((cpu_count() - 1, 1))
else:
self._nWorkers = nWorkers
manager = Manager()
self.sharedNamespace = manager.Namespace()
if hasattr(self, "fitWorkerID") == False or self.parallelWorkerIDs is None:
self.parallelWorkerIDs = manager.Queue()
for i in range(self._nWorkers):
self.parallelWorkerIDs.put((i))
super(SpatioTemporalESN, self).__init__(n_input=self._n_input, n_reservoir=n_reservoir, n_output=1,
spectralRadius=spectralRadius,
noiseLevel=noiseLevel, inputScaling=inputScaling,
leakingRate=leakingRate, reservoirDensity=reservoirDensity,
randomSeed=randomSeed, out_activation=out_activation,
out_inverse_activation=out_inverse_activation,
weightGeneration=weightGeneration, bias=bias, outputBias=outputBias,
outputInputScaling=outputInputScaling,
inputDensity=inputDensity, activation=activation,
activationDerivation=activationDerivation)
"""
def generate(self,
n,
inputData=None,
initialOutputData=None,
continuation=True,
initialData=None,
update_processor=lambda x: x,
verbose=0):
#initialOutputData is the output of the last step BEFORE the generation shall start, e.g. the last step of the training's output
#check the input data
#if (self.n_input != self.n_output):
# raise ValueError("n_input does not equal n_output. The generation mode uses its own output as its input. Therefore, n_input has to be equal to n_output - please adjust these numbers!")
if inputData is not None:
inputData = B.array(inputData)
if initialOutputData is not None:
initialOutputData = B.array(initialOutputData)
if initialData is not None:
initialData = B.array(initialData)
if initialOutputData is None and initialData is None:
raise ValueError(
"Either intitialOutputData or initialData must be different from None, as the network needs an initial output value"
)
if initialOutputData is None and initialData is not None:
initialOutputData = initialData[1][-1]
if inputData is not None:
inputData = B.array(inputData)
if initialData is not None:
initialData = B.array(initialData)
#let some input run through the ESN to initialize its states from a new starting value
if not continuation:
self._x = B.zeros(self._x.shape)
if initialData is not None:
if type(initialData) is tuple:
initialDataInput, initialDataOutput = initialData
if initialDataInput is not None and len(
initialDataInput) != len(initialDataOutput):
raise ValueError(
"Length of the inputData and the outputData of the initialData tuple do not match."
)
else:
raise ValueError(
"initialData has to be a tuple consisting out of the input and the output data."
)
for t in range(initialDataInput.shape[0]):
super(PredictionESN, self).update(initialDataInput[t],
initialDataOutput[t])
if self.n_input != 0:
if inputData is None:
raise ValueError("inputData must not be None.")
elif len(inputData) < n:
raise ValueError("Length of inputData has to be >= n.")
_, Y = self.propagate(inputData,
None,
verbose=verbose,
steps=n,
previousOutputData=initialOutputData)
Y = update_processor(Y)
#return the result
return Y.T
def fit(
self,
inputData,
outputData,
transientTime="AutoReduce",
transientTimeCalculationEpsilon=1e-3,
transientTimeCalculationLength=20,
verbose=0,
):
# check the input data
if inputData.shape[0] != outputData.shape[0]:
raise ValueError(
"Amount of input and output datasets is not equal - {0} != {1}"
.format(inputData.shape[0], outputData.shape[0]))
nSequences = inputData.shape[0]
trainingLength = inputData.shape[1]
self._x = B.zeros((self.n_reservoir, 1))
# Automatic transient time calculations
if transientTime == "Auto":
transientTime = self.calculateTransientTime(
inputData,
outputData,
transientTimeCalculationEpsilon,
transientTimeCalculationLength,
)
if transientTime == "AutoReduce":
if (inputData is None
and outputData.shape[1] == 1) or inputData.shape[1] == 1:
transientTime = self.calculateTransientTime(
inputData,
outputData,
transientTimeCalculationEpsilon,
transientTimeCalculationLength,
)
transientTime = self.reduceTransientTime(
inputData, outputData, transientTime)
else:
print(
"Transient time reduction is supported only for 1 dimensional input."
)
self._X = B.zeros((
1 + self.n_input + self.n_reservoir,
nSequences * (trainingLength - transientTime),
))
Y_target = B.zeros(
(self.n_output, (trainingLength - transientTime) * nSequences))
if verbose > 0:
bar = progressbar.ProgressBar(max_value=len(inputData),
redirect_stdout=True,
poll_interval=0.0001)
bar.update(0)
for n in range(len(inputData)):
self._x = B.zeros((self.n_reservoir, 1))
self._X[:, n * (trainingLength - transientTime):(n + 1) *
(trainingLength - transientTime), ] = self.propagate(
inputData[n], transientTime=transientTime, verbose=0)
# set the target values
Y_target[:, n * (trainingLength - transientTime):(n + 1) *
(trainingLength - transientTime), ] = np.tile(
self.out_inverse_activation(outputData[n]),
trainingLength - transientTime,
).T
if verbose > 0:
bar.update(n)
if verbose > 0:
bar.finish()
if self._solver == "pinv":
self._WOut = B.dot(Y_target, B.pinv(self._X))
# calculate the training prediction now
train_prediction = self.out_activation((B.dot(self._WOut,
self._X)).T)
elif self._solver == "lsqr":
X_T = self._X.T
self._WOut = B.dot(
B.dot(Y_target, X_T),
B.inv(
B.dot(self._X, X_T) + self._regressionParameters[0] *
B.identity(1 + self.n_input + self.n_reservoir)),
)
"""
#alternative represantation of the equation
Xt = X.T
A = np.dot(X, Y_target.T)
B = np.linalg.inv(np.dot(X, Xt) + regression_parameter*np.identity(1+self.n_input+self.n_reservoir))
self._WOut = np.dot(B, A)
self._WOut = self._WOut.T
"""
# calculate the training prediction now
train_prediction = self.out_activation(
B.dot(self._WOut, self._X).T)
elif self._solver in [
"sklearn_auto",
"sklearn_lsqr",
"sklearn_sag",
"sklearn_svd",
]:
mode = self._solver[8:]
params = self._regressionParameters
params["solver"] = mode
self._ridgeSolver = Ridge(**params)
self._ridgeSolver.fit(self._X.T, Y_target.T)
# calculate the training prediction now
train_prediction = self.out_activation(
self._ridgeSolver.predict(self._X.T))
elif self._solver in ["sklearn_svr", "sklearn_svc"]:
self._ridgeSolver = SVR(**self._regressionParameters)
self._ridgeSolver.fit(self._X.T, Y_target.T.flatten())
# calculate the training prediction now
train_prediction = self.out_activation(
self._ridgeSolver.predict(self._X.T))
train_prediction = np.mean(train_prediction, 0)
# calculate the training error now
training_error = B.sqrt(B.mean((train_prediction - outputData.T)**2))
return training_error
def fit_loop(self,
inputData,
outputData1,
outputData2,
transientTime="AutoReduce",
transientTimeCalculationEpsilon=1e-3,
transientTimeCalculationLength=20,
verbose=0):
#check the input data
if self.n_input != 0:
if len(inputData.shape) == 3 and len(outputData1.shape) > 1:
#multiple time series are used with a shape (timeseries, time, dimension) -> (timeseries, time, dimension)
if inputData.shape[0] != outputData1.shape[0]:
raise ValueError(
"Amount of input and output datasets is not equal - {0} != {1}"
.format(inputData.shape[0], outputData1.shape[0]))
if inputData.shape[1] != outputData1.shape[1]:
raise ValueError(
"Amount of input and output time steps is not equal - {0} != {1}"
.format(inputData.shape[1], outputData1.shape[1]))
else:
if inputData.shape[0] != outputData1.shape[0]:
raise ValueError(
"Amount of input and output time steps is not equal - {0} != {1}"
.format(inputData.shape[0], outputData1.shape[0]))
else:
if inputData is not None:
raise ValueError(
"n_input has been set to zero. Therefore, the given inputData will not be used."
)
inputData = B.array(inputData)
outputData1 = B.array(outputData1)
outputData2 = B.array(outputData2)
self._transientTime = transientTime
self._esn1.resetState()
self._esn2.resetState()
total_length = inputData.shape[0]
print("total_length ", total_length)
if (verbose > 0):
bar = progressbar.ProgressBar(max_value=total_length,
redirect_stdout=True,
poll_interval=0.0001)
bar.update(0)
#should be named aggregated_y
aggregated_y1 = B.empty((outputData1.shape))
aggregated_y2 = B.empty((outputData2.shape))
# X1, _ = self._esn1.propagate(inputData=inputData, outputData=outputData1, transientTime=transientTime, verbose=verbose-1)
# self._esn1._X = X1
# Y_target = self.out_inverse_activation(outputData1).T
# self._esn1._WOut = B.dot(Y_target, B.pinv(X1))
# y1 = self.out_activation((B.dot(self._esn1._WOut, X1)).T)
# training_error1 = B.sqrt(B.mean((y1.reshape((total_length, self._n_output1))- outputData1)**2))
# training_error2 = 0
y2 = self.out_activation(B.dot(self._esn2._WOut, self._esn2._X).T)
for i in range((total_length)):
#input: input + output from layer 2
inputDataWithFeedback = B.zeros((self.n_input + self._n_output2))
inputDataWithFeedback[:self.n_input] = inputData[i, :]
inputDataWithFeedback[self.n_input:] = y2
#update models
X1, untrained_y1 = self._esn1.propagateOneStep(
inputData=inputDataWithFeedback,
outputData=outputData1[i],
step=i,
transientTime=transientTime,
verbose=verbose - 1,
learn=True)
#self._esn1._X = X1
#y after model update (ideally we would use y before model update)
#y1 = self.out_activation((B.dot(self._esn1._WOut, X1)).T)
aggregated_y1[i, :] = untrained_y1.reshape(self._n_output1)
#output from 1st layer and correct output
# input2 = B.vstack((y1.reshape(self._n_output1), outputData1[i]))
# x2, y2 = self._esn2.propagateOneStep(inputData=input2, outputData=outputData2[i], step=i, transientTime=transientTime, verbose=verbose-1, learn=True)
# Y_target2[i,:] = y2.reshape(self._n_output2)
if (verbose > 0):
bar.update(i)
if (verbose > 0):
bar.finish()
self._training_res = aggregated_y1
training_error1 = B.sqrt(B.mean((aggregated_y1 - outputData1)**2))
training_error2 = B.sqrt(B.mean((aggregated_y2 - outputData2)**2))
print("training errors")
print(training_error1)
print(training_error2)
return training_error1, training_error2
def fit(self,
inputData,
outputData1,
outputData2,
transientTime="AutoReduce",
transientTimeCalculationEpsilon=1e-3,
transientTimeCalculationLength=20,
verbose=0):
#check the input data
if self.n_input != 0:
if len(inputData.shape) == 3 and len(outputData1.shape) > 1:
#multiple time series are used with a shape (timeseries, time, dimension) -> (timeseries, time, dimension)
if inputData.shape[0] != outputData1.shape[0]:
raise ValueError(
"Amount of input and output datasets is not equal - {0} != {1}"
.format(inputData.shape[0], outputData1.shape[0]))
if inputData.shape[1] != outputData1.shape[1]:
raise ValueError(
"Amount of input and output time steps is not equal - {0} != {1}"
.format(inputData.shape[1], outputData1.shape[1]))
else:
if inputData.shape[0] != outputData1.shape[0]:
raise ValueError(
"Amount of input and output time steps is not equal - {0} != {1}"
.format(inputData.shape[0], outputData1.shape[0]))
else:
if inputData is not None:
raise ValueError(
"n_input has been set to zero. Therefore, the given inputData will not be used."
)
inputData = B.array(inputData)
outputData1 = B.array(outputData1)
outputData2 = B.array(outputData2)
self._transientTime = transientTime
self._esn1.resetState()
self._esn2.resetState()
total_length = inputData.shape[0]
print("total_length ", total_length)
aggregated_y1 = B.empty((outputData1.shape))
aggregated_y2 = B.empty((outputData2.shape))
#put pixels and switch data together
inputDataWithFeedback = B.zeros(
(total_length, self.n_input + self._n_output2))
inputDataWithFeedback[:, :self.n_input] = inputData
inputDataWithFeedback[:, self.n_input:] = outputData2
X1, _ = self._esn1.propagate(inputData=inputDataWithFeedback,
outputData=outputData1,
transientTime=transientTime,
verbose=verbose - 1)
self._esn1._X = X1
Y_target = self.out_inverse_activation(outputData1).T
self._esn1._WOut = B.dot(Y_target, B.pinv(X1))
aggregated_y1 = self.out_activation((B.dot(self._esn1._WOut, X1)).T)
training_error1 = B.sqrt(
B.mean((aggregated_y1.reshape(
(total_length, self._n_output1)) - outputData1)**2))
training_error2 = 0
training_error1 = B.sqrt(B.mean((aggregated_y1 - outputData1)**2))
training_error2 = B.sqrt(B.mean((aggregated_y2 - outputData2)**2))
return training_error1, training_error2
def propagate(self, inputData, outputData=None, transientTime=0, verbose=0, x=None, steps="auto", feedbackData=None):
if x is None:
x = self._x
inputLength = steps
if inputData is None:
if outputData is not None:
inputLength = len(outputData)
else:
inputLength = len(inputData)
if inputLength == "auto":
raise ValueError("inputData and outputData are both None. Therefore, steps must not be `auto`.")
# define states' matrix
X = B.zeros((1 + self.n_input + self.n_reservoir, inputLength - transientTime))
if (verbose > 0):
bar = progressbar.ProgressBar(max_value=inputLength, redirect_stdout=True, poll_interval=0.0001)
bar.update(0)
if self._WFeedback is None:
#do not distinguish between whether inputData is None or not, as the feedback has been disabled
#therefore, the input has to be anything but None
for t in range(inputLength):
u, x = self.update(inputData[t], x=x)
if (t >= transientTime):
#add valueset to the states' matrix
X[:,t-transientTime] = B.vstack((B.array(self._outputBias), self._outputInputScaling*u, x))[:,0]
Y = B.dot(self._WOut, X)
if (verbose > 0):
bar.update(t)
else:
if outputData is None:
Y = B.empty((inputLength-transientTime, self.n_output))
if feedbackData is None:
feedbackData = B.zeros((1, self.n_output))
if inputData is None:
for t in range(inputLength):
self.update(None, feedbackData, x=x)
if (t >= transientTime):
#add valueset to the states' matrix
X[:,t-transientTime] = B.vstack((B.array(self._outputBias), x))[:,0]
if outputData is None:
#calculate the prediction using the trained model
if (self._solver in ["sklearn_auto", "sklearn_lsqr", "sklearn_sag", "sklearn_svd"]):
feedbackData = self._ridgeSolver.predict(B.vstack((B.array(self._outputBias), self._x)).T)
else:
feedbackData = B.dot(self._WOut, B.vstack((B.array(self._outputBias), self._x)))
if t >= transientTime:
Y[t-transientTime, :] = feedbackData
else:
feedbackData = outputData[t]
if (verbose > 0):
bar.update(t)
else:
for t in range(inputLength):
u = self.update(inputData[t], feedbackData, x=x)
if (t >= transientTime):
#add valueset to the states' matrix
X[:,t-transientTime] = B.vstack((B.array(self._outputBias), self._outputInputScaling*u, x))[:,0]
if outputData is None:
#calculate the prediction using the trained model
if (self._solver in ["sklearn_auto", "sklearn_lsqr", "sklearn_sag", "sklearn_svd"]):
feedbackData = self._ridgeSolver.predict(B.vstack((B.array(self._outputBias), self._outputInputScaling*u, self._x)).T)
else:
feedbackData = B.dot(self._WOut, B.vstack((B.array(self._outputBias), self._outputInputScaling*u, self._x)))
Y[t, :] = feedbackData.ravel()
else:
feedbackData = outputData[t]
if (verbose > 0):
bar.update(t)
if (verbose > 0):
bar.finish()
return X, Y
def __init__(
self,
n_input,
n_reservoir,
n_output,
spectralRadius=1.0,
noiseLevel=0.0,
inputScaling=None,
leakingRate=1.0,
feedbackScaling=1.0,
reservoirDensity=0.2,
randomSeed=None,
out_activation=lambda x: x,
out_inverse_activation=lambda x: x,
weightGeneration="naive",
bias=1.0,
outputBias=1.0,
feedback=False,
outputInputScaling=1.0,
inputDensity=1.0,
solver="pinv",
regressionParameters={},
activation=B.tanh,
activationDerivative=lambda x: 1.0 / B.cosh(x)**2,
):
""" ESN that predicts (steps of) a time series based on a time series.
Args:
n_input : Dimensionality of the input.
n_reservoir : Number of units in the reservoir.
n_output : Dimensionality of the output.
spectralRadius : Spectral radius of the reservoir's connection/weight matrix.
noiseLevel : Magnitude of noise that is added to the input while fitting to prevent overfitting.
inputScaling : Scaling factor of the input.
leakingRate : Convex combination factor between 0 and 1 that weights current and new state value.
feedbackScaling : Rescaling factor of the output-to-input feedback in the update process.
reservoirDensity : Percentage of non-zero weight connections in the reservoir.
randomSeed : Seed for random processes, e.g. weight initialization.
out_activation : Final activation function (i.e. activation function of the output).
out_inverse_activation : Inverse of the final activation function
weightGeneration : Algorithm to generate weight matrices. Choices: naive, SORM, advanced, custom
bias : Size of the bias added for the internal update process.
outputBias : Size of the bias added for the final linear regression of the output.
feedback : Include output-input feedback in the ESN.
outputInputScaling : Rescaling factor for the input of the ESN for the regression.
inputDensity : Percentage of non-zero weights in the input-to-reservoir weight matrix.
solver : Algorithm to find output matrix. Choices: pinv, lsqr.
regressionParameters : Arguments to the solving algorithm. For LSQR this controls the L2 regularization.
activation : (Non-linear) Activation function.
activationDerivative : Derivative of the activation function.
"""
super(PredictionESN, self).__init__(
n_input=n_input,
n_reservoir=n_reservoir,
n_output=n_output,
spectralRadius=spectralRadius,
noiseLevel=noiseLevel,
inputScaling=inputScaling,
leakingRate=leakingRate,
feedbackScaling=feedbackScaling,
reservoirDensity=reservoirDensity,
randomSeed=randomSeed,
feedback=feedback,
out_activation=out_activation,
out_inverse_activation=out_inverse_activation,
weightGeneration=weightGeneration,
bias=bias,
outputBias=outputBias,
outputInputScaling=outputInputScaling,
inputDensity=inputDensity,
activation=activation,
activationDerivative=activationDerivative,
)
self._solver = solver
self._regressionParameters = regressionParameters
self._x = B.zeros((self.n_reservoir, 1))
"""
def resetState(self, index=None):
if index is None:
self._x = B.zeros((self._nWorkers, self.n_reservoir, 1))
else:
self._x[index] = B.zeros((self.n_reservoir, 1))
def predict(self, inputData, transientTime=0, update_processor=lambda x: x, verbose=0):
rank = len(inputData.shape) - 1
if rank != self.n_inputDimensions:
raise ValueError(
"The `inputData` does not have a suitable shape. It has to have {0} spatial dimensions and 1 temporal dimension.".format(
self.n_inputDimensions))
manager = Manager()
predictQueue = manager.Queue()
# workaround as predict does not support batches atm
# add dummy dimension to let embedInputData work properly (is optimized to work for batches)
inputData = inputData.reshape(1, *inputData.shape)
modifiedInputData = self._embedInputData(inputData)
modifiedInputData = modifiedInputData[0]
inputData = inputData[0]
self.transientTime = transientTime
self.sharedNamespace.transientTime = transientTime
predictionOutput = B.zeros(np.insert(self.inputShape, 0, inputData.shape[0] - transientTime))
jobs = np.stack(np.meshgrid(*[np.arange(x) + self._filterWidth for x in inputData.shape[1:]]),
axis=rank).reshape(-1, rank).tolist()
nJobs = len(jobs)
self.resetState()
iterator = PredictionArrayIterator(modifiedInputData, jobs, self._filterWidth, self._stride, self)
pool = Pool(processes=self._nWorkers, initializer=SpatioTemporalESN._init_predictProcess,
initargs=[predictQueue, self])
pool.map_async(self._predictProcess, iterator, chunksize=200)#, chunksize=1)
def _processPoolWorkerResults():
nJobsDone = 0
if verbose > 0:
bar = progressbar.ProgressBar(max_value=nJobs, redirect_stdout=True, poll_interval=0.0001)
bar.update(0)
while nJobsDone < nJobs:
data = predictQueue.get()
# result of predicting
indices, prediction, state = data
id = self._uniqueIDFromIndices(indices)
self._xs[id] = state
# update the values
predictionOutput[tuple([Ellipsis] + indices)] = prediction
nJobsDone += 1
if verbose > 0:
bar.update(nJobsDone)
if verbose > 1:
print(nJobsDone)
if verbose > 0:
bar.finish()
_processPoolWorkerResults()
pool.close()
return predictionOutput
def __init__(
self,
inputShape,
n_reservoir,
filterSize=1,
stride=1,
borderMode="mirror",
nWorkers="auto",
spectralRadius=1.0,
noiseLevel=0.0,
inputScaling=None,
leakingRate=1.0,
reservoirDensity=0.2,
randomSeed=None,
averageOutputWeights=True,
out_activation=lambda x: x,
out_inverse_activation=lambda x: x,
weightGeneration="naive",
bias=1.0,
outputBias=1.0,
outputInputScaling=1.0,
inputDensity=1.0,
solver="pinv",
regressionParameters={},
activation=B.tanh,
activationDerivative=lambda x: 1.0 / B.cosh(x)**2,
chunkSize=16,
):
""" ESN that predicts (steps of) a spatio-temporal time series based on a time series.
Args:
inputShape : Shape of the input w/o the time axis, e.g. (W, H) for a 2D input.
n_reservoir : Number of units in the reservoir.
filterSize : Size of patches used to predict a single output element.
stride : Stride between different patches.
borderMode : How to handle border values. Choices: mirror, padding, edge, wrap.
nWorkers : Number of CPU threads executed in parallel to solve the problem.
spectralRadius : Spectral radius of the reservoir's connection/weight matrix.
noiseLevel : Magnitude of noise that is added to the input while fitting to prevent overfitting.
inputScaling : Scaling factor of the input.
leakingRate : Convex combination factor between 0 and 1 that weights current and new state value.
reservoirDensity : Percentage of non-zero weight connections in the reservoir.
randomSeed : Seed for random processes, e.g. weight initialization.
averageOutputWeights : Average output matrices after fitting across all pixels or use a distinct matrix
per pixel. The former assumes homogeneity of the problem across all pixels.
out_activation : Final activation function (i.e. activation function of the output).
out_inverse_activation : Inverse of the final activation function
weightGeneration : Algorithm to generate weight matrices. Choices: naive, SORM, advanced, custom
bias : Size of the bias added for the internal update process.
outputBias : Size of the bias added for the final linear regression of the output.
outputInputScaling : Rescaling factor for the input of the ESN for the regression.
inputDensity : Percentage of non-zero weights in the input-to-reservoir weight matrix.
solver : Algorithm to find output matrix. Choices: pinv, lsqr.
regressionParameters : Arguments to the solving algorithm. For LSQR this controls the L2 regularization.
activation : (Non-linear) Activation function.
activationDerivative : Derivative of the activation function.
chunkSize : Internal parameter for the multi-threading. For long time series this should be reduced to
avoid OOM errors/getting stuck and to reduce memory consumption.
"""
self._averageOutputWeights = averageOutputWeights
if averageOutputWeights and solver != "lsqr":
raise ValueError(
"`averageOutputWeights` can only be set to `True` when `solver` is set to `lsqr` (Ridge Regression)"
)
self._borderMode = borderMode
if not borderMode in ["mirror", "padding", "edge", "wrap"]:
raise ValueError(
"`borderMode` must be set to one of the following values: `mirror`, `padding`, `edge` or `wrap`."
)
self._regressionParameters = regressionParameters
self._solver = solver
n_inputDimensions = len(inputShape)
if filterSize % 2 == 0:
raise ValueError(
"filterSize has to be an odd number (1, 3, 5, ...).")
self._filterSize = filterSize
self._filterWidth = int(np.floor(filterSize / 2))
self._stride = stride
self._n_input = int(
np.power(np.ceil(filterSize / stride), n_inputDimensions))
self.n_inputDimensions = n_inputDimensions
self.inputShape = inputShape
if not self._averageOutputWeights:
self._WOuts = B.empty(
(np.prod(inputShape), 1, self._n_input + n_reservoir + 1))
self._WOut = None
else:
self._WOuts = None
self._WOut = B.zeros((1, self._n_input + n_reservoir + 1))
self._xs = B.empty((np.prod(inputShape), n_reservoir, 1))
if nWorkers == "auto":
self._nWorkers = np.max((cpu_count() - 1, 1))
else:
self._nWorkers = nWorkers
manager = Manager()
self.sharedNamespace = manager.Namespace()
if hasattr(self,
"fitWorkerID") == False or self.parallelWorkerIDs is None:
self.parallelWorkerIDs = manager.Queue()
for i in range(self._nWorkers):
self.parallelWorkerIDs.put((i))
self._chunkSize = chunkSize
super(SpatioTemporalESN, self).__init__(
n_input=self._n_input,
n_reservoir=n_reservoir,
n_output=1,
spectralRadius=spectralRadius,
noiseLevel=noiseLevel,
inputScaling=inputScaling,
leakingRate=leakingRate,
reservoirDensity=reservoirDensity,
randomSeed=randomSeed,
out_activation=out_activation,
out_inverse_activation=out_inverse_activation,
weightGeneration=weightGeneration,
bias=bias,
outputBias=outputBias,
outputInputScaling=outputInputScaling,
inputDensity=inputDensity,
activation=activation,
activationDerivative=activationDerivative,
)
"""
