def __init__(self,
n_input,
n_reservoir,
n_output,
spectralRadius=1.0,
noiseLevel=0.0,
inputScaling=None,
leakingRate=1.0,
feedbackScaling=1.0,
reservoirDensity=0.2,
randomSeed=None,
out_activation=lambda x: x,
out_inverse_activation=lambda x: x,
weightGeneration='naive',
bias=1.0,
outputBias=1.0,
feedback=False,
outputInputScaling=1.0,
inputDensity=1.0,
solver='pinv',
regressionParameters={},
activation=B.tanh,
activationDerivation=lambda x: 1.0 / B.cosh(x)**2):
super(PredictionESN,
self).__init__(n_input=n_input,
n_reservoir=n_reservoir,
n_output=n_output,
spectralRadius=spectralRadius,
noiseLevel=noiseLevel,
inputScaling=inputScaling,
leakingRate=leakingRate,
feedbackScaling=feedbackScaling,
reservoirDensity=reservoirDensity,
randomSeed=randomSeed,
feedback=feedback,
out_activation=out_activation,
out_inverse_activation=out_inverse_activation,
weightGeneration=weightGeneration,
bias=bias,
outputBias=outputBias,
outputInputScaling=outputInputScaling,
inputDensity=inputDensity,
activation=activation,
activationDerivation=activationDerivation)
self._solver = solver
self._regressionParameters = regressionParameters
self._x = B.zeros((self.n_reservoir, 1))
#self._WOut = np.random.rand(n_output, n_reservoir+1)#bias term
self._WOut = np.random.rand(n_output,
1 + n_input + n_reservoir) #bias term
#this seems to have been defined somewhere already
self._X = B.zeros((1 + n_input + n_reservoir, 1))
"""
def __init__(self, inputShape, n_reservoir,
filterSize=1, stride=1, borderMode="mirror", nWorkers="auto",
spectralRadius=1.0, noiseLevel=0.0, inputScaling=None,
leakingRate=1.0, reservoirDensity=0.2, randomSeed=None, averageOutputWeights=True,
out_activation=lambda x: x, out_inverse_activation=lambda x: x,
weightGeneration='naive', bias=1.0, outputBias=1.0,
outputInputScaling=1.0, inputDensity=1.0, solver='pinv', regressionParameters={}, activation=B.tanh,
activationDerivation=lambda x: 1.0 / B.cosh(x) ** 2):
self._averageOutputWeights = averageOutputWeights
if averageOutputWeights and solver != "lsqr":
raise ValueError(
"`averageOutputWeights` can only be set to `True` when `solver` is set to `lsqr` (Ridge Regression)")
self._borderMode = borderMode
if not borderMode in ["mirror", "padding", "edge", "wrap"]:
raise ValueError(
"`borderMode` must be set to one of the following values: `mirror`, `padding`, `edge` or `wrap`.")
self._regressionParameters = regressionParameters
self._solver = solver
n_inputDimensions = len(inputShape)
if filterSize % 2 == 0:
raise ValueError("filterSize has to be an odd number (1, 3, 5, ...).")
self._filterSize = filterSize
self._filterWidth = int(np.floor(filterSize / 2))
self._stride = stride
self._n_input = int(np.power(np.ceil(filterSize / stride), n_inputDimensions))
self.n_inputDimensions = n_inputDimensions
self.inputShape = inputShape
if not self._averageOutputWeights:
self._WOuts = B.empty((np.prod(inputShape), 1, self._n_input + n_reservoir + 1))
self._WOut = None
else:
self._WOuts = None
self._WOut = B.zeros((1, self._n_input + n_reservoir + 1))
self._xs = B.empty((np.prod(inputShape), n_reservoir, 1))
if nWorkers == "auto":
self._nWorkers = np.max((cpu_count() - 1, 1))
else:
self._nWorkers = nWorkers
manager = Manager()
self.sharedNamespace = manager.Namespace()
if hasattr(self, "fitWorkerID") == False or self.parallelWorkerIDs is None:
self.parallelWorkerIDs = manager.Queue()
for i in range(self._nWorkers):
self.parallelWorkerIDs.put((i))
super(SpatioTemporalESN, self).__init__(n_input=self._n_input, n_reservoir=n_reservoir, n_output=1,
spectralRadius=spectralRadius,
noiseLevel=noiseLevel, inputScaling=inputScaling,
leakingRate=leakingRate, reservoirDensity=reservoirDensity,
randomSeed=randomSeed, out_activation=out_activation,
out_inverse_activation=out_inverse_activation,
weightGeneration=weightGeneration, bias=bias, outputBias=outputBias,
outputInputScaling=outputInputScaling,
inputDensity=inputDensity, activation=activation,
activationDerivation=activationDerivation)
"""
def __init__(
self,
n_input,
n_reservoir,
n_output,
spectralRadius=1.0,
noiseLevel=0.0,
inputScaling=None,
leakingRate=1.0,
reservoirDensity=0.2,
randomSeed=None,
out_activation=lambda x: x,
out_inverse_activation=lambda x: x,
weightGeneration="naive",
bias=1.0,
outputBias=1.0,
outputInputScaling=1.0,
inputDensity=1.0,
solver="pinv",
regressionParameters={},
activation=B.tanh,
activationDerivative=lambda x: 1.0 / B.cosh(x)**2,
):
""" ESN that predicts a single value from a time series
Args:
n_input : Dimensionality of the input.
n_reservoir : Number of units in the reservoir.
n_output : Dimensionality of the output.
spectralRadius : Spectral radius of the reservoir's connection/weight matrix.
noiseLevel : Magnitude of noise that is added to the input while fitting to prevent overfitting.
inputScaling : Scaling factor of the input.
leakingRate : Convex combination factor between 0 and 1 that weights current and new state value.
reservoirDensity : Percentage of non-zero weight connections in the reservoir.
randomSeed : Seed for random processes, e.g. weight initialization.
out_activation : Final activation function (i.e. activation function of the output).
out_inverse_activation : Inverse of the final activation function
weightGeneration : Algorithm to generate weight matrices. Choices: naive, SORM, advanced, custom
bias : Size of the bias added for the internal update process.
outputBias : Size of the bias added for the final linear regression of the output.
outputInputScaling : Rescaling factor for the input of the ESN for the regression.
inputDensity : Percentage of non-zero weights in the input-to-reservoir weight matrix.
solver : Algorithm to find output matrix. Choices: pinv, lsqr.
regressionParameters : Arguments to the solving algorithm. For LSQR this controls the L2 regularization.
activation : (Non-linear) Activation function.
activationDerivative : Derivative of the activation function.
"""
super(RegressionESN, self).__init__(
n_input=n_input,
n_reservoir=n_reservoir,
n_output=n_output,
spectralRadius=spectralRadius,
noiseLevel=noiseLevel,
inputScaling=inputScaling,
leakingRate=leakingRate,
reservoirDensity=reservoirDensity,
randomSeed=randomSeed,
out_activation=out_activation,
out_inverse_activation=out_inverse_activation,
weightGeneration=weightGeneration,
bias=bias,
outputBias=outputBias,
outputInputScaling=outputInputScaling,
inputDensity=inputDensity,
activation=activation,
activationDerivative=activationDerivative,
)
self._solver = solver
self._regressionParameters = regressionParameters
"""
def __init__(self,
n_input,
n_output1,
n_output2,
n_reservoir,
spectralRadius=1.0,
noiseLevel=0.0,
inputScaling=None,
leakingRate=1.0,
feedbackScaling=1.0,
reservoirDensity=0.2,
randomSeed=None,
out_activation=lambda x: x,
out_inverse_activation=lambda x: x,
weightGeneration='naive',
bias=1.0,
outputBias=1.0,
outputInputScaling=1.0,
inputDensity=1.0,
solver='pinv',
regressionParameters={},
activation=B.tanh,
activationDerivation=lambda x: 1.0 / B.cosh(x)**2):
#probably not needed
super(StackedESN,
self).__init__(n_input=n_input,
n_reservoir=n_reservoir,
n_output=n_output1,
spectralRadius=spectralRadius,
noiseLevel=noiseLevel,
inputScaling=inputScaling,
leakingRate=leakingRate,
feedbackScaling=feedbackScaling,
reservoirDensity=reservoirDensity,
randomSeed=randomSeed,
feedback=False,
out_activation=out_activation,
out_inverse_activation=out_inverse_activation,
weightGeneration=weightGeneration,
bias=bias,
outputBias=outputBias,
outputInputScaling=outputInputScaling,
inputDensity=inputDensity,
activation=activation,
activationDerivation=activationDerivation)
#layers
self._esn1 = PredictionESN(
n_input=(n_input + n_output2),
n_reservoir=n_reservoir,
n_output=n_output1,
spectralRadius=spectralRadius,
noiseLevel=noiseLevel,
inputScaling=inputScaling,
leakingRate=leakingRate,
feedbackScaling=feedbackScaling,
reservoirDensity=reservoirDensity,
randomSeed=randomSeed,
feedback=False,
out_activation=out_activation,
out_inverse_activation=out_inverse_activation,
weightGeneration=weightGeneration,
bias=bias,
outputBias=outputBias,
outputInputScaling=outputInputScaling,
inputDensity=inputDensity,
activation=activation,
activationDerivation=activationDerivation)
#esn_2 takes coordinates in (estimated and actual)
self._esn2 = PredictionESN(
n_input=n_output1 * 2,
n_reservoir=n_reservoir,
n_output=n_output2,
spectralRadius=spectralRadius,
noiseLevel=noiseLevel,
inputScaling=inputScaling,
leakingRate=leakingRate,
feedbackScaling=feedbackScaling,
reservoirDensity=reservoirDensity,
randomSeed=randomSeed,
feedback=False,
out_activation=out_activation,
out_inverse_activation=out_inverse_activation,
weightGeneration=weightGeneration,
bias=bias,
outputBias=outputBias,
outputInputScaling=outputInputScaling,
inputDensity=inputDensity,
activation=activation,
activationDerivation=activationDerivation)
self._solver = solver
self._regressionParameters = regressionParameters
self._transientTime = 0
self._n_output1 = n_output1
self._n_output2 = n_output2
self._training_res = B.empty((2, 2))
"""
def __init__(self, n_input, n_reservoir, n_output,
spectralRadius=1.0, noiseLevel=0.01, inputScaling=None,
leakingRate=1.0, feedbackScaling = 1.0, reservoirDensity=0.2, randomSeed=None,
out_activation=lambda x: x, out_inverse_activation=lambda x: x,
weightGeneration='naive', bias=1.0, outputBias=1.0, outputInputScaling=1.0,
feedback=False, inputDensity=1.0, activation = B.tanh, activationDerivation=lambda x: 1.0/B.cosh(x)**2):
self.n_input = n_input
self.n_reservoir = n_reservoir
self.n_output = n_output
self._spectralRadius = spectralRadius
self._noiseLevel = noiseLevel
self._reservoirDensity = reservoirDensity
self._leakingRate = leakingRate
self._feedbackScaling = feedbackScaling
self.inputDensity = inputDensity
self._activation = activation
self._activationDerivation = activationDerivation
self._inputScaling = inputScaling
#this seems to be initialized somewhere (mystery)
#self._x = np.zeros(( 1+ n_reservoir, 1))
if self._inputScaling is None:
self._inputScaling = 1.0
if np.isscalar(self._inputScaling):
self._inputScaling = B.ones(n_input) * self._inputScaling
else:
if len(self._inputScaling) != self.n_input:
raise ValueError("Dimension of inputScaling ({0}) does not match the input data dimension ({1})".format(len(self._inputScaling), n_input))
self._inputScaling = inputScaling
self._expandedInputScaling = B.vstack((B.array(1.0), self._inputScaling.reshape(-1,1))).flatten()
self.out_activation = out_activation
self.out_inverse_activation = out_inverse_activation
if randomSeed is not None:
rnd.seed(randomSeed)
self._bias = bias
self._outputBias = outputBias
self._outputInputScaling = outputInputScaling
self._createReservoir(weightGeneration, feedback)
def __init__(
self,
n_input,
n_reservoir,
n_output,
spectralRadius=1.0,
noiseLevel=0.01,
inputScaling=None,
leakingRate=1.0,
feedbackScaling=1.0,
reservoirDensity=0.2,
randomSeed=None,
out_activation=lambda x: x,
out_inverse_activation=lambda x: x,
weightGeneration="naive",
bias=1.0,
outputBias=1.0,
outputInputScaling=1.0,
feedback=False,
inputDensity=1.0,
activation=B.tanh,
activationDerivative=lambda x: 1.0 / B.cosh(x)**2,
):
"""Implementation of a ESN.
Args:
n_input : Dimensionality of the input.
n_reservoir : Number of units in the reservoir.
n_output : Dimensionality of the output.
spectralRadius : Spectral radius of the reservoir's connection/weight matrix.
noiseLevel : Magnitude of noise that is added to the input while fitting to prevent overfitting.
inputScaling : Scaling factor of the input.
leakingRate : Convex combination factor between 0 and 1 that weights current and new state value.
feedbackScaling : Rescaling factor of the output-to-input feedback in the update process.
reservoirDensity : Percentage of non-zero weight connections in the reservoir.
randomSeed : Seed for random processes, e.g. weight initialization.
out_activation : Final activation function (i.e. activation function of the output).
out_inverse_activation : Inverse of the final activation function
weightGeneration : Algorithm to generate weight matrices. Choices: naive, SORM, advanced, custom
bias : Size of the bias added for the internal update process.
outputBias : Size of the bias added for the final linear regression of the output.
outputInputScaling : Rescaling factor for the input of the ESN for the regression.
feedback : Include output-input feedback in the ESN.
inputDensity : Percentage of non-zero weights in the input-to-reservoir weight matrix.
activation : (Non-linear) Activation function.
activationDerivative : Derivative of the activation function.
"""
self.n_input = n_input
self.n_reservoir = n_reservoir
self.n_output = n_output
self._spectralRadius = spectralRadius
self._noiseLevel = noiseLevel
self._reservoirDensity = reservoirDensity
self._leakingRate = leakingRate
self._feedbackScaling = feedbackScaling
self.inputDensity = inputDensity
self._activation = activation
self._activationDerivative = activationDerivative
self._inputScaling = inputScaling
if self._inputScaling is None:
self._inputScaling = 1.0
if np.isscalar(self._inputScaling):
self._inputScaling = B.ones(n_input) * self._inputScaling
else:
if len(self._inputScaling) != self.n_input:
raise ValueError(
"Dimension of inputScaling ({0}) does not match the input data dimension ({1})"
.format(len(self._inputScaling), n_input))
self._inputScaling = inputScaling
self._expandedInputScaling = B.vstack(
(B.array(1.0), self._inputScaling.reshape(-1, 1))).flatten()
self.out_activation = out_activation
self.out_inverse_activation = out_inverse_activation
if randomSeed is not None:
B.seed(randomSeed)
np.random.seed(randomSeed)
self._bias = bias
self._outputBias = outputBias
self._outputInputScaling = outputInputScaling
self._createReservoir(weightGeneration, feedback)
def __init__(
self,
inputShape,
n_reservoir,
filterSize=1,
stride=1,
borderMode="mirror",
nWorkers="auto",
spectralRadius=1.0,
noiseLevel=0.0,
inputScaling=None,
leakingRate=1.0,
reservoirDensity=0.2,
randomSeed=None,
averageOutputWeights=True,
out_activation=lambda x: x,
out_inverse_activation=lambda x: x,
weightGeneration="naive",
bias=1.0,
outputBias=1.0,
outputInputScaling=1.0,
inputDensity=1.0,
solver="pinv",
regressionParameters={},
activation=B.tanh,
activationDerivative=lambda x: 1.0 / B.cosh(x)**2,
chunkSize=16,
):
""" ESN that predicts (steps of) a spatio-temporal time series based on a time series.
Args:
inputShape : Shape of the input w/o the time axis, e.g. (W, H) for a 2D input.
n_reservoir : Number of units in the reservoir.
filterSize : Size of patches used to predict a single output element.
stride : Stride between different patches.
borderMode : How to handle border values. Choices: mirror, padding, edge, wrap.
nWorkers : Number of CPU threads executed in parallel to solve the problem.
spectralRadius : Spectral radius of the reservoir's connection/weight matrix.
noiseLevel : Magnitude of noise that is added to the input while fitting to prevent overfitting.
inputScaling : Scaling factor of the input.
leakingRate : Convex combination factor between 0 and 1 that weights current and new state value.
reservoirDensity : Percentage of non-zero weight connections in the reservoir.
randomSeed : Seed for random processes, e.g. weight initialization.
averageOutputWeights : Average output matrices after fitting across all pixels or use a distinct matrix
per pixel. The former assumes homogeneity of the problem across all pixels.
out_activation : Final activation function (i.e. activation function of the output).
out_inverse_activation : Inverse of the final activation function
weightGeneration : Algorithm to generate weight matrices. Choices: naive, SORM, advanced, custom
bias : Size of the bias added for the internal update process.
outputBias : Size of the bias added for the final linear regression of the output.
outputInputScaling : Rescaling factor for the input of the ESN for the regression.
inputDensity : Percentage of non-zero weights in the input-to-reservoir weight matrix.
solver : Algorithm to find output matrix. Choices: pinv, lsqr.
regressionParameters : Arguments to the solving algorithm. For LSQR this controls the L2 regularization.
activation : (Non-linear) Activation function.
activationDerivative : Derivative of the activation function.
chunkSize : Internal parameter for the multi-threading. For long time series this should be reduced to
avoid OOM errors/getting stuck and to reduce memory consumption.
"""
self._averageOutputWeights = averageOutputWeights
if averageOutputWeights and solver != "lsqr":
raise ValueError(
"`averageOutputWeights` can only be set to `True` when `solver` is set to `lsqr` (Ridge Regression)"
)
self._borderMode = borderMode
if not borderMode in ["mirror", "padding", "edge", "wrap"]:
raise ValueError(
"`borderMode` must be set to one of the following values: `mirror`, `padding`, `edge` or `wrap`."
)
self._regressionParameters = regressionParameters
self._solver = solver
n_inputDimensions = len(inputShape)
if filterSize % 2 == 0:
raise ValueError(
"filterSize has to be an odd number (1, 3, 5, ...).")
self._filterSize = filterSize
self._filterWidth = int(np.floor(filterSize / 2))
self._stride = stride
self._n_input = int(
np.power(np.ceil(filterSize / stride), n_inputDimensions))
self.n_inputDimensions = n_inputDimensions
self.inputShape = inputShape
if not self._averageOutputWeights:
self._WOuts = B.empty(
(np.prod(inputShape), 1, self._n_input + n_reservoir + 1))
self._WOut = None
else:
self._WOuts = None
self._WOut = B.zeros((1, self._n_input + n_reservoir + 1))
self._xs = B.empty((np.prod(inputShape), n_reservoir, 1))
if nWorkers == "auto":
self._nWorkers = np.max((cpu_count() - 1, 1))
else:
self._nWorkers = nWorkers
manager = Manager()
self.sharedNamespace = manager.Namespace()
if hasattr(self,
"fitWorkerID") == False or self.parallelWorkerIDs is None:
self.parallelWorkerIDs = manager.Queue()
for i in range(self._nWorkers):
self.parallelWorkerIDs.put((i))
self._chunkSize = chunkSize
super(SpatioTemporalESN, self).__init__(
n_input=self._n_input,
n_reservoir=n_reservoir,
n_output=1,
spectralRadius=spectralRadius,
noiseLevel=noiseLevel,
inputScaling=inputScaling,
leakingRate=leakingRate,
reservoirDensity=reservoirDensity,
randomSeed=randomSeed,
out_activation=out_activation,
out_inverse_activation=out_inverse_activation,
weightGeneration=weightGeneration,
bias=bias,
outputBias=outputBias,
outputInputScaling=outputInputScaling,
inputDensity=inputDensity,
activation=activation,
activationDerivative=activationDerivative,
)
"""