def __reservoirTrain__(self, x):
#Extract the parameters
meanDegree, beta = x
meanDegree = int(meanDegree)
# To get rid off the randomness in assigning weights, run it 10 times and take the average error
times = 100
cumulativeError = 0
for i in range(times):
# Input and weight connectivity Matrix
inputWeightMatrix = topology.ClassicInputTopology(
self.inputD, self.size).generateWeightMatrix()
reservoirWeightMatrix = topology.SmallWorldGraphs(
size=self.size, meanDegree=meanDegree,
beta=beta).generateWeightMatrix()
#Create the reservoir
res = classicESN.Reservoir(
size=self.size,
spectralRadius=self.spectralRadius,
inputScaling=self.inputScaling,
reservoirScaling=self.reservoirScaling,
leakingRate=self.leakingRate,
initialTransient=self.initialTransient,
inputData=self.trainingInputData,
outputData=self.trainingOutputData,
inputWeightRandom=inputWeightMatrix,
reservoirWeightRandom=reservoirWeightMatrix)
#Train the reservoir
res.trainReservoir()
# Warm up
predictedTrainingOutputData = res.predict(
self.trainingInputData[-self.initialTransient:])
#Predict for the validation data
predictedOutputData = util.predictFuture(res, self.initialSeed,
self.horizon)
gc.collect()
#Calcuate the regression error
errorFunction = metrics.MeanSquareError()
error = errorFunction.compute(self.validationOutputData,
predictedOutputData)
cumulativeError += error
regressionError = cumulativeError / times
#Return the error
print("\nThe Parameters: " + str(x) + " Regression error:" +
str(regressionError))
return regressionError
def evaluate(self, x):
# Extract the parameters
attachment = int(x[0, 0])
# To get rid off the randomness in assigning weights, run it 10 times and take the average error
times = 1
cumulativeError = 0
for i in range(times):
# Input and weight connectivity Matrix
inputWeightMatrix = topology.ClassicInputTopology(
self.inputD, self.size).generateWeightMatrix()
network = topology.ScaleFreeNetworks(size=self.size,
attachmentCount=attachment)
reservoirWeightMatrix = network.generateWeightMatrix()
# Create the reservoir
res = esn.Reservoir(size=self.size,
spectralRadius=self.spectralRadius,
inputScaling=self.inputScaling,
reservoirScaling=self.reservoirScaling,
leakingRate=self.leakingRate,
initialTransient=self.initialTransient,
inputData=self.trainingInputData,
outputData=self.trainingOutputData,
inputWeightRandom=inputWeightMatrix,
reservoirWeightRandom=reservoirWeightMatrix)
# Train the reservoir
res.trainReservoir()
# Warm up
predictedTrainingOutputData = res.predict(
self.trainingInputData[-self.initialTransient:])
# Predict for the validation data
predictedOutputData = util.predictFuture(res, self.initialSeed,
self.horizon)
gc.collect()
# Calculate the regression error
errorFunction = metrics.MeanSquareError()
error = errorFunction.compute(self.validationOutputData,
predictedOutputData)
cumulativeError += error
regressionError = cumulativeError / times
# Return the error
#print("Attachment: "+str(attachment) + "Error: "+str(regressionError))
return regressionError
def __collectOutputs__(self, featureIndices):
# Form the feature and target vectors
featureVectors = self.__formFeaturevectors__(self.inputData,
featureIndices)
targetVectors = self.outputData
# Append bias
featureVectors = np.hstack((np.ones(
(featureVectors.shape[0], 1)), featureVectors))
# Input weight matrix
inputSize = featureVectors.shape[1]
reservoirSize = self.lowerLayerParameters['size']
inputWeightRandom = topology.RandomInputTopology(
inputSize, reservoirSize, self.
lowerLayerParameters['inputConnectivity']).generateWeightMatrix()
reservoirWeightRandom = topology.RandomReservoirTopology(
reservoirSize, self.lowerLayerParameters['reservoirConnectivity']
).generateWeightMatrix()
# Generate the reservoir
res = esn.Reservoir(
size=self.lowerLayerParameters['size'],
spectralRadius=self.lowerLayerParameters['spectralRadius'],
inputScaling=self.lowerLayerParameters['inputScaling'],
reservoirScaling=self.lowerLayerParameters['reservoirScaling'],
leakingRate=self.lowerLayerParameters['leakingRate'],
initialTransient=self.lowerLayerParameters['initialTransient'],
inputData=featureVectors,
outputData=targetVectors,
inputWeightRandom=inputWeightRandom,
reservoirWeightRandom=reservoirWeightRandom,
reservoirActivationFunction=self.
lowerLayerParameters['reservoirActivation'],
outputActivationFunction=self.
lowerLayerParameters['outputActivation'])
# Train the reservoir
res.trainReservoir()
# Just assign the weights randomly
# Store the reservoir
self.lowerLayerNetworks.append(res)
# Collect the outputs
outputs = res.predict(featureVectors)
return outputs
def tuneTrainPredictGA(trainingInputData, trainingOutputData, validationOutputData,
initialInputSeedForValidation, testingData, size = 256,initialTransient=50,
spectralRadiusBound=(0.0,1.0),
inputScalingBound=(0.0,1.0),
reservoirScalingBound=(0.0,1.0),
leakingRateBound=(0.0,1.0),
reservoirTopology=None):
# Generate the input and reservoir weight matrices based on the reservoir topology
inputWeightMatrix = topology.ClassicInputTopology(inputSize=trainingInputData.shape[1], reservoirSize=size).generateWeightMatrix()
if reservoirTopology is None:
reservoirWeightMatrix = topology.ClassicReservoirTopology(size=size).generateWeightMatrix()
else: #TODO - think about matrix multiplication
reservoirWeightMatrix = reservoirTopology.generateWeightMatrix()
resTuner = tuner.ReservoirParameterTuner(size=size,
initialTransient=initialTransient,
trainingInputData=trainingInputData,
trainingOutputData=trainingOutputData,
initialSeed=initialInputSeedForValidation,
validationOutputData=validationOutputData,
spectralRadiusBound=spectralRadiusBound,
inputScalingBound=inputScalingBound,
reservoirScalingBound=reservoirScalingBound,
leakingRateBound=leakingRateBound,
inputWeightMatrix=inputWeightMatrix,
reservoirWeightMatrix=reservoirWeightMatrix)
spectralRadiusOptimum, inputScalingOptimum, reservoirScalingOptimum, leakingRateOptimum = resTuner.getOptimalParameters()
#Train
network = esn.Reservoir(size=size,
spectralRadius=spectralRadiusOptimum,
inputScaling=inputScalingOptimum,
reservoirScaling=reservoirScalingOptimum,
leakingRate=leakingRateOptimum,
initialTransient=initialTransient,
inputData=trainingInputData,
outputData=trainingOutputData,
inputWeightRandom=inputWeightMatrix,
reservoirWeightRandom=reservoirWeightMatrix)
network.trainReservoir()
warmupFeatureVectors, warmTargetVectors = formFeatureVectors(validationOutputData)
predictedWarmup = network.predict(warmupFeatureVectors[-initialTransient:])
initialInputSeedForTesing = validationOutputData[-1]
predictedOutputData = predictFuture(network, initialInputSeedForTesing, testingData.shape[0])[:,0]
return predictedOutputData
def __init__(self,
size,
initialTransient,
trainingInputData,
trainingOutputData,
initialSeed,
validationOutputData,
reservoirConnectivityBound=(0.1, 1.0),
minimizer=Minimizer.DifferentialEvolution,
initialGuess=0.5,
spectralRadius=0.79,
inputScaling=0.5,
reservoirScaling=0.5,
leakingRate=0.3):
self.size = size
self.initialTransient = initialTransient
self.trainingInputData = trainingInputData
self.trainingOutputData = trainingOutputData
self.initialSeed = initialSeed
self.validationOutputData = validationOutputData
self.reservoirConnectivityBound = reservoirConnectivityBound
self.horizon = self.validationOutputData.shape[0]
self.minimizer = minimizer
self.initialGuess = np.array([initialGuess])
# Input-to-reservoir is of Classic Type - Fully connected and maintained as constant
self.inputN, self.inputD = self.trainingInputData.shape
self.inputWeight = topology.ClassicInputTopology(
self.inputD, self.size).generateWeightMatrix()
# Other reservoir parameters are also kept constant
self.spectralRadius = spectralRadius
self.inputScaling = inputScaling
self.reservoirScaling = reservoirScaling
self.leakingRate = leakingRate
def runStandardESN():
standardError = 0
testPredictedOutputDataStandard = 0
for i in range(runTimes):
#Tune the standard reservoir
reservoirConnectivityBound = (0.1, 1.0)
resTuner = tuner.ESNConnTuner(
size=size,
initialTransient=initialTransient,
trainingInputData=trainingInputData,
trainingOutputData=trainingOutputData,
validationInputData=validationInputData,
validationOutputData=validationOutputData,
inputConnectivity=inputConnectivity,
reservoirConnectivityBound=reservoirConnectivityBound,
times=10)
reservoirConnectivityOptimum = resTuner.getOptimalParameters()
#Train the reservoir with optimal parameters
esn = ESN.EchoStateNetwork(
size=size,
inputData=trainingInputData,
outputData=trainingOutputData,
reservoirTopology=topology.RandomTopology(
size=size, connectivity=reservoirConnectivityOptimum),
inputConnectivity=inputConnectivity)
esn.trainReservoir()
#Warm up for the trained data
predictedTrainingOutputData = esn.predict(inputTrainingData)
#Predict for future
lastAvailablePoint = predictedTrainingOutputData[nTraining - 1, 0]
testingPredictedOutputData = []
for i in range(nTesting):
#Compose the query
query = [1.0]
query.append(lastAvailablePoint)
#Predict the next point
nextPoint = esn.predict(np.array(query).reshape(1, 2))[0, 0]
testingPredictedOutputData.append(nextPoint)
lastAvailablePoint = nextPoint
testingPredictedOutputData = np.array(
testingPredictedOutputData).reshape(nTesting, 1)
#Predict
testPredictedOutputDataStandard = minMax.inverse_transform(
testingPredictedOutputData)
actual = minMax.inverse_transform(testActualOutputData)
standardError += errorFunction.compute(
actual.reshape((actual.shape[0], 1)),
testPredictedOutputDataStandard.reshape(
(testPredictedOutputDataStandard.shape[0], 1)))
return testPredictedOutputDataStandard, (standardError / runTimes)
def __tune__(self):
# First tune for the input connectivity and the reservoir connectivity
connBounds = [
self.inputConnectivityBound, self.reservoirConnectivityBound
]
connResult = optimize.differential_evolution(self.__ESNConnTrain__,
bounds=connBounds,
maxiter=1)
self.inputConnectivityOptimum = connResult.x[0]
self.reservoirConnectivityOptimum = connResult.x[1]
# With tuned parameters, create the network with optimal connections and keep the connections as same
esn = EchoStateNetwork.EchoStateNetwork(
size=self.size,
inputData=self.trainingInputData,
outputData=self.trainingOutputData,
reservoirTopology=topology.RandomTopology(
size=self.size,
connectivity=self.reservoirConnectivityOptimum),
inputConnectivity=self.inputConnectivityOptimum)
self.inputWeightConn = esn.inputWeightRandom, esn.randomInputIndices
self.reservoirWeightConn = esn.reservoirWeightRandom, esn.randomReservoirIndices
# Tune the other parameters
bounds = [self.spectralRadiusBound, self.leakingRateBound]
result = optimize.differential_evolution(self.__ESNTrain__,
bounds=bounds,
maxiter=1)
return result.x[0], result.x[
1], self.inputWeightConn, self.reservoirWeightConn
def __tune__(self):
# First tune for the input connectivity and the reservoir connectivity
connBounds = [
self.inputConnectivityBound, self.meanDegreeBound, self.betaBound
]
connResult = optimize.differential_evolution(self.__ESNConnTrain__,
bounds=connBounds)
self.inputConnectivityOptimum = connResult.x[0]
self.meanDegreeOptimum = int(np.floor(connResult.x[1]))
self.betaOptimum = connResult.x[2]
# With tuned parameters, create the network with optimal connections and keep the connections as same
esn = EchoStateNetwork.EchoStateNetwork(
size=self.size,
inputData=self.trainingInputData,
outputData=self.trainingOutputData,
reservoirTopology=topology.SmallWorldGraphs(
size=self.size,
meanDegree=self.meanDegreeOptimum,
beta=self.betaOptimum),
inputConnectivity=self.inputConnectivityOptimum)
self.inputWeightConn = esn.inputWeightRandom, esn.randomInputIndices
self.reservoirWeightConn = esn.reservoirWeightRandom, esn.randomReservoirIndices
# Tune the other parameters
bounds = [
self.spectralRadiusBound, self.inputScalingBound,
self.reservoirScalingBound, self.leakingRateBound
]
result = optimize.differential_evolution(self.__ESNTrain__,
bounds=bounds)
return result.x[0], result.x[1], result.x[2], result.x[
3], self.inputWeightConn, self.reservoirWeightConn
def __ESNConnTrain__(self, x):
#Extract the parameters
inputConnectivity = x[0]
reservoirConnectivity = x[1]
reservoirTopology = topology.RandomTopology(
size=self.size, connectivity=reservoirConnectivity)
#Create the network
esn = EchoStateNetwork.EchoStateNetwork(
size=self.size,
inputData=self.trainingInputData,
outputData=self.trainingOutputData,
reservoirTopology=reservoirTopology,
inputConnectivity=inputConnectivity)
#Train the reservoir
esn.trainReservoir()
#Predict for the validation data
predictedOutputData = esn.predict(self.validationInputData)
#Calcuate the regression error
errorFunction = rmse.MeanSquareError()
regressionError = errorFunction.compute(self.validationOutputData,
predictedOutputData)
#Free the memory
gc.collect()
#Return the error
#print("Optimizing connectivity..")
#print("\nRegression error:"+str(regressionError)+"\n")
return regressionError
def trainReservoir(self):
# Features for the network in the higher layer
features = None
# Collect outputs from the lower layer
for i in range(self.lowerLayerCount):
if (features is None): # First time
features = self.__collectOutputs__(self.featureIndicesList[i])
else:
features = np.hstack(
(features,
self.__collectOutputs__(self.featureIndicesList[i])))
# Append bias
features = np.hstack((np.ones((features.shape[0], 1)), features))
# Generate the higher layer reservoir
# where features are the outputs of the lower layer networks
inputSize = features.shape[1]
reservoirSize = self.higherLayerParameters['size']
inputWeightRandom = topology.RandomInputTopology(
inputSize, reservoirSize, self.
higherLayerParameters['inputConnectivity']).generateWeightMatrix()
reservoirWeightRandom = topology.RandomReservoirTopology(
self.higherLayerParameters['size'],
self.higherLayerParameters['reservoirConnectivity']
).generateWeightMatrix()
self.higherLayerReservoir = esn.Reservoir(
size=self.higherLayerParameters['size'],
spectralRadius=self.higherLayerParameters['spectralRadius'],
inputScaling=self.higherLayerParameters['inputScaling'],
reservoirScaling=self.higherLayerParameters['reservoirScaling'],
leakingRate=self.higherLayerParameters['leakingRate'],
initialTransient=self.higherLayerParameters['initialTransient'],
inputData=features,
outputData=self.outputData,
inputWeightRandom=inputWeightRandom,
reservoirWeightRandom=reservoirWeightRandom,
reservoirActivationFunction=self.
higherLayerParameters['reservoirActivation'],
outputActivationFunction=self.
higherLayerParameters['outputActivation'])
# Train the composite network
self.higherLayerReservoir.trainReservoir()
def runErdosESN():
erdosError = 0
testPredictedOutputDataErdos = 0
for i in range(runTimes):
# Tune the Erdoys Renyi Network
probabilityBound = (
0.1, 1.0) #To avoid isolated bounds, keep the upper bound low
esnTuner = tuner.ESNErdosTuner(
size=size,
initialTransient=initialTransient,
trainingInputData=trainingInputData,
trainingOutputData=trainingOutputData,
validationInputData=validationInputData,
validationOutputData=validationOutputData,
inputConnectivity=inputConnectivity,
probabilityBound=probabilityBound,
times=10)
probabilityOptimum = esnTuner.getOptimalParameters()
res = ESN.EchoStateNetwork(
size=size,
inputData=trainingInputData,
outputData=trainingOutputData,
reservoirTopology=topology.ErdosRenyiTopology(
size=size, probability=probabilityOptimum),
inputConnectivity=inputConnectivity)
res.trainReservoir()
#Warm up using training data
trainingPredictedOutputData = res.predict(inputTrainingData)
#Predict for future
lastAvailablePoint = testInputData[0, 1]
testingPredictedOutputData = []
for i in range(nTesting):
#Compose the query
query = [1.0]
query.append(lastAvailablePoint)
#Predict the next point
nextPoint = res.predict(np.array(query).reshape(1, 2))[0, 0]
testingPredictedOutputData.append(nextPoint)
lastAvailablePoint = nextPoint
testingPredictedOutputData = np.array(
testingPredictedOutputData).reshape(nTesting, 1)
#De-normalize
actual = minMax.inverse_transform(testActualOutputData)
testPredictedOutputDataErdos = minMax.inverse_transform(
testingPredictedOutputData)
#Error
erdosError += errorFunction.compute(
actual.reshape((actual.shape[0], 1)),
testPredictedOutputDataErdos.reshape(
(testPredictedOutputDataErdos.shape[0], 1)))
return testPredictedOutputDataErdos, (erdosError / runTimes)
def runScaleFree():
scaleFreeError = 0
testPredictedOutputDataScaleFree = 0
for i in range(runTimes):
# Tune the scale free networks
attachmentBound = (1, size - 1)
esnTuner = tuner.ESNScaleFreeNetworksTuner(
size=size,
initialTransient=initialTransient,
trainingInputData=trainingInputData,
trainingOutputData=trainingOutputData,
validationInputData=validationInputData,
validationOutputData=validationOutputData,
inputConnectivity=inputConnectivity,
attachmentBound=attachmentBound,
times=5)
attachmentOptimum = esnTuner.getOptimalParameters()
res = ESN.EchoStateNetwork(
size=size,
inputData=trainingInputData,
outputData=trainingOutputData,
reservoirTopology=topology.ScaleFreeNetworks(
size=size, attachmentCount=attachmentOptimum),
inputConnectivity=inputConnectivity)
res.trainReservoir()
#Warm up using training data
trainingPredictedOutputData = res.predict(trainingInputData)
#Predict for future
lastAvailablePoint = testInputData[0, 1]
testingPredictedOutputData = []
for i in range(nTesting):
#Compose the query
query = [1.0]
query.append(lastAvailablePoint)
#Predict the next point
nextPoint = res.predict(np.array(query).reshape(1, 2))[0, 0]
testingPredictedOutputData.append(nextPoint)
lastAvailablePoint = nextPoint
testingPredictedOutputData = np.array(
testingPredictedOutputData).reshape(nTesting, 1)
#De-normalize
actual = minMax.inverse_transform(testActualOutputData)
testPredictedOutputDataScaleFree = minMax.inverse_transform(
testingPredictedOutputData)
#Error
scaleFreeError += errorFunction.compute(
actual.reshape((actual.shape[0], 1)),
testPredictedOutputDataScaleFree.reshape(
(testPredictedOutputDataScaleFree.shape[0], 1)))
return testPredictedOutputDataScaleFree, (scaleFreeError / runTimes)
def __createTransformer__(self, featureVectors):
# Append the bias
#featureVectors = np.hstack((np.ones((featureVectors.shape[0],1)),featureVectors))
featureVectors = featureVectors
targetVectors = self.outputData
# Input weight matrix
inputSize = featureVectors.shape[1]
reservoirSize = self.featureTransformerParameters['size']
inputWeightRandom = topology.RandomInputTopology(
inputSize, reservoirSize,
self.featureTransformerParameters['inputConnectivity']
).generateWeightMatrix()
reservoirWeightRandom = topology.RandomReservoirTopology(
reservoirSize,
self.featureTransformerParameters['reservoirConnectivity']
).generateWeightMatrix()
# Generate the reservoir
self.transformer = esn.Reservoir(
size=self.featureTransformerParameters['size'],
spectralRadius=self.featureTransformerParameters['spectralRadius'],
inputScaling=self.featureTransformerParameters['inputScaling'],
reservoirScaling=self.
featureTransformerParameters['reservoirScaling'],
leakingRate=self.featureTransformerParameters['leakingRate'],
initialTransient=self.
featureTransformerParameters['initialTransient'],
inputData=featureVectors,
outputData=targetVectors,
inputWeightRandom=inputWeightRandom,
reservoirWeightRandom=reservoirWeightRandom,
reservoirActivationFunction=self.
featureTransformerParameters['reservoirActivation'],
outputActivationFunction=self.
featureTransformerParameters['outputActivation'])
# Collect and return the internal state
internalStates = self.transformer.collectInternalStates(featureVectors)
return internalStates
def getNetworkStats(bestPopulation, type, size):
log = []
for item in bestPopulation:
averageDegree = 0.0
averagePathLength = 0.0
averageDiameter = 0.0
averageClusteringCoefficient = 0.0
# Run many times to get the average stats
times = 1
for i in range(times):
if(type == Topology.Random):
connectivity = item[0][0,0]
network = topology.RandomReservoirTopology(size=size, connectivity=connectivity)
elif(type == Topology.ErdosRenyi):
probability = item[0][0,0]
network = topology.ErdosRenyiTopology(size=size, probability=probability)
elif(type == Topology.ScaleFreeNetworks):
attachment = int(item[0][0,0])
network = topology.ScaleFreeNetworks(size=size, attachmentCount=attachment)
elif(type == Topology.SmallWorldGraphs):
meanDegree = item[0][0,0]
beta = item[0][1,0]
network = topology.SmallWorldGraphs(size=size, meanDegree=meanDegree, beta=beta)
averageDegree += network.networkStats.getAverageDegree()
averagePathLength += network.networkStats.getAveragePathLenth()
averageDiameter += network.networkStats.getDiameter()
averageClusteringCoefficient += network.networkStats.getAverageClusteringCoefficient()
stats = {}
stats["averageDegree"] = averageDegree/times
stats["averagePathLength"] = averagePathLength/times
stats["averageDiameter"] = averageDiameter/times
stats["averageClusteringCoefficient"] = averageClusteringCoefficient/times
log.append((item[0], item[1], stats))
return log
def trainReservoir(self):
# Feature transformation
features = self.__createTransformer__(self.inputData)
# Append bias
features = np.hstack((np.ones((features.shape[0], 1)), features))
# Generate the predictor
# where features are transformed using transformer(esn)
inputSize = features.shape[1]
reservoirSize = self.predictorParameters['size']
inputWeightRandom = topology.RandomInputTopology(
inputSize, reservoirSize, self.
predictorParameters['inputConnectivity']).generateWeightMatrix()
reservoirWeightRandom = topology.RandomReservoirTopology(
self.predictorParameters['size'],
self.predictorParameters['reservoirConnectivity']
).generateWeightMatrix()
self.predictor = esn.Reservoir(
size=self.predictorParameters['size'],
spectralRadius=self.predictorParameters['spectralRadius'],
inputScaling=self.predictorParameters['inputScaling'],
reservoirScaling=self.predictorParameters['reservoirScaling'],
leakingRate=self.predictorParameters['leakingRate'],
initialTransient=self.predictorParameters['initialTransient'],
inputData=features,
outputData=self.outputData,
inputWeightRandom=inputWeightRandom,
reservoirWeightRandom=reservoirWeightRandom,
reservoirActivationFunction=self.
predictorParameters['reservoirActivation'],
outputActivationFunction=self.
predictorParameters['outputActivation'])
# Train the predictor network
self.predictor.trainReservoir()
def __ESNConnTrain__(self, x):
#Extract the parameters
inputConnectivity = x[0]
meanDegree = int(np.floor(x[1]))
beta = x[2]
reservoirTopology = topology.SmallWorldGraphs(size=self.size,
meanDegree=meanDegree,
beta=beta)
#print("\nParameters:"+str(x))
cumRMSE = 0
times = 10
#Run many times - just to get rid of randomness in assigning random weights
for i in range(times):
#Create the network
esn = EchoStateNetwork.EchoStateNetwork(
size=self.size,
inputData=self.trainingInputData,
outputData=self.trainingOutputData,
reservoirTopology=reservoirTopology,
inputConnectivity=inputConnectivity)
#Train the reservoir
esn.trainReservoir()
#Predict for the validation data
predictedOutputData = esn.predict(self.validationInputData)
#Calcuate the regression error
errorFunction = rmse.MeanSquareError()
regressionError = errorFunction.compute(self.validationOutputData,
predictedOutputData)
cumRMSE += regressionError
#Free the memory
gc.collect()
regressionError = cumRMSE / times
#Return the error
#print("Regression error"+str(regressionError)+"\n")
return regressionError
def __ESNTrain__(self, x):
#print("\nOptimizing esn parameters:"+str(x))
#Extract the parameters
spectralRadius = x[0]
inputScaling = x[1]
reservoirScaling = x[2]
leakingRate = x[3]
#Create the reservoir
esn = EchoStateNetwork.EchoStateNetwork(
size=self.size,
inputData=self.trainingInputData,
outputData=self.trainingOutputData,
spectralRadius=spectralRadius,
inputScaling=inputScaling,
reservoirScaling=reservoirScaling,
leakingRate=leakingRate,
initialTransient=self.initialTransient,
inputWeightConn=self.inputWeightConn,
reservoirWeightConn=self.reservoirWeightConn,
reservoirTopology=topology.SmallWorldGraphs(
size=self.size,
meanDegree=self.meanDegreeOptimum,
beta=self.betaOptimum))
#Train the reservoir
esn.trainReservoir()
#Predict for the validation data
predictedOutputData = esn.predict(self.validationInputData)
#Calcuate the regression error
errorFunction = rmse.MeanSquareError()
regressionError = errorFunction.compute(self.validationOutputData,
predictedOutputData)
#Free the memory
gc.collect()
#Return the error
#print("Regression error"+str(regressionError)+"\n")
return regressionError
def __ESNConnTrain__(self, x):
#Extract the parameters
inputConnectivity = x[0]
probability = x[1]
reservoirTopology = topology.ErdosRenyiTopology(
size=self.size, probability=probability)
#print("\nInput:"+str(inputConnectivity)+" Probability:"+str(probability))
cumRMSE = 0
times = 10
#Run many times - just to get rid of randomness in assigning random weights
for i in range(times):
#Create the network
esn = EchoStateNetwork.EchoStateNetwork(
size=self.size,
inputData=self.trainingInputData,
outputData=self.trainingOutputData,
reservoirTopology=reservoirTopology,
inputConnectivity=inputConnectivity)
#Train the reservoir
esn.trainReservoir()
#Predict for the validation data
predictedOutputData = esn.predict(self.validationInputData)
#Calcuate the regression error
errorFunction = rmse.MeanSquareError()
regressionError = errorFunction.compute(self.validationOutputData,
predictedOutputData)
cumRMSE += regressionError
#Free the memory
gc.collect()
regressionError = cumRMSE / times
#Return the error
#print("\nOptimizing connectivity..")
#print("Regression error"+str(regressionError)+"\n")
return regressionError
def __ESNTrain__(self, x):
#Extract the parameters
attachment = int(np.floor(x[0]))
reservoirTopology = topology.ScaleFreeNetworks(
size=self.size, attachmentCount=attachment)
#print("\nOptimizing connectivity..")
print("\nAttachment:" + str(attachment))
cumRMSE = 0
times = self.times
for i in range(times):
#Create the network
esn = EchoStateNetwork.EchoStateNetwork(
size=self.size,
inputData=self.trainingInputData,
outputData=self.trainingOutputData,
reservoirTopology=reservoirTopology,
inputConnectivity=self.inputConnectivity)
#Train the reservoir
esn.trainReservoir()
#Predict for the validation data
predictedOutputData = esn.predict(self.validationInputData)
#Calcuate the regression error
errorFunction = rmse.MeanSquareError()
regressionError = errorFunction.compute(self.validationOutputData,
predictedOutputData)
cumRMSE += regressionError
#Free the memory
gc.collect()
regressionError = cumRMSE / times
#Return the error
print("\nRegression error:" + str(regressionError) + "\n")
return regressionError
def __ESNTrain__(self, x):
#Extract the parameters
spectralRadius = x[0]
leakingRate = x[1]
#Create the reservoir
esn = EchoStateNetwork.EchoStateNetwork(
size=self.size,
inputData=self.trainingInputData,
outputData=self.trainingOutputData,
spectralRadius=spectralRadius,
leakingRate=leakingRate,
initialTransient=self.initialTransient,
inputWeightConn=self.inputWeightConn,
reservoirWeightConn=self.reservoirWeightConn,
reservoirTopology=topology.RandomTopology(
self.size, self.reservoirConnectivityOptimum))
#Train the reservoir
esn.trainReservoir()
#Predict for the validation data
predictedOutputData = esn.predict(self.validationInputData)
#Calcuate the regression error
errorFunction = rmse.MeanSquareError()
regressionError = errorFunction.compute(self.validationOutputData,
predictedOutputData)
#Free the memory
gc.collect()
#Return the error
#print("Optimizing spectral radius and leaking rate...")
#print("\nRegression error"+str(regressionError)+"\n")
return regressionError
(nTraining, 1)), data[:nTraining].reshape((nTraining, 1))))
trainingOutputData = data[1:nTraining + 1].reshape((nTraining, 1))
#Testing data
nTesting = 300
testInputData = np.hstack((np.ones(
(nTesting, 1)), data[nTraining:nTraining + nTesting].reshape(
(nTesting, 1))))
testActualOutputData = data[nTraining + 1:nTraining + nTesting + 1].reshape(
(nTesting, 1))
#Tune and Train
size = 100
initialTransient = 5
inputConnectivity = 0.6
reservoirTopology = topology.RandomTopology(size=size, connectivity=0.9)
inputScalingBound = reservoirScalingBound = leakingRateBound = spectralRadiusBound = (
0.0, 1.0)
esnTuner = tuner.ESNTuner(size=size,
initialTransient=initialTransient,
trainingInputData=trainingInputData,
trainingOutputData=trainingOutputData,
validationInputData=trainingInputData,
validationOutputData=trainingOutputData,
inputScalingBound=inputScalingBound,
reservoirScalingBound=reservoirScalingBound,
leakingRateBound=leakingRateBound,
spectralRadiusBound=spectralRadiusBound,
reservoirTopology=reservoirTopology,
inputConnectivity=inputConnectivity)
# Step 3 - Rescale the series
normalizedSeries = util.scaleSeriesStandard(resampledSeries)
del resampledSeries
# Step 4 - Form feature and target vectors
depth = 1
featureVectors, targetVectors = util.formContinousFeatureAndTargetVectorsInOrder(
normalizedSeries, depth)
n = featureVectors.shape[0]
# Input-to-reservoir fully connected
size = 100
inputWeight = topology.ClassicInputTopology(
inputSize=featureVectors.shape[1],
reservoirSize=size).generateWeightMatrix()
# Reservoir-to-reservoir fully connected
#reservoirWeight = topology.ClassicReservoirTopology(size=size).generateWeightMatrix()
reservoirWeight = topology.SmallWorldGraphs(size=size,
meanDegree=int(size / 2),
beta=0.8).generateWeightMatrix()
res = ESN.Reservoir(size=size,
inputData=featureVectors,
outputData=targetVectors,
spectralRadius=1.5,
inputScaling=0.1,
reservoirScaling=0.5,
leakingRate=0.7,
data = data[:5000].reshape((5000, 1))
# Number of points - 5000
trainingData, testingData = util.splitData2(data, 0.4)
nTesting = testingData.shape[0]
# Form feature vectors
inputTrainingData, outputTrainingData = util.formFeatureVectors(trainingData)
# Tune the network
size = 256
initialTransient = 50
# Input-to-reservoir fully connected
inputWeight = topology.ClassicInputTopology(
inputSize=inputTrainingData.shape[1],
reservoirSize=size).generateWeightMatrix()
# Reservoir-to-reservoir fully connected
reservoirWeight = topology.ClassicReservoirTopology(
size=size).generateWeightMatrix()
res = ESN.Reservoir(size=size,
inputData=inputTrainingData,
outputData=outputTrainingData,
spectralRadius=0.79,
inputScaling=0.5,
reservoirScaling=0.5,
leakingRate=0.3,
initialTransient=initialTransient,
inputWeightRandom=inputWeight,
data = data[:5000].reshape((5000, 1))
# Number of points - 5000
trainingData, testingData = util.splitData2(data, 0.4)
nTesting = testingData.shape[0]
# Form feature vectors
inputTrainingData, outputTrainingData = util.formFeatureVectors(trainingData)
# Tune the network
size = 256
initialTransient = 50
# Input-to-reservoir 60% connected
inputWeight = topology.RandomInputTopology(
inputSize=inputTrainingData.shape[1],
reservoirSize=size,
inputConnectivity=1.0).generateWeightMatrix()
# Reservoir-to-reservoir 50% connected
reservoirWeight = topology.RandomReservoirTopology(
size=size, connectivity=0.92).generateWeightMatrix()
res = ESN.Reservoir(size=size,
inputData=inputTrainingData,
outputData=outputTrainingData,
spectralRadius=0.79,
inputScaling=0.5,
reservoirScaling=0.5,
leakingRate=0.3,
initialTransient=initialTransient,
inputWeightRandom=inputWeight,
# Remove the outliers
resampledSeries = util.detectAndRemoveOutliers(resampledSeries)
# Step 3 - Rescale the series
normalizedSeries = util.scaleSeriesStandard(resampledSeries)
del resampledSeries
# Step 4 - Split the data into training and testing series
trainingSeries, testingSeries = util.splitIntoTrainingAndTestingSeries(normalizedSeries, horizon)
# Step 4 - Form feature and target vectors
featureVectors, targetVectors = util.formContinousFeatureAndTargetVectors(trainingSeries, depth)
# Input-to-reservoir fully connected
size = 2000
inputConnMatrix = topology.RandomInputTopology(inputSize=featureVectors.shape[1], reservoirSize=size, inputConnectivity=0.7).generateConnectivityMatrix()
correlationMatrix = util.getCorrelationMatrix(featureVectors, targetVectors, size)
inputWeightMatrix = inputConnMatrix * correlationMatrix
# Reservoir-to-reservoir fully connected
reservoirWeight = topology.RandomReservoirTopology(size=size, connectivity=0.4).generateWeightMatrix()
res = ESN.Reservoir(size=size,
inputData=featureVectors,
outputData=targetVectors,
spectralRadius=0.85,
inputScaling=0.0019,
reservoirScaling=0.07,
leakingRate=0.17,
initialTransient=0,
for i in range(self.conn.shape[0]):
for j in range(self.conn.shape[0]):
if self.conn[i, j] == 1.0:
linksData += '{"source":' + str(i) + ',"target":' + str(
j) + ',"value":1},'
linksData = linksData[0:len(linksData) - 1]
self.data.write(linksData + ']')
self.data.write('}')
def createOutput(self):
self.__generateHTMLOutput()
self.__generateDataOutput()
if __name__ == '__main__':
network2 = topology.ErdosRenyiTopology(size=10, probability=0.2)
conn, indices = network2.generateConnectivityMatrix()
plot2 = NetworkPlot("ErdosRenyi_0.2.html", "", conn)
plot2.createOutput()
# network3 = topology.ErdosRenyiTopology(size = 10, probability=0.7)
# conn, indices = network3.generateConnectivityMatrix()
# plot3 = NetworkPlot("ErdosRenyi_0.7.html", "", conn)
# plot3.createOutput()
#
# network4 = topology.ErdosRenyiTopology(size = 10, probability=0.5)
# conn, indices = network4.generateConnectivityMatrix()
# plot4 = NetworkPlot("ErdosRenyi_0.5.html", "", conn)
# plot4.createOutput()
probabilityBound = (0.0, 0.9
) #To avoid isolated bounds, keep the upper bound low
errorFunction = rmse.MeanSquareError()
esnTuner = tuner.ESNErdosTuner(size=size,
initialTransient=initialTransient,
trainingInputData=inputTrainingData,
trainingOutputData=outputTrainingData,
validationInputData=inputTrainingData,
validationOutputData=outputTrainingData,
inputConnectivityBound=inputConnectivityBound,
probabilityBound=probabilityBound)
inputConnectivityOptimum, probabilityOptimum = esnTuner.getOptimalParameters()
res = esn.EchoStateNetwork(size=size,
inputData=inputTrainingData,
outputData=outputTrainingData,
reservoirTopology=topology.ErdosRenyiTopology(
size=size, probability=probabilityOptimum),
inputConnectivity=inputConnectivityOptimum)
res.trainReservoir()
#Warm up using training data
trainingPredictedOutputData = res.predict(inputTrainingData)
#Predict for future
lastAvailablePoint = testInputData[0, 1]
testingPredictedOutputData = []
for i in range(nTesting):
#Compose the query
query = [1.0]
query.append(lastAvailablePoint)
#Predict the next point
#Error
topologyError.append(
errorFunction.compute(
testActualOutputData.reshape((testActualOutputData.shape[0], 1)),
testPredictedOutputData.reshape(
(testPredictedOutputData.shape[0], 1))))
#Tune the standard reservoir
spectralRadiusBound = (0.0, 1.25)
inputScalingBound = (0.0, 1.0)
reservoirScalingBound = (0.0, 1.0)
leakingRateBound = (0.0, 1.0)
size = 100
initialTransient = 5
randomTopology = topology.RandomTopology(size=size, connectivity=0.6)
inputConnectivity = 0.8
resTuner = tuner.ESNTuner(size=size,
initialTransient=initialTransient,
trainingInputData=inputTrainingData,
trainingOutputData=outputTrainingData,
validationInputData=inputTrainingData,
validationOutputData=outputTrainingData,
spectralRadiusBound=spectralRadiusBound,
inputScalingBound=inputScalingBound,
reservoirScalingBound=reservoirScalingBound,
leakingRateBound=leakingRateBound,
reservoirTopology=randomTopology,
inputConnectivity=inputConnectivity)
spectralRadiusOptimum, inputScalingOptimum, reservoirScalingOptimum, leakingRateOptimum, inputWeightConn, reservoirConn = resTuner.getOptimalParameters(
import numpy as np
from reservoir import ReservoirTopology as topology
conn = topology.RandomTopology(size=5,
connectivity=0.6).generateConnectivityMatrix()
weight = np.random.rand(5, 5)
weight = conn * weight
scaling = 0.5
weight[weight != 0.0] = weight[weight != 0.0] - 0.5
print(weight)
size=size,
initialTransient=initialTransient,
trainingInputData=trainingInputData,
trainingOutputData=trainingOutputData,
validationInputData=trainingInputData,
validationOutputData=trainingOutputData,
inputConnectivityBound=inputConnectivityBound,
attachmentBound=attachmentBound)
inputConnectivityOptimum, attachmentOptimum = esnTuner.getOptimalParameters()
network = esn.EchoStateNetwork(size=size,
inputData=trainingInputData,
outputData=trainingOutputData,
reservoirTopology=topology.ScaleFreeNetworks(
size=size,
attachmentCount=attachmentOptimum),
inputConnectivity=inputConnectivityOptimum)
network.trainReservoir()
#Warm up for the trained data
predictedTrainingOutputData = network.predict(trainingInputData)
#Predict for future
lastAvailablePoint = predictedTrainingOutputData[nTraining - 1, 0]
testingPredictedOutputData = []
for i in range(nTesting):
#Compose the query
query = [1.0]
query.append(lastAvailablePoint)
