def tuneTrainPredictGA(trainingInputData, trainingOutputData, validationOutputData,
initialInputSeedForValidation, testingData, size = 256,initialTransient=50,
spectralRadiusBound=(0.0,1.0),
inputScalingBound=(0.0,1.0),
reservoirScalingBound=(0.0,1.0),
leakingRateBound=(0.0,1.0),
reservoirTopology=None):
# Generate the input and reservoir weight matrices based on the reservoir topology
inputWeightMatrix = topology.ClassicInputTopology(inputSize=trainingInputData.shape[1], reservoirSize=size).generateWeightMatrix()
if reservoirTopology is None:
reservoirWeightMatrix = topology.ClassicReservoirTopology(size=size).generateWeightMatrix()
else: #TODO - think about matrix multiplication
reservoirWeightMatrix = reservoirTopology.generateWeightMatrix()
resTuner = tuner.ReservoirParameterTuner(size=size,
initialTransient=initialTransient,
trainingInputData=trainingInputData,
trainingOutputData=trainingOutputData,
initialSeed=initialInputSeedForValidation,
validationOutputData=validationOutputData,
spectralRadiusBound=spectralRadiusBound,
inputScalingBound=inputScalingBound,
reservoirScalingBound=reservoirScalingBound,
leakingRateBound=leakingRateBound,
inputWeightMatrix=inputWeightMatrix,
reservoirWeightMatrix=reservoirWeightMatrix)
spectralRadiusOptimum, inputScalingOptimum, reservoirScalingOptimum, leakingRateOptimum = resTuner.getOptimalParameters()
#Train
network = esn.Reservoir(size=size,
spectralRadius=spectralRadiusOptimum,
inputScaling=inputScalingOptimum,
reservoirScaling=reservoirScalingOptimum,
leakingRate=leakingRateOptimum,
initialTransient=initialTransient,
inputData=trainingInputData,
outputData=trainingOutputData,
inputWeightRandom=inputWeightMatrix,
reservoirWeightRandom=reservoirWeightMatrix)
network.trainReservoir()
warmupFeatureVectors, warmTargetVectors = formFeatureVectors(validationOutputData)
predictedWarmup = network.predict(warmupFeatureVectors[-initialTransient:])
initialInputSeedForTesing = validationOutputData[-1]
predictedOutputData = predictFuture(network, initialInputSeedForTesing, testingData.shape[0])[:,0]
return predictedOutputData
nTesting = testingData.shape[0]
# Form feature vectors
inputTrainingData, outputTrainingData = util.formFeatureVectors(trainingData)
# Tune the network
size = 256
initialTransient = 50
# Input-to-reservoir fully connected
inputWeight = topology.ClassicInputTopology(
inputSize=inputTrainingData.shape[1],
reservoirSize=size).generateWeightMatrix()
# Reservoir-to-reservoir fully connected
reservoirWeight = topology.ClassicReservoirTopology(
size=size).generateWeightMatrix()
res = ESN.Reservoir(size=size,
inputData=inputTrainingData,
outputData=outputTrainingData,
spectralRadius=0.79,
inputScaling=0.5,
reservoirScaling=0.5,
leakingRate=0.3,
initialTransient=initialTransient,
inputWeightRandom=inputWeight,
reservoirWeightRandom=reservoirWeight)
res.trainReservoir()
#Warm up
predictedTrainingOutputData = res.predict(inputTrainingData)
# Form feature vectors
input_training, output_training = form_feature_vectors(training_data)
# Train an echo state network
size = 500
initial_transient = 50
reg_factor = 1e-4
leaking_rate = 0.3
spectral_radius = 0.79
# Input-to-reservoir fully connected
input_weight = topology.ClassicInputTopology(inputSize=input_training.shape[1], reservoirSize=size).generateWeightMatrix(scaling=1.0)
# Reservoir-to-reservoir fully connected
reservoir_weight = topology.ClassicReservoirTopology(size=size).generateWeightMatrix(scaling=1.0)
# Plot variables to hold data for comparison
plot_names = []
predicted = []
plot_names.append("actual")
predicted.append(testing_data)
# Train full batch - classic closed form solution
res = ESN.Reservoir(size=size,
input_data=torch.from_numpy(input_training.T),
output_data=torch.from_numpy(output_training.T),
leaking_rate=leaking_rate,
spectral_radius=spectral_radius,
input_weight=torch.from_numpy(input_weight),
def tuneTrainPredict(trainingInputData,
trainingOutputData,
validationOutputData,
initialInputSeedForValidation,
testingData,
size=256,
initialTransient=50,
spectralRadiusBound=(0.0, 1.0),
inputScalingBound=(0.0, 1.0),
reservoirScalingBound=(0.0, 1.0),
leakingRateBound=(0.0, 1.0),
reservoirTopology=None):
# Generate the input and reservoir weight matrices based on the reservoir topology
inputWeightMatrix = topology.ClassicInputTopology(
inputSize=trainingInputData.shape[1],
reservoirSize=size).generateWeightMatrix()
if reservoirTopology is None:
reservoirWeightMatrix = topology.ClassicReservoirTopology(
size=size).generateWeightMatrix()
else: #TODO - think about matrix multiplication
reservoirWeightMatrix = reservoirTopology.generateWeightMatrix()
resTuner = tuner.ReservoirParameterTuner(
size=size,
initialTransient=initialTransient,
trainingInputData=trainingInputData,
trainingOutputData=trainingOutputData,
initialSeed=initialInputSeedForValidation,
validationOutputData=validationOutputData,
spectralRadiusBound=spectralRadiusBound,
inputScalingBound=inputScalingBound,
reservoirScalingBound=reservoirScalingBound,
leakingRateBound=leakingRateBound,
inputWeightMatrix=inputWeightMatrix,
reservoirWeightMatrix=reservoirWeightMatrix,
minimizer=tuner.Minimizer.DifferentialEvolution)
spectralRadiusOptimum, inputScalingOptimum, reservoirScalingOptimum, leakingRateOptimum = resTuner.getOptimalParameters(
)
#Train
network = ESN.Reservoir(size=size,
spectralRadius=spectralRadiusOptimum,
inputScaling=inputScalingOptimum,
reservoirScaling=reservoirScalingOptimum,
leakingRate=leakingRateOptimum,
initialTransient=initialTransient,
inputData=trainingInputData,
outputData=trainingOutputData,
inputWeightRandom=inputWeightMatrix,
reservoirWeightRandom=reservoirWeightMatrix)
network.trainReservoir()
warmupFeatureVectors, warmTargetVectors = formFeatureVectors(
validationOutputData)
predictedWarmup = network.predict(warmupFeatureVectors[-initialTransient:])
initialInputSeedForTesing = validationOutputData[-1]
predictedOutputData = predictFuture(network, initialInputSeedForTesing,
testingData.shape[0])[:, 0]
cumError = 0
times = 100
for i in range(times):
# Run for many time and get the average regression error
regressionError = util.trainAndGetError(
size=size,
spectralRadius=spectralRadiusOptimum,
inputScaling=inputScalingOptimum,
reservoirScaling=reservoirScalingOptimum,
leakingRate=leakingRateOptimum,
initialTransient=initialTransient,
trainingInputData=trainingInputData,
trainingOutputData=trainingOutputData,
inputWeightMatrix=inputWeightMatrix,
reservoirWeightMatrix=reservoirWeightMatrix,
validationOutputData=validationOutputData,
horizon=testingData.shape[0],
testingActualOutputData=testingData)
cumError += regressionError
error = cumError / times
return predictedOutputData, error
def tuneConnectivity(trainingInputData,
trainingOutputData,
validationOutputData,
initialInputSeedForValidation,
horizon,
testingActualOutputData,
size=256,
initialTransient=50,
resTopology=Topology.Classic):
# Other reservoir parameters
spectralRadius = 0.79
inputScaling = 0.5
reservoirScaling = 0.5
leakingRate = 0.3
# Optimal Parameters List
optimalParameters = {}
if (resTopology == Topology.Classic):
# Run 100 times and get the average regression error
iterations = 1000
cumulativeError = 0.0
for i in range(iterations):
inputWeightMatrix = topology.ClassicInputTopology(
inputSize=trainingInputData.shape[1],
reservoirSize=size).generateWeightMatrix()
reservoirWeightMatrix = topology.ClassicReservoirTopology(
size=size).generateWeightMatrix()
error = trainAndGetError(
size=size,
spectralRadius=spectralRadius,
inputScaling=inputScaling,
reservoirScaling=reservoirScaling,
leakingRate=leakingRate,
initialTransient=initialTransient,
trainingInputData=trainingInputData,
trainingOutputData=trainingOutputData,
inputWeightMatrix=inputWeightMatrix,
reservoirWeightMatrix=reservoirWeightMatrix,
validationOutputData=validationOutputData,
horizon=horizon,
testingActualOutputData=testingActualOutputData)
# Calculate the error
cumulativeError += error
return cumulativeError / iterations, optimalParameters
elif (resTopology == Topology.Random):
resTuner = tuner.RandomConnectivityBruteTuner(
size=size,
initialTransient=initialTransient,
trainingInputData=trainingInputData,
trainingOutputData=trainingOutputData,
initialSeed=initialInputSeedForValidation,
validationOutputData=validationOutputData,
spectralRadius=spectralRadius,
inputScaling=inputScaling,
reservoirScaling=reservoirScaling,
leakingRate=leakingRate)
reservoirConnectivityOptimum = resTuner.getOptimalParameters()
optimalParameters[
"Optimal_Reservoir_Connectivity"] = reservoirConnectivityOptimum
# Run 100 times and get the average regression error
iterations = 1000
cumulativeError = 0.0
for i in range(iterations):
inputWeightMatrix = topology.ClassicInputTopology(
inputSize=trainingInputData.shape[1],
reservoirSize=size).generateWeightMatrix()
reservoirWeightMatrix = topology.RandomReservoirTopology(
size=size, connectivity=reservoirConnectivityOptimum
).generateWeightMatrix()
error = trainAndGetError(
size=size,
spectralRadius=spectralRadius,
inputScaling=inputScaling,
reservoirScaling=reservoirScaling,
leakingRate=leakingRate,
initialTransient=initialTransient,
trainingInputData=trainingInputData,
trainingOutputData=trainingOutputData,
inputWeightMatrix=inputWeightMatrix,
reservoirWeightMatrix=reservoirWeightMatrix,
validationOutputData=validationOutputData,
horizon=horizon,
testingActualOutputData=testingActualOutputData)
# Calculate the error
cumulativeError += error
return cumulativeError / iterations, optimalParameters
elif (resTopology == Topology.ErdosRenyi):
resTuner = tuner.ErdosRenyiConnectivityBruteTuner(
size=size,
initialTransient=initialTransient,
trainingInputData=trainingInputData,
trainingOutputData=trainingOutputData,
initialSeed=initialInputSeedForValidation,
validationOutputData=validationOutputData,
spectralRadius=spectralRadius,
inputScaling=inputScaling,
reservoirScaling=reservoirScaling,
leakingRate=leakingRate)
probabilityOptimum = resTuner.getOptimalParameters()
optimalParameters[
"Optimal_Connectivity_Probability"] = probabilityOptimum
# Run 100 times and get the average regression error
iterations = 1000
cumulativeError = 0.0
for i in range(iterations):
inputWeightMatrix = topology.ClassicInputTopology(
inputSize=trainingInputData.shape[1],
reservoirSize=size).generateWeightMatrix()
reservoirWeightMatrix = topology.ErdosRenyiTopology(
size=size,
probability=probabilityOptimum).generateWeightMatrix()
error = trainAndGetError(
size=size,
spectralRadius=spectralRadius,
inputScaling=inputScaling,
reservoirScaling=reservoirScaling,
leakingRate=leakingRate,
initialTransient=initialTransient,
trainingInputData=trainingInputData,
trainingOutputData=trainingOutputData,
inputWeightMatrix=inputWeightMatrix,
reservoirWeightMatrix=reservoirWeightMatrix,
validationOutputData=validationOutputData,
horizon=horizon,
testingActualOutputData=testingActualOutputData)
# Calculate the error
cumulativeError += error
return cumulativeError / iterations, optimalParameters
elif (resTopology == Topology.ScaleFreeNetworks):
resTuner = tuner.ScaleFreeNetworksConnectivityBruteTuner(
size=size,
initialTransient=initialTransient,
trainingInputData=trainingInputData,
trainingOutputData=trainingOutputData,
initialSeed=initialInputSeedForValidation,
validationOutputData=validationOutputData,
spectralRadius=spectralRadius,
inputScaling=inputScaling,
reservoirScaling=reservoirScaling,
leakingRate=leakingRate)
attachmentOptimum = resTuner.getOptimalParameters()
optimalParameters[
"Optimal_Preferential_Attachment"] = attachmentOptimum
# Run 100 times and get the average regression error
iterations = 1000
cumulativeError = 0.0
for i in range(iterations):
inputWeightMatrix = topology.ClassicInputTopology(
inputSize=trainingInputData.shape[1],
reservoirSize=size).generateWeightMatrix()
reservoirWeightMatrix = topology.ScaleFreeNetworks(
size=size,
attachmentCount=attachmentOptimum).generateWeightMatrix()
error = trainAndGetError(
size=size,
spectralRadius=spectralRadius,
inputScaling=inputScaling,
reservoirScaling=reservoirScaling,
leakingRate=leakingRate,
initialTransient=initialTransient,
trainingInputData=trainingInputData,
trainingOutputData=trainingOutputData,
inputWeightMatrix=inputWeightMatrix,
reservoirWeightMatrix=reservoirWeightMatrix,
validationOutputData=validationOutputData,
horizon=horizon,
testingActualOutputData=testingActualOutputData)
# Calculate the error
cumulativeError += error
return cumulativeError / iterations, optimalParameters
elif (resTopology == Topology.SmallWorldGraphs):
resTuner = tuner.SmallWorldGraphsConnectivityBruteTuner(
size=size,
initialTransient=initialTransient,
trainingInputData=trainingInputData,
trainingOutputData=trainingOutputData,
initialSeed=initialInputSeedForValidation,
validationOutputData=validationOutputData,
spectralRadius=spectralRadius,
inputScaling=inputScaling,
reservoirScaling=reservoirScaling,
leakingRate=leakingRate)
meanDegreeOptimum, betaOptimum = resTuner.getOptimalParameters()
optimalParameters["Optimal_MeanDegree"] = meanDegreeOptimum
optimalParameters["Optimal_Beta"] = betaOptimum
# Run 100 times and get the average regression error
iterations = 1000
cumulativeError = 0.0
for i in range(iterations):
inputWeightMatrix = topology.ClassicInputTopology(
inputSize=trainingInputData.shape[1],
reservoirSize=size).generateWeightMatrix()
reservoirWeightMatrix = topology.SmallWorldGraphs(
size=size, meanDegree=int(meanDegreeOptimum),
beta=betaOptimum).generateWeightMatrix()
error = trainAndGetError(
size=size,
spectralRadius=spectralRadius,
inputScaling=inputScaling,
reservoirScaling=reservoirScaling,
leakingRate=leakingRate,
initialTransient=initialTransient,
trainingInputData=trainingInputData,
trainingOutputData=trainingOutputData,
inputWeightMatrix=inputWeightMatrix,
reservoirWeightMatrix=reservoirWeightMatrix,
validationOutputData=validationOutputData,
horizon=horizon,
testingActualOutputData=testingActualOutputData)
# Calculate the error
cumulativeError += error
return cumulativeError / iterations, optimalParameters
errorFun = metrics.MeanSquareError()
# Number of iterations
iterations = 1
reservoirSize = 256
# Run Classic ESN
classicESNError = 0
print("\n Running Classic ESN Tuner..")
for i in range(iterations):
predictedOutputData, error = util.tuneTrainPredict(
trainingInputData=trainingInputData,
trainingOutputData=trainingOutputData,
validationOutputData=validationData,
initialInputSeedForValidation=initialSeedForValidation,
reservoirTopology=topology.ClassicReservoirTopology(
size=reservoirSize),
testingData=testingData)
classicESNPredicted = minMax.inverse_transform(predictedOutputData)
classicESNError += error
classicESNError = classicESNError / iterations
# Run random ESN Tuner
randomESNError = 0
connectivityOptimum = 0.72999999999999998
print("\n Running Random ESN Tuner..")
for i in range(iterations):
predictedOutputData, error = util.tuneTrainPredict(
trainingInputData=trainingInputData,
trainingOutputData=trainingOutputData,
validationOutputData=validationData,
initialInputSeedForValidation=initialSeedForValidation,
