def trainUntilConvergence(self, dataset=None, maxEpochs=None, verbose=None,
continueEpochs=10, validationProportion=0.25):
"""Train the module on the dataset until it converges.
Return the module with the parameters that gave the minimal validation
error.
If no dataset is given, the dataset passed during Trainer
initialization is used. validationProportion is the ratio of the dataset
that is used for the validation dataset.
If maxEpochs is given, at most that many epochs
are trained. Each time validation error hits a minimum, try for
continueEpochs epochs to find a better one."""
epochs = 0
if dataset == None:
dataset = self.ds
if verbose == None:
verbose = self.verbose
# Split the dataset randomly: validationProportion of the samples for
# validation.
trainingData, validationData = (
dataset.splitWithProportion(1 - validationProportion))
if not (len(trainingData) > 0 and len(validationData)):
raise ValueError("Provided dataset too small to be split into training " +
"and validation sets with proportion " + str(validationProportion))
self.ds = trainingData
bestweights = self.module.params.copy()
bestverr = self.testOnData(validationData)
trainingErrors = []
validationErrors = [bestverr]
while True:
trainingErrors.append(self.train())
validationErrors.append(self.testOnData(validationData))
if epochs == 0 or validationErrors[-1] < bestverr:
# one update is always done
bestverr = validationErrors[-1]
bestweights = self.module.params.copy()
if maxEpochs != None and epochs >= maxEpochs:
self.module.params[:] = bestweights
break
epochs += 1
if len(validationErrors) >= continueEpochs * 2:
# have the validation errors started going up again?
# compare the average of the last few to the previous few
old = validationErrors[-continueEpochs * 2:-continueEpochs]
new = validationErrors[-continueEpochs:]
if min(new) > max(old):
self.module.params[:] = bestweights
break
trainingErrors.append(self.testOnData(trainingData))
self.ds = dataset
if verbose:
print 'train-errors:', fListToString(trainingErrors, 6)
print 'valid-errors:', fListToString(validationErrors, 6)
return trainingErrors, validationErrors
def testSimple(self):
r = self.runSequences(num_actions=3, num_features=5, num_states=4, num_interactions=2000,
lr=0.1, _lambda=0.5, gamma=0.5)
if self.verbose:
for x, l in r:
print x
for a in l:
print fListToString(a[0], 2)
for _, l in r:
self.assertAlmostEquals(min(l[0][0]), max(l[0][0]), places=0)
self.assertAlmostEquals(min(l[1][0]), max(l[1][0]), places=0)
self.assertAlmostEquals(min(l[2][0]) + len(l[2][0]) - 1, max(l[2][0]), places=0)
self.assertAlmostEquals(min(l[3][0]), max(l[3][0]), places=0)
def testSingleAction(self):
r = self.runSequences(num_actions=1, r_states=map(array, [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0]]),
num_interactions=1000, lr=0.1, _lambda=0.5, gamma=0.5)
if self.verbose:
for x, l in r:
print x
for a in l:
print fListToString(a, 2)
for _, l in r:
self.assertAlmostEquals(min(l[0]), max(l[0]), places=0)
self.assertAlmostEquals(min(l[1]), max(l[1]), places=0)
self.assertAlmostEquals(min(l[2]), max(l[2]), places=0)
self.assertAlmostEquals(max(l[3]) - 1, min(l[3]), places=0)
def _evaluateSequence(self, f, seq, verbose = False):
"""Return the ponderated MSE over one sequence."""
totalError = 0.
ponderation = 0.
for input, target in seq:
res = f(input)
e = 0.5 * sum((target-res).flatten()**2)
totalError += e
ponderation += len(target)
if verbose:
print(( 'out: ', fListToString( list( res ) )))
print(( 'correct:', fListToString( target )))
print(( 'error: % .8f' % e))
return totalError, ponderation
def _evaluateSequence(self, f, seq, verbose = False):
""" return the importance-ponderated MSE over one sequence. """
totalError = 0
ponderation = 0.
for input, target, importance in seq:
res = f(input)
e = 0.5 * dot(importance.flatten(), ((target-res).flatten()**2))
totalError += e
ponderation += sum(importance)
if verbose:
print 'out: ', fListToString(list(res))
print 'correct: ', fListToString(target)
print 'importance:', fListToString(importance)
print 'error: % .8f' % e
return totalError, ponderation
def testSingleStateFullDiscounted(self):
r = self.runSequences(num_actions=4, num_features=3, num_states=1, num_interactions=500,
gamma=0, lr=0.25)
if self.verbose:
for x, l in r:
print x
for a in l:
print fListToString(a[0], 2)
for _, l in r:
self.assertAlmostEquals(min(l[0][0]), 1, places=0)
self.assertAlmostEquals(max(l[0][0]), 1, places=0)
self.assertAlmostEquals(2 * min(l[1][0]), 1, places=0)
self.assertAlmostEquals(2 * max(l[1][0]), 1, places=0)
self.assertAlmostEquals(min(l[2][0]), 0, places=0)
self.assertAlmostEquals(max(l[2][0]), len(l[2][0]) - 1, places=0)
self.assertAlmostEquals(min(l[3][0]), max(l[3][0]), places=0)
def _oneGeneration(self):
self.oldPops.append(self.pop)
self.generation += 1
fitnesses = self._evaluatePopulation()
# store best in hall of fame
besti = argmax(array(fitnesses))
best = self.pop[besti]
bestFits = sorted(fitnesses)[::-1][:self._numSelected()]
self.hallOfFame.append(best)
self.hallOfFitnesses.append(bestFits)
if self.verbose:
print 'Generation', self.generation
print ' relat. fits:', fListToString(sorted(fitnesses), 4)
if len(best.params) < 20:
print ' best params:', fListToString(best.params, 4)
self.pop = self._selectAndReproduce(self.pop, fitnesses)
def trainUntilConvergence(self, dataset=None, maxEpochs=None, verbose=None,
continueEpochs=10, validationProportion=0.25):
epochs = 0
if dataset == None:
dataset = self.ds
if verbose == None:
verbose = self.verbose
trainingData, validationData = (
dataset.splitWithProportion(1 - validationProportion))
if not (len(trainingData) > 0 and len(validationData)):
raise ValueError("Provided dataset too small to be split into training " +
"and validation sets with proportion " + str(validationProportion))
self.ds = trainingData
bestweights = self.module.params.copy()
bestverr = self.testOnData(validationData)
trainingErrors = []
validationErrors = [bestverr]
while True:
trainingErrors.append(self.train())
validationErrors.append(self.testOnData(validationData))
if epochs == 0 or validationErrors[-1] < bestverr:
bestverr = validationErrors[-1]
bestweights = self.module.params.copy()
if maxEpochs != None and epochs >= maxEpochs:
self.module.params[:] = bestweights
break
epochs += 1
if len(validationErrors) >= continueEpochs * 2:
old = validationErrors[-continueEpochs * 2:-continueEpochs]
new = validationErrors[-continueEpochs:]
if min(new) > max(old):
self.module.params[:] = bestweights
break
trainingErrors.append(self.testOnData(trainingData))
self.ds = dataset
if verbose:
print 'train-errors:', fListToString(trainingErrors, 6)
print 'valid-errors:', fListToString(validationErrors, 6)
return trainingErrors, validationErrors
def _updateShaping(self):
""" Daan: "This won't work. I like it!" """
assert self.numberOfCenters == 1
possible = self.shapingFunction.getPossibleParameters(self.windowSize)
matchValues = []
pdfs = [multivariateNormalPdf(s, self.mus[0], self.sigmas[0])
for s in self.samples]
for p in possible:
self.shapingFunction.setParameter(p)
transformedFitnesses = self.shapingFunction(self.fitnesses)
#transformedFitnesses /= sum(transformedFitnesses)
sumValue = sum([x * log(y) for x, y in zip(pdfs, transformedFitnesses) if y > 0])
normalization = sum([x * y for x, y in zip(pdfs, transformedFitnesses) if y > 0])
matchValues.append(sumValue / normalization)
self.shapingFunction.setParameter(possible[argmax(matchValues)])
if len(self.allsamples) % 100 == 0:
print possible[argmax(matchValues)]
print fListToString(matchValues, 3)
# any episodic task
task = BalanceTask()
# any neural network controller
net = buildNetwork(task.outdim, 1, task.indim)
# any optimization algorithm to be plugged in, for example:
# learner = CMAES(storeAllEvaluations = True)
# or:
learner = HillClimber(storeAllEvaluations = True)
# in a non-optimization case the agent would be a LearningAgent:
# agent = LearningAgent(net, ENAC())
# here it is an OptimizationAgent:
agent = OptimizationAgent(net, learner)
# the agent and task are linked in an Experiment
# and everything else happens under the hood.
exp = EpisodicExperiment(task, agent)
exp.doEpisodes(100)
print('Episodes learned from:', len(learner._allEvaluations))
n, fit = learner._bestFound()
print('Best fitness found:', fit)
print('with this network:')
print(n)
print('containing these parameters:')
print(fListToString(n.params, 4))
def trainUntilConvergence(self, dataset=None, maxEpochs=None, verbose=None,
continueEpochs=10, validationProportion=0.25,
trainingData=None, validationData=None,
convergence_threshold=10):
"""Train the module on the dataset until it converges.
Return the module with the parameters that gave the minimal validation
error.
If no dataset is given, the dataset passed during Trainer
initialization is used. validationProportion is the ratio of the dataset
that is used for the validation dataset.
If the training and validation data is already set, the splitPropotion is ignored
If maxEpochs is given, at most that many epochs
are trained. Each time validation error hits a minimum, try for
continueEpochs epochs to find a better one."""
epochs = 0
if dataset is None:
dataset = self.ds
if verbose is None:
verbose = self.verbose
if trainingData is None or validationData is None:
# Split the dataset randomly: validationProportion of the samples for
# validation.
trainingData, validationData = (
dataset.splitWithProportion(1 - validationProportion))
if not (len(trainingData) > 0 and len(validationData)):
raise ValueError("Provided dataset too small to be split into training " +
"and validation sets with proportion " + str(validationProportion))
self.ds = trainingData
bestweights = self.module.params.copy()
bestverr = self.testOnData(validationData)
bestepoch = 0
self.trainingErrors = []
self.validationErrors = [bestverr]
while True:
trainingError = self.train()
validationError = self.testOnData(validationData)
if isnan(trainingError) or isnan(validationError):
raise Exception("Training produced NaN results")
self.trainingErrors.append(trainingError)
self.validationErrors.append(validationError)
if epochs == 0 or self.validationErrors[-1] < bestverr:
# one update is always done
bestverr = self.validationErrors[-1]
bestweights = self.module.params.copy()
bestepoch = epochs
if maxEpochs != None and epochs >= maxEpochs:
self.module.params[:] = bestweights
break
epochs += 1
if len(self.validationErrors) >= continueEpochs * 2:
# have the validation errors started going up again?
# compare the average of the last few to the previous few
old = self.validationErrors[-continueEpochs * 2:-continueEpochs]
new = self.validationErrors[-continueEpochs:]
if min(new) > max(old):
self.module.params[:] = bestweights
break
lastnew = round(new[-1], convergence_threshold)
if sum(round(y, convergence_threshold) - lastnew for y in new) == 0:
self.module.params[:] = bestweights
break
#self.trainingErrors.append(self.testOnData(trainingData))
self.ds = dataset
if verbose:
print(('train-errors:', fListToString(self.trainingErrors, 6)))
print(('valid-errors:', fListToString(self.validationErrors, 6)))
return self.trainingErrors[:bestepoch], self.validationErrors[:1 + bestepoch]
from pybrain.utilities import fListToString
# TODO: convert to unittest
C = CompetitiveCoevolution(None, [1, 2, 3, 4, 5, 6, 7, 8], populationSize=4)
def b(x, y):
C.allResults[(x, y)] = [1, 1, 1, []]
C.allResults[(y, x)] = [-1, 1, -1, []]
if x not in C.allOpponents:
C.allOpponents[x] = []
if y not in C.allOpponents:
C.allOpponents[y] = []
C.allOpponents[x].append(y)
C.allOpponents[y].append(x)
b(1, 6)
b(1, 7)
b(8, 1)
b(5, 2)
b(6, 2)
b(8, 2)
b(3, 5)
b(3, 6)
b(3, 7)
b(4, 5)
b(4, 7)
b(8, 4)
print(C.pop)
print(C.parasitePop)
print(' ', fListToString(C._competitiveSharedFitness(C.pop, C.parasitePop), 2))
print('should be:', fListToString([0.83, 0.00, 1.33, 0.83], 2))
# any episodic task
task = BalanceTask()
# any neural network controller
net = buildNetwork(task.outdim, 1, task.indim)
# any optimization algorithm to be plugged in, for example:
# learner = CMAES(storeAllEvaluations = True)
# or:
learner = HillClimber(storeAllEvaluations = True)
# in a non-optimization case the agent would be a LearningAgent:
# agent = LearningAgent(net, ENAC())
# here it is an OptimizationAgent:
agent = OptimizationAgent(net, learner)
# the agent and task are linked in an Experiment
# and everything else happens under the hood.
exp = EpisodicExperiment(task, agent)
exp.doEpisodes(100)
print('Episodes learned from:', len(learner._allEvaluations))
n, fit = learner._bestFound()
print('Best fitness found:', fit)
print('with this network:')
print(n)
print('containing these parameters:')
print(fListToString(n.params, 4))
def trainUntilConvergence(self,
dataset=None,
maxEpochs=None,
verbose=None,
continueEpochs=10,
validationProportion=0.25,
trainingData=None,
validationData=None,
convergence_threshold=10):
"""Train the module on the dataset until it converges.
Return the module with the parameters that gave the minimal validation
error.
If no dataset is given, the dataset passed during Trainer
initialization is used. validationProportion is the ratio of the dataset
that is used for the validation dataset.
If the training and validation data is already set, the splitPropotion is ignored
If maxEpochs is given, at most that many epochs
are trained. Each time validation error hits a minimum, try for
continueEpochs epochs to find a better one."""
epochs = 0
if dataset is None:
dataset = self.ds
if verbose is None:
verbose = self.verbose
if trainingData is None or validationData is None:
# Split the dataset randomly: validationProportion of the samples for
# validation.
trainingData, validationData = (
dataset.splitWithProportion(1 - validationProportion))
if not (len(trainingData) > 0 and len(validationData)):
raise ValueError(
"Provided dataset too small to be split into training " +
"and validation sets with proportion " +
str(validationProportion))
self.ds = trainingData
bestweights = self.module.params.copy()
bestverr = self.testOnData(validationData)
bestepoch = 0
self.trainingErrors = []
self.validationErrors = [bestverr]
while True:
trainingError = self.train()
validationError = self.testOnData(validationData)
if isnan(trainingError) or isnan(validationError):
raise Exception("Training produced NaN results")
self.trainingErrors.append(trainingError)
self.validationErrors.append(validationError)
if epochs == 0 or self.validationErrors[-1] < bestverr:
# one update is always done
bestverr = self.validationErrors[-1]
bestweights = self.module.params.copy()
bestepoch = epochs
if maxEpochs != None and epochs >= maxEpochs:
self.module.params[:] = bestweights
break
epochs += 1
if len(self.validationErrors) >= continueEpochs * 2:
# have the validation errors started going up again?
# compare the average of the last few to the previous few
old = self.validationErrors[-continueEpochs *
2:-continueEpochs]
new = self.validationErrors[-continueEpochs:]
if min(new) > max(old):
self.module.params[:] = bestweights
break
lastnew = round(new[-1], convergence_threshold)
if sum(round(y, convergence_threshold) - lastnew
for y in new) == 0:
self.module.params[:] = bestweights
break
#self.trainingErrors.append(self.testOnData(trainingData))
self.ds = dataset
if verbose:
print(('train-errors:', fListToString(self.trainingErrors, 6)))
print(('valid-errors:', fListToString(self.validationErrors, 6)))
return self.trainingErrors[:
bestepoch], self.validationErrors[:1 +
bestepoch]
from pybrain.utilities import fListToString
# TODO: convert to unittest
C = CompetitiveCoevolution(None, [1, 2, 3, 4, 5, 6, 7, 8], populationSize=4)
def b(x, y):
C.allResults[(x, y)] = [1, 1, 1, []]
C.allResults[(y, x)] = [-1, 1, -1, []]
if x not in C.allOpponents:
C.allOpponents[x] = []
if y not in C.allOpponents:
C.allOpponents[y] = []
C.allOpponents[x].append(y)
C.allOpponents[y].append(x)
b(1, 6)
b(1, 7)
b(8, 1)
b(5, 2)
b(6, 2)
b(8, 2)
b(3, 5)
b(3, 6)
b(3, 7)
b(4, 5)
b(4, 7)
b(8, 4)
print(C.pop)
print(C.parasitePop)
print(' ', fListToString(C._competitiveSharedFitness(C.pop, C.parasitePop), 2))
print('should be:', fListToString([0.83, 0.00, 1.33, 0.83], 2))
# any episodic task task = BalanceTask() # any neural network controller net = buildNetwork(task.outdim, 1, task.indim) # any optimization algorithm to be plugged in, for example: # learner = CMAES(storeAllEvaluations = True) # or: learner = HillClimber(storeAllEvaluations = True) # in a non-optimization case the agent would be a LearningAgent: # agent = LearningAgent(net, ENAC()) # here it is an OptimizationAgent: agent = OptimizationAgent(net, learner) # the agent and task are linked in an Experiment # and everything else happens under the hood. exp = EpisodicExperiment(task, agent) exp.doEpisodes(100) print 'Episodes learned from:', len(learner._allEvaluations) n, fit = learner._bestFound() print 'Best fitness found:', fit print 'with this network:' print n print 'containing these parameters:' print fListToString(n.params, 4)
def trainUntilConvergence(self, datasetTrain=None, datasetTest=None, maxEpochs=None, verbose=None,
continueEpochs=10, validationProportion=0.25):
"""Train the module on the dataset until it converges.
Return the module with the parameters that gave the minimal validation
error.
If no dataset is given, the dataset passed during Trainer
initialization is used. validationProportion is the ratio of the dataset
that is used for the validation dataset.
If maxEpochs is given, at most that many epochs
are trained. Each time validation error hits a minimum, try for
continueEpochs epochs to find a better one."""
epochs = 0
if datasetTrain is None or datasetTest is None:
return
if verbose == None:
verbose = self.verbose
# Split the dataset randomly: validationProportion of the samples for
# validation.
trainingData = datasetTrain
validationData = datasetTest
# trainingData, validationData = (
# dataset.splitWithProportion(1 - validationProportion))
# if not (len(trainingData) > 0 and len(validationData)):
# raise ValueError("Provided dataset too small to be split into training " +
# "and validation sets with proportion " + str(validationProportion))
self.ds = trainingData
bestweights = self.module.params.copy()
bestverr = self.testOnData(validationData)
trainingErrors = []
validationErrors = [bestverr]
while True:
trainingErrors.append(self.train())
validationErrors.append(self.testOnData(validationData))
if validationErrors[-1] < bestverr:
bestverr = validationErrors[-1]
bestweights = self.module.params.copy()
if maxEpochs is not None and epochs >= maxEpochs:
self.module.params[:] = bestweights
break
epochs += 1
if len(validationErrors) >= continueEpochs * 2:
# have the validation errors started going up again?
# compare the average of the last few to the previous few
old = validationErrors[-continueEpochs * 2:-continueEpochs]
new = validationErrors[-continueEpochs:]
if min(new) > max(old):
self.module.params[:] = bestweights
break
# trainingErrors.append(self.testOnData(trainingData))
# self.ds = datasetTrain
if verbose:
print 'train-errors:', fListToString(trainingErrors, 6)
print 'valid-errors:', fListToString(validationErrors, 6)
return trainingErrors, validationErrors
def writeDoubles(self, node, l, precision=6):
self.addTextNode(node, fListToString(l, precision)[2:-1])
def writeDoubles(self, node, l, precision = 6):
self.addTextNode(node, fListToString(l, precision)[2:-1])
