def __call__(self):
t0 = time.time()
args = self.args
names_bounds = yield self.ev.setup().addErrback(oops)
self.p = Population(self.evaluate,
names_bounds[0],
names_bounds[1],
popsize=args.p)
yield self.p.setup().addErrback(oops)
reporter = Reporter(self.ev, self.p)
self.p.addCallback(reporter)
F = [float(x) for x in args.F.split(',')]
de = DifferentialEvolution(self.p,
CR=args.C,
F=F,
maxiter=args.m,
randomBase=not args.b,
uniform=args.u,
adaptive=not args.n,
bitterEnd=args.e,
logHandle=self.fh)
yield de()
yield self.shutdown()
msg(0, "Final population:\n{}", self.p)
msg(0, "Elapsed time: {:.2f} seconds", time.time() - t0, 0)
msg(None)
reactor.stop()
def setUp(self):
abort.restart()
from asynqueue.util import DeferredTracker
self.dt = DeferredTracker()
self.p = Population(self.fifthSecond, ["x"], [(-5, 5)])
self.de = de.DifferentialEvolution(self.p, maxiter=35)
self.de.fm = de.FManager(0.5, 0.5, self.p.Np, True)
return self.p.setup()
def makeDE(
self, Np, Nd,
slow=False, blank=False, randomBase=False, callback=None):
def slowAckley(X):
delay = np.random.sample()
return self.deferToDelay(delay).addCallback(lambda _: tb.ackley(X))
self.p = Population(
slowAckley if slow else tb.ackley,
[sub("x{:d}", k) for k in range(Nd)],
[(-5, 5)]*Nd, popsize=Np/Nd)
if callback: self.p.addCallback(callback)
self.de = de.DifferentialEvolution(
self.p, maxiter=35, randomBase=randomBase)
yield self.p.setup(blank=blank)
tb.evals_reset()
def main():
"""
The C{pv} script entry point.
"""
if not len(args):
raise RuntimeError("No pickle file specified!")
filePath = args[0]
p = Population.load(os.path.expanduser(filePath))
analyzer = p.history.a
if args.f: analyzer.filePath(args.f, dims=args.d)
names = args[1:] if len(args) > 1 else []
pt = None
if args.r:
pt = analyzer.plot(names,
maxRatio=args.r,
inPop=args.i,
noShow=True,
semilog=args.s,
dims=args.d)
if args.N:
if names:
for name in names:
pt = analyzer.plotCorrelated(name=name,
N=args.N,
noShow=True,
verbose=args.v,
dims=args.d)
else:
pt = analyzer.plotCorrelated(N=args.N,
noShow=True,
verbose=args.v,
dims=args.d)
if pt is None: raise RuntimeError("No analysis done!")
pt.showAll()
class Test_Abort(tb.TestCase):
def setUp(self):
abort.restart()
from asynqueue.util import DeferredTracker
self.dt = DeferredTracker()
self.p = Population(self.fifthSecond, ["x"], [(-5, 5)])
self.de = de.DifferentialEvolution(self.p, maxiter=35)
self.de.fm = de.FManager(0.5, 0.5, self.p.Np, True)
return self.p.setup()
def tearDown(self):
def done(null):
abort.shutdown()
self.de.shutdown()
return self.dt.deferToAll().addCallback(done)
def fifthSecond(self, x):
d = self.deferToDelay(0.2)
d.addCallback(lambda _: 1.23)
self.dt.put(d)
return d
@defer.inlineCallbacks
def test_abort_challenge_beforehand(self):
self.de.shutdown()
t0 = time.time()
yield self.de.challenge(1, 2)
self.assertLess(time.time()-t0, 0.01)
@defer.inlineCallbacks
def test_abort_challenge_midway(self):
t0 = time.time()
d = self.de.challenge(1, 2)
dd = self.deferToDelay(0.05).addCallback(lambda _: self.de.shutdown())
yield d
self.assertWithinFivePercent(time.time()-t0, 0.2)
yield dd
@defer.inlineCallbacks
def test_abort_call(self):
t0 = time.time()
d = self.de()
t = 0.1 + 0.3*np.random.random()
dd = self.deferToDelay(t).addCallback(lambda _: self.de.shutdown())
yield d
self.assertLess(time.time()-t0, t+0.25)
yield dd
class TestDifferentialEvolution(tb.TestCase):
timeout = 30
verbose = False
def setUp(self):
abort.restart()
@defer.inlineCallbacks
def makeDE(
self, Np, Nd,
slow=False, blank=False, randomBase=False, callback=None):
def slowAckley(X):
delay = np.random.sample()
return self.deferToDelay(delay).addCallback(lambda _: tb.ackley(X))
self.p = Population(
slowAckley if slow else tb.ackley,
[sub("x{:d}", k) for k in range(Nd)],
[(-5, 5)]*Nd, popsize=Np/Nd)
if callback: self.p.addCallback(callback)
self.de = de.DifferentialEvolution(
self.p, maxiter=35, randomBase=randomBase)
yield self.p.setup(blank=blank)
tb.evals_reset()
def tearDown(self):
abort.shutdown()
self.de.shutdown()
@defer.inlineCallbacks
def test_crossover(self):
def doCrossovers(CR):
N_allMutant = 0
self.de.CR = CR
for trial in range(N_runs):
i1 = tb.MockIndividual(self.de.p, np.ones(Nd))
self.de.crossover(i0, i1)
self.assertNotEqual(i1, i0)
if i1.equals(np.ones(Nd)):
N_allMutant += 1
return float(N_allMutant)/N_runs
N_runs = 1000
Np = 100; Nd = 4
yield self.makeDE(Np, Nd)
i0 = tb.MockIndividual(self.p, np.zeros(Nd))
self.assertEqual(doCrossovers(0), 0.0)
self.assertEqual(doCrossovers(1.0), 1.0)
# 1 2 3 4
# 1.0 0.5 0.5 0.5 x = 0.125
self.assertAlmostEqual(doCrossovers(0.5), 0.125, 1)
# 1 2 3 4
# 1.0 0.8 0.8 0.8 x = 0.729
self.assertAlmostEqual(doCrossovers(0.9), 0.729, 1)
@defer.inlineCallbacks
def test_challenge(self):
Np = 120; Nd = 5
yield self.makeDE(Np, Nd)
self.de.fm = de.FManager(0.5, 0.5, self.p.Np, True)
from ade import individual
for k in (10,11):
i = individual.Individual(self.p, np.zeros(Nd))
i.SSE = 0.00001*k
if k == 10:
i10 = i
self.de.p[k] = i
tb.evals_reset()
yield self.de.challenge(10, 11)
self.assertEqual(tb.evals(), 1)
self.assertEqual(self.p.kBest, 10)
self.assertEqual(self.p.individuals(10), i10)
@defer.inlineCallbacks
def test_challenge_realistic(self):
Np = 12; Nd = 5
N_won = 0; N_challenges = 25
yield self.makeDE(Np, Nd, slow=True)
self.de.fm = de.FManager(0.5, 0.5, self.p.Np, True)
from ade import individual
for kc in range(N_challenges):
kList = []; iList = []
for k in self.p.sample(2):
kList.append(k)
i = self.p.individuals(k)
i.update(10*np.random.sample(Nd)-5)
iList.append(i)
yield self.de.challenge(*kList)
iWinner = self.p.individuals(kList[0])
iParent = iList[0]
if iWinner is not iParent:
N_won += 1
self.assertLess(iWinner, iParent)
self.assertGreater(N_won, 0.2*N_challenges)
self.assertLess(N_won, 0.8*N_challenges)
msg("Best after challenges:\n{}", self.p.best())
@defer.inlineCallbacks
def test_call(self):
def callback(values, counter, SSE):
cbList.append((values, counter, SSE))
cbList = []
yield self.makeDE(20, 2, callback=callback)
self.assertEqual(cbList[0][1], 1)
self.assertGreater(len(cbList), 1)
p = yield self.de()
self.assertEqual(tb.EVAL_COUNT[0], 700)
self.assertGreater(cbList[-1][1], 600)
x = p.best()
self.assertAlmostEqual(x[0], 0, 4)
self.assertAlmostEqual(x[1], 0, 4)
@defer.inlineCallbacks
def test_call_genFunc(self):
def func(kg):
cbList.append(kg)
cbList = []
yield self.makeDE(20, 2)
yield self.de(func)
self.assertEqual(tb.EVAL_COUNT[0], 700)
self.assertEqual(len(np.unique(cbList)), len(cbList))
self.assertEqual(len(cbList), 35)
@defer.inlineCallbacks
def test_call_rb_0r25(self):
yield self.makeDE(20, 2, randomBase=0.25)
p = yield self.de()
self.assertEqual(tb.EVAL_COUNT[0], 700)
x = p.best()
self.assertAlmostEqual(x[0], 0, 4)
self.assertAlmostEqual(x[1], 0, 4)
@defer.inlineCallbacks
def test_call_challenge_failure(self):
def challengeWrapper(kt, kb):
count[0] += 1
if count[0] == 5:
return defer.fail(failure.Failure(FakeException()))
return orig_challenge(kt, kb).addCallback(done)
def done(result):
count[1] += 1
return result
count = [0, 0]
yield self.makeDE(20, 2)
orig_challenge = self.de.challenge
self.patch(self.de, 'challenge', challengeWrapper)
p = yield self.de()
self.assertEqual(tb.EVAL_COUNT[0], 0)
self.assertLess(count[1], 19)
self.assertIsInstance(p, Population)
# This little delay is needed for some reason to avoid the
# FakeException being logged as an error
yield self.deferToDelay(0.05)
self.flushLoggedErrors()
@defer.inlineCallbacks
def test_abort_during_setup(self):
d = self.makeDE(20, 2).addCallback(lambda _: self.de())
self.p.abort()
self.de.shutdown()
yield d
self.assertLess(len(self.p), 20)
@defer.inlineCallbacks
def test_abort_during_call(self):
def setupDone(null):
self.de()
return self.deferToDelay(0.1).addCallback(
lambda _: self.de.shutdown())
# TODO: Actually test that the full 35 generations don't get
# evolved, by capturing and examining STDOUT
d = self.makeDE(20, 2).addCallback(setupDone)
yield d
self.assertEqual(len(self.p), 20)
class Runner(object):
"""
I run everything to fit a curve to thermistor data using
asynchronous differential evolution.
Construct an instance of me with an instance of L{Args} that has
parsed command-line options, then have the Twisted reactor call
the instance when it starts. Then start the reactor and watch the
fun.
"""
def __init__(self, args):
"""
C{Runner(args)}
"""
self.args = args
self.ev = Evaluator()
N = args.N if args.N else ProcessQueue.cores() - 1
self.q = ProcessQueue(N, returnFailure=True)
self.fh = open("voc.log", 'w') if args.l else True
msg(self.fh)
@defer.inlineCallbacks
def shutdown(self):
"""
Call this to shut me down when I'm done. Shuts down my
C{ProcessQueue}, which can take a moment.
Repeated calls have no effect.
"""
if self.q is not None:
msg("Shutting down...")
yield self.q.shutdown()
msg("Task Queue is shut down")
self.q = None
msg("Goodbye")
def evaluate(self, values):
"""
The function that gets called with each combination of parameters
to be evaluated for fitness.
"""
if values is None:
return self.shutdown()
values = list(values)
if self.q: return self.q.call(self.ev, values)
@defer.inlineCallbacks
def __call__(self):
t0 = time.time()
args = self.args
names_bounds = yield self.ev.setup().addErrback(oops)
self.p = Population(self.evaluate,
names_bounds[0],
names_bounds[1],
popsize=args.p)
yield self.p.setup().addErrback(oops)
reporter = Reporter(self.ev, self.p)
self.p.addCallback(reporter)
F = [float(x) for x in args.F.split(',')]
de = DifferentialEvolution(self.p,
CR=args.C,
F=F,
maxiter=args.m,
randomBase=not args.b,
uniform=args.u,
adaptive=not args.n,
bitterEnd=args.e,
logHandle=self.fh)
yield de()
yield self.shutdown()
msg(0, "Final population:\n{}", self.p)
msg(0, "Elapsed time: {:.2f} seconds", time.time() - t0, 0)
msg(None)
reactor.stop()
def run(self):
return self().addErrback(oops)
class Runner(object):
"""
I run everything to fit a curve to thermistor data using
asynchronous differential evolution.
Construct an instance of me with an instance of L{Args} that has
parsed command-line options, then have the Twisted reactor call
the instance when it starts. Then start the reactor and watch the
fun.
@cvar targetFraction: Normally 3%, but set much lower in this
example (0.5%) because real improvements seem to be made in
this example even with low improvement scores. I think that
behavior has something to do with all the independent
parameters for six thermistors.
"""
targetFraction = 0.005
def __init__(self, args):
"""
C{Runner(args)}
"""
self.args = args
self.ev = Evaluator()
N = args.N if args.N else ProcessQueue.cores()-1
self.q = ProcessQueue(N, returnFailure=True)
self.fh = open("thermistor.log", 'w') if args.l else True
msg(self.fh)
@defer.inlineCallbacks
def shutdown(self):
"""
Call this to shut me down when I'm done. Shuts down my
C{ProcessQueue}, which can take a moment.
Repeated calls have no effect.
"""
if self.q is not None:
msg("Shutting down...")
yield self.q.shutdown()
msg("Task Queue is shut down")
self.q = None
msg("Goodbye")
def evaluate(self, values, xSSE=None):
"""
The function that gets called with each combination of parameters
to be evaluated for fitness.
"""
if values is None:
# Shutting down, don't try to evaluate
return defer.succeed(None)
values = list(values)
if self.q: return self.q.call(self.ev, values, xSSE)
@defer.inlineCallbacks
def __call__(self):
t0 = time.time()
args = self.args
names_bounds = yield self.ev.setup().addErrback(oops)
self.p = Population(
self.evaluate,
names_bounds[0], names_bounds[1],
popsize=args.p, targetFraction=self.targetFraction)
yield self.p.setup().addErrback(oops)
reporter = Reporter(self.ev, self.p)
self.p.addCallback(reporter)
F = [float(x) for x in args.F.split(',')]
de = DifferentialEvolution(
self.p,
CR=args.C, F=F, maxiter=args.m, xSSE=args.x,
randomBase=args.r, uniform=args.u, adaptive=not args.n,
bitterEnd=args.e, logHandle=self.fh, dwellByGrave=12)
yield de()
yield self.shutdown()
if len(args.P) > 1: self.p.save(args.P)
msg(0, "Final population:\n{}", self.p)
msg(0, "Elapsed time: {:.2f} seconds", time.time()-t0, 0)
reactor.stop()
def run(self):
return self().addErrback(oops)
@Slot()
def shutdownNow(self):
self.r.shutdown()
class Runner(object):
"""
"""
def __init__(self, history):
self.app = QtGui.QApplication([])
self.history = history
self.window = Window(parent=None, runner=self)
self.pt = self.window.pt
def update(self, name):
print "UPDATE", name
def shutdown(self):
self.app.exit()
def run(self):
self.app.installEventFilter(self.window)
self.window.show()
self.app.exec_()
if __name__ == '__main__':
p = Population.load("~/pfinder.dat")
Runner(p.history).run()