def testRecoveryTasks(self):
t = task.Task("i=16", resultNames='i')
t2 = task.Task("raise Exception", recoveryTask=t, retries=2)
d = self.tc.run(t2)
d.addCallback(self.tc.getTaskResult, block=True)
d.addCallback(lambda tr: tr.ns.i)
d = self.assertDeferredEquals(d, 16)
return d
def testRecoveryTasks(self):
self.addEngine(1)
t = task.Task("i=16", pull='i')
t2 = task.Task("raise Exception", recovery_task=t, retries = 2)
d = self.tc.run(t2)
d.addCallback(self.tc.get_task_result, block=True)
d.addCallback(lambda tr: tr.ns.i)
d.addCallback(lambda r: self.assertEquals(r, 16))
return d
def testInfiniteRecoveryLoop(self):
t = task.Task("raise Exception", retries=5)
t2 = task.Task("assert False", retries=2, recoveryTask=t)
t.recoveryTask = t2
d = self.tc.run(t)
d.addCallback(self.tc.getTaskResult, block=True)
d.addCallback(lambda tr: tr.ns.i)
d = self.assertDeferredRaises(d, AssertionError)
return d
def testTaskIDs(self):
self.addEngine(1)
d = self.tc.run(task.Task('a=5'))
d.addCallback(lambda r: self.assertEquals(r, 0))
d.addCallback(lambda r: self.tc.run(task.Task('a=5')))
d.addCallback(lambda r: self.assertEquals(r, 1))
d.addCallback(lambda r: self.tc.run(task.Task('a=5')))
d.addCallback(lambda r: self.assertEquals(r, 2))
d.addCallback(lambda r: self.tc.run(task.Task('a=5')))
d.addCallback(lambda r: self.assertEquals(r, 3))
return d
def testInfiniteRecoveryLoop(self):
self.addEngine(1)
t = task.Task("raise Exception", retries=5)
t2 = task.Task("assert False", retries=2, recovery_task=t)
t.recovery_task = t2
d = self.tc.run(t)
d.addCallback(self.tc.get_task_result, block=True)
d.addCallback(lambda tr: tr.ns.i)
d.addErrback(
lambda f: self.assertRaises(AssertionError, f.raiseException))
return d
def testSimpleRetries(self):
self.addEngine(1)
t = task.Task("i += 1\nassert i == 16", pull='i',retries=10)
t2 = task.Task("i += 1\nassert i == 16", pull='i',retries=10)
d = self.multiengine.execute('i=0', targets=0)
d.addCallback(lambda r: self.tc.run(t))
d.addCallback(self.tc.get_task_result, block=True)
d.addCallback(lambda tr: tr.ns.i)
d.addErrback(lambda f: self.assertRaises(AssertionError, f.raiseException))
d.addCallback(lambda r: self.tc.run(t2))
d.addCallback(self.tc.get_task_result, block=True)
d.addCallback(lambda tr: tr.ns.i)
d.addCallback(lambda r: self.assertEquals(r, 16))
return d
def testSimpleRetries(self):
d = self.me.execute(0, 'i=0')
t = task.Task("i += 1\nassert i == 16", resultNames='i', retries=10)
t2 = task.Task("i += 1\nassert i == 16", resultNames='i', retries=10)
d.addCallback(lambda r: self.tc.run(t))
d.addCallback(self.tc.getTaskResult, block=True)
d.addCallback(lambda tr: tr.ns.i)
d = self.assertDeferredRaises(d, AssertionError)
d.addCallback(lambda r: self.tc.run(t2))
d.addCallback(self.tc.getTaskResult, block=True)
d.addCallback(lambda tr: tr.ns.i)
d = self.assertDeferredEquals(d, 16)
return d
def testTaskIDs(self):
l = []
for i in range(16):
d = self.tc.run(task.Task('a=5'))
d = self.assertDeferredEquals(d, i)
l.append(d)
return defer.DeferredList(l)
def testSetupNS(self):
d = self.me.execute(0, 'a=0')
ns = dict(a=1, b=0)
t = task.Task("", setupNS=ns, resultNames=['a', 'b'])
d.addCallback(lambda r: self.tc.run(t))
d.addCallback(self.tc.getTaskResult, block=True)
d.addCallback(lambda tr: {'a': tr.ns.a, 'b': tr['b']})
d = self.assertDeferredEquals(d, ns)
return d
def testAbort(self):
"""Cannot do a proper abort test, because blocking execution prevents
abort from being called before task completes"""
t = task.Task('a=5')
d = self.tc.abort(0)
d = self.assertDeferredRaises(d, IndexError)
d.addCallback(lambda _: self.tc.run(t))
d.addCallback(self.tc.abort)
d = self.assertDeferredRaises(d, IndexError)
return d
def testSetupNS(self):
self.addEngine(1)
d = self.multiengine.execute('a=0', targets=0)
ns = dict(a=1, b=0)
t = task.Task("", push=ns, pull=['a','b'])
d.addCallback(lambda r: self.tc.run(t))
d.addCallback(self.tc.get_task_result, block=True)
d.addCallback(lambda tr: {'a':tr.ns.a, 'b':tr['b']})
d.addCallback(lambda r: self.assertEquals(r, ns))
return d
def testTaskResults(self):
t1 = task.Task('a=5', resultNames='a')
d = self.tc.run(t1)
d.addCallback(self.tc.getTaskResult, block=True)
d.addCallback(lambda tr:
(tr.ns.a, tr['a'], tr.failure, tr.raiseException()))
d = self.assertDeferredEquals(d, (5, 5, None, None))
t2 = task.Task('7=5')
d.addCallback(lambda r: self.tc.run(t2))
d.addCallback(self.tc.getTaskResult, block=True)
d.addCallback(lambda tr: tr.ns)
d = self.assertDeferredRaises(d, SyntaxError)
t3 = task.Task('', resultNames='b')
d.addCallback(lambda r: self.tc.run(t3))
d.addCallback(self.tc.getTaskResult, block=True)
d.addCallback(lambda tr: tr.ns)
d = self.assertDeferredRaises(d, NameError)
return d
def testTaskResults(self):
self.addEngine(1)
t1 = task.Task('a=5', pull='a')
d = self.tc.run(t1)
d.addCallback(self.tc.get_task_result, block=True)
d.addCallback(lambda tr: (tr.ns.a,tr['a'],tr.failure, tr.raiseException()))
d.addCallback(lambda r: self.assertEquals(r, (5,5,None,None)))
t2 = task.Task('7=5')
d.addCallback(lambda r: self.tc.run(t2))
d.addCallback(self.tc.get_task_result, block=True)
d.addCallback(lambda tr: tr.ns)
d.addErrback(lambda f: self.assertRaises(SyntaxError, f.raiseException))
t3 = task.Task('', pull='b')
d.addCallback(lambda r: self.tc.run(t3))
d.addCallback(self.tc.get_task_result, block=True)
d.addCallback(lambda tr: tr.ns)
d.addErrback(lambda f: self.assertRaises(NameError, f.raiseException))
return d
def testAbort(self):
"""Cannot do a proper abort test, because blocking execution prevents
abort from being called before task completes"""
self.addEngine(1)
t = task.Task('a=5')
d = self.tc.abort(0)
d.addErrback(lambda f: self.assertRaises(IndexError, f.raiseException))
d.addCallback(lambda _:self.tc.run(t))
d.addCallback(self.tc.abort)
d.addErrback(lambda f: self.assertRaises(IndexError, f.raiseException))
return d
def testClears(self):
self.addEngine(1)
t = task.Task('a=1', clear_before=True, pull='b', clear_after=True)
d = self.multiengine.execute('b=1', targets=0)
d.addCallback(lambda _: self.tc.run(t))
d.addCallback(lambda tid: self.tc.get_task_result(tid,block=True))
d.addCallback(lambda tr: tr.failure)
d.addErrback(lambda f: self.assertRaises(NameError, f.raiseException))
d.addCallback(lambda _:self.multiengine.pull('a', targets=0))
d.addErrback(lambda f: self.assertRaises(NameError, _raise_it, f))
return d
def testClears(self):
d = self.me.execute(0, 'b=1')
t = task.Task('a=1',
clearBefore=True,
resultNames='b',
clearAfter=True)
d.addCallback(lambda _: self.tc.run(t))
d.addCallback(self.tc.getTaskResult, block=True)
d.addCallback(lambda tr: tr.failure)
d = self.assertDeferredRaises(d, NameError) # check b for clearBefore
d.addCallback(lambda _: self.me.pull(0, 'a'))
d = self.assertDeferredRaises(d, NameError) # check a for clearAfter
return d
def irun(self, *args, **kwargs):
"""Run a task on the `TaskController`.
This method is a shorthand for run(task) and its arguments are simply
passed onto a `Task` object:
irun(*args, **kwargs) -> run(Task(*args, **kwargs))
:Parameters:
expression : str
A str that is valid python code that is the task.
pull : str or list of str
The names of objects to be pulled as results.
push : dict
A dict of objects to be pushed into the engines namespace before
execution of the expression.
clear_before : boolean
Should the engine's namespace be cleared before the task is run.
Default=False.
clear_after : boolean
Should the engine's namespace be cleared after the task is run.
Default=False.
retries : int
The number of times to resumbit the task if it fails. Default=0.
options : dict
Any other keyword options for more elaborate uses of tasks
:Returns: A `TaskResult` object.
"""
block = kwargs.pop('block', False)
if len(args) == 1 and isinstance(args[0], task.Task):
t = args[0]
else:
t = task.Task(*args, **kwargs)
taskid = self.run(t)
print "TaskID = %i"%taskid
if block:
return self.get_task_result(taskid, block)
else:
return taskid
def main():
parser = OptionParser()
parser.set_defaults(n=100)
parser.set_defaults(tmin=1)
parser.set_defaults(tmax=60)
parser.set_defaults(controller='localhost')
parser.set_defaults(meport=10111)
parser.set_defaults(tport=10114)
parser.add_option("-n",
type='int',
dest='n',
help='the number of tasks to run')
parser.add_option("-t",
type='float',
dest='tmin',
help='the minimum task length in seconds')
parser.add_option("-T",
type='float',
dest='tmax',
help='the maximum task length in seconds')
parser.add_option("-c",
type='string',
dest='controller',
help='the address of the controller')
parser.add_option(
"-p",
type='int',
dest='meport',
help="the port on which the controller listens for MultiEnginePB")
parser.add_option(
"-P",
type='int',
dest='tport',
help="the port on which the controller listens for TaskPB")
(opts, args) = parser.parse_args()
assert opts.tmax >= opts.tmin, "tmax must not be smaller than tmin"
rc = multienginepb.PBInteractiveMultiEngineClient(
(opts.controller, opts.meport))
rt = taskpb.PBConnectingTaskClient((opts.controller, opts.tport))
rc.connect()
rt.connect()
rc.block = True
nengines = len(rc.getIDs())
rc.executeAll('from IPython.genutils import time')
# the jobs should take a random time within a range
times = [
random.random() * (opts.tmax - opts.tmin) + opts.tmin
for i in range(opts.n)
]
tasks = [task.Task("time.sleep(%f)" % t) for t in times]
stime = sum(times)
print "executing %i tasks, totalling %.1f secs on %i engines" % (
opts.n, stime, nengines)
time.sleep(1)
start = time.time()
results = [rt.run(t) for t in tasks]
rt.barrier()
stop = time.time()
ptime = stop - start
scale = stime / ptime
print "executed %.1f secs in %.1f secs" % (stime, ptime)
print "%.3fx parallel performance on %i engines" % (scale, nengines)
print "%.1f%% of theoretical max" % (100 * scale / nengines)
