def testRecoveryTasks(self): t = task.Task("i=16", resultNames='i') t2 = task.Task("raise Exception", recoveryTask=t, retries=2) d = self.tc.run(t2) d.addCallback(self.tc.getTaskResult, block=True) d.addCallback(lambda tr: tr.ns.i) d = self.assertDeferredEquals(d, 16) return d
def testRecoveryTasks(self): self.addEngine(1) t = task.Task("i=16", pull='i') t2 = task.Task("raise Exception", recovery_task=t, retries = 2) d = self.tc.run(t2) d.addCallback(self.tc.get_task_result, block=True) d.addCallback(lambda tr: tr.ns.i) d.addCallback(lambda r: self.assertEquals(r, 16)) return d
def testInfiniteRecoveryLoop(self): t = task.Task("raise Exception", retries=5) t2 = task.Task("assert False", retries=2, recoveryTask=t) t.recoveryTask = t2 d = self.tc.run(t) d.addCallback(self.tc.getTaskResult, block=True) d.addCallback(lambda tr: tr.ns.i) d = self.assertDeferredRaises(d, AssertionError) return d
def testTaskIDs(self): self.addEngine(1) d = self.tc.run(task.Task('a=5')) d.addCallback(lambda r: self.assertEquals(r, 0)) d.addCallback(lambda r: self.tc.run(task.Task('a=5'))) d.addCallback(lambda r: self.assertEquals(r, 1)) d.addCallback(lambda r: self.tc.run(task.Task('a=5'))) d.addCallback(lambda r: self.assertEquals(r, 2)) d.addCallback(lambda r: self.tc.run(task.Task('a=5'))) d.addCallback(lambda r: self.assertEquals(r, 3)) return d
def testInfiniteRecoveryLoop(self): self.addEngine(1) t = task.Task("raise Exception", retries=5) t2 = task.Task("assert False", retries=2, recovery_task=t) t.recovery_task = t2 d = self.tc.run(t) d.addCallback(self.tc.get_task_result, block=True) d.addCallback(lambda tr: tr.ns.i) d.addErrback( lambda f: self.assertRaises(AssertionError, f.raiseException)) return d
def testSimpleRetries(self): self.addEngine(1) t = task.Task("i += 1\nassert i == 16", pull='i',retries=10) t2 = task.Task("i += 1\nassert i == 16", pull='i',retries=10) d = self.multiengine.execute('i=0', targets=0) d.addCallback(lambda r: self.tc.run(t)) d.addCallback(self.tc.get_task_result, block=True) d.addCallback(lambda tr: tr.ns.i) d.addErrback(lambda f: self.assertRaises(AssertionError, f.raiseException)) d.addCallback(lambda r: self.tc.run(t2)) d.addCallback(self.tc.get_task_result, block=True) d.addCallback(lambda tr: tr.ns.i) d.addCallback(lambda r: self.assertEquals(r, 16)) return d
def testSimpleRetries(self): d = self.me.execute(0, 'i=0') t = task.Task("i += 1\nassert i == 16", resultNames='i', retries=10) t2 = task.Task("i += 1\nassert i == 16", resultNames='i', retries=10) d.addCallback(lambda r: self.tc.run(t)) d.addCallback(self.tc.getTaskResult, block=True) d.addCallback(lambda tr: tr.ns.i) d = self.assertDeferredRaises(d, AssertionError) d.addCallback(lambda r: self.tc.run(t2)) d.addCallback(self.tc.getTaskResult, block=True) d.addCallback(lambda tr: tr.ns.i) d = self.assertDeferredEquals(d, 16) return d
def testTaskIDs(self): l = [] for i in range(16): d = self.tc.run(task.Task('a=5')) d = self.assertDeferredEquals(d, i) l.append(d) return defer.DeferredList(l)
def testSetupNS(self): d = self.me.execute(0, 'a=0') ns = dict(a=1, b=0) t = task.Task("", setupNS=ns, resultNames=['a', 'b']) d.addCallback(lambda r: self.tc.run(t)) d.addCallback(self.tc.getTaskResult, block=True) d.addCallback(lambda tr: {'a': tr.ns.a, 'b': tr['b']}) d = self.assertDeferredEquals(d, ns) return d
def testAbort(self): """Cannot do a proper abort test, because blocking execution prevents abort from being called before task completes""" t = task.Task('a=5') d = self.tc.abort(0) d = self.assertDeferredRaises(d, IndexError) d.addCallback(lambda _: self.tc.run(t)) d.addCallback(self.tc.abort) d = self.assertDeferredRaises(d, IndexError) return d
def testSetupNS(self): self.addEngine(1) d = self.multiengine.execute('a=0', targets=0) ns = dict(a=1, b=0) t = task.Task("", push=ns, pull=['a','b']) d.addCallback(lambda r: self.tc.run(t)) d.addCallback(self.tc.get_task_result, block=True) d.addCallback(lambda tr: {'a':tr.ns.a, 'b':tr['b']}) d.addCallback(lambda r: self.assertEquals(r, ns)) return d
def testTaskResults(self): t1 = task.Task('a=5', resultNames='a') d = self.tc.run(t1) d.addCallback(self.tc.getTaskResult, block=True) d.addCallback(lambda tr: (tr.ns.a, tr['a'], tr.failure, tr.raiseException())) d = self.assertDeferredEquals(d, (5, 5, None, None)) t2 = task.Task('7=5') d.addCallback(lambda r: self.tc.run(t2)) d.addCallback(self.tc.getTaskResult, block=True) d.addCallback(lambda tr: tr.ns) d = self.assertDeferredRaises(d, SyntaxError) t3 = task.Task('', resultNames='b') d.addCallback(lambda r: self.tc.run(t3)) d.addCallback(self.tc.getTaskResult, block=True) d.addCallback(lambda tr: tr.ns) d = self.assertDeferredRaises(d, NameError) return d
def testTaskResults(self): self.addEngine(1) t1 = task.Task('a=5', pull='a') d = self.tc.run(t1) d.addCallback(self.tc.get_task_result, block=True) d.addCallback(lambda tr: (tr.ns.a,tr['a'],tr.failure, tr.raiseException())) d.addCallback(lambda r: self.assertEquals(r, (5,5,None,None))) t2 = task.Task('7=5') d.addCallback(lambda r: self.tc.run(t2)) d.addCallback(self.tc.get_task_result, block=True) d.addCallback(lambda tr: tr.ns) d.addErrback(lambda f: self.assertRaises(SyntaxError, f.raiseException)) t3 = task.Task('', pull='b') d.addCallback(lambda r: self.tc.run(t3)) d.addCallback(self.tc.get_task_result, block=True) d.addCallback(lambda tr: tr.ns) d.addErrback(lambda f: self.assertRaises(NameError, f.raiseException)) return d
def testAbort(self): """Cannot do a proper abort test, because blocking execution prevents abort from being called before task completes""" self.addEngine(1) t = task.Task('a=5') d = self.tc.abort(0) d.addErrback(lambda f: self.assertRaises(IndexError, f.raiseException)) d.addCallback(lambda _:self.tc.run(t)) d.addCallback(self.tc.abort) d.addErrback(lambda f: self.assertRaises(IndexError, f.raiseException)) return d
def testClears(self): self.addEngine(1) t = task.Task('a=1', clear_before=True, pull='b', clear_after=True) d = self.multiengine.execute('b=1', targets=0) d.addCallback(lambda _: self.tc.run(t)) d.addCallback(lambda tid: self.tc.get_task_result(tid,block=True)) d.addCallback(lambda tr: tr.failure) d.addErrback(lambda f: self.assertRaises(NameError, f.raiseException)) d.addCallback(lambda _:self.multiengine.pull('a', targets=0)) d.addErrback(lambda f: self.assertRaises(NameError, _raise_it, f)) return d
def testClears(self): d = self.me.execute(0, 'b=1') t = task.Task('a=1', clearBefore=True, resultNames='b', clearAfter=True) d.addCallback(lambda _: self.tc.run(t)) d.addCallback(self.tc.getTaskResult, block=True) d.addCallback(lambda tr: tr.failure) d = self.assertDeferredRaises(d, NameError) # check b for clearBefore d.addCallback(lambda _: self.me.pull(0, 'a')) d = self.assertDeferredRaises(d, NameError) # check a for clearAfter return d
def irun(self, *args, **kwargs): """Run a task on the `TaskController`. This method is a shorthand for run(task) and its arguments are simply passed onto a `Task` object: irun(*args, **kwargs) -> run(Task(*args, **kwargs)) :Parameters: expression : str A str that is valid python code that is the task. pull : str or list of str The names of objects to be pulled as results. push : dict A dict of objects to be pushed into the engines namespace before execution of the expression. clear_before : boolean Should the engine's namespace be cleared before the task is run. Default=False. clear_after : boolean Should the engine's namespace be cleared after the task is run. Default=False. retries : int The number of times to resumbit the task if it fails. Default=0. options : dict Any other keyword options for more elaborate uses of tasks :Returns: A `TaskResult` object. """ block = kwargs.pop('block', False) if len(args) == 1 and isinstance(args[0], task.Task): t = args[0] else: t = task.Task(*args, **kwargs) taskid = self.run(t) print "TaskID = %i"%taskid if block: return self.get_task_result(taskid, block) else: return taskid
def main(): parser = OptionParser() parser.set_defaults(n=100) parser.set_defaults(tmin=1) parser.set_defaults(tmax=60) parser.set_defaults(controller='localhost') parser.set_defaults(meport=10111) parser.set_defaults(tport=10114) parser.add_option("-n", type='int', dest='n', help='the number of tasks to run') parser.add_option("-t", type='float', dest='tmin', help='the minimum task length in seconds') parser.add_option("-T", type='float', dest='tmax', help='the maximum task length in seconds') parser.add_option("-c", type='string', dest='controller', help='the address of the controller') parser.add_option( "-p", type='int', dest='meport', help="the port on which the controller listens for MultiEnginePB") parser.add_option( "-P", type='int', dest='tport', help="the port on which the controller listens for TaskPB") (opts, args) = parser.parse_args() assert opts.tmax >= opts.tmin, "tmax must not be smaller than tmin" rc = multienginepb.PBInteractiveMultiEngineClient( (opts.controller, opts.meport)) rt = taskpb.PBConnectingTaskClient((opts.controller, opts.tport)) rc.connect() rt.connect() rc.block = True nengines = len(rc.getIDs()) rc.executeAll('from IPython.genutils import time') # the jobs should take a random time within a range times = [ random.random() * (opts.tmax - opts.tmin) + opts.tmin for i in range(opts.n) ] tasks = [task.Task("time.sleep(%f)" % t) for t in times] stime = sum(times) print "executing %i tasks, totalling %.1f secs on %i engines" % ( opts.n, stime, nengines) time.sleep(1) start = time.time() results = [rt.run(t) for t in tasks] rt.barrier() stop = time.time() ptime = stop - start scale = stime / ptime print "executed %.1f secs in %.1f secs" % (stime, ptime) print "%.3fx parallel performance on %i engines" % (scale, nengines) print "%.1f%% of theoretical max" % (100 * scale / nengines)