def testRequestAndAccept(self):
task = MockTask(lambda x: 10*x, (2,), {}, 100, None)
worker = MockWorker()
worker.hired = False
self.af.request(worker, None)
d = defer.DeferredList([self.af.new(task), task.d])
d.addCallback(lambda _: self.failUnlessEqual(worker.ran, [task]))
return d
def testHireMultipleWorkersThenShutdown(self):
ID_1 = self.mgr.hire(MockWorker())
ID_2 = self.mgr.hire(MockWorker())
self.failIfEqual(ID_1, ID_2)
self.failUnlessEqual(len(self.mgr.workers), 2)
d = self.mgr.shutdown()
d.addCallback(lambda _: self.failUnlessEqual(self.mgr.workers, {}))
return d
def test_multipleWorkers(self):
N = 100
mutable = []
def checkResults(null):
self.failUnlessEqual(len(mutable), N)
self.failUnlessEqual(sum(mutable), sum([2 * x for x in xrange(N)]))
for runDelay in (0.05, 0.1, 0.4):
worker = MockWorker(runDelay)
ID = self.queue.attachWorker(worker)
worker.ID = ID
dList = []
for x in xrange(N):
d = self.queue.call(lambda y: 2 * y, x)
d.addCallback(lambda result: mutable.append(result))
dList.append(d)
d = defer.DeferredList(dList)
d.addCallback(checkResults)
return d
def testTerminateGracefully(self):
self._count = 0
worker = MockWorker()
workerID = self.mgr.hire(worker)
task = MockTask(lambda x: x, ('foo',), {}, 100, None)
d1 = self.mgr.assignment(task)
d1.addCallback(self._callback, "Assignment accepted", 1)
d2 = task.d
d2.addCallback(self._callback, "Task done", 2)
d3 = self.mgr.terminate(workerID)
d3.addCallback(self._callback, "Worker terminated", 3)
return defer.gatherResults([d1,d2,d3])
def testTerminateByCrashing(self):
def checkTask(null):
self.failIf(task.d.called)
self._count = 0
worker = MockWorker(runDelay=1.0)
workerID = self.mgr.hire(worker)
task = MockTask(lambda x: x, ('foo',), {}, 100, None)
d = self.mgr.assignment(task)
d.addCallback(self._callback, "Assignment accepted", 1)
d.addCallback(lambda _: self.mgr.terminate(workerID, crash=True))
d.addCallback(self._callback, "Worker terminated", 2, value=[task])
return d.addCallback(checkTask)
def testTerminateBeforeTimeout(self):
def checkTask(null):
self.failUnless(task.d.called)
self._count = 0
worker = MockWorker(runDelay=1.0)
workerID = self.mgr.hire(worker)
task = MockTask(lambda x: x, ('foo',), {}, 100, None)
d1 = self.mgr.assignment(task)
d1.addCallback(self._callback, "Assignment accepted", 1)
d2 = self.mgr.terminate(workerID, timeout=2.0)
d2.addCallback(self._callback, "Worker terminated", 2, value=[])
return defer.gatherResults([d1,d2]).addCallback(checkTask)
def test_multipleWorkers(self):
N = 100
mutable = []
def checkResults(null):
self.failUnlessEqual(len(mutable), N)
self.failUnlessEqual(
sum(mutable),
sum([2*x for x in xrange(N)]))
for runDelay in (0.05, 0.1, 0.4):
worker = MockWorker(runDelay)
ID = self.queue.attachWorker(worker)
worker.ID = ID
dList = []
for x in xrange(N):
d = self.queue.call(lambda y: 2*y, x)
d.addCallback(lambda result: mutable.append(result))
dList.append(d)
d = defer.DeferredList(dList)
d.addCallback(checkResults)
return d
def testMultipleWorkers(self):
N = 50
mutable = []
workerFast = MockWorker(0.1)
workerSlow = MockWorker(0.2)
def checkResults(null):
self.failUnlessEqual(len(mutable), N)
self.failUnlessApproximates(
2*len(workerSlow.ran), len(workerFast.ran), 2)
self.mgr.hire(workerFast)
self.mgr.hire(workerSlow)
dList = []
for null in xrange(N):
task = MockTask(lambda : mutable.append(None), (), {}, 100, None)
# For this test, we don't care about when assignments are accepted
self.mgr.assignment(task)
# We only care about when they are done
dList.append(task.d)
d = defer.DeferredList(dList)
d.addCallback(checkResults)
return d
def testRequestBasic(self):
OriginalAssignment = tasks.Assignment
class ModifiedAssignment(tasks.Assignment):
mutable = []
def accept(self, worker):
self.mutable.append(worker)
self.d.callback(None)
def finishUp(null, worker):
self.failUnlessEqual(ModifiedAssignment.mutable, [worker])
tasks.Assignment = OriginalAssignment
tasks.Assignment = ModifiedAssignment
task = MockTask(lambda x: 10*x, (2,), {}, 100, None)
worker = MockWorker()
worker.hired = False
self.af.request(worker, None)
for dList in worker.assignments.itervalues():
self.failUnlessEqual(
[isinstance(x, defer.Deferred) for x in dList], [True])
d = self.af.new(task)
d.addCallback(finishUp, worker)
return d
def test_oneWorker(self):
N = 30
mutable = []
def checkResults(null):
self.failUnlessEqual(len(mutable), N)
self.failUnlessEqual(sum(mutable), sum([2 * x for x in xrange(N)]))
worker = MockWorker(0.01)
self.queue.attachWorker(worker)
dList = []
for x in xrange(N):
d = self.queue.call(lambda y: 2 * y, x)
d.addCallback(lambda result: mutable.append(result))
dList.append(d)
d = defer.DeferredList(dList)
d.addCallback(checkResults)
return d
def testOneWorker(self):
worker = MockWorker(0.2)
N = 10
def completed(null):
self.failUnlessEqual(
[type(x) for x in worker.ran], [MockTask]*N)
self.mgr.hire(worker)
dList = []
for null in xrange(N):
task = MockTask(lambda x: x, ('foo',), {}, 100, None)
# For this test, we don't care about when assignments are accepted
self.mgr.assignment(task)
# We only care about when they are done
dList.append(task.d)
d = defer.DeferredList(dList)
d.addCallback(completed)
return d
def testHireInstanceQualifications(self):
worker = MockWorker()
worker.iQualified = ['bar']
workerID = self.mgr.hire(worker)
self._checkAssignments(workerID)
def testHireSetWorkerID(self):
worker = MockWorker()
workerID = self.mgr.hire(worker)
self.failUnlessEqual(getattr(worker, 'ID', None), workerID)
def test_oneTask(self):
worker = MockWorker(0.5)
self.queue.attachWorker(worker)
d = self.queue.call(lambda x: 2 * x, 15)
d.addCallback(self.failUnlessEqual, 30)
return d
