def test_task_registry(self):
r = TaskRegistry()
self.assertIsInstance(r, dict,
"TaskRegistry is mapping")
self.assertRegisterUnregisterCls(r, MockTask)
self.assertRegisterUnregisterCls(r, MockPeriodicTask)
r.register(MockPeriodicTask)
r.unregister(MockPeriodicTask.name)
self.assertNotIn(MockPeriodicTask, r)
r.register(MockPeriodicTask)
tasks = dict(r)
self.assertIsInstance(tasks.get(MockTask.name), MockTask)
self.assertIsInstance(tasks.get(MockPeriodicTask.name),
MockPeriodicTask)
self.assertIsInstance(r[MockTask.name], MockTask)
self.assertIsInstance(r[MockPeriodicTask.name],
MockPeriodicTask)
r.unregister(MockTask)
self.assertNotIn(MockTask.name, r)
r.unregister(MockPeriodicTask)
self.assertNotIn(MockPeriodicTask.name, r)
self.assertTrue(MockTask().run())
self.assertTrue(MockPeriodicTask().run())
class test_TaskBucket(Case):
def setUp(self):
self.registry = TaskRegistry()
self.task_classes = (TaskA, TaskB, TaskC)
for task_cls in self.task_classes:
self.registry.register(task_cls)
@skip_if_disabled
def test_get_nowait(self):
x = buckets.TaskBucket(task_registry=self.registry)
with self.assertRaises(buckets.Empty):
x.get_nowait()
@patch("celery.worker.buckets.sleep")
def test_get_block(self, sleep):
x = buckets.TaskBucket(task_registry=self.registry)
x.not_empty = Mock()
get = x._get = Mock()
remaining = [0]
def effect():
if get.call_count == 1:
raise Empty()
rem = remaining[0]
remaining[0] = 0
return rem, Mock()
get.side_effect = effect
with mock_context(Mock()) as context:
x.not_empty = context
x.wait = Mock()
x.get(block=True)
get.reset()
remaining[0] = 1
x.get(block=True)
def test_get_raises_rate(self):
x = buckets.TaskBucket(task_registry=self.registry)
x.buckets = {1: Mock()}
x.buckets[1].get_nowait.side_effect = buckets.RateLimitExceeded()
x.buckets[1].expected_time.return_value = 0
x._get()
@skip_if_disabled
def test_refresh(self):
reg = {}
x = buckets.TaskBucket(task_registry=reg)
reg["foo"] = "something"
x.refresh()
self.assertIn("foo", x.buckets)
self.assertTrue(x.get_bucket_for_type("foo"))
@skip_if_disabled
def test__get_queue_for_type(self):
x = buckets.TaskBucket(task_registry={})
x.buckets["foo"] = buckets.TokenBucketQueue(fill_rate=1)
self.assertIs(x._get_queue_for_type("foo"), x.buckets["foo"].queue)
x.buckets["bar"] = buckets.FastQueue()
self.assertIs(x._get_queue_for_type("bar"), x.buckets["bar"])
@skip_if_disabled
def test_update_bucket_for_type(self):
bucket = buckets.TaskBucket(task_registry=self.registry)
b = bucket._get_queue_for_type(TaskC.name)
self.assertIs(bucket.update_bucket_for_type(TaskC.name).queue, b)
self.assertIs(bucket.buckets[TaskC.name].queue, b)
@skip_if_disabled
def test_auto_add_on_missing_put(self):
reg = {}
b = buckets.TaskBucket(task_registry=reg)
reg["nonexisting.task"] = "foo"
b.put(MockJob(uuid(), "nonexisting.task", (), {}))
self.assertIn("nonexisting.task", b.buckets)
@skip_if_disabled
def test_auto_add_on_missing(self):
b = buckets.TaskBucket(task_registry=self.registry)
for task_cls in self.task_classes:
self.assertIn(task_cls.name, b.buckets.keys())
self.registry.register(TaskD)
self.assertTrue(b.get_bucket_for_type(TaskD.name))
self.assertIn(TaskD.name, b.buckets.keys())
self.registry.unregister(TaskD)
@skip_if_disabled
def test_has_rate_limits(self):
b = buckets.TaskBucket(task_registry=self.registry)
self.assertEqual(b.buckets[TaskA.name]._bucket.fill_rate, 10)
self.assertIsInstance(b.buckets[TaskB.name], buckets.Queue)
self.assertEqual(b.buckets[TaskC.name]._bucket.fill_rate, 1)
self.registry.register(TaskD)
b.init_with_registry()
try:
self.assertEqual(b.buckets[TaskD.name]._bucket.fill_rate,
1000 / 60.0)
finally:
self.registry.unregister(TaskD)
@skip_if_disabled
def test_on_empty_buckets__get_raises_empty(self):
b = buckets.TaskBucket(task_registry=self.registry)
with self.assertRaises(buckets.Empty):
b.get(block=False)
self.assertEqual(b.qsize(), 0)
@skip_if_disabled
def test_put__get(self):
b = buckets.TaskBucket(task_registry=self.registry)
job = MockJob(uuid(), TaskA.name, ["theqbf"], {"foo": "bar"})
b.put(job)
self.assertEqual(b.get(), job)
@skip_if_disabled
def test_fill_rate(self):
b = buckets.TaskBucket(task_registry=self.registry)
cjob = lambda i: MockJob(uuid(), TaskA.name, [i], {})
jobs = [cjob(i) for i in xrange(20)]
[b.put(job) for job in jobs]
self.assertEqual(b.qsize(), 20)
# 20 items should take at least one second to complete
time_start = time.time()
for i, job in enumerate(jobs):
sys.stderr.write(".")
self.assertEqual(b.get(), job)
self.assertGreater(time.time() - time_start, 1.5)
@skip_if_disabled
def test__very_busy_queue_doesnt_block_others(self):
b = buckets.TaskBucket(task_registry=self.registry)
cjob = lambda i, t: MockJob(uuid(), t.name, [i], {})
ajobs = [cjob(i, TaskA) for i in xrange(10)]
bjobs = [cjob(i, TaskB) for i in xrange(20)]
jobs = list(chain(*izip(bjobs, ajobs)))
for job in jobs:
b.put(job)
got_ajobs = 0
for job in (b.get() for i in xrange(20)):
if job.name == TaskA.name:
got_ajobs += 1
self.assertGreater(got_ajobs, 2)
@skip_if_disabled
def test_thorough__multiple_types(self):
self.registry.register(TaskD)
try:
b = buckets.TaskBucket(task_registry=self.registry)
cjob = lambda i, t: MockJob(uuid(), t.name, [i], {})
ajobs = [cjob(i, TaskA) for i in xrange(10)]
bjobs = [cjob(i, TaskB) for i in xrange(10)]
cjobs = [cjob(i, TaskC) for i in xrange(10)]
djobs = [cjob(i, TaskD) for i in xrange(10)]
# Spread the jobs around.
jobs = list(chain(*izip(ajobs, bjobs, cjobs, djobs)))
[b.put(job) for job in jobs]
for i, job in enumerate(jobs):
sys.stderr.write(".")
self.assertTrue(b.get(), job)
self.assertEqual(i + 1, len(jobs))
finally:
self.registry.unregister(TaskD)
@skip_if_disabled
def test_empty(self):
x = buckets.TaskBucket(task_registry=self.registry)
self.assertTrue(x.empty())
x.put(MockJob(uuid(), TaskC.name, [], {}))
self.assertFalse(x.empty())
x.clear()
self.assertTrue(x.empty())
@skip_if_disabled
def test_items(self):
x = buckets.TaskBucket(task_registry=self.registry)
x.buckets[TaskA.name].put(1)
x.buckets[TaskB.name].put(2)
x.buckets[TaskC.name].put(3)
self.assertEqual(sorted(x.items), [1, 2, 3])
class test_TaskBucket(Case):
def setUp(self):
self.registry = TaskRegistry()
self.task_classes = (TaskA, TaskB, TaskC)
for task_cls in self.task_classes:
self.registry.register(task_cls)
@skip_if_disabled
def test_get_nowait(self):
x = buckets.TaskBucket(task_registry=self.registry)
with self.assertRaises(buckets.Empty):
x.get_nowait()
@patch('celery.worker.buckets.sleep')
def test_get_block(self, sleep):
x = buckets.TaskBucket(task_registry=self.registry)
x.not_empty = Mock()
get = x._get = Mock()
remaining = [0]
def effect():
if get.call_count == 1:
raise Empty()
rem = remaining[0]
remaining[0] = 0
return rem, Mock()
get.side_effect = effect
with mock_context(Mock()) as context:
x.not_empty = context
x.wait = Mock()
x.get(block=True)
get.reset()
remaining[0] = 1
x.get(block=True)
def test_get_raises_rate(self):
x = buckets.TaskBucket(task_registry=self.registry)
x.buckets = {1: Mock()}
x.buckets[1].get_nowait.side_effect = buckets.RateLimitExceeded()
x.buckets[1].expected_time.return_value = 0
x._get()
@skip_if_disabled
def test_refresh(self):
reg = {}
x = buckets.TaskBucket(task_registry=reg)
reg['foo'] = 'something'
x.refresh()
self.assertIn('foo', x.buckets)
self.assertTrue(x.get_bucket_for_type('foo'))
@skip_if_disabled
def test__get_queue_for_type(self):
x = buckets.TaskBucket(task_registry={})
x.buckets['foo'] = buckets.TokenBucketQueue(fill_rate=1)
self.assertIs(x._get_queue_for_type('foo'), x.buckets['foo'].queue)
x.buckets['bar'] = buckets.FastQueue()
self.assertIs(x._get_queue_for_type('bar'), x.buckets['bar'])
@skip_if_disabled
def test_update_bucket_for_type(self):
bucket = buckets.TaskBucket(task_registry=self.registry)
b = bucket._get_queue_for_type(TaskC.name)
self.assertIs(bucket.update_bucket_for_type(TaskC.name).queue, b)
self.assertIs(bucket.buckets[TaskC.name].queue, b)
@skip_if_disabled
def test_auto_add_on_missing_put(self):
reg = {}
b = buckets.TaskBucket(task_registry=reg)
reg['nonexisting.task'] = 'foo'
b.put(MockJob(uuid(), 'nonexisting.task', (), {}))
self.assertIn('nonexisting.task', b.buckets)
@skip_if_disabled
def test_auto_add_on_missing(self):
b = buckets.TaskBucket(task_registry=self.registry)
for task_cls in self.task_classes:
self.assertIn(task_cls.name, b.buckets.keys())
self.registry.register(TaskD)
self.assertTrue(b.get_bucket_for_type(TaskD.name))
self.assertIn(TaskD.name, b.buckets.keys())
self.registry.unregister(TaskD)
@skip_if_disabled
def test_has_rate_limits(self):
b = buckets.TaskBucket(task_registry=self.registry)
self.assertEqual(b.buckets[TaskA.name]._bucket.fill_rate, 10)
self.assertIsInstance(b.buckets[TaskB.name], buckets.Queue)
self.assertEqual(b.buckets[TaskC.name]._bucket.fill_rate, 1)
self.registry.register(TaskD)
b.init_with_registry()
try:
self.assertEqual(b.buckets[TaskD.name]._bucket.fill_rate,
1000 / 60.0)
finally:
self.registry.unregister(TaskD)
@skip_if_disabled
def test_on_empty_buckets__get_raises_empty(self):
b = buckets.TaskBucket(task_registry=self.registry)
with self.assertRaises(buckets.Empty):
b.get(block=False)
self.assertEqual(b.qsize(), 0)
@skip_if_disabled
def test_put__get(self):
b = buckets.TaskBucket(task_registry=self.registry)
job = MockJob(uuid(), TaskA.name, ['theqbf'], {'foo': 'bar'})
b.put(job)
self.assertEqual(b.get(), job)
@skip_if_disabled
def test_fill_rate(self):
b = buckets.TaskBucket(task_registry=self.registry)
cjob = lambda i: MockJob(uuid(), TaskA.name, [i], {})
jobs = [cjob(i) for i in xrange(20)]
[b.put(job) for job in jobs]
self.assertEqual(b.qsize(), 20)
# 20 items should take at least one second to complete
time_start = time.time()
for i, job in enumerate(jobs):
sys.stderr.write('.')
self.assertEqual(b.get(), job)
self.assertGreater(time.time() - time_start, 1.5)
@skip_if_disabled
def test__very_busy_queue_doesnt_block_others(self):
b = buckets.TaskBucket(task_registry=self.registry)
cjob = lambda i, t: MockJob(uuid(), t.name, [i], {})
ajobs = [cjob(i, TaskA) for i in xrange(10)]
bjobs = [cjob(i, TaskB) for i in xrange(20)]
jobs = list(chain(*izip(bjobs, ajobs)))
for job in jobs:
b.put(job)
got_ajobs = 0
for job in (b.get() for i in xrange(20)):
if job.name == TaskA.name:
got_ajobs += 1
self.assertGreater(got_ajobs, 2)
@skip_if_disabled
def test_thorough__multiple_types(self):
self.registry.register(TaskD)
try:
b = buckets.TaskBucket(task_registry=self.registry)
cjob = lambda i, t: MockJob(uuid(), t.name, [i], {})
ajobs = [cjob(i, TaskA) for i in xrange(10)]
bjobs = [cjob(i, TaskB) for i in xrange(10)]
cjobs = [cjob(i, TaskC) for i in xrange(10)]
djobs = [cjob(i, TaskD) for i in xrange(10)]
# Spread the jobs around.
jobs = list(chain(*izip(ajobs, bjobs, cjobs, djobs)))
[b.put(job) for job in jobs]
for i, job in enumerate(jobs):
sys.stderr.write('.')
self.assertTrue(b.get(), job)
self.assertEqual(i + 1, len(jobs))
finally:
self.registry.unregister(TaskD)
@skip_if_disabled
def test_empty(self):
x = buckets.TaskBucket(task_registry=self.registry)
self.assertTrue(x.empty())
x.put(MockJob(uuid(), TaskC.name, [], {}))
self.assertFalse(x.empty())
x.clear()
self.assertTrue(x.empty())
@skip_if_disabled
def test_items(self):
x = buckets.TaskBucket(task_registry=self.registry)
x.buckets[TaskA.name].put(1)
x.buckets[TaskB.name].put(2)
x.buckets[TaskC.name].put(3)
self.assertEqual(sorted(x.items), [1, 2, 3])
