class TestScheduler(unittest.TestCase):
def setUp(self):
self.registry = TaskRegistry()
self.registry.register(DuePeriodicTask)
self.registry.register(PendingPeriodicTask)
self.scheduler = beat.Scheduler(self.registry,
max_interval=0.0001,
logger=log.get_default_logger())
def test_constructor(self):
s = beat.Scheduler()
self.assertTrue(isinstance(s.registry, TaskRegistry))
self.assertTrue(isinstance(s.schedule, dict))
self.assertTrue(isinstance(s.logger, logging.Logger))
self.assertEquals(s.max_interval, conf.CELERYBEAT_MAX_LOOP_INTERVAL)
def test_cleanup(self):
self.scheduler.schedule["fbz"] = beat.ScheduleEntry("fbz")
self.scheduler.cleanup()
self.assertTrue("fbz" not in self.scheduler.schedule)
def test_schedule_registry(self):
self.registry.register(AdditionalTask)
self.scheduler.schedule_registry()
self.assertTrue(AdditionalTask.name in self.scheduler.schedule)
def test_apply_async(self):
due_task = self.registry[DuePeriodicTask.name]
self.scheduler.apply_async(self.scheduler[due_task.name])
self.assertTrue(due_task.applied)
def test_apply_async_raises_SchedulingError_on_error(self):
self.registry.register(AdditionalTask)
self.scheduler.schedule_registry()
add_task = self.registry[AdditionalTask.name]
self.assertRaises(beat.SchedulingError,
self.scheduler.apply_async,
self.scheduler[add_task.name])
def test_is_due(self):
due = self.scheduler[DuePeriodicTask.name]
pending = self.scheduler[PendingPeriodicTask.name]
self.assertTrue(self.scheduler.is_due(due)[0])
self.assertFalse(self.scheduler.is_due(pending)[0])
def test_tick(self):
self.scheduler.schedule.pop(DuePeriodicTaskRaising.name, None)
self.registry.pop(DuePeriodicTaskRaising.name, None)
self.assertEquals(self.scheduler.tick(),
self.scheduler.max_interval)
def test_quick_schedulingerror(self):
self.registry.register(DuePeriodicTaskRaising)
self.scheduler.schedule_registry()
self.assertEquals(self.scheduler.tick(),
self.scheduler.max_interval)
class TestScheduler(unittest.TestCase):
def setUp(self):
self.registry = TaskRegistry()
self.registry.register(DuePeriodicTask)
self.registry.register(PendingPeriodicTask)
self.scheduler = beat.Scheduler(self.registry,
max_interval=0.0001,
logger=log.get_default_logger())
def test_constructor(self):
s = beat.Scheduler()
self.assertIsInstance(s.registry, TaskRegistry)
self.assertIsInstance(s.schedule, dict)
self.assertIsInstance(s.logger, logging.Logger)
self.assertEqual(s.max_interval, conf.CELERYBEAT_MAX_LOOP_INTERVAL)
def test_cleanup(self):
self.scheduler.schedule["fbz"] = beat.ScheduleEntry("fbz")
self.scheduler.cleanup()
self.assertNotIn("fbz", self.scheduler.schedule)
def test_schedule_registry(self):
self.registry.register(AdditionalTask)
self.scheduler.schedule_registry()
self.assertIn(AdditionalTask.name, self.scheduler.schedule)
def test_apply_async(self):
due_task = self.registry[DuePeriodicTask.name]
self.scheduler.apply_async(self.scheduler[due_task.name])
self.assertTrue(due_task.applied)
def test_apply_async_raises_SchedulingError_on_error(self):
self.registry.register(AdditionalTask)
self.scheduler.schedule_registry()
add_task = self.registry[AdditionalTask.name]
self.assertRaises(beat.SchedulingError,
self.scheduler.apply_async,
self.scheduler[add_task.name])
def test_is_due(self):
due = self.scheduler[DuePeriodicTask.name]
pending = self.scheduler[PendingPeriodicTask.name]
self.assertTrue(self.scheduler.is_due(due)[0])
self.assertFalse(self.scheduler.is_due(pending)[0])
def test_tick(self):
self.scheduler.schedule.pop(DuePeriodicTaskRaising.name, None)
self.registry.pop(DuePeriodicTaskRaising.name, None)
self.assertEqual(self.scheduler.tick(),
self.scheduler.max_interval)
def test_quick_schedulingerror(self):
self.registry.register(DuePeriodicTaskRaising)
self.scheduler.schedule_registry()
self.assertEqual(self.scheduler.tick(),
self.scheduler.max_interval)
class test_TaskBucket(unittest.TestCase):
def setUp(self):
self.registry = TaskRegistry()
self.task_classes = (TaskA, TaskB, TaskC)
for task_cls in self.task_classes:
self.registry.register(task_cls)
@skip_if_disabled
def test_get_nowait(self):
x = buckets.TaskBucket(task_registry=self.registry)
with self.assertRaises(buckets.Empty):
x.get_nowait()
@skip_if_disabled
def test_refresh(self):
reg = {}
x = buckets.TaskBucket(task_registry=reg)
reg["foo"] = "something"
x.refresh()
self.assertIn("foo", x.buckets)
self.assertTrue(x.get_bucket_for_type("foo"))
@skip_if_disabled
def test__get_queue_for_type(self):
x = buckets.TaskBucket(task_registry={})
x.buckets["foo"] = buckets.TokenBucketQueue(fill_rate=1)
self.assertIs(x._get_queue_for_type("foo"), x.buckets["foo"].queue)
x.buckets["bar"] = buckets.FastQueue()
self.assertIs(x._get_queue_for_type("bar"), x.buckets["bar"])
@skip_if_disabled
def test_update_bucket_for_type(self):
bucket = buckets.TaskBucket(task_registry=self.registry)
b = bucket._get_queue_for_type(TaskC.name)
self.assertIs(bucket.update_bucket_for_type(TaskC.name).queue, b)
self.assertIs(bucket.buckets[TaskC.name].queue, b)
@skip_if_disabled
def test_auto_add_on_missing_put(self):
reg = {}
b = buckets.TaskBucket(task_registry=reg)
reg["nonexisting.task"] = "foo"
b.put(MockJob(uuid(), "nonexisting.task", (), {}))
self.assertIn("nonexisting.task", b.buckets)
@skip_if_disabled
def test_auto_add_on_missing(self):
b = buckets.TaskBucket(task_registry=self.registry)
for task_cls in self.task_classes:
self.assertIn(task_cls.name, b.buckets.keys())
self.registry.register(TaskD)
self.assertTrue(b.get_bucket_for_type(TaskD.name))
self.assertIn(TaskD.name, b.buckets.keys())
self.registry.unregister(TaskD)
@skip_if_disabled
def test_has_rate_limits(self):
b = buckets.TaskBucket(task_registry=self.registry)
self.assertEqual(b.buckets[TaskA.name]._bucket.fill_rate, 10)
self.assertIsInstance(b.buckets[TaskB.name], buckets.Queue)
self.assertEqual(b.buckets[TaskC.name]._bucket.fill_rate, 1)
self.registry.register(TaskD)
b.init_with_registry()
try:
self.assertEqual(b.buckets[TaskD.name]._bucket.fill_rate,
1000 / 60.0)
finally:
self.registry.unregister(TaskD)
@skip_if_disabled
def test_on_empty_buckets__get_raises_empty(self):
b = buckets.TaskBucket(task_registry=self.registry)
with self.assertRaises(buckets.Empty):
b.get(block=False)
self.assertEqual(b.qsize(), 0)
@skip_if_disabled
def test_put__get(self):
b = buckets.TaskBucket(task_registry=self.registry)
job = MockJob(uuid(), TaskA.name, ["theqbf"], {"foo": "bar"})
b.put(job)
self.assertEqual(b.get(), job)
@skip_if_disabled
def test_fill_rate(self):
b = buckets.TaskBucket(task_registry=self.registry)
cjob = lambda i: MockJob(uuid(), TaskA.name, [i], {})
jobs = [cjob(i) for i in xrange(20)]
[b.put(job) for job in jobs]
self.assertEqual(b.qsize(), 20)
# 20 items should take at least one second to complete
time_start = time.time()
for i, job in enumerate(jobs):
sys.stderr.write(".")
self.assertEqual(b.get(), job)
self.assertGreater(time.time() - time_start, 1.5)
@skip_if_disabled
def test__very_busy_queue_doesnt_block_others(self):
b = buckets.TaskBucket(task_registry=self.registry)
cjob = lambda i, t: MockJob(uuid(), t.name, [i], {})
ajobs = [cjob(i, TaskA) for i in xrange(10)]
bjobs = [cjob(i, TaskB) for i in xrange(20)]
jobs = list(chain(*izip(bjobs, ajobs)))
for job in jobs:
b.put(job)
got_ajobs = 0
for job in (b.get() for i in xrange(20)):
if job.task_name == TaskA.name:
got_ajobs += 1
self.assertGreater(got_ajobs, 2)
@skip_if_disabled
def test_thorough__multiple_types(self):
self.registry.register(TaskD)
try:
b = buckets.TaskBucket(task_registry=self.registry)
cjob = lambda i, t: MockJob(uuid(), t.name, [i], {})
ajobs = [cjob(i, TaskA) for i in xrange(10)]
bjobs = [cjob(i, TaskB) for i in xrange(10)]
cjobs = [cjob(i, TaskC) for i in xrange(10)]
djobs = [cjob(i, TaskD) for i in xrange(10)]
# Spread the jobs around.
jobs = list(chain(*izip(ajobs, bjobs, cjobs, djobs)))
[b.put(job) for job in jobs]
for i, job in enumerate(jobs):
sys.stderr.write(".")
self.assertTrue(b.get(), job)
self.assertEqual(i + 1, len(jobs))
finally:
self.registry.unregister(TaskD)
@skip_if_disabled
def test_empty(self):
x = buckets.TaskBucket(task_registry=self.registry)
self.assertTrue(x.empty())
x.put(MockJob(uuid(), TaskC.name, [], {}))
self.assertFalse(x.empty())
x.clear()
self.assertTrue(x.empty())
@skip_if_disabled
def test_items(self):
x = buckets.TaskBucket(task_registry=self.registry)
x.buckets[TaskA.name].put(1)
x.buckets[TaskB.name].put(2)
x.buckets[TaskC.name].put(3)
self.assertEqual(sorted(x.items), [1, 2, 3])
class test_TaskBucket(Case):
def setUp(self):
self.registry = TaskRegistry()
self.task_classes = (TaskA, TaskB, TaskC)
for task_cls in self.task_classes:
self.registry.register(task_cls)
@skip_if_disabled
def test_get_nowait(self):
x = buckets.TaskBucket(task_registry=self.registry)
with self.assertRaises(buckets.Empty):
x.get_nowait()
@skip_if_disabled
def test_refresh(self):
reg = {}
x = buckets.TaskBucket(task_registry=reg)
reg["foo"] = "something"
x.refresh()
self.assertIn("foo", x.buckets)
self.assertTrue(x.get_bucket_for_type("foo"))
@skip_if_disabled
def test__get_queue_for_type(self):
x = buckets.TaskBucket(task_registry={})
x.buckets["foo"] = buckets.TokenBucketQueue(fill_rate=1)
self.assertIs(x._get_queue_for_type("foo"), x.buckets["foo"].queue)
x.buckets["bar"] = buckets.FastQueue()
self.assertIs(x._get_queue_for_type("bar"), x.buckets["bar"])
@skip_if_disabled
def test_update_bucket_for_type(self):
bucket = buckets.TaskBucket(task_registry=self.registry)
b = bucket._get_queue_for_type(TaskC.name)
self.assertIs(bucket.update_bucket_for_type(TaskC.name).queue, b)
self.assertIs(bucket.buckets[TaskC.name].queue, b)
@skip_if_disabled
def test_auto_add_on_missing_put(self):
reg = {}
b = buckets.TaskBucket(task_registry=reg)
reg["nonexisting.task"] = "foo"
b.put(MockJob(uuid(), "nonexisting.task", (), {}))
self.assertIn("nonexisting.task", b.buckets)
@skip_if_disabled
def test_auto_add_on_missing(self):
b = buckets.TaskBucket(task_registry=self.registry)
for task_cls in self.task_classes:
self.assertIn(task_cls.name, b.buckets.keys())
self.registry.register(TaskD)
self.assertTrue(b.get_bucket_for_type(TaskD.name))
self.assertIn(TaskD.name, b.buckets.keys())
self.registry.unregister(TaskD)
@skip_if_disabled
def test_has_rate_limits(self):
b = buckets.TaskBucket(task_registry=self.registry)
self.assertEqual(b.buckets[TaskA.name]._bucket.fill_rate, 10)
self.assertIsInstance(b.buckets[TaskB.name], buckets.Queue)
self.assertEqual(b.buckets[TaskC.name]._bucket.fill_rate, 1)
self.registry.register(TaskD)
b.init_with_registry()
try:
self.assertEqual(b.buckets[TaskD.name]._bucket.fill_rate,
1000 / 60.0)
finally:
self.registry.unregister(TaskD)
@skip_if_disabled
def test_on_empty_buckets__get_raises_empty(self):
b = buckets.TaskBucket(task_registry=self.registry)
with self.assertRaises(buckets.Empty):
b.get(block=False)
self.assertEqual(b.qsize(), 0)
@skip_if_disabled
def test_put__get(self):
b = buckets.TaskBucket(task_registry=self.registry)
job = MockJob(uuid(), TaskA.name, ["theqbf"], {"foo": "bar"})
b.put(job)
self.assertEqual(b.get(), job)
@skip_if_disabled
def test_fill_rate(self):
b = buckets.TaskBucket(task_registry=self.registry)
cjob = lambda i: MockJob(uuid(), TaskA.name, [i], {})
jobs = [cjob(i) for i in xrange(20)]
[b.put(job) for job in jobs]
self.assertEqual(b.qsize(), 20)
# 20 items should take at least one second to complete
time_start = time.time()
for i, job in enumerate(jobs):
sys.stderr.write(".")
self.assertEqual(b.get(), job)
self.assertGreater(time.time() - time_start, 1.5)
@skip_if_disabled
def test__very_busy_queue_doesnt_block_others(self):
b = buckets.TaskBucket(task_registry=self.registry)
cjob = lambda i, t: MockJob(uuid(), t.name, [i], {})
ajobs = [cjob(i, TaskA) for i in xrange(10)]
bjobs = [cjob(i, TaskB) for i in xrange(20)]
jobs = list(chain(*izip(bjobs, ajobs)))
for job in jobs:
b.put(job)
got_ajobs = 0
for job in (b.get() for i in xrange(20)):
if job.task_name == TaskA.name:
got_ajobs += 1
self.assertGreater(got_ajobs, 2)
@skip_if_disabled
def test_thorough__multiple_types(self):
self.registry.register(TaskD)
try:
b = buckets.TaskBucket(task_registry=self.registry)
cjob = lambda i, t: MockJob(uuid(), t.name, [i], {})
ajobs = [cjob(i, TaskA) for i in xrange(10)]
bjobs = [cjob(i, TaskB) for i in xrange(10)]
cjobs = [cjob(i, TaskC) for i in xrange(10)]
djobs = [cjob(i, TaskD) for i in xrange(10)]
# Spread the jobs around.
jobs = list(chain(*izip(ajobs, bjobs, cjobs, djobs)))
[b.put(job) for job in jobs]
for i, job in enumerate(jobs):
sys.stderr.write(".")
self.assertTrue(b.get(), job)
self.assertEqual(i + 1, len(jobs))
finally:
self.registry.unregister(TaskD)
@skip_if_disabled
def test_empty(self):
x = buckets.TaskBucket(task_registry=self.registry)
self.assertTrue(x.empty())
x.put(MockJob(uuid(), TaskC.name, [], {}))
self.assertFalse(x.empty())
x.clear()
self.assertTrue(x.empty())
@skip_if_disabled
def test_items(self):
x = buckets.TaskBucket(task_registry=self.registry)
x.buckets[TaskA.name].put(1)
x.buckets[TaskB.name].put(2)
x.buckets[TaskC.name].put(3)
self.assertEqual(sorted(x.items), [1, 2, 3])
class TestTaskBuckets(unittest.TestCase):
def setUp(self):
self.registry = TaskRegistry()
self.task_classes = (TaskA, TaskB, TaskC)
for task_cls in self.task_classes:
self.registry.register(task_cls)
@skip_if_disabled
def test_auto_add_on_missing(self):
b = buckets.TaskBucket(task_registry=self.registry)
for task_cls in self.task_classes:
self.assertIn(task_cls.name, b.buckets.keys())
self.registry.register(TaskD)
self.assertTrue(b.get_bucket_for_type(TaskD.name))
self.assertIn(TaskD.name, b.buckets.keys())
self.registry.unregister(TaskD)
@skip_if_disabled
def test_has_rate_limits(self):
b = buckets.TaskBucket(task_registry=self.registry)
self.assertEqual(b.buckets[TaskA.name].fill_rate, 10)
self.assertIsInstance(b.buckets[TaskB.name], buckets.Queue)
self.assertEqual(b.buckets[TaskC.name].fill_rate, 1)
self.registry.register(TaskD)
b.init_with_registry()
try:
self.assertEqual(b.buckets[TaskD.name].fill_rate, 1000 / 60.0)
finally:
self.registry.unregister(TaskD)
@skip_if_disabled
def test_on_empty_buckets__get_raises_empty(self):
b = buckets.TaskBucket(task_registry=self.registry)
self.assertRaises(buckets.QueueEmpty, b.get)
self.assertEqual(b.qsize(), 0)
@skip_if_disabled
def test_put__get(self):
b = buckets.TaskBucket(task_registry=self.registry)
job = MockJob(gen_unique_id(), TaskA.name, ["theqbf"], {"foo": "bar"})
b.put(job)
self.assertEqual(b.get(), job)
@skip_if_disabled
def test_fill_rate(self):
b = buckets.TaskBucket(task_registry=self.registry)
cjob = lambda i: MockJob(gen_unique_id(), TaskA.name, [i], {})
jobs = [cjob(i) for i in xrange(20)]
[b.put(job) for job in jobs]
self.assertEqual(b.qsize(), 20)
# 20 items should take at least one second to complete
time_start = time.time()
for i, job in enumerate(jobs):
sys.stderr.write("i")
self.assertEqual(b.get(), job)
self.assertGreater(time.time() - time_start, 1.5)
@skip_if_disabled
def test__very_busy_queue_doesnt_block_others(self):
b = buckets.TaskBucket(task_registry=self.registry)
cjob = lambda i, t: MockJob(gen_unique_id(), t.name, [i], {})
ajobs = [cjob(i, TaskA) for i in xrange(10)]
bjobs = [cjob(i, TaskB) for i in xrange(20)]
jobs = list(chain(*izip(bjobs, ajobs)))
map(b.put, jobs)
got_ajobs = 0
for job in (b.get() for i in xrange(20)):
if job.task_name == TaskA.name:
got_ajobs += 1
self.assertGreater(got_ajobs, 2)
@skip_if_disabled
def test_thorough__multiple_types(self):
self.registry.register(TaskD)
try:
b = buckets.TaskBucket(task_registry=self.registry)
cjob = lambda i, t: MockJob(gen_unique_id(), t.name, [i], {})
ajobs = [cjob(i, TaskA) for i in xrange(10)]
bjobs = [cjob(i, TaskB) for i in xrange(10)]
cjobs = [cjob(i, TaskC) for i in xrange(10)]
djobs = [cjob(i, TaskD) for i in xrange(10)]
# Spread the jobs around.
jobs = list(chain(*izip(ajobs, bjobs, cjobs, djobs)))
[b.put(job) for job in jobs]
for i, job in enumerate(jobs):
sys.stderr.write("0")
self.assertTrue(b.get(), job)
self.assertEqual(i+1, len(jobs))
finally:
self.registry.unregister(TaskD)