def test_requeue_delayed_jobs_put_back_ready_delayed_jobs_to_the_waiting_list(self):
queue = Queue.get_queue(name='test')
job1 = Job.add_job(identifier='job:1', queue_name='test', delayed_for=10)
job2 = Job.add_job(identifier='job:2', queue_name='test', delayed_for=1)
self.assertEqual(Queue.count_waiting_jobs('test'), 0)
self.assertEqual(Queue.count_delayed_jobs('test'), 2)
self.assertEqual(job1.status.hget(), STATUSES.DELAYED)
self.assertEqual(job2.status.hget(), STATUSES.DELAYED)
# must not move any jobs, too soon
queue.requeue_delayed_jobs()
self.assertEqual(Queue.count_waiting_jobs('test'), 0)
self.assertEqual(Queue.count_delayed_jobs('test'), 2)
self.assertEqual(job1.status.hget(), STATUSES.DELAYED)
self.assertEqual(job2.status.hget(), STATUSES.DELAYED)
sleep(1)
# now we should have one job in the waiting list
queue.requeue_delayed_jobs()
self.assertEqual(Queue.count_waiting_jobs('test'), 1)
self.assertEqual(Queue.count_delayed_jobs('test'), 1)
self.assertEqual(job1.status.hget(), STATUSES.DELAYED)
self.assertEqual(job2.status.hget(), STATUSES.WAITING)
waiting_jobs = queue.waiting.lmembers()
self.assertEqual(waiting_jobs, [job2.ident])
delayed_jobs = queue.delayed.zrange(0, -1, withscores=True)
self.assertEqual(delayed_jobs[0][0], job1.ident)
def test_a_job_in_error_could_be_requeued(self):
job = Job(identifier='job:1', status=STATUSES.ERROR)
job.requeue('test')
queue = Queue.get_queue('test')
self.assertEqual(queue.waiting.llen(), 1)
self.assertEqual(queue.delayed.zcard(), 0)
self.assertEqual(job.status.hget(), STATUSES.WAITING)
def test_requeue_delayed_jobs_put_back_ready_delayed_jobs_to_the_waiting_list(self):
queue = Queue.get_queue(name='test')
job1 = Job.add_job(identifier='job:1', queue_name='test', delayed_for=10)
job2 = Job.add_job(identifier='job:2', queue_name='test', delayed_for=1)
self.assertEqual(Queue.count_waiting_jobs('test'), 0)
self.assertEqual(Queue.count_delayed_jobs('test'), 2)
self.assertEqual(job1.status.hget(), STATUSES.DELAYED)
self.assertEqual(job2.status.hget(), STATUSES.DELAYED)
# must not move any jobs, too soon
queue.requeue_delayed_jobs()
self.assertEqual(Queue.count_waiting_jobs('test'), 0)
self.assertEqual(Queue.count_delayed_jobs('test'), 2)
self.assertEqual(job1.status.hget(), STATUSES.DELAYED)
self.assertEqual(job2.status.hget(), STATUSES.DELAYED)
sleep(1)
# now we should have one job in the waiting list
queue.requeue_delayed_jobs()
self.assertEqual(Queue.count_waiting_jobs('test'), 1)
self.assertEqual(Queue.count_delayed_jobs('test'), 1)
self.assertEqual(job1.status.hget(), STATUSES.DELAYED)
self.assertEqual(job2.status.hget(), STATUSES.WAITING)
waiting_jobs = queue.waiting.lmembers()
self.assertEqual(waiting_jobs, [job2.ident])
delayed_jobs = queue.delayed.zrange(0, -1, withscores=True)
self.assertEqual(delayed_jobs[0][0], job1.ident)
def test_job_error_method_should_be_called(self):
class ExceptionWithCode(Exception):
def __init__(self, message, code):
super(ExceptionWithCode, self).__init__(message)
self.message = message
self.code = code
def callback(job, queue):
raise ExceptionWithCode('foobar', 42)
job1 = Job.add_job(identifier='job:1', queue_name='test')
queue = Queue.get_queue(name='test')
worker = Worker(name='test', max_loops=1) # no callback
worker.run()
self.assertEqual(job1.status.hget(), STATUSES.ERROR)
self.assertIn(job1.get_pk(), queue.errors.lmembers())
self.assertEqual(len(Error.collection()), 1)
error = Error.get(identifier='job:1')
self.assertEqual(error.message.hget(), 'You must implement your own action')
self.assertEqual(error.code.hget(), None)
job2 = Job.add_job(identifier='job:2', queue_name='test')
queue = Queue.get_queue(name='test')
worker = Worker(name='test', max_loops=1, callback=callback) # callback with exception
worker.run()
self.assertEqual(job2.status.hget(), STATUSES.ERROR)
self.assertIn(job2.get_pk(), queue.errors.lmembers())
self.assertEqual(len(Error.collection()), 2)
error = Error.get(identifier='job:2')
self.assertEqual(error.message.hget(), 'foobar')
self.assertEqual(error.code.hget(), '42')
def test_a_job_in_error_could_be_requeued(self):
job = Job(identifier='job:1', status=STATUSES.ERROR)
job.requeue('test')
queue = Queue.get_queue('test')
self.assertEqual(queue.waiting.llen(), 1)
self.assertEqual(queue.delayed.zcard(), 0)
self.assertEqual(job.status.hget(), STATUSES.WAITING)
def test_get_from_ident_should_return_a_job(self):
job = Job.add_job(identifier='job:1', queue_name='test')
test_job = Job.get_from_ident('limpyd_jobs.models.Job:%s' % job.pk.get())
self.assertEqual(job.ident, test_job.ident)
job = JobFooBar.add_job(identifier='foobar:1', queue_name='foobar')
test_job = Job.get_from_ident('tests.models.JobFooBar:%s' % job.pk.get())
self.assertEqual(job.ident, test_job.ident)
def test_get_model_repr_should_return_the_string_repr_of_the_model(self):
self.assertEqual(Job.get_model_repr(), 'limpyd_jobs.models.Job')
job = Job()
self.assertEqual(job.get_model_repr(), 'limpyd_jobs.models.Job')
self.assertEqual(JobFooBar.get_model_repr(), 'tests.models.JobFooBar')
job = JobFooBar()
self.assertEqual(job.get_model_repr(), 'tests.models.JobFooBar')
def test_get_from_ident_should_return_a_job(self):
job = Job.add_job(identifier='job:1', queue_name='test')
test_job = Job.get_from_ident('limpyd_jobs.models.Job:%s' % job.pk.get())
self.assertEqual(job.ident, test_job.ident)
job = JobFooBar.add_job(identifier='foobar:1', queue_name='foobar')
test_job = Job.get_from_ident('tests.models.JobFooBar:%s' % job.pk.get())
self.assertEqual(job.ident, test_job.ident)
def test_requeuing_a_job_with_a_datetime_should_put_it_in_the_delayed_lits(self):
job = Job(identifier='job:1', status=STATUSES.ERROR)
queue = Queue.get_queue('test')
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(queue.delayed.zcard(), 0)
job.requeue('test', delayed_until=datetime.utcnow()+timedelta(seconds=5))
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(queue.delayed.zcard(), 1)
self.assertEqual(job.status.hget(), STATUSES.DELAYED)
def test_requeuing_a_job_with_a_datetime_should_put_it_in_the_delayed_lits(self):
job = Job(identifier='job:1', status=STATUSES.ERROR)
queue = Queue.get_queue('test')
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(queue.delayed.zcard(), 0)
job.requeue('test', delayed_until=datetime.utcnow()+timedelta(seconds=5))
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(queue.delayed.zcard(), 1)
self.assertEqual(job.status.hget(), STATUSES.DELAYED)
def test_prepending_an_existing_job_should_move_it_at_the_beginning(self):
queue = Queue.get_queue(name='test', priority=1)
job1 = Job.add_job(identifier='job:1', queue_name='test', priority=1)
job2 = Job.add_job(identifier='job:2', queue_name='test', priority=1)
self.assertEqual(queue.waiting.lmembers(), [job1.ident, job2.ident])
Job.add_job(identifier='job:2', queue_name='test', priority=1, prepend=True)
self.assertEqual(queue.waiting.lmembers(), [job2.ident, job1.ident])
def test_prepending_an_existing_job_should_move_it_at_the_beginning(self):
queue = Queue.get_queue(name='test', priority=1)
job1 = Job.add_job(identifier='job:1', queue_name='test', priority=1)
job2 = Job.add_job(identifier='job:2', queue_name='test', priority=1)
self.assertEqual(queue.waiting.lmembers(), [job1.ident, job2.ident])
Job.add_job(identifier='job:2', queue_name='test', priority=1, prepend=True)
self.assertEqual(queue.waiting.lmembers(), [job2.ident, job1.ident])
def test_enqueue_or_delay_should_delay_if_in_the_future(self):
queue = Queue.get_queue(name='test', priority=1)
job = Job(identifier='job:1')
delayed_until = datetime.utcnow() + timedelta(seconds=5)
job.enqueue_or_delay('test', 1, delayed_until=delayed_until)
self.assertEqual(queue.delayed.zcard(), 1)
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(job.delayed_until.hget(), str(delayed_until))
self.assertEqual(job.status.hget(), STATUSES.DELAYED)
self.assertEqual(job.priority.hget(), '1')
def test_enqueue_or_delay_should_delay_if_in_the_future(self):
queue = Queue.get_queue(name='test', priority=1)
job = Job(identifier='job:1')
delayed_until = datetime.utcnow() + timedelta(seconds=5)
job.enqueue_or_delay('test', 1, delayed_until=delayed_until)
self.assertEqual(queue.delayed.zcard(), 1)
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(job.delayed_until.hget(), str(delayed_until))
self.assertEqual(job.status.hget(), STATUSES.DELAYED)
self.assertEqual(job.priority.hget(), '1')
def test_first_delayed_should_return_the_first_delayed_job_to_be_ready(self):
Job.add_job(identifier='job:1', queue_name='test', delayed_for=5)
job2 = Job.add_job(identifier='job:2', queue_name='test', delayed_for=1)
queue = Queue.get_queue(name='test')
attended_timestamp = datetime_to_score(parse(job2.delayed_until.hget()))
job_ident, timestamp = queue.first_delayed
self.assertEqual(job_ident, job2.ident)
self.assertEqual(timestamp, attended_timestamp)
timestamp = queue.first_delayed_time
self.assertEqual(timestamp, attended_timestamp)
def test_run_ended_method_should_be_called(self):
class TestWorker(Worker):
passed = False
def run_ended(self):
super(TestWorker, self).run_ended()
self.passed = True
Job.add_job('job:1', 'test')
worker = TestWorker('test', max_loops=1) # one loop to run only one job
worker.run()
self.assertEqual(worker.passed, True)
def test_first_delayed_should_return_the_first_delayed_job_to_be_ready(self):
Job.add_job(identifier='job:1', queue_name='test', delayed_for=5)
job2 = Job.add_job(identifier='job:2', queue_name='test', delayed_for=1)
queue = Queue.get_queue(name='test')
attended_timestamp = datetime_to_score(parse(job2.delayed_until.hget()))
job_ident, timestamp = queue.first_delayed
self.assertEqual(job_ident, job2.ident)
self.assertEqual(timestamp, attended_timestamp)
timestamp = queue.first_delayed_time
self.assertEqual(timestamp, attended_timestamp)
def test_adding_a_job_with_delayed_until_not_in_the_future_should_add_the_job_in_the_waiting_list(self):
queue = Queue.get_queue(name='test')
job = Job.add_job(identifier='job:1', queue_name='test', delayed_until=datetime.utcnow() - timedelta(seconds=5))
self.assertEqual(job.status.hget(), STATUSES.WAITING)
self.assertEqual(queue.waiting.llen(), 1)
self.assertEqual(queue.delayed.zcard(), 0)
def test_add_error_method_should_add_an_error_instance(self):
e = ErrorTests.ExceptionWithCode('the answer', 42)
job = Job.add_job(identifier='job:1', queue_name='test')
error1 = Error.add_error(queue_name='test', job=job, error=e)
self.assertEqual(list(Error.collection()), [error1.pk.get()])
Error.add_error(queue_name='test', job=job, error=e)
self.assertEqual(len(Error.collection()), 2)
def test_wait_for_job_should_respond_with_queue_and_job_when_a_queue_is_not_empty(self):
queue = Queue.get_queue('test')
worker = Worker('test')
worker.test_content = None
class Thread(threading.Thread):
def run(self):
worker.update_keys()
queue, job = worker.wait_for_job()
worker.test_content = queue, job
# start listening
thread = Thread()
thread.start()
time.sleep(0.1)
# no job, nothing received
self.assertIsNone(worker.test_content)
# add a job
job = Job.add_job(queue_name='test', identifier='job:1')
time.sleep(0.1)
# info should be received
self.assertEqual(worker.test_content[0].get_pk(), queue.get_pk())
self.assertTrue(isinstance(worker.test_content[0], Queue))
self.assertEqual(worker.test_content[1].get_pk(), job.get_pk())
self.assertTrue(isinstance(worker.test_content[1], Job))
# job must no be in the queue anymore
self.assertNotIn(job.get_pk(), queue.waiting.lmembers())
def test_add_error_method_should_add_an_error_instance(self):
e = ErrorTests.ExceptionWithCode('the answer', 42)
job = Job.add_job(identifier='job:1', queue_name='test')
error1 = Error.add_error(queue_name='test', job=job, error=e)
self.assertListEqual(list(Error.collection()), [error1.pk.get()])
Error.add_error(queue_name='test', job=job, error=e)
self.assertEqual(len(Error.collection()), 2)
def test_adding_a_job_with_delayed_until_not_in_the_future_should_add_the_job_in_the_waiting_list(self):
queue = Queue.get_queue(name='test')
job = Job.add_job(identifier='job:1', queue_name='test', delayed_until=datetime.utcnow() - timedelta(seconds=5))
self.assertEqual(job.status.hget(), STATUSES.WAITING)
self.assertEqual(queue.waiting.llen(), 1)
self.assertEqual(queue.delayed.zcard(), 0)
def test_enqueue_job_should_add_the_job_to_the_waiting_list(self):
job = Job(identifier='job:1')
queue = Queue.get_queue(name='test')
queue.enqueue_job(job)
waiting_jobs = queue.waiting.lmembers()
self.assertEqual(waiting_jobs, [job.ident])
# test a delayed job
delayed_until = datetime.utcnow() + timedelta(seconds=1)
job2 = Job(identifier='job:2', delayed_until=str(delayed_until))
queue.delay_job(job2, delayed_until)
waiting_jobs = queue.waiting.lmembers()
self.assertEqual(waiting_jobs, [job.ident])
def test_job_started_method_should_be_called(self):
class TestWorker(Worker):
passed = False
def execute():
pass
def job_started(self, job, queue):
super(TestWorker, self).job_started(job, queue)
self.passed = True
Job.add_job(identifier='job:1', queue_name='test')
worker = TestWorker(name='test', max_loops=1)
worker.run()
self.assertTrue(worker.passed)
def test_send_stop_signal_should_stop_worker(self):
class TestWorker(Worker):
def execute(self, job, queue):
"""
Simulate a signal by directly calling the signal handler
"""
self.catch_end_signal(signal.SIGINT, None)
Job.add_job('job:1', 'test')
Job.add_job('job:2', 'test')
queue = Queue.get_queue('test')
worker = TestWorker('test', max_loops=2)
worker.run()
self.assertEqual(worker.end_signal_caught, True)
self.assertEqual(queue.success.llen(), 1)
self.assertEqual(queue.waiting.llen(), 1)
def test_adding_an_existing_job_with_higher_priority_should_change_its_queue(self):
job1 = Job.add_job(identifier='job:1', queue_name='test', priority=1)
queue1 = Queue.get_queue(name='test', priority=1)
job2 = Job.add_job(identifier='job:1', queue_name='test', priority=2)
queue2 = Queue.get_queue(name='test', priority=2)
# only one job
self.assertEqual(job1.ident, job2.ident)
# not anymore in queue with priority 1
self.assertEqual(queue1.waiting.llen(), 0)
# but now in queue with priority 2
self.assertEqual(queue2.waiting.llen(), 1)
# the new prioriy must be stored, and status should still be waiting
self.assert_job_status_and_priority(job1, STATUSES.WAITING, '2')
# idem for job2 (which is, in fact, job1)
self.assert_job_status_and_priority(job2, STATUSES.WAITING, '2')
def test_adding_an_existing_job_with_lower_priority_should_do_nothing(self):
job1 = Job.add_job(identifier='job:1', queue_name='test', priority=3)
job2 = Job.add_job(identifier='job:1', queue_name='test', priority=1)
# only one job
self.assertEqual(job1.ident, job2.ident)
# is in high priority queue
queue = Queue.get_queue(name='test', priority=3)
self.assertEqual(queue.waiting.llen(), 1)
# nothing in high priority queue
queue = Queue.get_queue(name='test', priority=1)
self.assertEqual(queue.waiting.llen(), 0)
# we should still have original priority and status
self.assert_job_status_and_priority(job1, STATUSES.WAITING, '3')
# idem for job2 (which is, in fact, job1)
self.assert_job_status_and_priority(job2, STATUSES.WAITING, '3')
def test_adding_a_job_with_delayed_for_should_add_the_job_in_the_delayed_list(self):
queue = Queue.get_queue(name='test')
job = Job.add_job(identifier='job:1', queue_name='test', delayed_for=5)
self.assertEqual(job.status.hget(), STATUSES.DELAYED)
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(queue.delayed.zcard(), 1)
job2 = Job.add_job(identifier='job:2', queue_name='test', delayed_for=5.5)
self.assertEqual(job2.status.hget(), STATUSES.DELAYED)
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(queue.delayed.zcard(), 2)
job3 = Job.add_job(identifier='job:3', queue_name='test', delayed_for=timedelta(seconds=5))
self.assertEqual(job3.status.hget(), STATUSES.DELAYED)
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(queue.delayed.zcard(), 3)
def test_enqueue_job_can_prepend_jobs(self):
queue = Queue.get_queue(name='test')
job = Job(identifier='job:1')
queue.enqueue_job(job)
waiting_jobs = queue.waiting.lmembers()
self.assertEqual(waiting_jobs, [job.ident])
job2 = Job(identifier='job:2')
queue.enqueue_job(job2)
waiting_jobs = queue.waiting.lmembers()
self.assertEqual(waiting_jobs, [job.ident, job2.ident])
job3 = Job(identifier='job:3')
queue.enqueue_job(job3, prepend=True)
waiting_jobs = queue.waiting.lmembers()
self.assertEqual(waiting_jobs, [job3.ident, job.ident, job2.ident])
def test_adding_an_existing_job_with_higher_priority_should_change_its_queue(self):
job1 = Job.add_job(identifier='job:1', queue_name='test', priority=1)
queue1 = Queue.get_queue(name='test', priority=1)
job2 = Job.add_job(identifier='job:1', queue_name='test', priority=2)
queue2 = Queue.get_queue(name='test', priority=2)
# only one job
self.assertEqual(job1.ident, job2.ident)
# not anymore in queue with priority 1
self.assertEqual(queue1.waiting.llen(), 0)
# but now in queue with priority 2
self.assertEqual(queue2.waiting.llen(), 1)
# the new prioriy must be stored, and status should still be waiting
self.assert_job_status_and_priority(job1, STATUSES.WAITING, '2')
# idem for job2 (which is, in fact, job1)
self.assert_job_status_and_priority(job2, STATUSES.WAITING, '2')
def test_adding_a_job_with_delayed_for_should_add_the_job_in_the_delayed_list(self):
queue = Queue.get_queue(name='test')
job = Job.add_job(identifier='job:1', queue_name='test', delayed_for=5)
self.assertEqual(job.status.hget(), STATUSES.DELAYED)
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(queue.delayed.zcard(), 1)
job2 = Job.add_job(identifier='job:2', queue_name='test', delayed_for=5.5)
self.assertEqual(job2.status.hget(), STATUSES.DELAYED)
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(queue.delayed.zcard(), 2)
job3 = Job.add_job(identifier='job:3', queue_name='test', delayed_for=timedelta(seconds=5))
self.assertEqual(job3.status.hget(), STATUSES.DELAYED)
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(queue.delayed.zcard(), 3)
def test_adding_an_existing_job_with_lower_priority_should_do_nothing(self):
job1 = Job.add_job(identifier='job:1', queue_name='test', priority=3)
job2 = Job.add_job(identifier='job:1', queue_name='test', priority=1)
# only one job
self.assertEqual(job1.ident, job2.ident)
# is in high priority queue
queue = Queue.get_queue(name='test', priority=3)
self.assertEqual(queue.waiting.llen(), 1)
# nothing in high priority queue
queue = Queue.get_queue(name='test', priority=1)
self.assertEqual(queue.waiting.llen(), 0)
# we should still have original priority and status
self.assert_job_status_and_priority(job1, STATUSES.WAITING, '3')
# idem for job2 (which is, in fact, job1)
self.assert_job_status_and_priority(job2, STATUSES.WAITING, '3')
def test_count_delayed_jobs_should_return_the_number_of_delayed_jobs(self):
self.assertEqual(Queue.count_delayed_jobs('test'), 0)
Job.add_job(identifier='job:1', queue_name='test', delayed_for=5)
self.assertEqual(Queue.count_delayed_jobs('test'), 1)
Job.add_job(identifier='job:2', queue_name='test')
self.assertEqual(Queue.count_delayed_jobs('test'), 1)
Job.add_job(identifier='foo:1', queue_name='foo', delayed_for=5)
Job.add_job(identifier='bar:1', queue_name='bar', delayed_for=5)
self.assertEqual(Queue.count_delayed_jobs(['foo', 'bar']), 2)
self.assertEqual(Queue.count_delayed_jobs(['test', 'foo', 'bar']), 3)
def test_count_delayed_jobs_should_return_the_number_of_delayed_jobs(self):
self.assertEqual(Queue.count_delayed_jobs('test'), 0)
Job.add_job(identifier='job:1', queue_name='test', delayed_for=5)
self.assertEqual(Queue.count_delayed_jobs('test'), 1)
Job.add_job(identifier='job:2', queue_name='test')
self.assertEqual(Queue.count_delayed_jobs('test'), 1)
Job.add_job(identifier='foo:1', queue_name='foo', delayed_for=5)
Job.add_job(identifier='bar:1', queue_name='bar', delayed_for=5)
self.assertEqual(Queue.count_delayed_jobs(['foo', 'bar']), 2)
self.assertEqual(Queue.count_delayed_jobs(['test', 'foo', 'bar']), 3)
def test_using_a_subclass_of_queue_should_work(self):
class TestQueue(Queue):
namespace = 'test_using_a_subclass_of_queue_should_work'
class TestJob(Job):
namespace = 'test_using_a_subclass_of_queue_should_work'
# not specifying queue_model should use the default Queue model
default_queue = Queue.get_queue('test1')
queue = TestQueue.get_queue('test1')
Job.add_job(identifier='job:1', queue_name='test1')
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(default_queue.waiting.llen(), 1)
# idem with a subclass of job
default_queue = Queue.get_queue('test2')
queue = TestQueue.get_queue('test2')
TestJob.add_job(identifier='job:2', queue_name='test2')
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(default_queue.waiting.llen(), 1)
# specifiying a queue_model in add_job should use the wanted queue
default_queue = Queue.get_queue('test3')
queue = TestQueue.get_queue('test3')
Job.add_job(identifier='job:3', queue_name='test3', queue_model=TestQueue)
self.assertEqual(queue.waiting.llen(), 1)
self.assertEqual(default_queue.waiting.llen(), 0)
# idem with a subclass of job
default_queue = Queue.get_queue('test4')
queue = TestQueue.get_queue('test4')
TestJob.add_job(identifier='job:4', queue_name='test4', queue_model=TestQueue)
self.assertEqual(queue.waiting.llen(), 1)
self.assertEqual(default_queue.waiting.llen(), 0)
# now test with a queue_model defined in the job class
class TestJobWithQueueModel(Job):
namespace = 'test_using_a_subclass_of_queue_should_work'
queue_model = TestQueue
default_queue = Queue.get_queue('test5')
queue = TestQueue.get_queue('test5')
TestJobWithQueueModel.add_job(identifier='job:5', queue_name='test5')
self.assertEqual(queue.waiting.llen(), 1)
self.assertEqual(default_queue.waiting.llen(), 0)
def test_using_a_subclass_of_queue_should_work(self):
class TestQueue(Queue):
namespace = 'test_using_a_subclass_of_queue_should_work'
class TestJob(Job):
namespace = 'test_using_a_subclass_of_queue_should_work'
# not specifying queue_model should use the default Queue model
default_queue = Queue.get_queue('test1')
queue = TestQueue.get_queue('test1')
Job.add_job(identifier='job:1', queue_name='test1')
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(default_queue.waiting.llen(), 1)
# idem with a subclass of job
default_queue = Queue.get_queue('test2')
queue = TestQueue.get_queue('test2')
TestJob.add_job(identifier='job:2', queue_name='test2')
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(default_queue.waiting.llen(), 1)
# specifiying a queue_model in add_job should use the wanted queue
default_queue = Queue.get_queue('test3')
queue = TestQueue.get_queue('test3')
Job.add_job(identifier='job:3', queue_name='test3', queue_model=TestQueue)
self.assertEqual(queue.waiting.llen(), 1)
self.assertEqual(default_queue.waiting.llen(), 0)
# idem with a subclass of job
default_queue = Queue.get_queue('test4')
queue = TestQueue.get_queue('test4')
TestJob.add_job(identifier='job:4', queue_name='test4', queue_model=TestQueue)
self.assertEqual(queue.waiting.llen(), 1)
self.assertEqual(default_queue.waiting.llen(), 0)
# now test with a queue_model defined in the job class
class TestJobWithQueueModel(Job):
namespace = 'test_using_a_subclass_of_queue_should_work'
queue_model = TestQueue
default_queue = Queue.get_queue('test5')
queue = TestQueue.get_queue('test5')
TestJobWithQueueModel.add_job(identifier='job:5', queue_name='test5')
self.assertEqual(queue.waiting.llen(), 1)
self.assertEqual(default_queue.waiting.llen(), 0)
def test_delay_job_should_add_the_job_to_the_delayed_sortedset(self):
delayed_until = datetime.utcnow() + timedelta(seconds=1)
timestamp = datetime_to_score(delayed_until)
job = Job(identifier='job:1', delayed_until=str(delayed_until))
queue = Queue.get_queue(name='test')
queue.delay_job(job, delayed_until)
delayed_jobs = queue.delayed.zrange(0, -1, withscores=True)
self.assertEqual(delayed_jobs, [(job.ident, timestamp)])
# test a non-delayed job
job2 = Job(identifier='job:2')
queue.enqueue_job(job2)
delayed_jobs = queue.delayed.zrange(0, -1, withscores=True)
self.assertEqual(delayed_jobs, [(job.ident, timestamp)])
def test_add_error_should_store_the_name_of_the_exception(self):
e = Exception('foo')
job = Job.add_job(identifier='job:1', queue_name='test')
error = Error.add_error(queue_name='test', job=job, error=e)
self.assertEqual(error.type.hget(), 'Exception')
e = ErrorTests.ExceptionWithCode('the answer', 42)
error = Error.add_error(queue_name='test', job=job, error=e)
self.assertEqual(error.type.hget(), 'ExceptionWithCode')
def test_add_error_should_store_the_name_of_the_exception(self):
e = Exception('foo')
job = Job.add_job(identifier='job:1', queue_name='test')
error = Error.add_error(queue_name='test', job=job, error=e)
self.assertEqual(error.type.hget(), 'Exception')
e = ErrorTests.ExceptionWithCode('the answer', 42)
error = Error.add_error(queue_name='test', job=job, error=e)
self.assertEqual(error.type.hget(), 'ExceptionWithCode')
def test_new_error_save_date_and_time_appart(self):
e = ErrorTests.ExceptionWithCode('the answer', 42)
when = datetime(2012, 9, 29, 22, 58, 56)
job = Job.add_job(identifier='job:1', queue_name='test')
error = Error.add_error(queue_name='test', job=job, error=e, when=when)
self.assertEqual(error.date.hget(), '2012-09-29')
self.assertEqual(error.time.hget(), '22:58:56')
self.assertEqual(error.datetime, when)
self.assertEqual(list(Error.collection(date='2012-09-29')), [error.pk.get()])
def test_add_error_should_store_the_traceback(self):
job = Job.add_job(identifier='job:1', queue_name='test')
try:
foo = bar
except Exception as e:
trace = traceback.format_exc()
error = Error.add_error(queue_name='test', job=job, error=e, trace=trace)
self.assertRegexpMatches(error.traceback.hget(), "NameError:[^\n]+bar[^\n]+ is not defined")
def test_new_error_save_job_pk_and_identifier_appart(self):
e = ErrorTests.ExceptionWithCode('the answer', 42)
when = datetime(2012, 9, 29, 22, 58, 56)
job = Job.add_job(identifier='job:1', queue_name='test')
error = Error.add_error(queue_name='test', job=job, error=e, when=when)
self.assertEqual(error.job_model_repr.hget(), job.get_model_repr())
self.assertEqual(error.job_pk.hget(), job.pk.get())
self.assertEqual(error.identifier.hget(), 'job:1')
self.assertListEqual(list(Error.collection(job_pk=job.pk.get())), [error.pk.get()])
self.assertListEqual(list(Error.collection(identifier=job.identifier.hget())), [error.pk.get()])
def test_add_error_should_store_the_traceback(self):
job = Job.add_job(identifier='job:1', queue_name='test')
try:
foo = bar
except Exception as e:
trace = traceback.format_exc()
error = Error.add_error(queue_name='test', job=job, error=e, trace=trace)
self.assertRegexpMatches(error.traceback.hget(), "NameError:[^\n]+bar[^\n]+ is not defined")
def test_new_error_save_job_pk_and_identifier_appart(self):
e = ErrorTests.ExceptionWithCode('the answer', 42)
when = datetime(2012, 9, 29, 22, 58, 56)
job = Job.add_job(identifier='job:1', queue_name='test')
error = Error.add_error(queue_name='test', job=job, error=e, when=when)
self.assertEqual(error.job_model_repr.hget(), job.get_model_repr())
self.assertEqual(error.job_pk.hget(), job.pk.get())
self.assertEqual(error.identifier.hget(), 'job:1')
self.assertEqual(list(Error.collection(job_pk=job.pk.get())), [error.pk.get()])
self.assertEqual(list(Error.collection(identifier=job.identifier.hget())), [error.pk.get()])
def test_new_error_save_date_and_time_appart(self):
e = ErrorTests.ExceptionWithCode('the answer', 42)
when = datetime(2012, 9, 29, 22, 58, 56)
job = Job.add_job(identifier='job:1', queue_name='test')
error = Error.add_error(queue_name='test', job=job, error=e, when=when)
self.assertEqual(error.date.hget(), '2012-09-29')
self.assertEqual(error.time.hget(), '22:58:56')
self.assertEqual(error.date_time.hget(), '2012-09-29 22:58:56')
self.assertEqual(error.datetime, when)
self.assertListEqual(list(Error.collection(date='2012-09-29')), [error.pk.get()])
self.assertListEqual(list(Error.collection(date_time__date__gt='2012-09-01')), [error.pk.get()])
def test_get_job_method_should_return_a_job_based_on_its_pk(self):
worker = Worker('test')
job = Job.add_job(queue_name='test', identifier='job:1')
# a job...
test_job = worker.get_job(job.get_pk())
self.assertTrue(isinstance(test_job, Job))
self.assertEqual(test_job.get_pk(), job.get_pk())
# not a job...
with self.assertRaises(ValueError):
worker.get_job('foo')
def test_callback_should_be_called(self):
result = {}
def callback(job, queue):
result['job'] = job.get_pk()
result['queue'] = queue.get_pk()
job = Job.add_job(identifier='job:1', queue_name='test')
queue = Queue.get_queue(name='test')
worker = Worker(name='test', callback=callback, max_loops=1)
worker.run()
self.assertEqual(result, {'job': job.get_pk(), 'queue': queue.get_pk()})
def test_get_model_repr_should_return_the_string_repr_of_the_model(self):
self.assertEqual(Job.get_model_repr(), 'limpyd_jobs.models.Job')
job = Job()
self.assertEqual(job.get_model_repr(), 'limpyd_jobs.models.Job')
self.assertEqual(JobFooBar.get_model_repr(), 'tests.models.JobFooBar')
job = JobFooBar()
self.assertEqual(job.get_model_repr(), 'tests.models.JobFooBar')
def test_duration_property_should_return_elapsed_time_during_start_and_end(self):
job = Job(identifier='job:1')
self.assertIsNone(job.duration)
start = datetime.utcnow()
job.start.hset(str(start))
self.assertIsNone(job.duration)
end = start + timedelta(seconds=5)
job.end.hset(str(end))
self.assertEqual(job.duration, timedelta(seconds=5))
job.start.delete()
self.assertIsNone(job.duration)
def test_extended_error_can_accept_other_fields(self):
class ExtendedError(Error):
namespace = 'test-errortests'
foo = fields.StringField()
bar = fields.StringField()
e = ErrorTests.ExceptionWithCode('the answer', 42)
job = Job.add_job(identifier='job:1', queue_name='test')
# create a new error
error = ExtendedError.add_error(queue_name='test', job=job,
error=e, foo='FOO', bar='BAR')
self.assertEqual(error.foo.get(), 'FOO')
self.assertEqual(error.bar.get(), 'BAR')
def test_extended_error_can_accept_other_fields(self):
class ExtendedError(Error):
namespace = 'test-errortests'
foo = fields.StringField()
bar = fields.StringField()
e = ErrorTests.ExceptionWithCode('the answer', 42)
job = Job.add_job(identifier='job:1', queue_name='test')
# create a new error
error = ExtendedError.add_error(queue_name='test', job=job,
error=e, foo='FOO', bar='BAR')
self.assertEqual(error.foo.get(), 'FOO')
self.assertEqual(error.bar.get(), 'BAR')
def test_requeue_delayed_jobs_should_abort_if_another_thread_works_on_it(self):
queue = Queue.get_queue(name='test')
# simulate a lock on this queue
lock_key = queue.make_key(
queue._name,
queue.pk.get(),
"requeue_all_delayed_ready_jobs",
)
queue.get_connection().set(lock_key, 1)
Job.add_job(identifier='job:1', queue_name='test', delayed_for=0.1)
self.assertEqual(Queue.count_waiting_jobs('test'), 0)
self.assertEqual(Queue.count_delayed_jobs('test'), 1)
# wait until the job should be ready
sleep(0.5)
queue.requeue_delayed_jobs()
# the requeue must have done nothing
self.assertEqual(Queue.count_waiting_jobs('test'), 0)
self.assertEqual(Queue.count_delayed_jobs('test'), 1)
def test_requeue_delayed_jobs_should_abort_if_another_thread_works_on_it(self):
queue = Queue.get_queue(name='test')
# simulate a lock on this queue
lock_key = queue.make_key(
queue._name,
queue.pk.get(),
"requeue_all_delayed_ready_jobs",
)
queue.get_connection().set(lock_key, 1)
Job.add_job(identifier='job:1', queue_name='test', delayed_for=0.1)
self.assertEqual(Queue.count_waiting_jobs('test'), 0)
self.assertEqual(Queue.count_delayed_jobs('test'), 1)
# wait until the job should be ready
sleep(0.5)
queue.requeue_delayed_jobs()
# the requeue must have done nothing
self.assertEqual(Queue.count_waiting_jobs('test'), 0)
self.assertEqual(Queue.count_delayed_jobs('test'), 1)
def test_enqueue_or_delay_should_enqueue_if_not_in_the_future(self):
queue = Queue.get_queue(name='test', priority=1)
job = Job(identifier='job:1')
job.enqueue_or_delay('test', 1)
self.assertEqual(queue.delayed.zcard(), 0)
self.assertEqual(queue.waiting.llen(), 1)
self.assertEqual(job.delayed_until.hget(), None)
self.assertEqual(job.status.hget(), STATUSES.WAITING)
self.assertEqual(job.priority.hget(), '1')
job2 = Job(identifier='job:2')
delayed_until = datetime.utcnow() - timedelta(seconds=5) # the past !
job2.enqueue_or_delay('test', 1, delayed_until=delayed_until)
self.assertEqual(queue.delayed.zcard(), 0)
self.assertEqual(queue.waiting.llen(), 2)
self.assertEqual(job2.delayed_until.hget(), None)
self.assertEqual(job2.status.hget(), STATUSES.WAITING)
self.assertEqual(job2.priority.hget(), '1')
def test_adding_a_job_should_create_a_queue_with_the_job(self):
job = Job.add_job(identifier='job:1', queue_name='test', priority=5)
# count queues
keys = Queue.get_waiting_keys('test')
self.assertEqual(len(keys), 1)
# get the new queue, should not create it (number of keys should be 1)
queue = Queue.get_queue(name='test', priority=5)
keys = Queue.get_waiting_keys('test')
self.assertEqual(len(keys), 1)
# check that the job is in the queue
jobs = queue.waiting.lrange(0, -1)
self.assertEqual(jobs, [job.ident])
# ... with the correct status and priority
self.assert_job_status_and_priority(job, STATUSES.WAITING, '5')
def test_adding_a_job_should_create_a_queue_with_the_job(self):
job = Job.add_job(identifier='job:1', queue_name='test', priority=5)
# count queues
keys = Queue.get_waiting_keys('test')
self.assertEqual(len(keys), 1)
# get the new queue, should not create it (number of keys should be 1)
queue = Queue.get_queue(name='test', priority=5)
keys = Queue.get_waiting_keys('test')
self.assertEqual(len(keys), 1)
# check that the job is in the queue
jobs = queue.waiting.lrange(0, -1)
self.assertEqual(jobs, [job.ident])
# ... with the correct status and priority
self.assert_job_status_and_priority(job, STATUSES.WAITING, '5')
def test_job_success_method_should_be_called(self):
class TestWorker(Worker):
passed = None
def execute(self, job, queue):
return 42
def job_success(self, job, queue, job_result, message=None):
super(TestWorker, self).job_success(job, queue, job_result, message)
self.passed = job_result
job = Job.add_job(identifier='job:1', queue_name='test')
queue = Queue.get_queue(name='test')
worker = TestWorker(name='test', max_loops=1)
worker.run()
self.assertEqual(job.status.hget(), STATUSES.SUCCESS)
self.assertIn(job.get_pk(), queue.success.lmembers())
self.assertEqual(worker.passed, 42)
def test_enqueue_or_delay_without_queue_name_should_fail_if_no_class_one(self):
job = Job.add_job(identifier='job:1', queue_name='test')
with self.assertRaises(LimpydJobsException):
job.enqueue_or_delay(delayed_until=datetime.utcnow()+timedelta(seconds=5))
def test_add_error_can_accept_an_exception_without_code(self):
e = Exception('no code')
job = Job.add_job(identifier='job:1', queue_name='test')
error = Error.add_error(queue_name='test', job=job, error=e)
self.assertEqual(error.code.hget(), None)
def get_job(self, job_ident):
"""
Return a job based on its representation (class + pk).
"""
return Job.get_from_ident(job_ident)
