def test_requeue_delayed_jobs_put_back_ready_delayed_jobs_to_the_waiting_list(self):
queue = Queue.get_queue(name='test')
job1 = Job.add_job(identifier='job:1', queue_name='test', delayed_for=10)
job2 = Job.add_job(identifier='job:2', queue_name='test', delayed_for=1)
self.assertEqual(Queue.count_waiting_jobs('test'), 0)
self.assertEqual(Queue.count_delayed_jobs('test'), 2)
self.assertEqual(job1.status.hget(), STATUSES.DELAYED)
self.assertEqual(job2.status.hget(), STATUSES.DELAYED)
# must not move any jobs, too soon
queue.requeue_delayed_jobs()
self.assertEqual(Queue.count_waiting_jobs('test'), 0)
self.assertEqual(Queue.count_delayed_jobs('test'), 2)
self.assertEqual(job1.status.hget(), STATUSES.DELAYED)
self.assertEqual(job2.status.hget(), STATUSES.DELAYED)
sleep(1)
# now we should have one job in the waiting list
queue.requeue_delayed_jobs()
self.assertEqual(Queue.count_waiting_jobs('test'), 1)
self.assertEqual(Queue.count_delayed_jobs('test'), 1)
self.assertEqual(job1.status.hget(), STATUSES.DELAYED)
self.assertEqual(job2.status.hget(), STATUSES.WAITING)
waiting_jobs = queue.waiting.lmembers()
self.assertEqual(waiting_jobs, [job2.ident])
delayed_jobs = queue.delayed.zrange(0, -1, withscores=True)
self.assertEqual(delayed_jobs[0][0], job1.ident)
def test_requeue_delayed_jobs_put_back_ready_delayed_jobs_to_the_waiting_list(self):
queue = Queue.get_queue(name='test')
job1 = Job.add_job(identifier='job:1', queue_name='test', delayed_for=10)
job2 = Job.add_job(identifier='job:2', queue_name='test', delayed_for=1)
self.assertEqual(Queue.count_waiting_jobs('test'), 0)
self.assertEqual(Queue.count_delayed_jobs('test'), 2)
self.assertEqual(job1.status.hget(), STATUSES.DELAYED)
self.assertEqual(job2.status.hget(), STATUSES.DELAYED)
# must not move any jobs, too soon
queue.requeue_delayed_jobs()
self.assertEqual(Queue.count_waiting_jobs('test'), 0)
self.assertEqual(Queue.count_delayed_jobs('test'), 2)
self.assertEqual(job1.status.hget(), STATUSES.DELAYED)
self.assertEqual(job2.status.hget(), STATUSES.DELAYED)
sleep(1)
# now we should have one job in the waiting list
queue.requeue_delayed_jobs()
self.assertEqual(Queue.count_waiting_jobs('test'), 1)
self.assertEqual(Queue.count_delayed_jobs('test'), 1)
self.assertEqual(job1.status.hget(), STATUSES.DELAYED)
self.assertEqual(job2.status.hget(), STATUSES.WAITING)
waiting_jobs = queue.waiting.lmembers()
self.assertEqual(waiting_jobs, [job2.ident])
delayed_jobs = queue.delayed.zrange(0, -1, withscores=True)
self.assertEqual(delayed_jobs[0][0], job1.ident)
def test_job_error_method_should_be_called(self):
class ExceptionWithCode(Exception):
def __init__(self, message, code):
super(ExceptionWithCode, self).__init__(message)
self.message = message
self.code = code
def callback(job, queue):
raise ExceptionWithCode('foobar', 42)
job1 = Job.add_job(identifier='job:1', queue_name='test')
queue = Queue.get_queue(name='test')
worker = Worker(name='test', max_loops=1) # no callback
worker.run()
self.assertEqual(job1.status.hget(), STATUSES.ERROR)
self.assertIn(job1.get_pk(), queue.errors.lmembers())
self.assertEqual(len(Error.collection()), 1)
error = Error.get(identifier='job:1')
self.assertEqual(error.message.hget(), 'You must implement your own action')
self.assertEqual(error.code.hget(), None)
job2 = Job.add_job(identifier='job:2', queue_name='test')
queue = Queue.get_queue(name='test')
worker = Worker(name='test', max_loops=1, callback=callback) # callback with exception
worker.run()
self.assertEqual(job2.status.hget(), STATUSES.ERROR)
self.assertIn(job2.get_pk(), queue.errors.lmembers())
self.assertEqual(len(Error.collection()), 2)
error = Error.get(identifier='job:2')
self.assertEqual(error.message.hget(), 'foobar')
self.assertEqual(error.code.hget(), '42')
def test_prepending_an_existing_job_should_move_it_at_the_beginning(self):
queue = Queue.get_queue(name='test', priority=1)
job1 = Job.add_job(identifier='job:1', queue_name='test', priority=1)
job2 = Job.add_job(identifier='job:2', queue_name='test', priority=1)
self.assertEqual(queue.waiting.lmembers(), [job1.ident, job2.ident])
Job.add_job(identifier='job:2', queue_name='test', priority=1, prepend=True)
self.assertEqual(queue.waiting.lmembers(), [job2.ident, job1.ident])
def test_prepending_an_existing_job_should_move_it_at_the_beginning(self):
queue = Queue.get_queue(name='test', priority=1)
job1 = Job.add_job(identifier='job:1', queue_name='test', priority=1)
job2 = Job.add_job(identifier='job:2', queue_name='test', priority=1)
self.assertEqual(queue.waiting.lmembers(), [job1.ident, job2.ident])
Job.add_job(identifier='job:2', queue_name='test', priority=1, prepend=True)
self.assertEqual(queue.waiting.lmembers(), [job2.ident, job1.ident])
def test_run_ended_method_should_be_called(self):
class TestWorker(Worker):
passed = False
def run_ended(self):
super(TestWorker, self).run_ended()
self.passed = True
Job.add_job('job:1', 'test')
worker = TestWorker('test', max_loops=1) # one loop to run only one job
worker.run()
self.assertEqual(worker.passed, True)
def test_first_delayed_should_return_the_first_delayed_job_to_be_ready(self):
Job.add_job(identifier='job:1', queue_name='test', delayed_for=5)
job2 = Job.add_job(identifier='job:2', queue_name='test', delayed_for=1)
queue = Queue.get_queue(name='test')
attended_timestamp = datetime_to_score(parse(job2.delayed_until.hget()))
job_ident, timestamp = queue.first_delayed
self.assertEqual(job_ident, job2.ident)
self.assertEqual(timestamp, attended_timestamp)
timestamp = queue.first_delayed_time
self.assertEqual(timestamp, attended_timestamp)
def test_first_delayed_should_return_the_first_delayed_job_to_be_ready(self):
Job.add_job(identifier='job:1', queue_name='test', delayed_for=5)
job2 = Job.add_job(identifier='job:2', queue_name='test', delayed_for=1)
queue = Queue.get_queue(name='test')
attended_timestamp = datetime_to_score(parse(job2.delayed_until.hget()))
job_ident, timestamp = queue.first_delayed
self.assertEqual(job_ident, job2.ident)
self.assertEqual(timestamp, attended_timestamp)
timestamp = queue.first_delayed_time
self.assertEqual(timestamp, attended_timestamp)
def test_adding_a_job_with_delayed_until_not_in_the_future_should_add_the_job_in_the_waiting_list(self):
queue = Queue.get_queue(name='test')
job = Job.add_job(identifier='job:1', queue_name='test', delayed_until=datetime.utcnow() - timedelta(seconds=5))
self.assertEqual(job.status.hget(), STATUSES.WAITING)
self.assertEqual(queue.waiting.llen(), 1)
self.assertEqual(queue.delayed.zcard(), 0)
def test_add_error_method_should_add_an_error_instance(self):
e = ErrorTests.ExceptionWithCode('the answer', 42)
job = Job.add_job(identifier='job:1', queue_name='test')
error1 = Error.add_error(queue_name='test', job=job, error=e)
self.assertListEqual(list(Error.collection()), [error1.pk.get()])
Error.add_error(queue_name='test', job=job, error=e)
self.assertEqual(len(Error.collection()), 2)
def test_job_started_method_should_be_called(self):
class TestWorker(Worker):
passed = False
def execute():
pass
def job_started(self, job, queue):
super(TestWorker, self).job_started(job, queue)
self.passed = True
Job.add_job(identifier='job:1', queue_name='test')
worker = TestWorker(name='test', max_loops=1)
worker.run()
self.assertTrue(worker.passed)
def test_adding_a_job_with_delayed_until_not_in_the_future_should_add_the_job_in_the_waiting_list(self):
queue = Queue.get_queue(name='test')
job = Job.add_job(identifier='job:1', queue_name='test', delayed_until=datetime.utcnow() - timedelta(seconds=5))
self.assertEqual(job.status.hget(), STATUSES.WAITING)
self.assertEqual(queue.waiting.llen(), 1)
self.assertEqual(queue.delayed.zcard(), 0)
def test_add_error_method_should_add_an_error_instance(self):
e = ErrorTests.ExceptionWithCode('the answer', 42)
job = Job.add_job(identifier='job:1', queue_name='test')
error1 = Error.add_error(queue_name='test', job=job, error=e)
self.assertEqual(list(Error.collection()), [error1.pk.get()])
Error.add_error(queue_name='test', job=job, error=e)
self.assertEqual(len(Error.collection()), 2)
def test_wait_for_job_should_respond_with_queue_and_job_when_a_queue_is_not_empty(self):
queue = Queue.get_queue('test')
worker = Worker('test')
worker.test_content = None
class Thread(threading.Thread):
def run(self):
worker.update_keys()
queue, job = worker.wait_for_job()
worker.test_content = queue, job
# start listening
thread = Thread()
thread.start()
time.sleep(0.1)
# no job, nothing received
self.assertIsNone(worker.test_content)
# add a job
job = Job.add_job(queue_name='test', identifier='job:1')
time.sleep(0.1)
# info should be received
self.assertEqual(worker.test_content[0].get_pk(), queue.get_pk())
self.assertTrue(isinstance(worker.test_content[0], Queue))
self.assertEqual(worker.test_content[1].get_pk(), job.get_pk())
self.assertTrue(isinstance(worker.test_content[1], Job))
# job must no be in the queue anymore
self.assertNotIn(job.get_pk(), queue.waiting.lmembers())
def test_send_stop_signal_should_stop_worker(self):
class TestWorker(Worker):
def execute(self, job, queue):
"""
Simulate a signal by directly calling the signal handler
"""
self.catch_end_signal(signal.SIGINT, None)
Job.add_job('job:1', 'test')
Job.add_job('job:2', 'test')
queue = Queue.get_queue('test')
worker = TestWorker('test', max_loops=2)
worker.run()
self.assertEqual(worker.end_signal_caught, True)
self.assertEqual(queue.success.llen(), 1)
self.assertEqual(queue.waiting.llen(), 1)
def test_get_from_ident_should_return_a_job(self):
job = Job.add_job(identifier='job:1', queue_name='test')
test_job = Job.get_from_ident('limpyd_jobs.models.Job:%s' % job.pk.get())
self.assertEqual(job.ident, test_job.ident)
job = JobFooBar.add_job(identifier='foobar:1', queue_name='foobar')
test_job = Job.get_from_ident('tests.models.JobFooBar:%s' % job.pk.get())
self.assertEqual(job.ident, test_job.ident)
def test_get_from_ident_should_return_a_job(self):
job = Job.add_job(identifier='job:1', queue_name='test')
test_job = Job.get_from_ident('limpyd_jobs.models.Job:%s' % job.pk.get())
self.assertEqual(job.ident, test_job.ident)
job = JobFooBar.add_job(identifier='foobar:1', queue_name='foobar')
test_job = Job.get_from_ident('tests.models.JobFooBar:%s' % job.pk.get())
self.assertEqual(job.ident, test_job.ident)
def test_adding_a_job_with_delayed_for_should_add_the_job_in_the_delayed_list(self):
queue = Queue.get_queue(name='test')
job = Job.add_job(identifier='job:1', queue_name='test', delayed_for=5)
self.assertEqual(job.status.hget(), STATUSES.DELAYED)
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(queue.delayed.zcard(), 1)
job2 = Job.add_job(identifier='job:2', queue_name='test', delayed_for=5.5)
self.assertEqual(job2.status.hget(), STATUSES.DELAYED)
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(queue.delayed.zcard(), 2)
job3 = Job.add_job(identifier='job:3', queue_name='test', delayed_for=timedelta(seconds=5))
self.assertEqual(job3.status.hget(), STATUSES.DELAYED)
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(queue.delayed.zcard(), 3)
def test_adding_a_job_with_delayed_for_should_add_the_job_in_the_delayed_list(self):
queue = Queue.get_queue(name='test')
job = Job.add_job(identifier='job:1', queue_name='test', delayed_for=5)
self.assertEqual(job.status.hget(), STATUSES.DELAYED)
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(queue.delayed.zcard(), 1)
job2 = Job.add_job(identifier='job:2', queue_name='test', delayed_for=5.5)
self.assertEqual(job2.status.hget(), STATUSES.DELAYED)
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(queue.delayed.zcard(), 2)
job3 = Job.add_job(identifier='job:3', queue_name='test', delayed_for=timedelta(seconds=5))
self.assertEqual(job3.status.hget(), STATUSES.DELAYED)
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(queue.delayed.zcard(), 3)
def test_adding_an_existing_job_with_higher_priority_should_change_its_queue(self):
job1 = Job.add_job(identifier='job:1', queue_name='test', priority=1)
queue1 = Queue.get_queue(name='test', priority=1)
job2 = Job.add_job(identifier='job:1', queue_name='test', priority=2)
queue2 = Queue.get_queue(name='test', priority=2)
# only one job
self.assertEqual(job1.ident, job2.ident)
# not anymore in queue with priority 1
self.assertEqual(queue1.waiting.llen(), 0)
# but now in queue with priority 2
self.assertEqual(queue2.waiting.llen(), 1)
# the new prioriy must be stored, and status should still be waiting
self.assert_job_status_and_priority(job1, STATUSES.WAITING, '2')
# idem for job2 (which is, in fact, job1)
self.assert_job_status_and_priority(job2, STATUSES.WAITING, '2')
def test_adding_an_existing_job_with_higher_priority_should_change_its_queue(self):
job1 = Job.add_job(identifier='job:1', queue_name='test', priority=1)
queue1 = Queue.get_queue(name='test', priority=1)
job2 = Job.add_job(identifier='job:1', queue_name='test', priority=2)
queue2 = Queue.get_queue(name='test', priority=2)
# only one job
self.assertEqual(job1.ident, job2.ident)
# not anymore in queue with priority 1
self.assertEqual(queue1.waiting.llen(), 0)
# but now in queue with priority 2
self.assertEqual(queue2.waiting.llen(), 1)
# the new prioriy must be stored, and status should still be waiting
self.assert_job_status_and_priority(job1, STATUSES.WAITING, '2')
# idem for job2 (which is, in fact, job1)
self.assert_job_status_and_priority(job2, STATUSES.WAITING, '2')
def test_adding_an_existing_job_with_lower_priority_should_do_nothing(self):
job1 = Job.add_job(identifier='job:1', queue_name='test', priority=3)
job2 = Job.add_job(identifier='job:1', queue_name='test', priority=1)
# only one job
self.assertEqual(job1.ident, job2.ident)
# is in high priority queue
queue = Queue.get_queue(name='test', priority=3)
self.assertEqual(queue.waiting.llen(), 1)
# nothing in high priority queue
queue = Queue.get_queue(name='test', priority=1)
self.assertEqual(queue.waiting.llen(), 0)
# we should still have original priority and status
self.assert_job_status_and_priority(job1, STATUSES.WAITING, '3')
# idem for job2 (which is, in fact, job1)
self.assert_job_status_and_priority(job2, STATUSES.WAITING, '3')
def test_adding_an_existing_job_with_lower_priority_should_do_nothing(self):
job1 = Job.add_job(identifier='job:1', queue_name='test', priority=3)
job2 = Job.add_job(identifier='job:1', queue_name='test', priority=1)
# only one job
self.assertEqual(job1.ident, job2.ident)
# is in high priority queue
queue = Queue.get_queue(name='test', priority=3)
self.assertEqual(queue.waiting.llen(), 1)
# nothing in high priority queue
queue = Queue.get_queue(name='test', priority=1)
self.assertEqual(queue.waiting.llen(), 0)
# we should still have original priority and status
self.assert_job_status_and_priority(job1, STATUSES.WAITING, '3')
# idem for job2 (which is, in fact, job1)
self.assert_job_status_and_priority(job2, STATUSES.WAITING, '3')
def test_count_delayed_jobs_should_return_the_number_of_delayed_jobs(self):
self.assertEqual(Queue.count_delayed_jobs('test'), 0)
Job.add_job(identifier='job:1', queue_name='test', delayed_for=5)
self.assertEqual(Queue.count_delayed_jobs('test'), 1)
Job.add_job(identifier='job:2', queue_name='test')
self.assertEqual(Queue.count_delayed_jobs('test'), 1)
Job.add_job(identifier='foo:1', queue_name='foo', delayed_for=5)
Job.add_job(identifier='bar:1', queue_name='bar', delayed_for=5)
self.assertEqual(Queue.count_delayed_jobs(['foo', 'bar']), 2)
self.assertEqual(Queue.count_delayed_jobs(['test', 'foo', 'bar']), 3)
def test_count_delayed_jobs_should_return_the_number_of_delayed_jobs(self):
self.assertEqual(Queue.count_delayed_jobs('test'), 0)
Job.add_job(identifier='job:1', queue_name='test', delayed_for=5)
self.assertEqual(Queue.count_delayed_jobs('test'), 1)
Job.add_job(identifier='job:2', queue_name='test')
self.assertEqual(Queue.count_delayed_jobs('test'), 1)
Job.add_job(identifier='foo:1', queue_name='foo', delayed_for=5)
Job.add_job(identifier='bar:1', queue_name='bar', delayed_for=5)
self.assertEqual(Queue.count_delayed_jobs(['foo', 'bar']), 2)
self.assertEqual(Queue.count_delayed_jobs(['test', 'foo', 'bar']), 3)
def test_using_a_subclass_of_queue_should_work(self):
class TestQueue(Queue):
namespace = 'test_using_a_subclass_of_queue_should_work'
class TestJob(Job):
namespace = 'test_using_a_subclass_of_queue_should_work'
# not specifying queue_model should use the default Queue model
default_queue = Queue.get_queue('test1')
queue = TestQueue.get_queue('test1')
Job.add_job(identifier='job:1', queue_name='test1')
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(default_queue.waiting.llen(), 1)
# idem with a subclass of job
default_queue = Queue.get_queue('test2')
queue = TestQueue.get_queue('test2')
TestJob.add_job(identifier='job:2', queue_name='test2')
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(default_queue.waiting.llen(), 1)
# specifiying a queue_model in add_job should use the wanted queue
default_queue = Queue.get_queue('test3')
queue = TestQueue.get_queue('test3')
Job.add_job(identifier='job:3', queue_name='test3', queue_model=TestQueue)
self.assertEqual(queue.waiting.llen(), 1)
self.assertEqual(default_queue.waiting.llen(), 0)
# idem with a subclass of job
default_queue = Queue.get_queue('test4')
queue = TestQueue.get_queue('test4')
TestJob.add_job(identifier='job:4', queue_name='test4', queue_model=TestQueue)
self.assertEqual(queue.waiting.llen(), 1)
self.assertEqual(default_queue.waiting.llen(), 0)
# now test with a queue_model defined in the job class
class TestJobWithQueueModel(Job):
namespace = 'test_using_a_subclass_of_queue_should_work'
queue_model = TestQueue
default_queue = Queue.get_queue('test5')
queue = TestQueue.get_queue('test5')
TestJobWithQueueModel.add_job(identifier='job:5', queue_name='test5')
self.assertEqual(queue.waiting.llen(), 1)
self.assertEqual(default_queue.waiting.llen(), 0)
def test_using_a_subclass_of_queue_should_work(self):
class TestQueue(Queue):
namespace = 'test_using_a_subclass_of_queue_should_work'
class TestJob(Job):
namespace = 'test_using_a_subclass_of_queue_should_work'
# not specifying queue_model should use the default Queue model
default_queue = Queue.get_queue('test1')
queue = TestQueue.get_queue('test1')
Job.add_job(identifier='job:1', queue_name='test1')
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(default_queue.waiting.llen(), 1)
# idem with a subclass of job
default_queue = Queue.get_queue('test2')
queue = TestQueue.get_queue('test2')
TestJob.add_job(identifier='job:2', queue_name='test2')
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(default_queue.waiting.llen(), 1)
# specifiying a queue_model in add_job should use the wanted queue
default_queue = Queue.get_queue('test3')
queue = TestQueue.get_queue('test3')
Job.add_job(identifier='job:3', queue_name='test3', queue_model=TestQueue)
self.assertEqual(queue.waiting.llen(), 1)
self.assertEqual(default_queue.waiting.llen(), 0)
# idem with a subclass of job
default_queue = Queue.get_queue('test4')
queue = TestQueue.get_queue('test4')
TestJob.add_job(identifier='job:4', queue_name='test4', queue_model=TestQueue)
self.assertEqual(queue.waiting.llen(), 1)
self.assertEqual(default_queue.waiting.llen(), 0)
# now test with a queue_model defined in the job class
class TestJobWithQueueModel(Job):
namespace = 'test_using_a_subclass_of_queue_should_work'
queue_model = TestQueue
default_queue = Queue.get_queue('test5')
queue = TestQueue.get_queue('test5')
TestJobWithQueueModel.add_job(identifier='job:5', queue_name='test5')
self.assertEqual(queue.waiting.llen(), 1)
self.assertEqual(default_queue.waiting.llen(), 0)
def test_add_error_should_store_the_name_of_the_exception(self):
e = Exception('foo')
job = Job.add_job(identifier='job:1', queue_name='test')
error = Error.add_error(queue_name='test', job=job, error=e)
self.assertEqual(error.type.hget(), 'Exception')
e = ErrorTests.ExceptionWithCode('the answer', 42)
error = Error.add_error(queue_name='test', job=job, error=e)
self.assertEqual(error.type.hget(), 'ExceptionWithCode')
def test_new_error_save_date_and_time_appart(self):
e = ErrorTests.ExceptionWithCode('the answer', 42)
when = datetime(2012, 9, 29, 22, 58, 56)
job = Job.add_job(identifier='job:1', queue_name='test')
error = Error.add_error(queue_name='test', job=job, error=e, when=when)
self.assertEqual(error.date.hget(), '2012-09-29')
self.assertEqual(error.time.hget(), '22:58:56')
self.assertEqual(error.datetime, when)
self.assertEqual(list(Error.collection(date='2012-09-29')), [error.pk.get()])
def test_add_error_should_store_the_name_of_the_exception(self):
e = Exception('foo')
job = Job.add_job(identifier='job:1', queue_name='test')
error = Error.add_error(queue_name='test', job=job, error=e)
self.assertEqual(error.type.hget(), 'Exception')
e = ErrorTests.ExceptionWithCode('the answer', 42)
error = Error.add_error(queue_name='test', job=job, error=e)
self.assertEqual(error.type.hget(), 'ExceptionWithCode')
def test_add_error_should_store_the_traceback(self):
job = Job.add_job(identifier='job:1', queue_name='test')
try:
foo = bar
except Exception as e:
trace = traceback.format_exc()
error = Error.add_error(queue_name='test', job=job, error=e, trace=trace)
self.assertRegexpMatches(error.traceback.hget(), "NameError:[^\n]+bar[^\n]+ is not defined")
def test_new_error_save_job_pk_and_identifier_appart(self):
e = ErrorTests.ExceptionWithCode('the answer', 42)
when = datetime(2012, 9, 29, 22, 58, 56)
job = Job.add_job(identifier='job:1', queue_name='test')
error = Error.add_error(queue_name='test', job=job, error=e, when=when)
self.assertEqual(error.job_model_repr.hget(), job.get_model_repr())
self.assertEqual(error.job_pk.hget(), job.pk.get())
self.assertEqual(error.identifier.hget(), 'job:1')
self.assertListEqual(list(Error.collection(job_pk=job.pk.get())), [error.pk.get()])
self.assertListEqual(list(Error.collection(identifier=job.identifier.hget())), [error.pk.get()])
def test_add_error_should_store_the_traceback(self):
job = Job.add_job(identifier='job:1', queue_name='test')
try:
foo = bar
except Exception as e:
trace = traceback.format_exc()
error = Error.add_error(queue_name='test', job=job, error=e, trace=trace)
self.assertRegexpMatches(error.traceback.hget(), "NameError:[^\n]+bar[^\n]+ is not defined")
def test_new_error_save_job_pk_and_identifier_appart(self):
e = ErrorTests.ExceptionWithCode('the answer', 42)
when = datetime(2012, 9, 29, 22, 58, 56)
job = Job.add_job(identifier='job:1', queue_name='test')
error = Error.add_error(queue_name='test', job=job, error=e, when=when)
self.assertEqual(error.job_model_repr.hget(), job.get_model_repr())
self.assertEqual(error.job_pk.hget(), job.pk.get())
self.assertEqual(error.identifier.hget(), 'job:1')
self.assertEqual(list(Error.collection(job_pk=job.pk.get())), [error.pk.get()])
self.assertEqual(list(Error.collection(identifier=job.identifier.hget())), [error.pk.get()])
def test_new_error_save_date_and_time_appart(self):
e = ErrorTests.ExceptionWithCode('the answer', 42)
when = datetime(2012, 9, 29, 22, 58, 56)
job = Job.add_job(identifier='job:1', queue_name='test')
error = Error.add_error(queue_name='test', job=job, error=e, when=when)
self.assertEqual(error.date.hget(), '2012-09-29')
self.assertEqual(error.time.hget(), '22:58:56')
self.assertEqual(error.date_time.hget(), '2012-09-29 22:58:56')
self.assertEqual(error.datetime, when)
self.assertListEqual(list(Error.collection(date='2012-09-29')), [error.pk.get()])
self.assertListEqual(list(Error.collection(date_time__date__gt='2012-09-01')), [error.pk.get()])
def test_get_job_method_should_return_a_job_based_on_its_pk(self):
worker = Worker('test')
job = Job.add_job(queue_name='test', identifier='job:1')
# a job...
test_job = worker.get_job(job.get_pk())
self.assertTrue(isinstance(test_job, Job))
self.assertEqual(test_job.get_pk(), job.get_pk())
# not a job...
with self.assertRaises(ValueError):
worker.get_job('foo')
def test_callback_should_be_called(self):
result = {}
def callback(job, queue):
result['job'] = job.get_pk()
result['queue'] = queue.get_pk()
job = Job.add_job(identifier='job:1', queue_name='test')
queue = Queue.get_queue(name='test')
worker = Worker(name='test', callback=callback, max_loops=1)
worker.run()
self.assertEqual(result, {'job': job.get_pk(), 'queue': queue.get_pk()})
def test_extended_error_can_accept_other_fields(self):
class ExtendedError(Error):
namespace = 'test-errortests'
foo = fields.StringField()
bar = fields.StringField()
e = ErrorTests.ExceptionWithCode('the answer', 42)
job = Job.add_job(identifier='job:1', queue_name='test')
# create a new error
error = ExtendedError.add_error(queue_name='test', job=job,
error=e, foo='FOO', bar='BAR')
self.assertEqual(error.foo.get(), 'FOO')
self.assertEqual(error.bar.get(), 'BAR')
def test_extended_error_can_accept_other_fields(self):
class ExtendedError(Error):
namespace = 'test-errortests'
foo = fields.StringField()
bar = fields.StringField()
e = ErrorTests.ExceptionWithCode('the answer', 42)
job = Job.add_job(identifier='job:1', queue_name='test')
# create a new error
error = ExtendedError.add_error(queue_name='test', job=job,
error=e, foo='FOO', bar='BAR')
self.assertEqual(error.foo.get(), 'FOO')
self.assertEqual(error.bar.get(), 'BAR')
def test_requeue_delayed_jobs_should_abort_if_another_thread_works_on_it(self):
queue = Queue.get_queue(name='test')
# simulate a lock on this queue
lock_key = queue.make_key(
queue._name,
queue.pk.get(),
"requeue_all_delayed_ready_jobs",
)
queue.get_connection().set(lock_key, 1)
Job.add_job(identifier='job:1', queue_name='test', delayed_for=0.1)
self.assertEqual(Queue.count_waiting_jobs('test'), 0)
self.assertEqual(Queue.count_delayed_jobs('test'), 1)
# wait until the job should be ready
sleep(0.5)
queue.requeue_delayed_jobs()
# the requeue must have done nothing
self.assertEqual(Queue.count_waiting_jobs('test'), 0)
self.assertEqual(Queue.count_delayed_jobs('test'), 1)
def test_requeue_delayed_jobs_should_abort_if_another_thread_works_on_it(self):
queue = Queue.get_queue(name='test')
# simulate a lock on this queue
lock_key = queue.make_key(
queue._name,
queue.pk.get(),
"requeue_all_delayed_ready_jobs",
)
queue.get_connection().set(lock_key, 1)
Job.add_job(identifier='job:1', queue_name='test', delayed_for=0.1)
self.assertEqual(Queue.count_waiting_jobs('test'), 0)
self.assertEqual(Queue.count_delayed_jobs('test'), 1)
# wait until the job should be ready
sleep(0.5)
queue.requeue_delayed_jobs()
# the requeue must have done nothing
self.assertEqual(Queue.count_waiting_jobs('test'), 0)
self.assertEqual(Queue.count_delayed_jobs('test'), 1)
def test_adding_a_job_should_create_a_queue_with_the_job(self):
job = Job.add_job(identifier='job:1', queue_name='test', priority=5)
# count queues
keys = Queue.get_waiting_keys('test')
self.assertEqual(len(keys), 1)
# get the new queue, should not create it (number of keys should be 1)
queue = Queue.get_queue(name='test', priority=5)
keys = Queue.get_waiting_keys('test')
self.assertEqual(len(keys), 1)
# check that the job is in the queue
jobs = queue.waiting.lrange(0, -1)
self.assertEqual(jobs, [job.ident])
# ... with the correct status and priority
self.assert_job_status_and_priority(job, STATUSES.WAITING, '5')
def test_adding_a_job_should_create_a_queue_with_the_job(self):
job = Job.add_job(identifier='job:1', queue_name='test', priority=5)
# count queues
keys = Queue.get_waiting_keys('test')
self.assertEqual(len(keys), 1)
# get the new queue, should not create it (number of keys should be 1)
queue = Queue.get_queue(name='test', priority=5)
keys = Queue.get_waiting_keys('test')
self.assertEqual(len(keys), 1)
# check that the job is in the queue
jobs = queue.waiting.lrange(0, -1)
self.assertEqual(jobs, [job.ident])
# ... with the correct status and priority
self.assert_job_status_and_priority(job, STATUSES.WAITING, '5')
def test_job_success_method_should_be_called(self):
class TestWorker(Worker):
passed = None
def execute(self, job, queue):
return 42
def job_success(self, job, queue, job_result, message=None):
super(TestWorker, self).job_success(job, queue, job_result, message)
self.passed = job_result
job = Job.add_job(identifier='job:1', queue_name='test')
queue = Queue.get_queue(name='test')
worker = TestWorker(name='test', max_loops=1)
worker.run()
self.assertEqual(job.status.hget(), STATUSES.SUCCESS)
self.assertIn(job.get_pk(), queue.success.lmembers())
self.assertEqual(worker.passed, 42)
def test_error_model_with_additional_fields(self):
class TestError(Error):
foo = fields.HashableField()
class TestWorker(Worker):
error_model = TestError
def additional_error_fields(self, job, queue, exception):
return {
'foo': 'Error on queue %s for job %s' % (
job.get_pk(), queue.get_pk()
)
}
job = Job.add_job(identifier='job:1', queue_name='test')
queue = Queue.get_queue(name='test')
worker = TestWorker(name='test', max_loops=1) # no callback
worker.run()
error = TestError.get(identifier='job:1')
attended_foo = 'Error on queue %s for job %s' % (job.get_pk(), queue.get_pk())
self.assertEqual(error.foo.hget(), attended_foo)
def test_job_skipped_method_should_be_called(self):
class TestWorker(Worker):
passed = False
def execute():
pass
def job_skipped(self, job, queue):
super(TestWorker, self).job_skipped(job, queue)
self.passed = job._status
for status in ('CANCELED', 'RUNNING', 'SUCCESS', 'ERROR'):
job = Job.add_job(identifier='job:1', queue_name='test')
job.status.hset(STATUSES[status])
worker = TestWorker(name='test', max_loops=1)
worker.run()
self.assertEqual(worker.passed, STATUSES[status])
# job ignored: keep its status, and removed from queue
queue = Queue.get_queue('test')
self.assertEqual(queue.waiting.llen(), 0)
self.assertEqual(queue.success.llen(), 0)
self.assertEqual(queue.errors.llen(), 0)
self.assertEqual(job.status.hget(), STATUSES[status])
def test_add_error_can_accept_an_exception_without_code(self):
e = Exception('no code')
job = Job.add_job(identifier='job:1', queue_name='test')
error = Error.add_error(queue_name='test', job=job, error=e)
self.assertEqual(error.code.hget(), None)
def test_enqueue_or_delay_without_queue_name_should_fail_if_no_class_one(self):
job = Job.add_job(identifier='job:1', queue_name='test')
with self.assertRaises(LimpydJobsException):
job.enqueue_or_delay(delayed_until=datetime.utcnow()+timedelta(seconds=5))
def test_ident_should_return_a_string_repr_of_the_job(self):
job = Job.add_job(identifier='job:1', queue_name='test')
self.assertEqual(job.ident, 'limpyd_jobs.models.Job:%s' % job.pk.get())
job = JobFooBar.add_job(identifier='foobar:1', queue_name='foobar')
self.assertEqual(job.ident, 'tests.models.JobFooBar:%s' % job.pk.get())
def test_enqueue_or_delay_without_queue_name_should_fail_if_no_class_one(self):
job = Job.add_job(identifier='job:1', queue_name='test')
with self.assertRaises(LimpydJobsException):
job.enqueue_or_delay(delayed_until=datetime.utcnow()+timedelta(seconds=5))
def test_duration_should_compute_end_start_difference(self):
start = datetime.utcnow()
job = Job.add_job(identifier='job:1', queue_name='test', priority=1)
job.hmset(start=str(start), end=str(start + timedelta(seconds=2)))
duration = job.duration
self.assertEqual(duration, timedelta(seconds=2))
def test_a_job_not_in_error_couldnt_be_requeued(self):
job = Job.add_job(identifier='job:1', queue_name='test')
with self.assertRaises(LimpydJobsException):
job.requeue('test')
def test_run_method_should_raise_by_default(self):
job = Job.add_job(identifier='job:1', queue_name='test')
queue = Queue.get_queue('test')
with self.assertRaises(NotImplementedError):
job.run(queue)
def test_requeue_without_queue_name_should_fail_if_no_class_one(self):
job = Job.add_job(identifier='job:1', queue_name='test')
with self.assertRaises(LimpydJobsException):
job.requeue()
def test_delayed_for_and_delayed_until_should_be_exclusive(self):
with self.assertRaises(ValueError):
Job.add_job(identifier='job:1', queue_name='test', delayed_for=5,
delayed_until=datetime.utcnow()+timedelta(seconds=5))
def test_invalid_type_for_delayed_until_argument_of_add_job_should_raise(self):
with self.assertRaises(ValueError):
Job.add_job(identifier='job:1', queue_name='test', delayed_until='foo')
def test_add_job_without_queue_name_should_fail_if_no_class_one(self):
with self.assertRaises(LimpydJobsException):
Job.add_job(identifier='job:1')
def test_add_error_can_accept_an_exception_without_code(self):
e = Exception('no code')
job = Job.add_job(identifier='job:1', queue_name='test')
error = Error.add_error(queue_name='test', job=job, error=e)
self.assertEqual(error.code.hget(), None)
