class TestWorkerWithJobs(TransactionTestCase):
def setUp(self):
self.fooq, self.barq = Queue('foo'), Queue('bar')
self.w = Worker.create([self.fooq, self.barq])
self.fooq.enqueue(say_hello, name='Frank')
def test_worker_with_jobs(self):
self.assertEqual(self.w.work(burst=True), True,
'Expected at least some work done.')
class TestWorkerWithJobs(TransactionTestCase):
def setUp(self):
self.fooq, self.barq = Queue('foo'), Queue('bar')
self.w = Worker.create([self.fooq, self.barq])
self.fooq.enqueue(say_hello, name='Frank')
def test_worker_with_jobs(self):
self.assertEquals(self.w.work(burst=True), True,
'Expected at least some work done.')
class TestDequeueAnySingle(TransactionTestCase):
def setUp(self):
self.fooq = Queue('foo')
self.barq = Queue('bar')
# Enqueue a single item
self.barq.enqueue(say_hello)
def test_dequeue_any_single(self):
job, queue = PQ.dequeue_any([self.fooq, self.barq], None)
self.assertEqual(job.func, say_hello)
self.assertEqual(queue, self.barq)
class TestWorkFails(TransactionTestCase):
def setUp(self):
self.q = Queue()
self.fq = get_failed_queue()
self.w = Worker.create([self.q])
self.job = self.q.enqueue(div_by_zero)
self.enqueued_at = self.job.enqueued_at
def test_work_fails(self):
"""Failing jobs are put on the failed queue."""
self.w.work(burst=True) # should silently pass
# Postconditions
self.assertEquals(self.q.count, 0)
self.assertEquals(self.fq.count, 1)
# Check the job
job = Job.objects.get(id=self.job.id)
self.assertEquals(job.origin, self.q.name)
# Should be the original enqueued_at date, not the date of enqueueing
# to the failed queue
self.assertEquals(job.enqueued_at, self.enqueued_at)
self.assertIsNotNone(job.exc_info) # should contain exc_info
class TestWorkerCustomExcHandling(TransactionTestCase):
def setUp(self):
self.q = Queue()
self.fq = get_failed_queue()
def black_hole(job, *exc_info):
# Don't fall through to default behaviour of moving to failed queue
return False
self.black_hole = black_hole
self.job = self.q.enqueue(div_by_zero)
def test_custom_exc_handling(self):
"""Custom exception handling."""
w = Worker.create([self.q], exc_handler=self.black_hole)
w.work(burst=True) # should silently pass
# Postconditions
self.assertEquals(self.q.count, 0)
self.assertEquals(self.fq.count, 0)
# Check the job
job = Job.objects.get(id=self.job.id)
self.assertEquals(job.status, Job.FAILED)
class TestWorkerTimeouts(TransactionTestCase):
def setUp(self):
self.sentinel_file = '/tmp/.rq_sentinel'
self.q = Queue()
self.fq = get_failed_queue()
self.w = Worker.create([self.q])
def test_timeouts(self):
"""Worker kills jobs after timeout."""
# Put it on the queue with a timeout value
jobr = self.q.enqueue(create_file_after_timeout,
args=(self.sentinel_file, 4),
timeout=1)
self.assertEquals(os.path.exists(self.sentinel_file), False)
self.w.work(burst=True)
self.assertEquals(os.path.exists(self.sentinel_file), False)
job = Job.objects.get(id=jobr.id)
self.assertIn('JobTimeoutException', job.exc_info)
def tearDown(self):
try:
os.unlink(self.sentinel_file)
except OSError as e:
if e.errno == 2:
pass
class TestWorkerCustomExcHandling(TransactionTestCase):
def setUp(self):
self.q = Queue()
self.fq = get_failed_queue()
def black_hole(job, *exc_info):
# Don't fall through to default behaviour of moving to failed queue
return False
self.black_hole = black_hole
self.job = self.q.enqueue(div_by_zero)
def test_custom_exc_handling(self):
"""Custom exception handling."""
w = Worker.create([self.q], exc_handler=self.black_hole)
w.work(burst=True) # should silently pass
# Postconditions
self.assertEquals(self.q.count, 0)
self.assertEquals(self.fq.count, 0)
# Check the job
job = Job.objects.get(id=self.job.id)
self.assertEquals(job.status, Job.FAILED)
class TestWorkerTimeouts(TransactionTestCase):
def setUp(self):
self.sentinel_file = '/tmp/.rq_sentinel'
self.q = Queue()
self.fq = get_failed_queue()
self.w = Worker.create([self.q])
def test_timeouts(self):
"""Worker kills jobs after timeout."""
# Put it on the queue with a timeout value
jobr = self.q.enqueue(
create_file_after_timeout,
args=(self.sentinel_file, 4),
timeout=1)
self.assertEquals(os.path.exists(self.sentinel_file), False)
self.w.work(burst=True)
self.assertEquals(os.path.exists(self.sentinel_file), False)
job = Job.objects.get(id=jobr.id)
self.assertIn('JobTimeoutException', job.exc_info)
def tearDown(self):
try:
os.unlink(self.sentinel_file)
except OSError as e:
if e.errno == 2:
pass
class TestWorkFails(TransactionTestCase):
def setUp(self):
self.q = Queue()
self.fq = get_failed_queue()
self.w = Worker.create([self.q])
self.job = self.q.enqueue(div_by_zero)
self.enqueued_at = self.job.enqueued_at
def test_work_fails(self):
"""Failing jobs are put on the failed queue."""
self.w.work(burst=True) # should silently pass
# Postconditions
self.assertEquals(self.q.count, 0)
self.assertEquals(self.fq.count, 1)
# Check the job
job = Job.objects.get(id=self.job.id)
self.assertEquals(job.origin, self.q.name)
# Should be the original enqueued_at date, not the date of enqueueing
# to the failed queue
self.assertEquals(job.enqueued_at, self.enqueued_at)
self.assertIsNotNone(job.exc_info) # should contain exc_info
def setUp(self):
q = Queue()
q.enqueue(say_hello, kwargs={'name':'bob'}, result_ttl=1) # expires
q.enqueue(say_hello, kwargs={'name':'polly'}) # won't expire in this test lifecycle
q.enqueue(say_hello, kwargs={'name':'frank'}, result_ttl=-1) # never expires
w = Worker.create([q])
w.work(burst=True)
q.enqueue(say_hello, kwargs={'name':'david'}) # hasn't run yet
self.q = q
class TestQueueInstanceMethods(TransactionTestCase):
def setUp(self):
self.q = Queue()
def test_enqueue(self): # noqa
"""Enqueueing job onto queues."""
# say_hello spec holds which queue this is sent to
job = self.q.enqueue(say_hello, 'Nick', foo='bar')
self.assertEqual(job.queue, self.q)
class TestDequeueAnyMultiple(TransactionTestCase):
def setUp(self):
self.fooq = Queue('foo')
self.barq = Queue('bar')
# Enqueue items on both queues
self.barq.enqueue(say_hello, 'for Bar')
self.fooq.enqueue(say_hello, 'for Foo')
def test_dequeue_any_multiple(self):
job, queue = PQ.dequeue_any([self.fooq, self.barq], None)
self.assertEqual(queue, self.fooq)
self.assertEqual(job.func, say_hello)
self.assertEqual(job.origin, self.fooq.name)
self.assertEqual(
job.args[0], 'for Foo', 'Foo should be dequeued first.')
job, queue = PQ.dequeue_any([self.fooq, self.barq], None)
self.assertEqual(queue, self.barq)
self.assertEqual(job.func, say_hello)
self.assertEqual(job.origin, self.barq.name)
self.assertEqual(
job.args[0], 'for Bar', 'Bar should be dequeued second.')
class TestWorkerDeletesExpiredTTL(TransactionTestCase):
def setUp(self):
self.q = Queue()
self.w = Worker.create([self.q])
self.job = self.q.enqueue(say_hello, args=('Bob', ), result_ttl=1)
self.w.work(burst=True)
def test_worker_deletes_expired_ttl(self):
"""Ensure that Worker deletes expired jobs"""
time.sleep(1)
self.w.work(burst=True)
with self.assertRaises(Job.DoesNotExist) as exc:
rjob = Job.objects.get(id=self.job.id)
class TestDequeueInstanceMethods(TransactionTestCase):
def setUp(self):
self.q = Queue()
self.c = Calculator(2)
self.result = self.q.enqueue(self.c.calculate, 3, 4)
def test_dequeue_instance_method(self):
"""Dequeueing instance method jobs from queues."""
job = self.q.dequeue()
self.assertEqual(job.func.__name__, 'calculate')
self.assertEqual(job.args, (3, 4))
class TestWorkViaStringArg(TransactionTestCase):
def setUp(self):
self.q = Queue('foo')
self.w = Worker.create([self.q])
self.job = self.q.enqueue('test_pq.fixtures.say_hello', name='Frank')
def test_work_via_string_argument(self):
"""Worker processes work fed via string arguments."""
self.assertEquals(self.w.work(burst=True), True,
'Expected at least some work done.')
job = Job.objects.get(id=self.job.id)
self.assertEquals(job.result, 'Hi there, Frank!')
class TestWorkerDeletesExpiredTTL(TransactionTestCase):
def setUp(self):
self.q = Queue()
self.w = Worker.create([self.q])
self.job = self.q.enqueue(say_hello, args=('Bob',), result_ttl=1)
self.w.work(burst=True)
def test_worker_deletes_expired_ttl(self):
"""Ensure that Worker deletes expired jobs"""
time.sleep(1)
self.w.work(burst=True)
with self.assertRaises(Job.DoesNotExist):
Job.objects.get(id=self.job.id)
class TestWorkViaStringArg(TransactionTestCase):
def setUp(self):
self.q = Queue('foo')
self.w = Worker.create([self.q])
self.job = self.q.enqueue('test_pq.fixtures.say_hello', name='Frank')
def test_work_via_string_argument(self):
"""Worker processes work fed via string arguments."""
self.assertEquals(self.w.work(burst=True), True,
'Expected at least some work done.')
job = Job.objects.get(id=self.job.id)
self.assertEquals(job.result, 'Hi there, Frank!')
class TestEnqueueAsyncFalse(TestCase):
def setUp(self):
self.q = Queue()
def test_enqueue_async_false(self):
job = self.q.enqueue(some_calculation, args=(2, 3), async=False)
self.assertEqual(job.result, 6)
def test_enqueue_call_async_false(self):
job = self.q.enqueue_call(some_calculation, args=(2, 3), async=False)
self.assertEqual(job.result, 6)
def test_schedule_call_async_false(self):
job = self.q.enqueue_call(some_calculation, args=(2, 3), async=False)
self.assertEqual(job.result, 6)
class TestWorkerSetsResultTTL(TransactionTestCase):
def setUp(self):
self.q = Queue()
self.w = Worker.create([self.q])
@params((10, 10), (-1, -1), (0, None))
def test_worker_sets_result_ttl(self, ttl, outcome):
"""Ensure that Worker properly sets result_ttl for individual jobs or deletes them."""
job = self.q.enqueue(say_hello, args=('Frank', ), result_ttl=ttl)
self.w.work(burst=True)
try:
rjob = Job.objects.get(id=job.id)
result_ttl = rjob.result_ttl
except Job.DoesNotExist:
result_ttl = None
self.assertEqual(result_ttl, outcome)
class TestWorkerSetsResultTTL(TransactionTestCase):
def setUp(self):
self.q = Queue()
self.w = Worker.create([self.q])
@params((10,10), (-1,-1), (0, None))
def test_worker_sets_result_ttl(self, ttl, outcome):
"""Ensure that Worker properly sets result_ttl for individual jobs or deletes them."""
job = self.q.enqueue(say_hello, args=('Frank',), result_ttl=ttl)
self.w.work(burst=True)
try:
rjob = Job.objects.get(id=job.id)
result_ttl = rjob.result_ttl
except Job.DoesNotExist:
result_ttl = None
self.assertEqual(result_ttl, outcome)
class Test_get_job_or_promise(TransactionTestCase):
"""Test the Job._get_job_or_promise classmethod"""
def setUp(self):
self.q = Queue(scheduled=True)
# simulate the default worker timeout
self.timeout = 60
future = now() + timedelta(seconds=self.timeout/2)
# enqueue a job for 30 seconds time in the future
self.job = self.q.schedule_call(future, do_nothing)
def test_get_job_or_promise(self):
"""Test get a promise of a job in the future"""
job, promise, timeout = Job._get_job_or_promise(
self.q.connection, self.q, self.timeout)
self.assertLessEqual(timeout, self.timeout)
self.assertIsNone(job)
self.assertEqual(promise, self.q.name)
def test_get_no_job_no_promise(self):
"""Test get no job and no promise"""
# job is in the future beyond the current
# worker timeout
job, promise, timeout = Job._get_job_or_promise(
self.q.connection, self.q, 1)
self.assertEqual(timeout, 1)
self.assertIsNone(job)
self.assertIsNone(promise)
def test_get_earlier_job_no_promise(self):
"""Test get earlier job and no promise"""
# Job enqueue after the first scheduled job
# but to be exec ahead of the scheduled job
now_job = self.q.enqueue(do_nothing)
job, promise, timeout = Job._get_job_or_promise(
self.q.connection, self.q, 60)
# timeout should remain the same
self.assertEqual(timeout, 60)
self.assertEqual(now_job.id, job.id)
self.assertIsNone(promise)
class Test_get_job_or_promise(TransactionTestCase):
"""Test the Job._get_job_or_promise classmethod"""
def setUp(self):
self.q = Queue()
# simulate the default worker timeout
self.timeout = 60
future = now() + timedelta(seconds=self.timeout / 2)
# enqueue a job for 30 seconds time in the future
self.job = self.q.schedule_call(future, do_nothing)
def test_get_job_or_promise(self):
"""Test get a promise of a job in the future"""
job, promise, timeout = Job._get_job_or_promise(
self.q.connection, self.q, self.timeout)
self.assertLessEqual(timeout, self.timeout)
self.assertIsNone(job)
self.assertEqual(promise, self.q.name)
def test_get_no_job_no_promise(self):
"""Test get no job and no promise"""
# job is in the future beyond the current
# worker timeout
job, promise, timeout = Job._get_job_or_promise(
self.q.connection, self.q, 1)
self.assertEqual(timeout, 1)
self.assertIsNone(job)
self.assertIsNone(promise)
def test_get_earlier_job_no_promise(self):
"""Test get earlier job and no promise"""
# Job enqueue after the first scheduled job
# but to be exec ahead of the scheduled job
now_job = self.q.enqueue(do_nothing)
job, promise, timeout = Job._get_job_or_promise(
self.q.connection, self.q, 60)
# timeout should remain the same
self.assertEqual(timeout, 60)
self.assertEqual(now_job.id, job.id)
self.assertIsNone(promise)
class TestDequeue(TransactionTestCase):
def setUp(self):
self.q = Queue()
self.result = self.q.enqueue(say_hello, 'Rick', foo='bar')
# self.result2 = q.enqueue(c.calculate, 3, 4)
# self.c = Calculator(2)
def test_dequeue(self):
"""Dequeueing jobs from queues."""
# Dequeue a job (not a job ID) off the queue
self.assertEqual(self.q.count, 1)
job = self.q.dequeue()
self.assertEqual(job.id, self.result.id)
self.assertEqual(job.func, say_hello)
self.assertEqual(job.origin, self.q.name)
self.assertEqual(job.args[0], 'Rick')
self.assertEqual(job.kwargs['foo'], 'bar')
# ...and assert the queue count when down
self.assertEqual(self.q.count, 0)
def test_async_false(self): """Executes a job immediately if async=False.""" q = Queue(async=False) job = q.enqueue(some_calculation, args=(2, 3)) self.assertEqual(job.result, 6)