Ejemplo n.º 1
0
    def test_work_fails(self):
        """Failing jobs are put on the failed queue."""
        q = Queue()
        failed_q = get_failed_queue()

        # Preconditions
        self.assertEquals(failed_q.count, 0)
        self.assertEquals(q.count, 0)

        # Action
        job = q.enqueue(div_by_zero)
        self.assertEquals(q.count, 1)

        # keep for later
        enqueued_at_date = strip_microseconds(job.enqueued_at)

        w = Worker([q])
        w.work(burst=True)  # should silently pass

        # Postconditions
        self.assertEquals(q.count, 0)
        self.assertEquals(failed_q.count, 1)

        # Check the job
        job = Job.fetch(job.id)
        self.assertEquals(job.origin, q.name)

        # Should be the original enqueued_at date, not the date of enqueueing
        # to the failed queue
        self.assertEquals(job.enqueued_at, enqueued_at_date)
        self.assertIsNotNone(job.exc_info)  # should contain exc_info
Ejemplo n.º 2
0
    def test_work_is_unreadable(self):
        """Unreadable jobs are put on the failed queue."""
        q = Queue()
        failed_q = get_failed_queue()

        self.assertEquals(failed_q.count, 0)
        self.assertEquals(q.count, 0)

        # NOTE: We have to fake this enqueueing for this test case.
        # What we're simulating here is a call to a function that is not
        # importable from the worker process.
        job = Job.create(func=div_by_zero, args=(3,))
        job.save()
        data = self.testconn.hget(job.key, 'data')
        invalid_data = data.replace(b'div_by_zero', b'nonexisting')
        assert data != invalid_data
        self.testconn.hset(job.key, 'data', invalid_data)

        # We use the low-level internal function to enqueue any data (bypassing
        # validity checks)
        q.push_job_id(job.id)

        self.assertEquals(q.count, 1)

        # All set, we're going to process it
        w = Worker([q])
        w.work(burst=True)   # should silently pass
        self.assertEquals(q.count, 0)
        self.assertEquals(failed_q.count, 1)
Ejemplo n.º 3
0
 def test_work_via_string_argument(self):
     """Worker processes work fed via string arguments."""
     q = Queue("foo")
     w = Worker([q])
     job = q.enqueue("tests.fixtures.say_hello", name="Frank")
     self.assertEquals(w.work(burst=True), True, "Expected at least some work done.")
     self.assertEquals(job.result, "Hi there, Frank!")
Ejemplo n.º 4
0
    def test_worker_sets_job_status(self):
        """Ensure that worker correctly sets job status."""
        q = Queue()
        w = Worker([q])

        job = q.enqueue(say_hello)
        self.assertEqual(job.get_status(), Status.QUEUED)
        self.assertEqual(job.is_queued, True)
        self.assertEqual(job.is_finished, False)
        self.assertEqual(job.is_failed, False)

        w.work(burst=True)
        job = Job.fetch(job.id)
        self.assertEqual(job.get_status(), Status.FINISHED)
        self.assertEqual(job.is_queued, False)
        self.assertEqual(job.is_finished, True)
        self.assertEqual(job.is_failed, False)

        # Failed jobs should set status to "failed"
        job = q.enqueue(div_by_zero, args=(1,))
        w.work(burst=True)
        job = Job.fetch(job.id)
        self.assertEqual(job.get_status(), Status.FAILED)
        self.assertEqual(job.is_queued, False)
        self.assertEqual(job.is_finished, False)
        self.assertEqual(job.is_failed, True)
Ejemplo n.º 5
0
    def test_work_is_unreadable(self):
        """Unreadable jobs are put on the failed queue."""
        q = Queue()
        failed_q = get_failed_queue()

        self.assertEquals(failed_q.count, 0)
        self.assertEquals(q.count, 0)

        # NOTE: We have to fake this enqueueing for this test case.
        # What we're simulating here is a call to a function that is not
        # importable from the worker process.
        job = Job.create(func=div_by_zero, args=(3, ))
        job.save()
        data = self.testconn.hget(job.key, 'data')
        invalid_data = data.replace(b'div_by_zero', b'nonexisting')
        assert data != invalid_data
        self.testconn.hset(job.key, 'data', invalid_data)

        # We use the low-level internal function to enqueue any data (bypassing
        # validity checks)
        q.push_job_id(job.id)

        self.assertEquals(q.count, 1)

        # All set, we're going to process it
        w = Worker([q])
        w.work(burst=True)  # should silently pass
        self.assertEquals(q.count, 0)
        self.assertEquals(failed_q.count, 1)
Ejemplo n.º 6
0
    def test_custom_exc_handling(self):
        """Custom exception handling."""
        def black_hole(job, *exc_info):
            # Don't fall through to default behaviour (moving to failed queue)
            return False

        q = Queue()
        failed_q = get_failed_queue()

        # Preconditions
        self.assertEquals(failed_q.count, 0)
        self.assertEquals(q.count, 0)

        # Action
        job = q.enqueue(div_by_zero)
        self.assertEquals(q.count, 1)

        w = Worker([q], exc_handler=black_hole)
        w.work(burst=True)  # should silently pass

        # Postconditions
        self.assertEquals(q.count, 0)
        self.assertEquals(failed_q.count, 0)

        # Check the job
        job = Job.fetch(job.id)
        self.assertEquals(job.is_failed, True)
Ejemplo n.º 7
0
 def test_worker_ttl(self):
     """Worker ttl."""
     w = Worker([])
     w.register_birth()  # ugly: our test should only call public APIs
     [worker_key] = self.testconn.smembers(Worker.redis_workers_keys)
     self.assertIsNotNone(self.testconn.ttl(worker_key))
     w.register_death()
Ejemplo n.º 8
0
    def test_work_fails(self):
        """Failing jobs are put on the failed queue."""
        q = Queue()
        failed_q = get_failed_queue()

        # Preconditions
        self.assertEquals(failed_q.count, 0)
        self.assertEquals(q.count, 0)

        # Action
        job = q.enqueue(div_by_zero)
        self.assertEquals(q.count, 1)

        # keep for later
        enqueued_at_date = strip_microseconds(job.enqueued_at)

        w = Worker([q])
        w.work(burst=True)  # should silently pass

        # Postconditions
        self.assertEquals(q.count, 0)
        self.assertEquals(failed_q.count, 1)

        # Check the job
        job = Job.fetch(job.id)
        self.assertEquals(job.origin, q.name)

        # Should be the original enqueued_at date, not the date of enqueueing
        # to the failed queue
        self.assertEquals(job.enqueued_at, enqueued_at_date)
        self.assertIsNotNone(job.exc_info)  # should contain exc_info
Ejemplo n.º 9
0
    def test_timeouts(self):
        """Worker kills jobs after timeout."""
        sentinel_file = '/tmp/.rq_sentinel'

        q = Queue()
        w = Worker([q])

        # Put it on the queue with a timeout value
        res = q.enqueue(create_file_after_timeout,
                        args=(sentinel_file, 4),
                        timeout=1)

        try:
            os.unlink(sentinel_file)
        except OSError as e:
            if e.errno == 2:
                pass

        self.assertEquals(os.path.exists(sentinel_file), False)
        w.work(burst=True)
        self.assertEquals(os.path.exists(sentinel_file), False)

        # TODO: Having to do the manual refresh() here is really ugly!
        res.refresh()
        self.assertIn('JobTimeoutException', as_text(res.exc_info))
Ejemplo n.º 10
0
    def test_worker_sets_job_status(self):
        """Ensure that worker correctly sets job status."""
        q = Queue()
        w = Worker([q])

        job = q.enqueue(say_hello)
        self.assertEqual(job.get_status(), Status.QUEUED)
        self.assertEqual(job.is_queued, True)
        self.assertEqual(job.is_finished, False)
        self.assertEqual(job.is_failed, False)

        w.work(burst=True)
        job = Job.fetch(job.id)
        self.assertEqual(job.get_status(), Status.FINISHED)
        self.assertEqual(job.is_queued, False)
        self.assertEqual(job.is_finished, True)
        self.assertEqual(job.is_failed, False)

        # Failed jobs should set status to "failed"
        job = q.enqueue(div_by_zero, args=(1, ))
        w.work(burst=True)
        job = Job.fetch(job.id)
        self.assertEqual(job.get_status(), Status.FAILED)
        self.assertEqual(job.is_queued, False)
        self.assertEqual(job.is_finished, False)
        self.assertEqual(job.is_failed, True)
Ejemplo n.º 11
0
    def test_timeouts(self):
        """Worker kills jobs after timeout."""
        sentinel_file = '/tmp/.rq_sentinel'

        q = Queue()
        w = Worker([q])

        # Put it on the queue with a timeout value
        res = q.enqueue(create_file_after_timeout,
                        args=(sentinel_file, 4),
                        timeout=1)

        try:
            os.unlink(sentinel_file)
        except OSError as e:
            if e.errno == 2:
                pass

        self.assertEquals(os.path.exists(sentinel_file), False)
        w.work(burst=True)
        self.assertEquals(os.path.exists(sentinel_file), False)

        # TODO: Having to do the manual refresh() here is really ugly!
        res.refresh()
        self.assertIn('JobTimeoutException', as_text(res.exc_info))
Ejemplo n.º 12
0
    def test_custom_exc_handling(self):
        """Custom exception handling."""
        def black_hole(job, *exc_info):
            # Don't fall through to default behaviour (moving to failed queue)
            return False

        q = Queue()
        failed_q = get_failed_queue()

        # Preconditions
        self.assertEquals(failed_q.count, 0)
        self.assertEquals(q.count, 0)

        # Action
        job = q.enqueue(div_by_zero)
        self.assertEquals(q.count, 1)

        w = Worker([q], exc_handler=black_hole)
        w.work(burst=True)  # should silently pass

        # Postconditions
        self.assertEquals(q.count, 0)
        self.assertEquals(failed_q.count, 0)

        # Check the job
        job = Job.fetch(job.id)
        self.assertEquals(job.is_failed, True)
Ejemplo n.º 13
0
 def test_worker_ttl(self):
     """Worker ttl."""
     w = Worker([])
     w.register_birth()  # ugly: our test should only call public APIs
     [worker_key] = self.testconn.smembers(Worker.redis_workers_keys)
     self.assertIsNotNone(self.testconn.ttl(worker_key))
     w.register_death()
Ejemplo n.º 14
0
 def test_work_via_string_argument(self):
     """Worker processes work fed via string arguments."""
     q = Queue('foo')
     w = Worker([q])
     job = q.enqueue('tests.fixtures.say_hello', name='Frank')
     self.assertEquals(w.work(burst=True), True,
                       'Expected at least some work done.')
     self.assertEquals(job.result, 'Hi there, Frank!')
Ejemplo n.º 15
0
    def test_work_and_quit(self):
        """Worker processes work, then quits."""
        fooq, barq = Queue("foo"), Queue("bar")
        w = Worker([fooq, barq])
        self.assertEquals(w.work(burst=True), False, "Did not expect any work on the queue.")

        fooq.enqueue(say_hello, name="Frank")
        self.assertEquals(w.work(burst=True), True, "Expected at least some work done.")
Ejemplo n.º 16
0
 def test_work_via_string_argument(self):
     """Worker processes work fed via string arguments."""
     q = Queue('foo')
     w = Worker([q])
     job = q.enqueue('tests.fixtures.say_hello', name='Frank')
     self.assertEquals(w.work(burst=True), True,
                       'Expected at least some work done.')
     self.assertEquals(job.result, 'Hi there, Frank!')
Ejemplo n.º 17
0
    def test_work_and_quit(self):
        """Worker processes work, then quits."""
        fooq, barq = Queue('foo'), Queue('bar')
        w = Worker([fooq, barq])
        self.assertEquals(w.work(burst=True), False,
                          'Did not expect any work on the queue.')

        fooq.enqueue(say_hello, name='Frank')
        self.assertEquals(w.work(burst=True), True,
                          'Expected at least some work done.')
Ejemplo n.º 18
0
    def test_get_current_job(self):
        """Ensure worker.get_current_job() works properly"""
        q = Queue()
        worker = Worker([q])
        job = q.enqueue_call(say_hello)

        self.assertEqual(self.testconn.hget(worker.key, "current_job"), None)
        worker.set_current_job_id(job.id)
        self.assertEqual(worker.get_current_job_id(), as_text(self.testconn.hget(worker.key, "current_job")))
        self.assertEqual(worker.get_current_job(), job)
Ejemplo n.º 19
0
    def test_work_and_quit(self):
        """Worker processes work, then quits."""
        fooq, barq = Queue('foo'), Queue('bar')
        w = Worker([fooq, barq])
        self.assertEquals(w.work(burst=True), False,
                          'Did not expect any work on the queue.')

        fooq.enqueue(say_hello, name='Frank')
        self.assertEquals(w.work(burst=True), True,
                          'Expected at least some work done.')
    def test_gevent_num(self):
        """ensure gevent num is registered."""
        q = Queue()
        w = GeventWorker(q)
        w.heartbeat()
        self.assertEqual(self.testconn.hget(w.key, 'curr_pool_len'), '0')

        job = q.enqueue(say_hello)
        w.execute_job(job, q)
        w.heartbeat()
        self.assertEqual(self.testconn.hget(w.key, 'curr_pool_len'), '1')
        w.register_death()
Ejemplo n.º 21
0
    def test_get_current_job(self):
        """Ensure worker.get_current_job() works properly"""
        q = Queue()
        worker = Worker([q])
        job = q.enqueue_call(say_hello)

        self.assertEqual(self.testconn.hget(worker.key, 'current_job'), None)
        worker.set_current_job_id(job.id)
        self.assertEqual(
            worker.get_current_job_id(),
            as_text(self.testconn.hget(worker.key, 'current_job')))
        self.assertEqual(worker.get_current_job(), job)
Ejemplo n.º 22
0
    def test_job_dependency(self):
        """Enqueue dependent jobs only if their parents don't fail"""
        q = Queue()
        w = Worker([q])
        parent_job = q.enqueue(say_hello)
        job = q.enqueue_call(say_hello, depends_on=parent_job)
        w.work(burst=True)
        job = Job.fetch(job.id)
        self.assertEqual(job.get_status(), Status.FINISHED)

        parent_job = q.enqueue(div_by_zero)
        job = q.enqueue_call(say_hello, depends_on=parent_job)
        w.work(burst=True)
        job = Job.fetch(job.id)
        self.assertNotEqual(job.get_status(), Status.FINISHED)
Ejemplo n.º 23
0
    def test_job_dependency(self):
        """Enqueue dependent jobs only if their parents don't fail"""
        q = Queue()
        w = Worker([q])
        parent_job = q.enqueue(say_hello)
        job = q.enqueue_call(say_hello, depends_on=parent_job)
        w.work(burst=True)
        job = Job.fetch(job.id)
        self.assertEqual(job.get_status(), Status.FINISHED)

        parent_job = q.enqueue(div_by_zero)
        job = q.enqueue_call(say_hello, depends_on=parent_job)
        w.work(burst=True)
        job = Job.fetch(job.id)
        self.assertNotEqual(job.get_status(), Status.FINISHED)
    def test_gevent_num(self):
        """ensure gevent num is registered."""
        q = Queue()
        w = GeventWorker(q)
        w.heartbeat()
        self.assertEqual(self.testconn.hget(w.key, 'curr_pool_len'), '0')

        job = q.enqueue(say_hello)
        w.execute_job(job, q)
        w.heartbeat()
        self.assertEqual(self.testconn.hget(w.key, 'curr_pool_len'), '1')
        w.register_death()
Ejemplo n.º 25
0
    def test_worker_sets_result_ttl(self):
        """Ensure that Worker properly sets result_ttl for individual jobs."""
        q = Queue()
        job = q.enqueue(say_hello, args=('Frank', ), result_ttl=10)
        w = Worker([q])
        w.work(burst=True)
        self.assertNotEqual(self.testconn._ttl(job.key), 0)

        # Job with -1 result_ttl don't expire
        job = q.enqueue(say_hello, args=('Frank', ), result_ttl=-1)
        w = Worker([q])
        w.work(burst=True)
        self.assertEqual(self.testconn._ttl(job.key), -1)

        # Job with result_ttl = 0 gets deleted immediately
        job = q.enqueue(say_hello, args=('Frank', ), result_ttl=0)
        w = Worker([q])
        w.work(burst=True)
        self.assertEqual(self.testconn.get(job.key), None)
Ejemplo n.º 26
0
    def test_worker_sets_result_ttl(self):
        """Ensure that Worker properly sets result_ttl for individual jobs."""
        q = Queue()
        job = q.enqueue(say_hello, args=('Frank',), result_ttl=10)
        w = Worker([q])
        w.work(burst=True)
        self.assertNotEqual(self.testconn._ttl(job.key), 0)

        # Job with -1 result_ttl don't expire
        job = q.enqueue(say_hello, args=('Frank',), result_ttl=-1)
        w = Worker([q])
        w.work(burst=True)
        self.assertEqual(self.testconn._ttl(job.key), -1)

        # Job with result_ttl = 0 gets deleted immediately
        job = q.enqueue(say_hello, args=('Frank',), result_ttl=0)
        w = Worker([q])
        w.work(burst=True)
        self.assertEqual(self.testconn.get(job.key), None)
Ejemplo n.º 27
0
    def test_cancelled_jobs_arent_executed(self):  # noqa
        """Cancelling jobs."""

        SENTINEL_FILE = '/tmp/rq-tests.txt'

        try:
            # Remove the sentinel if it is leftover from a previous test run
            os.remove(SENTINEL_FILE)
        except OSError as e:
            if e.errno != 2:
                raise

        q = Queue()
        job = q.enqueue(create_file, SENTINEL_FILE)

        # Here, we cancel the job, so the sentinel file may not be created
        self.testconn.delete(job.key)

        w = Worker([q])
        w.work(burst=True)
        assert q.count == 0

        # Should not have created evidence of execution
        self.assertEquals(os.path.exists(SENTINEL_FILE), False)
Ejemplo n.º 28
0
    def test_cancelled_jobs_arent_executed(self):  # noqa
        """Cancelling jobs."""

        SENTINEL_FILE = '/tmp/rq-tests.txt'

        try:
            # Remove the sentinel if it is leftover from a previous test run
            os.remove(SENTINEL_FILE)
        except OSError as e:
            if e.errno != 2:
                raise

        q = Queue()
        job = q.enqueue(create_file, SENTINEL_FILE)

        # Here, we cancel the job, so the sentinel file may not be created
        self.testconn.delete(job.key)

        w = Worker([q])
        w.work(burst=True)
        assert q.count == 0

        # Should not have created evidence of execution
        self.assertEquals(os.path.exists(SENTINEL_FILE), False)
Ejemplo n.º 29
0
from config import REDIS_HOST, REDIS_PORT
from rq_gevent_worker import GeventWorker
from redis.client import StrictRedis
from rq import Connection, Queue
from logger import worker_logger

if __name__ == '__main__':
    # Tell rq what Redis connection to use
    with Connection(connection=StrictRedis(host=REDIS_HOST, port=REDIS_PORT)):
        q = Queue(connection=StrictRedis(host=REDIS_HOST, port=REDIS_PORT))
        waiting_jobs = q.get_jobs()
        for job in waiting_jobs:
            q.remove(job)
        gevent_worker = GeventWorker(q)
        gevent_worker.log = worker_logger
        gevent_worker.work()
 def test_register_worker(self):
     """ensure pool size is registered."""
     w = GeventWorker(Queue(), pool_size=10)
     w.register_birth()
     self.assertEqual(self.testconn.hget(w.key, 'pool_size'), '10')
    def test_sequence(self):
        w = GeventWorker(self.create_queue_with_two_jobs(), pool_size=1)
        with chronic.Timer('default'):
            w.work(burst=True)

        self.assertGreater(chronic.timings['default']['total_elapsed'], 2.0)
    def test_sequence(self):
        w = GeventWorker(self.create_queue_with_two_jobs(), pool_size=1)
        with chronic.Timer('default'):
            w.work(burst=True)

        self.assertGreater(chronic.timings['default']['total_elapsed'], 2.0)
    def test_concurrent(self):
        w = GeventWorker(self.create_queue_with_two_jobs(), pool_size=2)
        with chronic.Timer('default'):
            w.work(burst=True)

        self.assertLess(chronic.timings['default']['total_elapsed'], 1.5)
    def test_concurrent(self):
        w = GeventWorker(self.create_queue_with_two_jobs(), pool_size=2)
        with chronic.Timer('default'):
            w.work(burst=True)

        self.assertLess(chronic.timings['default']['total_elapsed'], 1.5)
 def test_register_worker(self):
     """ensure pool size is registered."""
     w = GeventWorker(Queue(), name="some_name", pool_size=10)
     w.register_birth()
     self.assertEqual(self.testconn.hget(w.key, 'pool_size'), '10')