Exemplo n.º 1
0
def test_worker_sets_job_status(loop):
    """Ensure that worker correctly sets job status."""

    q = Queue()
    w = Worker([q])

    job = yield from q.enqueue(say_hello)
    assert (yield from job.get_status()) == JobStatus.QUEUED
    assert (yield from job.is_queued)
    assert not (yield from job.is_finished)
    assert not (yield from job.is_failed)

    yield from w.work(burst=True, loop=loop)
    job = yield from Job.fetch(job.id)
    assert (yield from job.get_status()) == JobStatus.FINISHED
    assert not (yield from job.is_queued)
    assert (yield from job.is_finished)
    assert not (yield from job.is_failed)

    # Failed jobs should set status to "failed"
    job = yield from q.enqueue(div_by_zero, args=(1,))
    yield from w.work(burst=True, loop=loop)
    job = yield from Job.fetch(job.id)
    assert (yield from job.get_status()) == JobStatus.FAILED
    assert not (yield from job.is_queued)
    assert not (yield from job.is_finished)
    assert (yield from job.is_failed)
Exemplo n.º 2
0
def test_empty_remove_jobs(redis):
    """Emptying a queue deletes the associated job objects."""

    q = Queue('example')
    job = yield from q.enqueue(lambda x: x)
    assert (yield from Job.exists(job.id))
    yield from q.empty()
    assert not (yield from Job.exists(job.id))
Exemplo n.º 3
0
def test_data_property_sets_job_properties():
    """Job tuple gets derived lazily from data property."""

    job = Job()
    job.data = dumps(('foo', None, (1, 2, 3), {'bar': 'qux'}))

    assert job.func_name == 'foo'
    assert not job.instance
    assert job.args == (1, 2, 3)
    assert job.kwargs == {'bar': 'qux'}
Exemplo n.º 4
0
def test_job_properties_set_data_property():
    """Data property gets derived from the job tuple."""

    job = Job()
    job.func_name = 'foo'
    fname, instance, args, kwargs = loads(job.data)

    assert fname == job.func_name
    assert not instance
    assert args == ()
    assert kwargs == {}
Exemplo n.º 5
0
def test_result_ttl_is_persisted(redis):
    """Ensure that job's result_ttl is set properly"""

    job = Job.create(func=say_hello, args=('Lionel',), result_ttl=10)
    yield from job.save()
    yield from Job.fetch(job.id, connection=redis)
    assert job.result_ttl == 10

    job = Job.create(func=say_hello, args=('Lionel',))
    yield from job.save()
    yield from Job.fetch(job.id, connection=redis)
    assert not job.result_ttl
Exemplo n.º 6
0
def test_get_job_ttl():
    """Getting job TTL."""

    ttl = 1

    job = Job.create(func=say_hello, ttl=ttl)
    yield from job.save()
    assert job.get_ttl() == ttl

    job = Job.create(func=say_hello)
    yield from job.save()
    assert not job.get_ttl()
Exemplo n.º 7
0
def test_custom_meta_is_persisted(redis):
    """Additional meta data on jobs are stored persisted correctly."""

    job = Job.create(func=say_hello, args=('Lionel',))
    job.meta['foo'] = 'bar'
    yield from job.save()

    raw_data = yield from redis.hget(job.key, 'meta')
    assert loads(raw_data)['foo'] == 'bar'

    job2 = yield from Job.fetch(job.id)
    assert job2.meta['foo'] == 'bar'
Exemplo n.º 8
0
def test_store_then_fetch():
    """Store, then fetch."""

    job = Job.create(func=some_calculation, args=(3, 4), kwargs=dict(z=2))
    yield from job.save()

    job2 = yield from Job.fetch(job.id)
    assert job.func == job2.func
    assert job.args == job2.args
    assert job.kwargs == job2.kwargs

    # Mathematical equation
    assert job == job2
Exemplo n.º 9
0
def test_description_is_persisted(redis):
    """Ensure that job's custom description is set properly."""

    job = Job.create(func=say_hello, args=('Lionel',),
                     description='Say hello!')
    yield from job.save()
    yield from Job.fetch(job.id, connection=redis)
    assert job.description == 'Say hello!'

    # Ensure job description is constructed from function call string
    job = Job.create(func=say_hello, args=('Lionel',))
    yield from job.save()
    yield from Job.fetch(job.id, connection=redis)
    assert job.description == "fixtures.say_hello('Lionel')"
Exemplo n.º 10
0
def test_get_result_ttl():
    """Getting job result TTL."""

    job_result_ttl = 1
    default_ttl = 2

    job = Job.create(func=say_hello, result_ttl=job_result_ttl)
    yield from job.save()
    assert job.get_result_ttl(default_ttl=default_ttl) == job_result_ttl
    assert job.get_result_ttl() == job_result_ttl

    job = Job.create(func=say_hello)
    yield from job.save()
    assert job.get_result_ttl(default_ttl=default_ttl) == default_ttl
    assert not job.get_result_ttl()
Exemplo n.º 11
0
def test_work_is_unreadable(redis, loop):
    """Unreadable jobs are put on the failed queue."""

    q = Queue()
    failed_q = get_failed_queue()

    assert (yield from failed_q.count) == 0
    assert (yield from q.count) == 0

    # NOTE: We have to fake this enqueueing for this test case.
    # What we're simulating here is a call to a function that is not
    # importable from the worker process.
    job = Job.create(func=say_hello, args=(3,))
    yield from job.save()

    # NOTE: replacement and original strings must have the same length
    data = yield from redis.hget(job.key, 'data')
    invalid_data = data.replace(b'say_hello', b'fake_attr')
    assert data != invalid_data
    yield from redis.hset(job.key, 'data', invalid_data)

    # We use the low-level internal function to enqueue any data
    # (bypassing validity checks)
    yield from q.push_job_id(job.id)

    assert (yield from q.count) == 1

    # All set, we're going to process it
    w = Worker([q])
    yield from w.work(burst=True, loop=loop)  # Should silently pass
    assert (yield from q.count) == 0
    assert (yield from failed_q.count) == 1
Exemplo n.º 12
0
def test_job_dependency(loop):
    """Enqueue dependent jobs only if their parents don't fail."""

    q = Queue()
    w = Worker([q])
    parent_job = yield from q.enqueue(say_hello)
    job = yield from q.enqueue_call(say_hello, depends_on=parent_job)
    yield from w.work(burst=True, loop=loop)
    job = yield from Job.fetch(job.id)
    assert (yield from job.get_status()) == JobStatus.FINISHED

    parent_job = yield from q.enqueue(div_by_zero)
    job = yield from q.enqueue_call(say_hello, depends_on=parent_job)
    yield from w.work(burst=True, loop=loop)
    job = yield from Job.fetch(job.id)
    assert (yield from job.get_status()) != JobStatus.FINISHED
Exemplo n.º 13
0
def test_remove():
    """Ensure queue.remove properly removes Job from queue."""

    connection = object()
    sentinel = []

    class Protocol:
        @staticmethod
        @asyncio.coroutine
        def cancel_job(redis, name, id):
            assert redis is connection
            assert name == 'example'
            assert id == '56e6ba45-1aa3-4724-8c9f-51b7b0031cee'
            sentinel.append(1)

    class TestQueue(Queue):
        protocol = Protocol()

    q = TestQueue(connection, 'example')

    job = Job(connection=connection,
              id='56e6ba45-1aa3-4724-8c9f-51b7b0031cee',
              func=say_hello,
              args=(),
              kwargs={},
              description='fixtures.say_hello()',
              timeout=180,
              result_ttl=5000,
              origin='default',
              created_at=datetime(2016, 4, 5, 22, 40, 35))

    yield from q.remove(job)
    yield from q.remove(job.id)
    assert len(sentinel) == 2
Exemplo n.º 14
0
def test_unicode():
    """Unicode in job description."""

    job = Job.create('myfunc', args=[12, "☃"],
                     kwargs=dict(snowman="☃", null=None))
    expected_string = "myfunc(12, '☃', null=None, snowman='☃')"
    assert job.description, expected_string
Exemplo n.º 15
0
def test_custom_exc_handling(loop):
    """Custom exception handling."""

    @asyncio.coroutine
    def black_hole(job, *exc_info):
        # Don't fall through to default behaviour (moving to failed
        # queue)
        return False

    q = Queue()
    failed_q = get_failed_queue()

    # Preconditions
    assert not (yield from failed_q.count)
    assert not (yield from q.count)

    # Action
    job = yield from q.enqueue(div_by_zero)
    assert (yield from q.count) == 1

    w = Worker([q], exception_handlers=black_hole)
    yield from w.work(burst=True, loop=loop)  # Should silently pass

    # Postconditions
    assert not (yield from q.count)
    assert not (yield from failed_q.count)

    # Check the job
    job = yield from Job.fetch(job.id)
    assert job.is_failed
Exemplo n.º 16
0
def test_work_fails(loop):
    """Failing jobs are put on the failed queue."""

    q = Queue()
    failed_q = get_failed_queue()

    # Preconditions
    assert not (yield from failed_q.count)
    assert not (yield from q.count)

    # Action
    job = yield from q.enqueue(div_by_zero)
    assert (yield from q.count) == 1

    # keep for later
    enqueued_at_date = strip_microseconds(job.enqueued_at)

    w = Worker([q])
    yield from w.work(burst=True, loop=loop)  # Should silently pass

    # Postconditions
    assert not (yield from q.count)
    assert (yield from failed_q.count) == 1
    assert not (yield from w.get_current_job_id())

    # Check the job
    job = yield from Job.fetch(job.id)
    assert job.origin == q.name

    # Should be the original enqueued_at date, not the date of enqueueing
    # to the failed queue
    assert job.enqueued_at == enqueued_at_date
    assert job.exc_info  # should contain exc_info
Exemplo n.º 17
0
def test_create_job_from_callable_class():
    """Creation of jobs using a callable class specifier."""

    kallable = CallableObject()
    job = Job.create(func=kallable)

    assert job.func == kallable.__call__
    assert job.instance == kallable
Exemplo n.º 18
0
def test_cleanup(redis):
    """Test that jobs and results are expired properly."""

    job = Job.create(func=say_hello)
    yield from job.save()

    # Jobs with negative TTLs don't expire
    yield from job.cleanup(ttl=-1)
    assert (yield from redis.ttl(job.key)) == -1

    # Jobs with positive TTLs are eventually deleted
    yield from job.cleanup(ttl=100)
    assert (yield from redis.ttl(job.key)) == 100

    # Jobs with 0 TTL are immediately deleted
    yield from job.cleanup(ttl=0)
    with pytest.raises(NoSuchJobError):
        yield from Job.fetch(job.id, redis)
Exemplo n.º 19
0
def test_create_job_from_string_function():
    """Creation of jobs using string specifier."""

    job = Job.create(func='fixtures.say_hello', args=('World',))

    # Job data is set
    assert job.func == say_hello
    assert not job.instance
    assert job.args == ('World',)
Exemplo n.º 20
0
def test_persistence_of_parent_job():
    """Storing jobs with parent job, either instance or key."""

    parent_job = Job.create(func=some_calculation)
    yield from parent_job.save()

    job = Job.create(func=some_calculation, depends_on=parent_job)
    yield from job.save()

    stored_job = yield from Job.fetch(job.id)
    assert stored_job._dependency_id == parent_job.id
    assert (yield from stored_job.dependency) == parent_job

    job = Job.create(func=some_calculation, depends_on=parent_job.id)
    yield from job.save()
    stored_job = yield from Job.fetch(job.id)

    assert stored_job._dependency_id == parent_job.id
    assert (yield from stored_job.dependency) == parent_job
Exemplo n.º 21
0
def test_create_instance_method_job():
    """Creation of jobs for instance methods."""

    n = Number(2)
    job = Job.create(func=n.div, args=(4,))

    # Job data is set
    assert job.func == n.div
    assert job.instance == n
    assert job.args == (4,)
Exemplo n.º 22
0
def test_delete(redis):
    """job.delete() deletes itself & dependents mapping from Redis."""

    queue = Queue(connection=redis)
    job = yield from queue.enqueue(say_hello)
    job2 = Job.create(func=say_hello, depends_on=job)
    yield from job2.register_dependency()
    yield from job.delete()

    assert not (yield from redis.exists(job.key))
    assert not (yield from redis.exists(job.dependents_key))
    assert job.id not in (yield from queue.get_job_ids())
Exemplo n.º 23
0
def test_save(redis):
    """Storing jobs."""

    job = Job.create(func=some_calculation, args=(3, 4), kwargs=dict(z=2))

    # Saving creates a Redis hash
    assert not (yield from redis.exists(job.key))
    yield from job.save()
    assert (yield from redis.type(job.key)) == b'hash'

    # Saving writes pickled job data
    unpickled_data = loads((yield from redis.hget(job.key, 'data')))
    assert unpickled_data[0] == 'fixtures.some_calculation'
Exemplo n.º 24
0
def test_register_dependency(redis):
    """Ensure dependency registration works properly."""

    origin = 'some_queue'
    registry = DeferredJobRegistry(origin, redis)

    job = Job.create(func=say_hello, origin=origin)
    job._dependency_id = 'id'
    yield from job.save()

    assert not (yield from registry.get_job_ids())
    yield from job.register_dependency()
    assert as_text((yield from redis.spop('rq:job:id:dependents'))) == job.id
    assert (yield from registry.get_job_ids()) == [job.id]
Exemplo n.º 25
0
def test_enqueue_sets_metadata():
    """Enqueueing job onto queues modifies meta data."""

    q = Queue()
    job = Job.create(func=say_hello, args=('Nick',), kwargs=dict(foo='bar'))

    # Preconditions
    assert not job.enqueued_at

    # Action
    yield from q.enqueue_job(job)

    # Postconditions
    assert job.enqueued_at
Exemplo n.º 26
0
def test_persistence_of_typical_jobs(redis):
    """Storing typical jobs."""

    job = Job.create(func=some_calculation, args=(3, 4), kwargs=dict(z=2))
    yield from job.save()

    expected_date = strip_microseconds(job.created_at)
    stored_date = (yield from redis.hget(job.key, 'created_at')) \
        .decode('utf-8')
    assert stored_date == utcformat(expected_date)

    # ... and no other keys are stored
    assert sorted((yield from redis.hkeys(job.key))) \
        == [b'created_at', b'data', b'description']
Exemplo n.º 27
0
def test_fetching_unreadable_data(redis):
    """Fetching succeeds on unreadable data, but lazy props fail."""

    # Set up
    job = Job.create(func=some_calculation, args=(3, 4), kwargs=dict(z=2))
    yield from job.save()

    # Just replace the data hkey with some random noise
    yield from redis.hset(job.key, 'data', 'this is no pickle string')
    yield from job.refresh()

    for attr in ('func_name', 'instance', 'args', 'kwargs'):
        with pytest.raises(UnpickleError):
            getattr(job, attr)
Exemplo n.º 28
0
def test_job_is_unimportable(redis):
    """Jobs that cannot be imported throw exception on access."""

    job = Job.create(func=say_hello, args=('Lionel',))
    yield from job.save()

    # Now slightly modify the job to make it unimportable (this is
    # equivalent to a worker not having the most up-to-date source
    # code and unable to import the function)
    data = yield from redis.hget(job.key, 'data')
    unimportable_data = data.replace(b'say_hello', b'nay_hello')
    yield from redis.hset(job.key, 'data', unimportable_data)

    yield from job.refresh()
    with pytest.raises(AttributeError):
        job.func  # accessing the func property should fail
Exemplo n.º 29
0
def test_fetch(redis):
    """Fetching jobs."""

    yield from redis.hset('rq:job:some_id', 'data',
                          "(S'fixtures.some_calculation'\n"
                          "N(I3\nI4\nt(dp1\nS'z'\nI2\nstp2\n.")
    yield from redis.hset('rq:job:some_id', 'created_at',
                          '2012-02-07T22:13:24Z')

    # Fetch returns a job
    job = yield from Job.fetch('some_id')

    assert job.id == 'some_id'
    assert job.func_name == 'fixtures.some_calculation'
    assert not job.instance
    assert job.args == (3, 4)
    assert job.kwargs == dict(z=2)
    assert job.created_at == datetime(2012, 2, 7, 22, 13, 24)
Exemplo n.º 30
0
def test_create_typical_job():
    """Creation of jobs for function calls."""
    job = Job.create(func=some_calculation,
                     args=(3, 4), kwargs=dict(z=2))

    # Jobs have a random UUID
    assert job.id
    assert job.created_at
    assert job.description
    assert not job.instance

    # Job data is set...
    assert job.func == some_calculation
    assert job.args == (3, 4)
    assert job.kwargs == {'z': 2}

    # ...but metadata is not
    assert not job.origin
    assert not job.enqueued_at
    assert not (yield from job.result)
Exemplo n.º 31
0
def test_fetching_can_fail():
    """Fetching fails for non-existing jobs."""

    with pytest.raises(NoSuchJobError):
        yield from Job.fetch('b4a44d44-da16-4620-90a6-798e8cd72ca0')