Exemplo n.º 1
0
def test_work_is_unreadable(redis, loop):
    """Unreadable jobs are put on the failed queue."""

    q = Queue()
    failed_q = get_failed_queue()

    assert (yield from failed_q.count) == 0
    assert (yield from q.count) == 0

    # NOTE: We have to fake this enqueueing for this test case.
    # What we're simulating here is a call to a function that is not
    # importable from the worker process.
    job = Job.create(func=say_hello, args=(3,))
    yield from job.save()

    # NOTE: replacement and original strings must have the same length
    data = yield from redis.hget(job.key, 'data')
    invalid_data = data.replace(b'say_hello', b'fake_attr')
    assert data != invalid_data
    yield from redis.hset(job.key, 'data', invalid_data)

    # We use the low-level internal function to enqueue any data
    # (bypassing validity checks)
    yield from q.push_job_id(job.id)

    assert (yield from q.count) == 1

    # All set, we're going to process it
    w = Worker([q])
    yield from w.work(burst=True, loop=loop)  # Should silently pass
    assert (yield from q.count) == 0
    assert (yield from failed_q.count) == 1
Exemplo n.º 2
0
def test_work_fails(loop):
    """Failing jobs are put on the failed queue."""

    q = Queue()
    failed_q = get_failed_queue()

    # Preconditions
    assert not (yield from failed_q.count)
    assert not (yield from q.count)

    # Action
    job = yield from q.enqueue(div_by_zero)
    assert (yield from q.count) == 1

    # keep for later
    enqueued_at_date = strip_microseconds(job.enqueued_at)

    w = Worker([q])
    yield from w.work(burst=True, loop=loop)  # Should silently pass

    # Postconditions
    assert not (yield from q.count)
    assert (yield from failed_q.count) == 1
    assert not (yield from w.get_current_job_id())

    # Check the job
    job = yield from Job.fetch(job.id)
    assert job.origin == q.name

    # Should be the original enqueued_at date, not the date of enqueueing
    # to the failed queue
    assert job.enqueued_at == enqueued_at_date
    assert job.exc_info  # should contain exc_info
Exemplo n.º 3
0
def test_custom_exc_handling(loop):
    """Custom exception handling."""

    @asyncio.coroutine
    def black_hole(job, *exc_info):
        # Don't fall through to default behaviour (moving to failed
        # queue)
        return False

    q = Queue()
    failed_q = get_failed_queue()

    # Preconditions
    assert not (yield from failed_q.count)
    assert not (yield from q.count)

    # Action
    job = yield from q.enqueue(div_by_zero)
    assert (yield from q.count) == 1

    w = Worker([q], exception_handlers=black_hole)
    yield from w.work(burst=True, loop=loop)  # Should silently pass

    # Postconditions
    assert not (yield from q.count)
    assert not (yield from failed_q.count)

    # Check the job
    job = yield from Job.fetch(job.id)
    assert job.is_failed
Exemplo n.º 4
0
def test_worker_sets_job_status(loop):
    """Ensure that worker correctly sets job status."""

    q = Queue()
    w = Worker([q])

    job = yield from q.enqueue(say_hello)
    assert (yield from job.get_status()) == JobStatus.QUEUED
    assert (yield from job.is_queued)
    assert not (yield from job.is_finished)
    assert not (yield from job.is_failed)

    yield from w.work(burst=True, loop=loop)
    job = yield from Job.fetch(job.id)
    assert (yield from job.get_status()) == JobStatus.FINISHED
    assert not (yield from job.is_queued)
    assert (yield from job.is_finished)
    assert not (yield from job.is_failed)

    # Failed jobs should set status to "failed"
    job = yield from q.enqueue(div_by_zero, args=(1,))
    yield from w.work(burst=True, loop=loop)
    job = yield from Job.fetch(job.id)
    assert (yield from job.get_status()) == JobStatus.FAILED
    assert not (yield from job.is_queued)
    assert not (yield from job.is_finished)
    assert (yield from job.is_failed)
Exemplo n.º 5
0
def test_worker_ttl(redis):
    """Worker ttl."""

    w = Worker([])
    yield from w.register_birth()
    [worker_key] = yield from redis.smembers(Worker.redis_workers_keys)
    assert (yield from redis.ttl(worker_key))
    yield from w.register_death()
Exemplo n.º 6
0
def test_work_via_string_argument(loop):
    """Worker processes work fed via string arguments."""

    q = Queue('foo')
    w = Worker([q])
    job = yield from q.enqueue('fixtures.say_hello', name='Frank')
    assert (yield from w.work(burst=True, loop=loop))
    assert (yield from job.result) == 'Hi there, Frank!'
Exemplo n.º 7
0
def test_work_and_quit(loop):
    """Worker processes work, then quits."""

    fooq, barq = Queue('foo'), Queue('bar')
    w = Worker([fooq, barq])
    assert not (yield from w.work(burst=True, loop=loop))

    yield from fooq.enqueue(say_hello, name='Frank')
    assert (yield from w.work(burst=True, loop=loop))
Exemplo n.º 8
0
def test_worker_calls_clean_registries(redis, loop):
    """Worker calls clean_registries when run."""

    queue = Queue(connection=redis)
    registry = StartedJobRegistry(connection=redis)
    yield from redis.zadd(registry.key, 1, 'foo')

    worker = Worker(queue, connection=redis)
    yield from worker.work(burst=True, loop=loop)
    assert not (yield from redis.zcard(registry.key))
Exemplo n.º 9
0
def test_work_unicode_friendly(loop):
    """Worker processes work with unicode description, then quits."""

    q = Queue('foo')
    w = Worker([q])
    job = yield from q.enqueue(
        'fixtures.say_hello', name='Adam', description='你好 世界!')
    assert (yield from w.work(burst=True, loop=loop))
    assert (yield from job.result) == 'Hi there, Adam!'
    assert job.description == '你好 世界!'
Exemplo n.º 10
0
def test_worker_sets_death():
    """Ensure worker correctly sets worker death date."""

    q = Queue()
    w = Worker([q])

    yield from w.register_death()

    death_date = yield from w.death_date
    assert death_date
    assert type(death_date).__name__ == 'datetime'
Exemplo n.º 11
0
def test_should_run_maintenance_tasks():
    """Workers should run maintenance tasks on startup and every hour."""

    queue = Queue()
    worker = Worker(queue)
    assert worker.should_run_maintenance_tasks

    worker.last_cleaned_at = utcnow()
    assert not worker.should_run_maintenance_tasks
    worker.last_cleaned_at = utcnow() - timedelta(seconds=3700)
    assert worker.should_run_maintenance_tasks
Exemplo n.º 12
0
def test_worker_sets_birth():
    """Ensure worker correctly sets worker birth date."""

    q = Queue()
    w = Worker([q])

    yield from w.register_birth()

    birth_date = yield from w.birth_date
    assert birth_date
    assert type(birth_date).__name__ == 'datetime'
Exemplo n.º 13
0
def test_get_current_job(redis):
    """Ensure worker.get_current_job() works properly."""

    q = Queue()
    worker = Worker([q])
    job = yield from q.enqueue_call(say_hello)

    assert not (yield from redis.hget(worker.key, 'current_job'))
    yield from worker.set_current_job_id(job.id)
    current_id = as_text((yield from redis.hget(worker.key, 'current_job')))
    assert (yield from worker.get_current_job_id()) == current_id
    assert (yield from worker.get_current_job()) == job
Exemplo n.º 14
0
def test_prepare_job_execution(redis):
    """Prepare job execution does the necessary bookkeeping."""

    queue = Queue(connection=redis)
    job = yield from queue.enqueue(say_hello)
    worker = Worker([queue])
    yield from worker.prepare_job_execution(job)

    # Updates working queue
    registry = StartedJobRegistry(connection=redis)
    assert (yield from registry.get_job_ids()) == [job.id]

    # Updates worker statuses
    assert worker.get_state() == 'busy'
    assert (yield from worker.get_current_job_id()) == job.id
Exemplo n.º 15
0
def test_job_dependency(loop):
    """Enqueue dependent jobs only if their parents don't fail."""

    q = Queue()
    w = Worker([q])
    parent_job = yield from q.enqueue(say_hello)
    job = yield from q.enqueue_call(say_hello, depends_on=parent_job)
    yield from w.work(burst=True, loop=loop)
    job = yield from Job.fetch(job.id)
    assert (yield from job.get_status()) == JobStatus.FINISHED

    parent_job = yield from q.enqueue(div_by_zero)
    job = yield from q.enqueue_call(say_hello, depends_on=parent_job)
    yield from w.work(burst=True, loop=loop)
    job = yield from Job.fetch(job.id)
    assert (yield from job.get_status()) != JobStatus.FINISHED
Exemplo n.º 16
0
def test_cancelled_jobs_arent_executed(redis, loop):
    """Cancelling jobs."""

    q = Queue()
    job = yield from q.enqueue(touch_a_mock)

    # Here, we cancel the job, so the sentinel file may not be created
    yield from redis.delete(job.key)

    w = Worker([q])
    yield from w.work(burst=True, loop=loop)
    assert not (yield from q.count)

    # Should not have created evidence of execution
    assert not mock.call_count
    mock.reset_mock()
Exemplo n.º 17
0
def test_timeouts(set_loop):
    """Worker kills jobs after timeout."""

    q = Queue()
    w = Worker([q])

    # Put it on the queue with a timeout value
    res = yield from q.enqueue(
        touch_a_mock_after_timeout, args=(4,), timeout=1)

    assert not mock.call_count
    yield from w.work(burst=True)
    assert not mock.call_count

    # TODO: Having to do the manual refresh() here is really ugly!
    yield from res.refresh()
    assert 'JobTimeoutException' in as_text(res.exc_info)
    mock.reset_mock()
Exemplo n.º 18
0
def test_clean_queue_registries(redis):
    """Worker.clean_registries sets last_cleaned_at and cleans registries."""

    foo_queue = Queue('foo', connection=redis)
    foo_registry = StartedJobRegistry('foo', connection=redis)
    yield from redis.zadd(foo_registry.key, 1, 'foo')
    assert (yield from redis.zcard(foo_registry.key)) == 1

    bar_queue = Queue('bar', connection=redis)
    bar_registry = StartedJobRegistry('bar', connection=redis)
    (yield from redis.zadd(bar_registry.key, 1, 'bar'))
    assert (yield from redis.zcard(bar_registry.key)) == 1

    worker = Worker([foo_queue, bar_queue])
    assert not worker.last_cleaned_at
    yield from worker.clean_registries()
    assert worker.last_cleaned_at
    assert not (yield from redis.zcard(foo_registry.key))
    assert not (yield from redis.zcard(bar_registry.key))
Exemplo n.º 19
0
def test_job_times(loop):
    """Job times are set correctly."""

    q = Queue('foo')
    w = Worker([q])
    before = utcnow().replace(microsecond=0)
    job = yield from q.enqueue(say_hello)

    assert job.enqueued_at
    assert not job.started_at
    assert not job.ended_at
    assert (yield from w.work(burst=True, loop=loop))
    assert (yield from job.result) == 'Hi there, Stranger!'

    after = utcnow().replace(microsecond=0)
    yield from job.refresh()

    assert before <= job.enqueued_at <= after
    assert before <= job.started_at <= after
    assert before <= job.ended_at <= after
Exemplo n.º 20
0
def test_suspend_worker_execution(redis, loop):
    """Test Pause Worker Execution."""

    q = Queue()
    w = Worker([q])
    yield from q.enqueue(touch_a_mock)

    yield from suspend(redis)

    yield from w.work(burst=True, loop=loop)
    assert (yield from q.count) == 1
    assert not mock.call_count

    yield from resume(redis)

    yield from w.work(burst=True, loop=loop)
    assert not (yield from q.count)
    assert mock.call_count

    mock.reset_mock()
Exemplo n.º 21
0
def test_suspend_with_duration(redis, loop):
    """Test worker execution will continue after specified duration."""

    q = Queue()
    w = Worker([q])
    for i in range(5):
        yield from q.enqueue(do_nothing)

    # This suspends workers for working for 2 second
    yield from suspend(redis, 2)

    # So when this burst of work happens the queue should remain at 5
    yield from w.work(burst=True, loop=loop)
    assert (yield from q.count) == 5

    yield from asyncio.sleep(3, loop=loop)

    # The suspension should be expired now, and a burst of work should
    # now clear the queue
    yield from w.work(burst=True, loop=loop)
    assert (yield from q.count) == 0
Exemplo n.º 22
0
def test_worker_sets_result_ttl(redis, loop):
    """Ensure that Worker properly sets result_ttl for individual jobs."""

    q = Queue()
    job = yield from q.enqueue(say_hello, args=('Frank',), result_ttl=10)
    w = Worker([q])
    yield from w.work(burst=True, loop=loop)
    assert (yield from redis.ttl(job.key))

    # Job with -1 result_ttl don't expire
    job = yield from q.enqueue(say_hello, args=('Frank',), result_ttl=-1)
    w = Worker([q])
    yield from w.work(burst=True, loop=loop)
    assert (yield from redis.ttl(job.key)) == -1

    # Job with result_ttl = 0 gets deleted immediately
    job = yield from q.enqueue(say_hello, args=('Frank',), result_ttl=0)
    w = Worker([q])
    yield from w.work(burst=True, loop=loop)
    assert not (yield from redis.get(job.key))