def test_enqueue():
"""Enqueueing job onto queues."""
connection = object()
uuids = []
class Protocol:
@staticmethod
@asyncio.coroutine
def enqueue_job(redis,
queue,
id,
data,
description,
timeout,
created_at,
*,
result_ttl=unset,
dependency_id=unset,
at_front=False):
assert redis is connection
assert queue == 'example'
assert isinstance(id, str)
assert data == b'\x80\x04\x952\x00\x00\x00\x00\x00\x00\x00(\x8c\x12fixtures.say_hello\x94N\x8c\x04Nick\x94\x85\x94}\x94\x8c\x03foo\x94\x8c\x03bar\x94st\x94.' # noqa
assert description == "fixtures.say_hello('Nick', foo='bar')"
assert timeout == 180
assert created_at == utcformat(utcnow())
assert result_ttl is unset
assert dependency_id is unset
assert at_front is False
uuids.append(id)
return JobStatus.QUEUED, utcnow()
class TestQueue(Queue):
protocol = Protocol()
q = TestQueue(connection, 'example')
job = yield from q.enqueue(say_hello, 'Nick', foo='bar')
assert job.connection is connection
assert job.id == uuids.pop(0)
assert job.func == say_hello
assert job.args == ('Nick', )
assert job.kwargs == {'foo': 'bar'}
assert job.description == "fixtures.say_hello('Nick', foo='bar')"
assert job.timeout == 180
assert job.result_ttl is None # TODO: optional?
assert job.origin == q.name
assert helpers.strip_microseconds(
job.created_at) == helpers.strip_microseconds(utcnow())
assert helpers.strip_microseconds(
job.enqueued_at) == helpers.strip_microseconds(utcnow())
assert job.status == JobStatus.QUEUED
assert job.dependency_id is None
def enqueue_job(redis, queue, id, data, description, timeout,
created_at, *, result_ttl=unset, dependency_id=unset,
at_front=False):
assert redis is connection
assert queue == 'example'
assert isinstance(id, str)
assert data == b'\x80\x04\x952\x00\x00\x00\x00\x00\x00\x00(\x8c\x12fixtures.say_hello\x94N\x8c\x04Nick\x94\x85\x94}\x94\x8c\x03foo\x94\x8c\x03bar\x94st\x94.' # noqa
assert description == "fixtures.say_hello('Nick', foo='bar')"
assert timeout == 180
assert created_at == utcformat(utcnow())
assert result_ttl is unset
assert dependency_id is unset
assert at_front is False
uuids.append(id)
return JobStatus.QUEUED, utcnow()
def test_enqueue_call():
"""Enqueueing job onto queues."""
connection = object()
uuids = []
class Protocol:
@staticmethod
@asyncio.coroutine
def enqueue_job(redis, queue, id, data, description, timeout,
created_at, *, ttl=None, result_ttl=unset,
dependency_id=unset, at_front=False,
meta=None):
assert redis is connection
assert queue == 'example'
assert isinstance(id, str)
assert data == b'\x80\x04\x952\x00\x00\x00\x00\x00\x00\x00(\x8c\x12fixtures.say_hello\x94N\x8c\x04Nick\x94\x85\x94}\x94\x8c\x03foo\x94\x8c\x03bar\x94st\x94.' # noqa
assert description == "fixtures.say_hello('Nick', foo='bar')"
assert timeout == 180
assert created_at == utcformat(utcnow())
assert not ttl
assert result_ttl is unset
assert dependency_id is unset
assert at_front is False
assert not meta
uuids.append(id)
return JobStatus.QUEUED, utcnow()
class TestQueue(Queue):
protocol = Protocol()
q = TestQueue(connection, 'example')
job = yield from q.enqueue_call(say_hello, args=('Nick',), kwargs={'foo': 'bar'})
assert job.connection is connection
assert job.id == uuids.pop(0)
assert job.func == say_hello
assert job.args == ('Nick',)
assert job.kwargs == {'foo': 'bar'}
assert job.description == "fixtures.say_hello('Nick', foo='bar')"
assert job.timeout == 180
assert job.result_ttl is None # TODO: optional?
assert job.origin == q.name
assert helpers.strip_microseconds(job.created_at) == helpers.strip_microseconds(utcnow())
assert helpers.strip_microseconds(job.enqueued_at) == helpers.strip_microseconds(utcnow())
assert job.status == JobStatus.QUEUED
assert job.dependency_id is None
def test_enqueue_call_dependency_job():
"""Pass dependency as job instance. Create deferred job."""
dependencies = [unset, 'foo']
replies = [(JobStatus.QUEUED, utcnow()), (JobStatus.DEFERRED, None)]
class Protocol:
@staticmethod
@asyncio.coroutine
def enqueue_job(redis,
queue,
id,
data,
description,
timeout,
created_at,
*,
result_ttl=unset,
dependency_id=unset,
at_front=False):
assert dependency_id == dependencies.pop(0)
return replies.pop(0)
class TestQueue(Queue):
protocol = Protocol()
q = TestQueue(None)
job_foo = yield from q.enqueue_call(say_hello, job_id='foo')
job_bar = yield from q.enqueue_call(say_hello,
job_id='bar',
depends_on=job_foo)
assert job_bar.status == JobStatus.DEFERRED
assert not job_bar.enqueued_at
def test_worker_death_sets_death_date(redis):
"""Set worker death date."""
yield from worker_birth(redis, 'foo', ['bar', 'baz'])
yield from worker_death(redis, 'foo')
death = utcformat(utcnow()).encode()
assert (yield from redis.hget(worker_key('foo'), 'death')) == death
def test_enqueue_call_dependency_job():
"""Pass dependency as job instance. Create deferred job."""
dependencies = [unset, 'foo']
replies = [(JobStatus.QUEUED, utcnow()),
(JobStatus.DEFERRED, None)]
class Protocol:
@staticmethod
@asyncio.coroutine
def enqueue_job(redis, queue, id, data, description, timeout,
created_at, *, ttl=None, result_ttl=unset,
dependency_id=unset, at_front=False,
meta=None):
assert dependency_id == dependencies.pop(0)
return replies.pop(0)
class TestQueue(Queue):
protocol = Protocol()
q = TestQueue(None)
job_foo = yield from q.enqueue_call(say_hello, job_id='foo')
job_bar = yield from q.enqueue_call(say_hello, job_id='bar', depends_on=job_foo)
assert job_bar.status == JobStatus.DEFERRED
assert not job_bar.enqueued_at
def enqueue_job(redis, queue, id, data, description, timeout,
created_at, *, ttl=None, result_ttl=unset,
dependency_id=unset, at_front=False,
meta=None):
_, _, _, kwargs = pickle.loads(data)
assert kwargs == {}
return JobStatus.QUEUED, utcnow()
def test_worker_death_sets_death_date(redis):
"""Set worker death date."""
yield from worker_birth(redis, 'foo', ['bar', 'baz'])
yield from worker_death(redis, 'foo')
death = utcformat(utcnow()).encode()
assert (yield from redis.hget(worker_key('foo'), 'death')) == death
def test_worker_shutdown_requested(redis):
"""Set worker shutdown requested date."""
yield from worker_birth(redis, 'foo', ['bar', 'baz'])
yield from worker_shutdown_requested(redis, 'foo')
shutdown = yield from redis.hget(worker_key('foo'),
'shutdown_requested_date')
assert shutdown == utcformat(utcnow()).encode()
def test_worker_shutdown_requested(redis):
"""Set worker shutdown requested date."""
yield from worker_birth(redis, 'foo', ['bar', 'baz'])
yield from worker_shutdown_requested(redis, 'foo')
shutdown = yield from redis.hget(
worker_key('foo'), 'shutdown_requested_date')
assert shutdown == utcformat(utcnow()).encode()
def test_start_job_sets_started_time(redis):
"""Start job sets it started at time."""
yield from enqueue_job(redis=redis, **stubs.job)
stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue)
stored_id = stored_id.decode()
queue = stored_spec[b'origin'].decode()
timeout = stored_spec[b'timeout']
yield from start_job(redis, queue, stored_id, timeout)
started_at = yield from redis.hget(job_key(stubs.job_id), 'started_at')
assert started_at == utcformat(utcnow()).encode()
def test_start_job_sets_started_time(redis):
"""Start job sets it started at time."""
yield from enqueue_job(redis=redis, **stubs.job)
stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue)
stored_id = stored_id.decode()
queue = stored_spec[b'origin'].decode()
timeout = stored_spec[b'timeout']
yield from start_job(redis, queue, stored_id, timeout)
started_at = yield from redis.hget(job_key(stubs.job_id), 'started_at')
assert started_at == utcformat(utcnow()).encode()
def job(redis, id):
return {
b'created_at': b'2016-04-05T22:40:35Z',
b'data': b'\x80\x04\x950\x00\x00\x00\x00\x00\x00\x00(\x8c\x19fixtures.some_calculation\x94NK\x03K\x04\x86\x94}\x94\x8c\x01z\x94K\x02st\x94.', # noqa
b'description': b'fixtures.some_calculation(3, 4, z=2)',
b'timeout': 180,
b'result_ttl': 5000,
b'status': JobStatus.QUEUED.encode(),
b'origin': stubs.queue.encode(),
b'enqueued_at': utcformat(utcnow()).encode(),
}
def test_finish_job_sets_ended_at(redis):
"""Finish job sets ended at field."""
yield from enqueue_job(redis=redis, **stubs.job)
stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue)
stored_id = stored_id.decode()
queue = stored_spec[b'origin'].decode()
timeout = stored_spec[b'timeout']
yield from start_job(redis, queue, stored_id, timeout)
yield from finish_job(redis, queue, stored_id)
ended_at = yield from redis.hget(job_key(stubs.job_id), 'ended_at')
assert ended_at == utcformat(utcnow()).encode()
def test_finish_job_sets_ended_at(redis):
"""Finish job sets ended at field."""
yield from enqueue_job(redis=redis, **stubs.job)
stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue)
stored_id = stored_id.decode()
queue = stored_spec[b'origin'].decode()
timeout = stored_spec[b'timeout']
yield from start_job(redis, queue, stored_id, timeout)
yield from finish_job(redis, queue, stored_id)
ended_at = yield from redis.hget(job_key(stubs.job_id), 'ended_at')
assert ended_at == utcformat(utcnow()).encode()
def test_finish_job_dependents_enqueue_date(redis):
"""Finish job will enqueue its dependents with proper date field."""
yield from enqueue_job(redis=redis, **stubs.job)
yield from enqueue_job(redis=redis, **stubs.child_job)
stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue)
stored_id = stored_id.decode()
queue = stored_spec[b'origin'].decode()
timeout = stored_spec[b'timeout']
yield from start_job(redis, queue, stored_id, timeout)
yield from finish_job(redis, queue, stored_id)
enqueued_at = yield from redis.hget(job_key(stubs.job_id), 'enqueued_at')
assert enqueued_at == utcformat(utcnow()).encode()
def test_finish_job_dependents_enqueue_date(redis):
"""Finish job will enqueue its dependents with proper date field."""
yield from enqueue_job(redis=redis, **stubs.job)
yield from enqueue_job(redis=redis, **stubs.child_job)
stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue)
stored_id = stored_id.decode()
queue = stored_spec[b'origin'].decode()
timeout = stored_spec[b'timeout']
yield from start_job(redis, queue, stored_id, timeout)
yield from finish_job(redis, queue, stored_id)
enqueued_at = yield from redis.hget(job_key(stubs.job_id), 'enqueued_at')
assert enqueued_at == utcformat(utcnow()).encode()
def enqueue_job(redis,
queue,
id,
data,
description,
timeout,
created_at,
*,
result_ttl=unset,
dependency_id=unset,
at_front=False):
assert result_ttl == 7
return JobStatus.QUEUED, utcnow()
def enqueue_job(redis,
queue,
id,
data,
description,
timeout,
created_at,
*,
result_ttl=unset,
dependency_id=unset,
at_front=False):
_, _, _, kwargs = pickle.loads(data)
assert kwargs == {}
return JobStatus.QUEUED, utcnow()
def enqueue_job(redis,
queue,
id,
data,
description,
timeout,
created_at,
*,
result_ttl=unset,
dependency_id=unset,
at_front=False):
assert redis is connection
assert queue == 'example'
assert isinstance(id, str)
assert data == b'\x80\x04\x952\x00\x00\x00\x00\x00\x00\x00(\x8c\x12fixtures.say_hello\x94N\x8c\x04Nick\x94\x85\x94}\x94\x8c\x03foo\x94\x8c\x03bar\x94st\x94.' # noqa
assert description == "fixtures.say_hello('Nick', foo='bar')"
assert timeout == 180
assert created_at == utcformat(utcnow())
assert result_ttl is unset
assert dependency_id is unset
assert at_front is False
uuids.append(id)
return JobStatus.QUEUED, utcnow()
def job(redis, id):
assert redis is connection
assert id == stubs.job_id
return {
b'created_at': b'2016-04-05T22:40:35Z',
b'data':
b'\x80\x04\x950\x00\x00\x00\x00\x00\x00\x00(\x8c\x19fixtures.some_calculation\x94NK\x03K\x04\x86\x94}\x94\x8c\x01z\x94K\x02st\x94.', # noqa
b'description': b'fixtures.some_calculation(3, 4, z=2)',
b'timeout': 180,
b'result_ttl': 5000,
b'status': JobStatus.QUEUED.encode(),
b'origin': stubs.queue.encode(),
b'enqueued_at': utcformat(utcnow()).encode(),
}
def test_dequeue_job(redis):
"""Dequeueing jobs from queues."""
yield from enqueue_job(redis=redis, result_ttl=5000, **stubs.job)
stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue)
assert not (yield from queue_length(redis, stubs.queue))
assert stored_id == stubs.job_id.encode()
assert stored_spec == {
b'created_at': b'2016-04-05T22:40:35Z',
b'data': b'\x80\x04\x950\x00\x00\x00\x00\x00\x00\x00(\x8c\x19fixtures.some_calculation\x94NK\x03K\x04\x86\x94}\x94\x8c\x01z\x94K\x02st\x94.', # noqa
b'description': b'fixtures.some_calculation(3, 4, z=2)',
b'timeout': 180,
b'result_ttl': 5000,
b'status': JobStatus.QUEUED.encode(),
b'origin': stubs.queue.encode(),
b'enqueued_at': utcformat(utcnow()).encode(),
}
def test_dequeue_job(redis):
"""Dequeueing jobs from queues."""
yield from enqueue_job(redis=redis, result_ttl=5000, **stubs.job)
stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue)
assert not (yield from queue_length(redis, stubs.queue))
assert stored_id == stubs.job_id.encode()
assert stored_spec == {
b'created_at': b'2016-04-05T22:40:35Z',
b'data':
b'\x80\x04\x950\x00\x00\x00\x00\x00\x00\x00(\x8c\x19fixtures.some_calculation\x94NK\x03K\x04\x86\x94}\x94\x8c\x01z\x94K\x02st\x94.', # noqa
b'description': b'fixtures.some_calculation(3, 4, z=2)',
b'timeout': 180,
b'result_ttl': 5000,
b'status': JobStatus.QUEUED.encode(),
b'origin': stubs.queue.encode(),
b'enqueued_at': utcformat(utcnow()).encode(),
}
def test_fail_job_sets_ended_at(redis):
"""Failed job should have ended at time."""
yield from fail_job(redis, stubs.queue, stubs.job_id, stubs.job_exc_info)
ended_at = yield from redis.hget(job_key(stubs.job_id), 'ended_at')
assert ended_at == utcformat(utcnow()).encode()
def enqueue_job(redis, queue, id, data, description, timeout,
created_at, *, ttl=None, result_ttl=unset,
dependency_id=unset, at_front=False,
meta=None):
assert meta == stubs.job_meta
return JobStatus.QUEUED, utcnow()
def test_fail_job_sets_ended_at(redis):
"""Failed job should have ended at time."""
yield from fail_job(redis, stubs.queue, stubs.job_id, stubs.job_exc_info)
ended_at = yield from redis.hget(job_key(stubs.job_id), 'ended_at')
assert ended_at == utcformat(utcnow()).encode()
def enqueue_job(redis, queue, id, data, description, timeout,
created_at, *, result_ttl=unset, dependency_id=unset,
at_front=False):
assert result_ttl == 7
return JobStatus.QUEUED, utcnow()
