def test_enqueue_job_dependents_set(redis): """Add job to the dependents set if its dependency isn't finished yet.""" yield from enqueue_job(redis=redis, **stubs.job) yield from enqueue_job(redis=redis, **stubs.child_job) assert (yield from redis.smembers(dependents( stubs.job_id))) == [stubs.child_job_id.encode()]
def test_empty_queue_removes_dependents(redis): """Remove dependent jobs for jobs from cleaned queue.""" yield from enqueue_job(redis=redis, **stubs.job) yield from enqueue_job(redis=redis, **stubs.child_job) yield from empty_queue(redis, stubs.queue) assert not (yield from redis.smembers(dependents(stubs.job_id)))
def test_enqueue_job_defer_dependent(redis): """Defer dependent job. It shouldn't present in the queue.""" yield from enqueue_job(redis=redis, **stubs.job) yield from enqueue_job(redis=redis, **stubs.child_job) assert (yield from redis.lrange(queue_key(stubs.queue), 0, -1)) == [stubs.job_id.encode()]
def test_enqueue_job_defer_without_date(redis): """Enqueue job with dependency doesn't set enqueued_at date.""" yield from enqueue_job(redis=redis, **stubs.job) yield from enqueue_job(redis=redis, **stubs.child_job) assert not (yield from redis.hexists(job_key(stubs.child_job_id), 'enqueued_at'))
def test_enqueue_job_dependent_status(redis): """Set DEFERRED status to jobs with unfinished dependencies.""" yield from enqueue_job(redis=redis, **stubs.job) yield from enqueue_job(redis=redis, **stubs.child_job) assert (yield from job_status(redis, stubs.child_job_id)) == JobStatus.DEFERRED.encode()
def test_enqueue_job_deferred_job_registry(redis): """Add job with unfinished dependency to deferred job registry.""" yield from enqueue_job(redis=redis, **stubs.job) yield from enqueue_job(redis=redis, **stubs.child_job) assert (yield from deferred_jobs(redis, stubs.queue)) == [stubs.child_job_id.encode()]
def test_finish_job_cleanup_dependents(redis): """Finish job will cleanup parent job dependents set.""" yield from enqueue_job(redis=redis, **stubs.job) yield from enqueue_job(redis=redis, **stubs.child_job) stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue) stored_id = stored_id.decode() queue = stored_spec[b'origin'].decode() timeout = stored_spec[b'timeout'] yield from start_job(redis, queue, stored_id, timeout) yield from finish_job(redis, queue, stored_id) assert not (yield from redis.smembers(dependents(stubs.job_id)))
def test_finish_job_dependents_defered_registry(redis): """Finish job will remove its dependents from deferred job registries.""" yield from enqueue_job(redis=redis, **stubs.job) yield from enqueue_job(redis=redis, **stubs.child_job) stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue) stored_id = stored_id.decode() queue = stored_spec[b'origin'].decode() timeout = stored_spec[b'timeout'] yield from start_job(redis, queue, stored_id, timeout) yield from finish_job(redis, queue, stored_id) assert not (yield from deferred_jobs(redis, queue))
def test_finish_job_enqueue_dependents(redis): """Finish job will enqueue its dependents.""" yield from enqueue_job(redis=redis, **stubs.job) yield from enqueue_job(redis=redis, **stubs.child_job) stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue) stored_id = stored_id.decode() queue = stored_spec[b'origin'].decode() timeout = stored_spec[b'timeout'] yield from start_job(redis, queue, stored_id, timeout) yield from finish_job(redis, queue, stored_id) assert (yield from redis.lrange(queue_key(stubs.queue), 0, -1)) == [stubs.child_job_id.encode()]
def test_finish_job_enqueue_dependents_status(redis): """Finish job will set dependents status to QUEUED.""" yield from enqueue_job(redis=redis, **stubs.job) yield from enqueue_job(redis=redis, **stubs.child_job) stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue) stored_id = stored_id.decode() queue = stored_spec[b'origin'].decode() timeout = stored_spec[b'timeout'] yield from start_job(redis, queue, stored_id, timeout) yield from finish_job(redis, queue, stored_id) assert (yield from redis.hget(job_key(stubs.child_job_id), 'status')) == JobStatus.QUEUED.encode()
def test_enqueue_job_finished_dependency(redis): """Enqueue job immediately if its dependency already finished.""" yield from enqueue_job(redis=redis, **stubs.job) stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue) stored_id = stored_id.decode() queue = stored_spec[b'origin'].decode() timeout = stored_spec[b'timeout'] yield from start_job(redis, queue, stored_id, timeout) yield from finish_job(redis, queue, stored_id) yield from enqueue_job(redis=redis, **stubs.child_job) assert (yield from redis.lrange(queue_key(stubs.queue), 0, -1)) == [stubs.child_job_id.encode()]
def test_finish_job_dependents_enqueue_date(redis): """Finish job will enqueue its dependents with proper date field.""" yield from enqueue_job(redis=redis, **stubs.job) yield from enqueue_job(redis=redis, **stubs.child_job) stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue) stored_id = stored_id.decode() queue = stored_spec[b'origin'].decode() timeout = stored_spec[b'timeout'] yield from start_job(redis, queue, stored_id, timeout) yield from finish_job(redis, queue, stored_id) enqueued_at = yield from redis.hget(job_key(stubs.job_id), 'enqueued_at') assert enqueued_at == utcformat(utcnow()).encode()
def test_enqueue_job_at_front(redis): """Enqueue job at front must add its id to the front of the queue.""" yield from redis.lpush(queue_key(stubs.queue), 'xxx') yield from enqueue_job(redis=redis, at_front=True, **stubs.job) queue_content = [stubs.job_id.encode(), b'xxx'] assert (yield from redis.lrange(queue_key(stubs.queue), 0, -1)) == queue_content
def test_enqueue_job_store_job_hash(redis): """Storing jobs.""" yield from enqueue_job(redis=redis, **stubs.job) assert (yield from redis.type(job_key(stubs.job_id))) == b'hash' assert (yield from redis.hget(job_key(stubs.job_id), 'data')) == stubs.job_data
def test_enqueue_job_add_job_key_to_the_queue(redis): """Enqueue job must add its id to the queue.""" yield from enqueue_job(redis=redis, **stubs.job) queue_content = [stubs.job_id.encode()] assert (yield from redis.lrange(queue_key(stubs.queue), 0, -1)) == queue_content
def test_fail_job_set_status(redis): """Failed job should have corresponding status.""" yield from enqueue_job(redis=redis, **stubs.job) yield from fail_job(redis, stubs.queue, stubs.job_id, stubs.job_exc_info) assert (yield from job_status(redis, stubs.job_id)) == JobStatus.FAILED.encode()
def test_dequeue_job_no_such_job(redis): """Silently skip job ids from queue if there is no such job hash.""" yield from redis.rpush(queue_key(stubs.queue), 'foo') # Job id without hash. yield from enqueue_job(redis=redis, **stubs.job) stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue) assert stored_id == stubs.job_id.encode()
def test_requeue_job_enqueue_into_origin(redis): """Requeue existing job puts it into jobs origin queue.""" yield from enqueue_job(redis=redis, **stubs.job) stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue) yield from fail_job(redis, stubs.queue, stubs.job_id, stubs.job_exc_info) yield from requeue_job(redis, stored_id.decode()) assert stubs.job_id.encode() in (yield from jobs(redis, stubs.queue))
def test_requeue_job_clean_exc_info(redis): """Requeue existing job cleanup exception information.""" yield from enqueue_job(redis=redis, **stubs.job) stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue) yield from fail_job(redis, stubs.queue, stubs.job_id, stubs.job_exc_info) yield from requeue_job(redis, stored_id.decode()) assert not (yield from redis.hget(job_key(stubs.job_id), 'exc_info'))
def test_requeue_job_set_status(redis): """Requeue existing job set corresponding job status.""" yield from enqueue_job(redis=redis, **stubs.job) stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue) yield from fail_job(redis, stubs.queue, stubs.job_id, stubs.job_exc_info) yield from requeue_job(redis, stored_id.decode()) assert (yield from job_status(redis, stubs.job_id)) == JobStatus.QUEUED.encode()
def test_compact_queue(redis): """Compact queue. Clean non existing jobs.""" yield from redis.rpush(queue_key(stubs.queue), 'foo') yield from enqueue_job(redis=redis, **stubs.job) yield from redis.rpush(queue_key(stubs.queue), 'bar') yield from compact_queue(redis, stubs.queue) assert (yield from jobs(redis, stubs.queue)) == [stubs.job_id.encode()]
def test_start_job_sets_job_status(redis): """Start job sets corresponding job status.""" yield from enqueue_job(redis=redis, **stubs.job) stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue) stored_id = stored_id.decode() queue = stored_spec[b'origin'].decode() timeout = stored_spec[b'timeout'] yield from start_job(redis, queue, stored_id, timeout) assert (yield from job_status(redis, stubs.job_id)) == JobStatus.STARTED.encode()
def test_finish_job_started_registry(redis): """Finish job removes job from started job registry.""" yield from enqueue_job(redis=redis, **stubs.job) stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue) stored_id = stored_id.decode() queue = stored_spec[b'origin'].decode() timeout = stored_spec[b'timeout'] yield from start_job(redis, queue, stored_id, timeout) yield from finish_job(redis, queue, stored_id) assert not (yield from started_jobs(redis, stubs.queue))
def test_finish_job_sets_corresponding_status(redis): """Finish job sets corresponding status.""" yield from enqueue_job(redis=redis, **stubs.job) stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue) stored_id = stored_id.decode() queue = stored_spec[b'origin'].decode() timeout = stored_spec[b'timeout'] yield from start_job(redis, queue, stored_id, timeout) yield from finish_job(redis, queue, stored_id) assert (yield from job_status(redis, stubs.job_id)) == JobStatus.FINISHED.encode()
def test_fail_job_removes_from_started_registry(redis): """Fail job remove given job from started registry.""" yield from enqueue_job(redis=redis, **stubs.job) stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue) stored_id = stored_id.decode() queue = stored_spec[b'origin'].decode() timeout = stored_spec[b'timeout'] yield from start_job(redis, queue, stored_id, timeout) yield from fail_job(redis, stubs.queue, stubs.job_id, stubs.job_exc_info) assert stubs.job_id.encode() not in (yield from started_jobs(redis, queue))
def test_finish_job_sets_results_ttl(redis): """Finish job sets results TTL.""" yield from enqueue_job(redis=redis, **stubs.job) stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue) stored_id = stored_id.decode() queue = stored_spec[b'origin'].decode() timeout = stored_spec[b'timeout'] yield from start_job(redis, queue, stored_id, timeout) yield from finish_job(redis, queue, stored_id) assert (yield from redis.ttl(job_key(stubs.job_id))) == 500
def test_start_job_sets_started_time(redis): """Start job sets it started at time.""" yield from enqueue_job(redis=redis, **stubs.job) stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue) stored_id = stored_id.decode() queue = stored_spec[b'origin'].decode() timeout = stored_spec[b'timeout'] yield from start_job(redis, queue, stored_id, timeout) started_at = yield from redis.hget(job_key(stubs.job_id), 'started_at') assert started_at == utcformat(utcnow()).encode()
def test_start_job_persist_job(redis): """Start job set persistence to the job hash during job execution.""" yield from enqueue_job(redis=redis, **stubs.job) yield from redis.expire(job_key(stubs.job_id), 3) stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue) stored_id = stored_id.decode() queue = stored_spec[b'origin'].decode() timeout = stored_spec[b'timeout'] yield from start_job(redis, queue, stored_id, timeout) assert (yield from redis.ttl(job_key(stubs.job_id))) == -1
def test_start_job_add_job_to_registry(redis): """Start job will add it to started job registry.""" yield from enqueue_job(redis=redis, **stubs.job) stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue) stored_id = stored_id.decode() queue = stored_spec[b'origin'].decode() timeout = stored_spec[b'timeout'] yield from start_job(redis, queue, stored_id, timeout) score = current_timestamp() + stored_spec[b'timeout'] + 60 started = yield from redis.zrange(started_registry(queue), withscores=True) assert started == [stubs.job_id.encode(), score]
def test_finish_job_non_expired_job(redis): """Finish job persist non expired job.""" yield from enqueue_job(redis=redis, result_ttl=None, **stubs.job) stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue) queue = stored_spec[b'origin'].decode() stored_id = stored_id.decode() timeout = stored_spec[b'timeout'] result_ttl = stored_spec[b'result_ttl'] yield from start_job(redis, queue, stored_id, timeout) yield from finish_job(redis, queue, stored_id, result_ttl=result_ttl) assert (yield from redis.ttl(job_key(stubs.job_id))) == -1
def test_finish_job_finished_registry(redis): """Finish job add job to the finished job registry.""" yield from enqueue_job(redis=redis, **stubs.job) stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue) stored_id = stored_id.decode() queue = stored_spec[b'origin'].decode() timeout = stored_spec[b'timeout'] yield from start_job(redis, queue, stored_id, timeout) yield from finish_job(redis, queue, stored_id) finish = yield from redis.zrange(finished_registry(queue), withscores=True) assert finish == [stubs.job_id.encode(), current_timestamp() + 500]
def test_finish_job_remove_results_zero_ttl(redis): """Finish job removes jobs with zero TTL.""" yield from enqueue_job(redis=redis, result_ttl=0, **stubs.job) stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue) queue = stored_spec[b'origin'].decode() stored_id = stored_id.decode() timeout = stored_spec[b'timeout'] result_ttl = stored_spec[b'result_ttl'] yield from start_job(redis, queue, stored_id, timeout) yield from finish_job(redis, queue, stored_id, result_ttl=result_ttl) assert not (yield from redis.exists(job_key(stubs.job_id)))
def test_finish_job_sets_ended_at(redis): """Finish job sets ended at field.""" yield from enqueue_job(redis=redis, **stubs.job) stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue) stored_id = stored_id.decode() queue = stored_spec[b'origin'].decode() timeout = stored_spec[b'timeout'] yield from start_job(redis, queue, stored_id, timeout) yield from finish_job(redis, queue, stored_id) ended_at = yield from redis.hget(job_key(stubs.job_id), 'ended_at') assert ended_at == utcformat(utcnow()).encode()
def test_finish_job_finished_registry_negative_ttl(redis): """Don't use current timestamp in job score with empty result TTL.""" yield from enqueue_job(redis=redis, result_ttl=None, **stubs.job) stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue) queue = stored_spec[b'origin'].decode() stored_id = stored_id.decode() timeout = stored_spec[b'timeout'] result_ttl = stored_spec[b'result_ttl'] yield from start_job(redis, queue, stored_id, timeout) yield from finish_job(redis, queue, stored_id, result_ttl=result_ttl) finish = yield from redis.zrange(finished_registry(queue), withscores=True) assert finish == [stubs.job_id.encode(), -1]
def test_dequeue_job(redis): """Dequeueing jobs from queues.""" yield from enqueue_job(redis=redis, result_ttl=5000, **stubs.job) stored_id, stored_spec = yield from dequeue_job(redis, stubs.queue) assert not (yield from queue_length(redis, stubs.queue)) assert stored_id == stubs.job_id.encode() assert stored_spec == { b'created_at': b'2016-04-05T22:40:35Z', b'data': b'\x80\x04\x950\x00\x00\x00\x00\x00\x00\x00(\x8c\x19fixtures.some_calculation\x94NK\x03K\x04\x86\x94}\x94\x8c\x01z\x94K\x02st\x94.', # noqa b'description': b'fixtures.some_calculation(3, 4, z=2)', b'timeout': 180, b'result_ttl': 5000, b'status': JobStatus.QUEUED.encode(), b'origin': stubs.queue.encode(), b'enqueued_at': utcformat(utcnow()).encode(), }
def test_requeue_job_error_on_non_failed_job(redis): """Throw error if anyone tries to requeue non failed job.""" yield from enqueue_job(redis=redis, **stubs.job) with pytest.raises(InvalidOperationError): yield from requeue_job(redis, stubs.job_id)
def test_cancel_job(redis): """Remove jobs from queue.""" yield from enqueue_job(redis=redis, **stubs.job) yield from cancel_job(redis, stubs.queue, stubs.job_id) assert not (yield from queue_length(redis, stubs.queue))