def test_job_execution(self): """Job is removed from StartedJobRegistry after execution.""" registry = StartedJobRegistry(connection=self.testconn) queue = Queue(connection=self.testconn) worker = Worker([queue]) job = queue.enqueue(say_hello) self.assertTrue(job.is_queued) worker.prepare_job_execution(job) self.assertIn(job.id, registry.get_job_ids()) self.assertTrue(job.is_started) worker.perform_job(job, queue) self.assertNotIn(job.id, registry.get_job_ids()) self.assertTrue(job.is_finished) # Job that fails job = queue.enqueue(div_by_zero) worker.prepare_job_execution(job) self.assertIn(job.id, registry.get_job_ids()) worker.perform_job(job, queue) self.assertNotIn(job.id, registry.get_job_ids())
def test_job_access_within_job_function(self): """The current job is accessible within the job function.""" q = Queue() q.enqueue(fixtures.access_self) # access_self calls get_current_job() and asserts w = Worker([q]) w.work(burst=True) assert get_failed_queue(self.testconn).count == 0
def test_register_dependency(self): """Ensure job creation and deletion works properly with DeferredJobRegistry.""" queue = Queue(connection=self.testconn) job = queue.enqueue(say_hello) job2 = queue.enqueue(say_hello, depends_on=job) registry = DeferredJobRegistry(connection=self.testconn) self.assertEqual(registry.get_job_ids(), [job2.id]) # When deleted, job removes itself from DeferredJobRegistry job2.delete() self.assertEqual(registry.get_job_ids(), [])
def test_jobs_are_put_in_registry(self): """Completed jobs are added to FinishedJobRegistry.""" self.assertEqual(self.registry.get_job_ids(), []) queue = Queue(connection=self.testconn) worker = Worker([queue]) # Completed jobs are put in FinishedJobRegistry job = queue.enqueue(say_hello) worker.perform_job(job) self.assertEqual(self.registry.get_job_ids(), [job.id]) # Failed jobs are not put in FinishedJobRegistry failed_job = queue.enqueue(div_by_zero) worker.perform_job(failed_job) self.assertEqual(self.registry.get_job_ids(), [job.id])
def test_create_and_cancel_job(self): """test creating and using cancel_job deletes job properly""" queue = Queue(connection=self.testconn) job = queue.enqueue(fixtures.say_hello) self.assertEqual(1, len(queue.get_jobs())) cancel_job(job.id) self.assertEqual(0, len(queue.get_jobs()))
def test_default_failure_ttl(self): """Job TTL defaults to DEFAULT_FAILURE_TTL""" queue = Queue(connection=self.testconn) job = queue.enqueue(say_hello) registry = FailedJobRegistry(connection=self.testconn) key = registry.key timestamp = current_timestamp() registry.add(job) self.assertLess( self.testconn.zscore(key, job.id), timestamp + DEFAULT_FAILURE_TTL + 2 ) self.assertGreater( self.testconn.zscore(key, job.id), timestamp + DEFAULT_FAILURE_TTL - 2 ) timestamp = current_timestamp() ttl = 5 registry.add(job, ttl=5) self.assertLess( self.testconn.zscore(key, job.id), timestamp + ttl + 2 ) self.assertGreater( self.testconn.zscore(key, job.id), timestamp + ttl - 2 )
def test_get_call_string_unicode(self): """test call string with unicode keyword arguments""" queue = Queue(connection=self.testconn) job = queue.enqueue(fixtures.echo, arg_with_unicode=fixtures.UnicodeStringObject()) self.assertIsNotNone(job.get_call_string()) job.perform()
def test_create_job_with_id(self): """test creating jobs with a custom ID""" queue = Queue(connection=self.testconn) job = queue.enqueue(fixtures.say_hello, job_id="1234") self.assertEqual(job.id, "1234") job.perform() self.assertRaises(TypeError, queue.enqueue, fixtures.say_hello, job_id=1234)
def test_invalid_job(self): """Requeuing a job that's not in FailedJobRegistry raises an error.""" queue = Queue(connection=self.testconn) job = queue.enqueue(say_hello) registry = FailedJobRegistry(connection=self.testconn) with self.assertRaises(InvalidJobOperation): registry.requeue(job)
def test_job_access_within_job_function(self): """The current job is accessible within the job function.""" q = Queue() job = q.enqueue(fixtures.access_self) w = Worker([q]) w.work(burst=True) # access_self calls get_current_job() and executes successfully self.assertEqual(job.get_status(), JobStatus.FINISHED)
def test_get_expiration_time(self): """registry.get_expiration_time() returns correct datetime objects""" registry = StartedJobRegistry(connection=self.testconn) queue = Queue(connection=self.testconn) job = queue.enqueue(say_hello) registry.add(job, 5) self.assertEqual(registry.get_expiration_time(job), (datetime.utcnow() + timedelta(seconds=5)).replace(microsecond=0))
def test_add_and_remove(self): """Adding and removing job to StartedJobRegistry.""" timestamp = current_timestamp() queue = Queue(connection=self.testconn) job = queue.enqueue(say_hello) # Test that job is added with the right score self.registry.add(job, 1000) self.assertLess(self.testconn.zscore(self.registry.key, job.id), timestamp + 1002) # Ensure that a timeout of -1 results in a score of inf self.registry.add(job, -1) self.assertEqual(self.testconn.zscore(self.registry.key, job.id), float('inf')) # Ensure that job is removed from sorted set, but job key is not deleted self.registry.remove(job) self.assertIsNone(self.testconn.zscore(self.registry.key, job.id)) self.assertTrue(self.testconn.exists(job.key)) self.registry.add(job, -1) # registry.remove() also accepts job.id self.registry.remove(job.id) self.assertIsNone(self.testconn.zscore(self.registry.key, job.id)) self.registry.add(job, -1) # delete_job = True deletes job key self.registry.remove(job, delete_job=True) self.assertIsNone(self.testconn.zscore(self.registry.key, job.id)) self.assertFalse(self.testconn.exists(job.key)) job = queue.enqueue(say_hello) self.registry.add(job, -1) # delete_job = True also works with job.id self.registry.remove(job.id, delete_job=True) self.assertIsNone(self.testconn.zscore(self.registry.key, job.id)) self.assertFalse(self.testconn.exists(job.key))
def test_requeue_with_serializer(self): """FailedJobRegistry.requeue works properly (with serializer)""" queue = Queue(connection=self.testconn, serializer=JSONSerializer) job = queue.enqueue(div_by_zero, failure_ttl=5) worker = Worker([queue], serializer=JSONSerializer) worker.work(burst=True) registry = FailedJobRegistry(connection=worker.connection, serializer=JSONSerializer) self.assertTrue(job in registry) registry.requeue(job.id) self.assertFalse(job in registry) self.assertIn(job.id, queue.get_job_ids()) job.refresh() self.assertEqual(job.get_status(), JobStatus.QUEUED) self.assertEqual(job.started_at, None) self.assertEqual(job.ended_at, None) worker.work(burst=True) self.assertTrue(job in registry) # Should also work with job instance registry.requeue(job) self.assertFalse(job in registry) self.assertIn(job.id, queue.get_job_ids()) job.refresh() self.assertEqual(job.get_status(), JobStatus.QUEUED) worker.work(burst=True) self.assertTrue(job in registry) # requeue_job should work the same way requeue_job(job.id, connection=self.testconn, serializer=JSONSerializer) self.assertFalse(job in registry) self.assertIn(job.id, queue.get_job_ids()) job.refresh() self.assertEqual(job.get_status(), JobStatus.QUEUED) worker.work(burst=True) self.assertTrue(job in registry) # And so does job.requeue() job.requeue() self.assertFalse(job in registry) self.assertIn(job.id, queue.get_job_ids()) job.refresh() self.assertEqual(job.get_status(), JobStatus.QUEUED)
def test_add_and_remove_with_serializer(self): """Adding and removing job to StartedJobRegistry (with serializer).""" # delete_job = True also works with job.id and custom serializer queue = Queue(connection=self.testconn, serializer=JSONSerializer) registry = StartedJobRegistry(connection=self.testconn, serializer=JSONSerializer) job = queue.enqueue(say_hello) registry.add(job, -1) registry.remove(job.id, delete_job=True) self.assertIsNone(self.testconn.zscore(registry.key, job.id)) self.assertFalse(self.testconn.exists(job.key))
def test_delete(self): """job.delete() deletes itself & dependents mapping from Redis.""" queue = Queue(connection=self.testconn) job = queue.enqueue(fixtures.say_hello) job2 = Job.create(func=fixtures.say_hello, depends_on=job) job2.register_dependency() job.delete() self.assertFalse(self.testconn.exists(job.key)) self.assertFalse(self.testconn.exists(job.dependents_key)) self.assertNotIn(job.id, queue.get_job_ids())
def test_never_expire_during_execution(self): """Test what happens when job expires during execution""" ttl = 1 queue = Queue(connection=self.testconn) job = queue.enqueue(fixtures.long_running_job, args=(2,), ttl=ttl) self.assertEqual(job.get_ttl(), ttl) job.save() job.perform() self.assertEqual(job.get_ttl(), -1) self.assertTrue(job.exists(job.id)) self.assertEqual(job.result, 'Done sleeping...')
def test_jobs_are_put_in_registry(self): """Completed jobs are added to FinishedJobRegistry.""" self.assertEqual(self.registry.get_job_ids(), []) queue = Queue(connection=self.testconn) worker = Worker([queue]) # Completed jobs are put in FinishedJobRegistry job = queue.enqueue(say_hello) worker.perform_job(job, queue) self.assertEqual(self.registry.get_job_ids(), [job.id]) # When job is deleted, it should be removed from FinishedJobRegistry self.assertEqual(job.get_status(), JobStatus.FINISHED) job.delete() self.assertEqual(self.registry.get_job_ids(), []) # Failed jobs are not put in FinishedJobRegistry failed_job = queue.enqueue(div_by_zero) worker.perform_job(failed_job, queue) self.assertEqual(self.registry.get_job_ids(), [])
def test_cancel(self): """job.cancel() deletes itself & dependents mapping from Redis.""" queue = Queue(connection=self.testconn) job = queue.enqueue(say_hello) job2 = Job.create(func=say_hello, depends_on=job) job2.register_dependency() job.cancel() self.assertFalse(self.testconn.exists(job.key)) self.assertFalse(self.testconn.exists(job.dependents_key)) self.assertNotIn(job.id, queue.get_job_ids())
def test_never_expire_during_execution(self): """Test what happens when job expires during execution""" ttl = 1 queue = Queue(connection=self.testconn) job = queue.enqueue(fixtures.long_running_job, args=(2, ), ttl=ttl) self.assertEqual(job.get_ttl(), ttl) job.save() job.perform() self.assertEqual(job.get_ttl(), -1) self.assertTrue(job.exists(job.id)) self.assertEqual(job.result, 'Done sleeping...')
def test_worker_handle_job_failure(self): """Failed jobs are added to FailedJobRegistry""" q = Queue(connection=self.testconn) w = Worker([q]) registry = FailedJobRegistry(connection=w.connection) timestamp = current_timestamp() job = q.enqueue(div_by_zero, failure_ttl=5) w.handle_job_failure(job) # job is added to FailedJobRegistry with default failure ttl self.assertIn(job.id, registry.get_job_ids()) self.assertLess(self.testconn.zscore(registry.key, job.id), timestamp + DEFAULT_FAILURE_TTL + 5) # job is added to FailedJobRegistry with specified ttl job = q.enqueue(div_by_zero, failure_ttl=5) w.handle_job_failure(job) self.assertLess(self.testconn.zscore(registry.key, job.id), timestamp + 7)
def test_get_expiration_time(self): """registry.get_expiration_time() returns correct datetime objects""" registry = StartedJobRegistry(connection=self.testconn) queue = Queue(connection=self.testconn) job = queue.enqueue(say_hello) registry.add(job, 5) time = registry.get_expiration_time(job) expected_time = (datetime.utcnow() + timedelta(seconds=5)).replace(microsecond=0) self.assertGreaterEqual(time, expected_time - timedelta(seconds=2)) self.assertLessEqual(time, expected_time + timedelta(seconds=2))
def test_contains(self): registry = StartedJobRegistry(connection=self.testconn) queue = Queue(connection=self.testconn) job = queue.enqueue(say_hello) self.assertFalse(job in registry) self.assertFalse(job.id in registry) registry.add(job, 5) self.assertTrue(job in registry) self.assertTrue(job.id in registry)
def test_job_execution(self): """Job is removed from StartedJobRegistry after execution.""" registry = StartedJobRegistry(connection=self.testconn) queue = Queue(connection=self.testconn) worker = Worker([queue]) job = queue.enqueue(say_hello) worker.prepare_job_execution(job) self.assertIn(job.id, registry.get_job_ids()) worker.perform_job(job, queue) self.assertNotIn(job.id, registry.get_job_ids()) # Job that fails job = queue.enqueue(div_by_zero) worker.prepare_job_execution(job) self.assertIn(job.id, registry.get_job_ids()) worker.perform_job(job, queue) self.assertNotIn(job.id, registry.get_job_ids())
def test_job_deletion(self): """Ensure job is removed from StartedJobRegistry when deleted.""" registry = StartedJobRegistry(connection=self.testconn) queue = Queue(connection=self.testconn) worker = Worker([queue]) job = queue.enqueue(say_hello) self.assertTrue(job.is_queued) worker.prepare_job_execution(job) self.assertIn(job.id, registry.get_job_ids()) job.delete() self.assertNotIn(job.id, registry.get_job_ids())
def test_requeue(self): """FailedJobRegistry.requeue works properly""" queue = Queue(connection=self.testconn) job = queue.enqueue(div_by_zero, failure_ttl=5) worker = Worker([queue]) worker.work(burst=True) registry = FailedJobRegistry(connection=worker.connection) self.assertTrue(job in registry) registry.requeue(job.id) self.assertFalse(job in registry) self.assertIn(job.id, queue.get_job_ids()) job.refresh() self.assertEqual(job.get_status(), JobStatus.QUEUED) worker.work(burst=True) self.assertTrue(job in registry) # Should also work with job instance registry.requeue(job) self.assertFalse(job in registry) self.assertIn(job.id, queue.get_job_ids()) job.refresh() self.assertEqual(job.get_status(), JobStatus.QUEUED) worker.work(burst=True) self.assertTrue(job in registry) # requeue_job should work the same way requeue_job(job.id, connection=self.testconn) self.assertFalse(job in registry) self.assertIn(job.id, queue.get_job_ids()) job.refresh() self.assertEqual(job.get_status(), JobStatus.QUEUED) worker.work(burst=True) self.assertTrue(job in registry) # And so does job.requeue() job.requeue() self.assertFalse(job in registry) self.assertIn(job.id, queue.get_job_ids()) job.refresh() self.assertEqual(job.get_status(), JobStatus.QUEUED)
def test_status_deferred_job(self): # Create an asynchronous queue. # The name `separate_queue` used here is to ensure the queue isn't # used anywhere else. queue = Queue("separate_queue", connection=self.connection) # add job to the queue job = queue.enqueue(dummy_job) self.assertEqual(job.get_status(), "queued") # force the job status to be "started" job.set_status("deferred") self.assertTrue(job.is_deferred) # test rv = check_status(job, self.scheduler) self.assertEqual(rv, "deferred")
def test_status_failed_job(self): # Create an asynchronous queue. # The name `separate_queue` used here is to ensure the queue isn't used # anywhere else. queue = Queue("separate_queue", connection=self.connection) worker = SimpleWorker([queue], connection=queue.connection) # this job will fail job = queue.enqueue(dummy_fail_job) self.assertEqual(job.get_status(), "queued") # run the worker worker.work(burst=True) # test rv = check_status(job, self.scheduler) self.assertEqual(rv, "failed")
def test_cleanup_moves_jobs_to_failed_job_registry(self): """Moving expired jobs to FailedJobRegistry.""" queue = Queue(connection=self.testconn) failed_job_registry = FailedJobRegistry(connection=self.testconn) job = queue.enqueue(say_hello) self.testconn.zadd(self.registry.key, {job.id: 2}) # Job has not been moved to FailedJobRegistry self.registry.cleanup(1) self.assertNotIn(job, failed_job_registry) self.assertIn(job, self.registry) self.registry.cleanup() self.assertIn(job.id, failed_job_registry) self.assertNotIn(job, self.registry) job.refresh() self.assertEqual(job.get_status(), JobStatus.FAILED)
def test_job_access_within_job_function(self): """The current job is accessible within the job function.""" # Executing the job function from outside of RQ throws an exception self.assertIsNone(get_current_job()) # Executing the job function from within the job works (and in # this case leads to the job ID being returned) job = Job.create(func=access_self) job.save() id = job.perform() self.assertEqual(job.id, id) self.assertEqual(job.func, access_self) # Ensure that get_current_job also works from within synchronous jobs queue = Queue(async=False) job = queue.enqueue(access_self) id = job.perform() self.assertEqual(job.id, id) self.assertEqual(job.func, access_self)
def test_cleanup(self): """Moving expired jobs to FailedQueue.""" failed_queue = FailedQueue(connection=self.testconn) self.assertTrue(failed_queue.is_empty()) queue = Queue(connection=self.testconn) job = queue.enqueue(say_hello) self.testconn.zadd(self.registry.key, 2, job.id) self.registry.cleanup(1) self.assertNotIn(job.id, failed_queue.job_ids) self.assertEqual(self.testconn.zscore(self.registry.key, job.id), 2) self.registry.cleanup() self.assertIn(job.id, failed_queue.job_ids) self.assertEqual(self.testconn.zscore(self.registry.key, job.id), None) job.refresh() self.assertEqual(job.get_status(), JobStatus.FAILED)
def test_cleanup(self): """Moving expired jobs to FailedQueue.""" failed_queue = FailedQueue(connection=self.testconn) self.assertTrue(failed_queue.is_empty()) queue = Queue(connection=self.testconn) job = queue.enqueue(say_hello) self.testconn.zadd(self.registry.key, 2, job.id) self.registry.cleanup(1) self.assertNotIn(job.id, failed_queue.job_ids) self.assertEqual(self.testconn.zscore(self.registry.key, job.id), 2) self.registry.cleanup() self.assertIn(job.id, failed_queue.job_ids) self.assertEqual(self.testconn.zscore(self.registry.key, job.id), None) job.refresh() self.assertEqual(job.status, JobStatus.FAILED)
def test_default_failure_ttl(self): """Job TTL defaults to DEFAULT_FAILURE_TTL""" queue = Queue(connection=self.testconn) job = queue.enqueue(say_hello) registry = FailedJobRegistry(connection=self.testconn) key = registry.key timestamp = current_timestamp() registry.add(job) self.assertLess(self.testconn.zscore(key, job.id), timestamp + DEFAULT_FAILURE_TTL + 2) self.assertGreater(self.testconn.zscore(key, job.id), timestamp + DEFAULT_FAILURE_TTL - 2) timestamp = current_timestamp() ttl = 5 registry.add(job, ttl=5) self.assertLess(self.testconn.zscore(key, job.id), timestamp + ttl + 2) self.assertGreater(self.testconn.zscore(key, job.id), timestamp + ttl - 2)
def test_cleanup_handles_retries(self): """Expired jobs should also be retried""" queue = Queue(connection=self.testconn) registry = StartedJobRegistry(connection=self.testconn) failed_job_registry = FailedJobRegistry(connection=self.testconn) job = queue.enqueue(say_hello, retry=Retry(max=1)) # Add job to StartedJobRegistry with past expiration time self.testconn.zadd(registry.key, {job.id: 2}) registry.cleanup() self.assertEqual(len(queue), 2) self.assertEqual(job.get_status(), JobStatus.QUEUED) self.assertNotIn(job, failed_job_registry) self.testconn.zadd(registry.key, {job.id: 2}) # Job goes to FailedJobRegistry because it's only retried once registry.cleanup() self.assertEqual(len(queue), 2) self.assertEqual(job.get_status(), JobStatus.FAILED) self.assertIn(job, failed_job_registry)
def test_retry_interval(self): """Retries with intervals are scheduled""" connection = self.testconn queue = Queue(connection=connection) retry = Retry(max=1, interval=5) job = queue.enqueue(div_by_zero, retry=retry) worker = Worker([queue]) registry = queue.scheduled_job_registry # If job if configured to retry with interval, it will be scheduled, # not directly put back in the queue queue.empty() worker.handle_job_failure(job, queue) job.refresh() self.assertEqual(job.get_status(), JobStatus.SCHEDULED) self.assertEqual(job.retries_left, 0) self.assertEqual(len(registry), 1) self.assertEqual(queue.job_ids, []) # Scheduled time is roughly 5 seconds from now scheduled_time = registry.get_scheduled_time(job) now = datetime.now(timezone.utc) self.assertTrue(now + timedelta(seconds=4) < scheduled_time < now + timedelta(seconds=6))
def test_create_job_with_ttl_should_expire(self): """test if a job created with ttl expires [issue502]""" queue = Queue(connection=self.testconn) queue.enqueue(fixtures.say_hello, job_id="1234", ttl=1) time.sleep(1) self.assertEqual(0, len(queue.get_jobs()))
def test_create_job_with_ttl_should_have_ttl_after_enqueued(self): """test creating jobs with ttl and checks if get_jobs returns it properly [issue502]""" queue = Queue(connection=self.testconn) queue.enqueue(fixtures.say_hello, job_id="1234", ttl=10) job = queue.get_jobs()[0] self.assertEqual(job.ttl, 10)
def _add_jobs_to_queue(self, queue_name, num): queue = Queue(queue_name, connection=FakeStrictRedis()) for _ in range(num): queue.enqueue(self._dummy_func)
from redis import Redis from rq.queue import Queue from rq.serializers import JSONSerializer from . import settings from .logger import logger logger.info("Redis: %s:%s", settings.REDIS_HOST, settings.REDIS_PORT) redis_queue = Queue( name="email", connection=Redis(host=settings.REDIS_HOST, port=settings.REDIS_PORT), serializer=JSONSerializer, ) logger.info("Connection done") logger.info("Creating job") redis_queue.enqueue( "mail_job.send_email", kwargs=dict(recipients=["*****@*****.**"], subject="Test mail service", body="Hello!"), ) logger.info("Created job")
def test_job_access_within_job_function(self): """The current job is accessible within the job function.""" q = Queue() q.enqueue(fixtures.access_self) # access_self calls get_current_job() and asserts w = Worker([q]) w.work(burst=True)
def test_ttl_via_enqueue(self): ttl = 1 queue = Queue(connection=self.testconn) job = queue.enqueue(fixtures.say_hello, ttl=ttl) self.assertEqual(job.get_ttl(), ttl)
def test_job_access_within_synchronous_job_function(self): queue = Queue(async=False) queue.enqueue(fixtures.access_self)
def test_job_async_status_finished(self): queue = Queue(async=False) job = queue.enqueue(fixtures.say_hello) self.assertEqual(job.result, 'Hi there, Stranger!') self.assertEqual(job.get_status(), JobStatus.FINISHED)