def test_get_queue_for_job_without_job_queue_name(self):
"""
Tests that scheduler gets the scheduler queue for the job
when queue name is not provided for that job.
"""
queue = Queue('scheduler_foo', connection=self.testconn)
scheduler = Scheduler(connection=self.testconn, queue=queue)
job = scheduler._create_job(say_hello)
self.assertEqual(scheduler.get_queue_for_job(job), queue)
def requeue(*args, **kwargs):
"""Inverval check for scheduled jobs."""
job = args[0]
queue = Scheduler.get_queue_for_job(build_scheduler, job)
scheduler_log.info(
f"job {job.id}:{job.func_name} re/queued to {queue.name}")
return queue
def test_get_queue_for_job_with_job_queue_name(self):
"""
Tests that scheduler gets the correct queue for the job when
queue_name is provided.
"""
queue = Queue('scheduler_foo', connection=self.testconn)
job_queue = Queue('job_foo', connection=self.testconn)
scheduler = Scheduler(connection=self.testconn, queue=queue)
job = scheduler._create_job(say_hello, queue_name='job_foo')
self.assertEqual(scheduler.get_queue_for_job(job), job_queue)
def test_enqueue_job_with_job_queue_name(self):
"""
Ensure that job is enqueued correctly when queue_name is provided
at job creation
"""
queue = Queue('foo', connection=self.testconn)
job_queue = Queue('job_foo', connection=self.testconn)
scheduler = Scheduler(connection=self.testconn, queue=queue)
job = scheduler._create_job(say_hello, queue_name='job_foo')
self.assertEqual(scheduler.get_queue_for_job(job), job_queue)
scheduler.enqueue_job(job)
self.assertTrue(job.enqueued_at is not None)
self.assertIn(job, job_queue.jobs)
self.assertIn(job_queue, Queue.all())
def test_enqueue_job_with_scheduler_queue(self):
"""
Ensure that job is enqueued correctly when the scheduler is bound
to a queue object and job queue name is not provided.
"""
queue = Queue('foo', connection=self.testconn)
scheduler = Scheduler(connection=self.testconn, queue=queue)
job = scheduler._create_job(say_hello)
scheduler_queue = scheduler.get_queue_for_job(job)
self.assertEqual(queue, scheduler_queue)
scheduler.enqueue_job(job)
self.assertTrue(job.enqueued_at is not None)
self.assertIn(job, queue.jobs)
self.assertIn(queue, Queue.all())
def test_enqueue_job_with_queue(self):
"""
Ensure that job is enqueued correctly when the scheduler is bound
to a queue object.
"""
queue = Queue('foo', connection=self.testconn)
scheduler = Scheduler(connection=self.testconn, queue=queue)
job = scheduler._create_job(say_hello)
scheduler_queue = scheduler.get_queue_for_job(job)
self.assertEqual(queue, scheduler_queue)
scheduler.enqueue_job(job)
self.assertTrue(job.enqueued_at is not None)
self.assertIn(job, queue.jobs)
self.assertIn(queue, Queue.all())
def test_enqueue_job(self):
"""
When scheduled job is enqueued, make sure:
- Job is removed from the sorted set of scheduled jobs
- "enqueued_at" attribute is properly set
- Job appears in the right queue
"""
now = datetime.now()
queue_name = 'foo'
scheduler = Scheduler(connection=self.testconn, queue_name=queue_name)
job = scheduler.enqueue_at(now, say_hello)
scheduler.enqueue_job(job)
self.assertNotIn(job, self.testconn.zrange(scheduler.scheduled_jobs_key, 0, 10))
job = Job.fetch(job.id, connection=self.testconn)
self.assertTrue(job.enqueued_at is not None)
queue = scheduler.get_queue_for_job(job)
self.assertIn(job, queue.jobs)
queue = Queue.from_queue_key('rq:queue:{0}'.format(queue_name))
self.assertIn(job, queue.jobs)
def schedule():
"""Creates scheduler object."""
setup_loghandlers(level=DEPLOYMENT_LOG_LEVEL)
build_scheduler = Scheduler(connection=WorkerQueues.connection)
scheduler_log.info("scheduler created")
cleanup_interval = int(os.getenv("RENKU_SVC_CLEANUP_INTERVAL", 60))
scheduler_log.info("cleanup interval set to {}".format(cleanup_interval))
def requeue(*args, **kwargs):
"""Inverval check for scheduled jobs."""
job = args[0]
queue = Scheduler.get_queue_for_job(build_scheduler, job)
scheduler_log.info(
f"job {job.id}:{job.func_name} re/queued to {queue.name}")
return queue
# NOTE: Patch scheduler to have requeing information on INFO log level.
build_scheduler.get_queue_for_job = requeue
build_scheduler.schedule(
scheduled_time=datetime.utcnow(),
queue_name=CLEANUP_QUEUE_FILES,
func=cache_files_cleanup,
interval=cleanup_interval,
result_ttl=cleanup_interval + 1,
)
build_scheduler.schedule(
scheduled_time=datetime.utcnow(),
queue_name=CLEANUP_QUEUE_PROJECTS,
func=cache_project_cleanup,
interval=cleanup_interval,
result_ttl=cleanup_interval + 1,
)
scheduler_log.info(f"log level set to {DEPLOYMENT_LOG_LEVEL}")
yield build_scheduler
