def test_no_two_schedulers_acquire_lock(self):
"""
Ensure that no two schedulers can acquire the lock at the
same time. When removing the lock, only the scheduler which
originally acquired the lock can remove the lock.
"""
key = '%s_lock' % Scheduler.scheduler_key
self.assertNotIn(key, tl(self.testconn.keys('*')))
scheduler1 = Scheduler(connection=self.testconn, interval=20)
scheduler2 = Scheduler(connection=self.testconn, interval=20)
self.assertTrue(scheduler1.acquire_lock())
self.assertFalse(scheduler2.acquire_lock())
self.assertIn(key, tl(self.testconn.keys('*')))
scheduler2.remove_lock()
self.assertIn(key, tl(self.testconn.keys('*')))
scheduler1.remove_lock()
self.assertNotIn(key, tl(self.testconn.keys('*')))
def test_no_two_schedulers_acquire_lock(self):
"""
Ensure that no two schedulers can acquire the lock at the
same time. When removing the lock, only the scheduler which
originally acquired the lock can remove the lock.
"""
key = '%s_lock' % Scheduler.scheduler_key
self.assertNotIn(key, tl(self.testconn.keys('*')))
scheduler1 = Scheduler(connection=self.testconn, interval=20)
scheduler2 = Scheduler(connection=self.testconn, interval=20)
self.assertTrue(scheduler1.acquire_lock())
self.assertFalse(scheduler2.acquire_lock())
self.assertIn(key, tl(self.testconn.keys('*')))
scheduler2.remove_lock()
self.assertIn(key, tl(self.testconn.keys('*')))
scheduler1.remove_lock()
self.assertNotIn(key, tl(self.testconn.keys('*')))
def test_multiple_schedulers_are_running_simultaneously(self):
"""
Even though only 1 Schedulder holds the lock and performs the scheduling.
Multiple schedulders are still registered to take over in case the original
scheduler goes down.
"""
lock_key = Scheduler.scheduler_lock_key
self.assertNotIn(lock_key, tl(self.testconn.keys('*')))
scheduler1 = Scheduler(connection=self.testconn, interval=20)
scheduler2 = Scheduler(connection=self.testconn, interval=20)
scheduler1.register_birth()
self.assertIn(scheduler1.key, tl(self.testconn.keys('*')))
scheduler2.register_birth()
self.assertIn(scheduler2.key, tl(self.testconn.keys('*')))
scheduler1.acquire_lock()
scheduler2.acquire_lock()
self.assertIn(scheduler1.key, tl(self.testconn.keys('*')))
self.assertIn(scheduler2.key, tl(self.testconn.keys('*')))
def test_acquire_lock(self):
"""
When scheduler acquires a lock, besides creating a key, it should
also set an expiry that's a few seconds longer than it's polling
interval so it automatically expires if scheduler is unexpectedly
terminated.
"""
key = '%s_lock' % Scheduler.scheduler_key
self.assertNotIn(key, tl(self.testconn.keys('*')))
scheduler = Scheduler(connection=self.testconn, interval=20)
self.assertTrue(scheduler.acquire_lock())
self.assertIn(key, tl(self.testconn.keys('*')))
self.assertEqual(self.testconn.ttl(key), 30)
scheduler.remove_lock()
self.assertNotIn(key, tl(self.testconn.keys('*')))
def test_acquire_lock(self):
"""
When scheduler acquires a lock, besides creating a key, it should
also set an expiry that's a few seconds longer than it's polling
interval so it automatically expires if scheduler is unexpectedly
terminated.
"""
key = '%s_lock' % Scheduler.scheduler_key
self.assertNotIn(key, tl(self.testconn.keys('*')))
scheduler = Scheduler(connection=self.testconn, interval=20)
self.assertTrue(scheduler.acquire_lock())
self.assertIn(key, tl(self.testconn.keys('*')))
self.assertEqual(self.testconn.ttl(key), 30)
scheduler.remove_lock()
self.assertNotIn(key, tl(self.testconn.keys('*')))
class QueueScheduler:
def __init__(self, queue_name, app):
self.app = app
self.logger = self.app.logger.bind(queue_name=queue_name)
self.scheduler = Scheduler(queue_name=queue_name, connection=app.redis)
def move_jobs(self):
if self.scheduler.acquire_lock():
try:
jobs = self.scheduler.get_jobs()
self.logger.debug("Lock acquired. Enqueuing scheduled jobs...",
jobs=jobs)
self.scheduler.enqueue_jobs()
finally:
self.scheduler.remove_lock()
else:
self.logger.debug(
"Lock could not be acquired. Enqueuing scheduled jobs skipped. Trying again next cycle."
)
def test_small_float_interval(self):
"""
Test that scheduler accepts 'interval' of type float, less than 1 second.
"""
key = Scheduler.scheduler_key
lock_key = '%s_lock' % Scheduler.scheduler_key
self.assertNotIn(key, tl(self.testconn.keys('*')))
scheduler = Scheduler(connection=self.testconn,
interval=0.1) # testing interval = 0.1 second
self.assertEqual(scheduler._interval, 0.1)
#acquire lock
self.assertTrue(scheduler.acquire_lock())
self.assertIn(lock_key, tl(self.testconn.keys('*')))
self.assertEqual(self.testconn.ttl(lock_key), 10) # int(0.1) + 10 = 10
#enqueue a job
now = datetime.utcnow()
job = scheduler.enqueue_at(now, say_hello)
self.assertIn(job, self.scheduler.get_jobs_to_queue())
self.assertEqual(len(list(self.scheduler.get_jobs())), 1)
#remove the lock
scheduler.remove_lock()
#test that run works with the small floating-point interval
def send_stop_signal():
"""
Sleep for 1 second, then send a INT signal to ourself, so the
signal handler installed by scheduler.run() is called.
"""
time.sleep(1)
os.kill(os.getpid(), signal.SIGINT)
thread = Thread(target=send_stop_signal)
thread.start()
self.assertRaises(SystemExit, scheduler.run)
thread.join()
#all jobs must have been scheduled during 1 second
self.assertEqual(len(list(scheduler.get_jobs())), 0)
def test_lock_handover_between_multiple_schedulers(self):
lock_key = Scheduler.scheduler_lock_key
self.assertNotIn(lock_key, tl(self.testconn.keys('*')))
scheduler1 = Scheduler(connection=self.testconn, interval=20)
scheduler2 = Scheduler(connection=self.testconn, interval=20)
scheduler1.register_birth()
scheduler1.acquire_lock()
scheduler2.register_birth()
scheduler2.acquire_lock()
# Both schedulers are still active/registered
self.assertIn(scheduler1.key, tl(self.testconn.keys('*')))
self.assertIn(scheduler2.key, tl(self.testconn.keys('*')))
scheduler1.remove_lock()
self.assertNotIn(lock_key, tl(self.testconn.keys('*')))
scheduler2.acquire_lock()
self.assertIn(lock_key, tl(self.testconn.keys('*')))
def test_small_float_interval(self):
"""
Test that scheduler accepts 'interval' of type float, less than 1 second.
"""
key = Scheduler.scheduler_key
lock_key = '%s_lock' % Scheduler.scheduler_key
self.assertNotIn(key, tl(self.testconn.keys('*')))
scheduler = Scheduler(connection=self.testconn, interval=0.1) # testing interval = 0.1 second
self.assertEqual(scheduler._interval, 0.1)
#acquire lock
self.assertTrue(scheduler.acquire_lock())
self.assertIn(lock_key, tl(self.testconn.keys('*')))
self.assertEqual(self.testconn.ttl(lock_key), 10) # int(0.1) + 10 = 10
#enqueue a job
now = datetime.utcnow()
job = scheduler.enqueue_at(now, say_hello)
self.assertIn(job, self.scheduler.get_jobs_to_queue())
self.assertEqual(len(list(self.scheduler.get_jobs())), 1)
#remove the lock
scheduler.remove_lock()
#test that run works with the small floating-point interval
def send_stop_signal():
"""
Sleep for 1 second, then send a INT signal to ourself, so the
signal handler installed by scheduler.run() is called.
"""
time.sleep(1)
os.kill(os.getpid(), signal.SIGINT)
thread = Thread(target=send_stop_signal)
thread.start()
self.assertRaises(SystemExit, scheduler.run)
thread.join()
#all jobs must have been scheduled during 1 second
self.assertEqual(len(list(scheduler.get_jobs())), 0)
