def test_queue_system_lock(self):
"""Test queue system lock."""
with FreezeTime(datetime.datetime(2014, 1, 1)):
# Queue three tasks
for i in range(0, 3):
task = Task(self.tiger, long_task_ok, queue='a')
task.delay()
self._ensure_queues(queued={'a': 3})
# Ensure we can process one
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(queued={'a': 2})
# Set system lock so no processing should occur for 10 seconds
self.tiger.set_queue_system_lock('a', 10)
lock_timeout = self.tiger.get_queue_system_lock('a')
assert lock_timeout == time.time() + 10
# Confirm tasks don't get processed within the system lock timeout
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 9)):
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(queued={'a': 2})
# 10 seconds in the future the lock should have expired
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 10)):
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(queued={'a': 1})
def test_queue_system_lock(self):
"""Test queue system lock."""
with FreezeTime(datetime.datetime(2014, 1, 1)):
# Queue three tasks
for i in range(0, 3):
task = Task(self.tiger, long_task_ok, queue='a')
task.delay()
self._ensure_queues(queued={'a': 3})
# Ensure we can process one
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(queued={'a': 2})
# Set system lock so no processing should occur for 10 seconds
self.tiger.set_queue_system_lock('a', 10)
lock_timeout = self.tiger.get_queue_system_lock('a')
assert lock_timeout == time.time() + 10
# Confirm tasks don't get processed within the system lock timeout
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 9)):
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(queued={'a': 2})
# 10 seconds in the future the lock should have expired
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 10)):
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(queued={'a': 1})
def test_max_workers(self):
"""Test Single Worker Queue."""
# Queue three tasks
for i in range(0, 3):
task = Task(self.tiger, long_task_ok, queue='a')
task.delay()
self._ensure_queues(queued={'a': 3})
# Start two workers and wait until they start processing.
worker1 = Process(target=external_worker,
kwargs={'max_workers_per_queue': 2})
worker2 = Process(target=external_worker,
kwargs={'max_workers_per_queue': 2})
worker1.start()
worker2.start()
# Wait for both tasks to start
wait_for_long_task()
wait_for_long_task()
# Verify they both are active
self._ensure_queues(active={'a': 2}, queued={'a': 1})
# This worker should fail to get the queue lock and exit immediately
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(active={'a': 2}, queued={'a': 1})
# Wait for external workers
worker1.join()
worker2.join()
def test_max_workers(self):
"""Test Single Worker Queue."""
# Queue three tasks
for i in range(0, 3):
task = Task(self.tiger, long_task_killed, queue='a')
task.delay()
self._ensure_queues(queued={'a': 3})
# Start two workers and wait until they start processing.
worker1 = Process(target=external_worker,
kwargs={'max_workers_per_queue': 2})
worker2 = Process(target=external_worker,
kwargs={'max_workers_per_queue': 2})
worker1.start()
worker2.start()
time.sleep(DELAY)
# This worker should fail to get the queue lock and exit immediately
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(active={'a': 2}, queued={'a': 1})
# Wait for external workers
worker1.join()
worker2.join()
def external_worker(n=None, patch_config=None, max_workers_per_queue=None):
"""
Runs a worker. To be used with multiprocessing.Pool.map.
"""
tiger = get_tiger()
if patch_config:
tiger.config.update(patch_config)
worker = Worker(tiger)
if max_workers_per_queue is not None:
worker.max_workers_per_queue = max_workers_per_queue
worker.run(once=True, force_once=True)
def external_worker(n=None, patch_config=None, max_workers_per_queue=None):
"""
Runs a worker. To be used with multiprocessing.Pool.map.
"""
tiger = get_tiger()
if patch_config:
tiger.config.update(patch_config)
worker = Worker(tiger)
if max_workers_per_queue is not None:
worker.max_workers_per_queue = max_workers_per_queue
worker.run(once=True)
