def test_file_args_task(self):
# Use a temp file to communicate since we're forking.
tmpfile = tempfile.NamedTemporaryFile()
worker = Worker(self.tiger)
self.tiger.delay(file_args_task, args=(tmpfile.name, ))
queues = self._ensure_queues(queued={'default': 1})
task = queues['queued']['default'][0]
assert task['func'] == 'tests.tasks:file_args_task'
worker.run(once=True)
self._ensure_queues(queued={'default': 0})
json_data = tmpfile.read().decode('utf8')
assert json.loads(json_data) == {'args': [], 'kwargs': {}}
tmpfile.seek(0)
self.tiger.delay(file_args_task,
args=(tmpfile.name, 123, 'args'),
kwargs={'more': [1, 2, 3]})
self._ensure_queues(queued={'default': 1})
worker.run(once=True)
self._ensure_queues(queued={'default': 0})
json_data = tmpfile.read().decode('utf8')
assert json.loads(json_data) == {
'args': [123, 'args'],
'kwargs': {
'more': [1, 2, 3]
}
}
def test_file_args_task(self):
# Use a temp file to communicate since we're forking.
tmpfile = tempfile.NamedTemporaryFile()
worker = Worker(self.tiger)
self.tiger.delay(file_args_task, args=(tmpfile.name,))
queues = self._ensure_queues(queued={'default': 1})
task = queues['queued']['default'][0]
assert task['func'] == 'tests.tasks:file_args_task'
worker.run(once=True)
self._ensure_queues(queued={'default': 0})
json_data = tmpfile.read().decode('utf8')
assert json.loads(json_data) == {
'args': [],
'kwargs': {}
}
tmpfile.seek(0)
self.tiger.delay(file_args_task, args=(tmpfile.name, 123, 'args'),
kwargs={'more': [1, 2, 3]})
self._ensure_queues(queued={'default': 1})
worker.run(once=True)
self._ensure_queues(queued={'default': 0})
json_data = tmpfile.read().decode('utf8')
assert json.loads(json_data) == {
'args': [123, 'args'],
'kwargs': {'more': [1, 2, 3]}
}
def external_worker(n=None):
"""
Runs a worker. To be used with multiprocessing.Pool.map.
"""
tiger = get_tiger()
worker = Worker(tiger)
worker.run(once=True)
def test_max_workers(self):
"""Test Single Worker Queue."""
# Queue three tasks
for i in range(0, 3):
task = Task(self.tiger, long_task_ok, queue='a')
task.delay()
self._ensure_queues(queued={'a': 3})
# Start two workers and wait until they start processing.
worker1 = Process(target=external_worker,
kwargs={'max_workers_per_queue': 2})
worker2 = Process(target=external_worker,
kwargs={'max_workers_per_queue': 2})
worker1.start()
worker2.start()
# Wait for both tasks to start
wait_for_long_task()
wait_for_long_task()
# Verify they both are active
self._ensure_queues(active={'a': 2}, queued={'a': 1})
# This worker should fail to get the queue lock and exit immediately
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(active={'a': 2}, queued={'a': 1})
# Wait for external workers
worker1.join()
worker2.join()
def external_worker(n=None):
"""
Runs a worker. To be used with multiprocessing.Pool.map.
"""
tiger = get_tiger()
worker = Worker(tiger)
worker.run(once=True)
def test_max_workers(self):
"""Test Single Worker Queue."""
# Queue three tasks
for i in range(0, 3):
task = Task(self.tiger, long_task_killed, queue='a')
task.delay()
self._ensure_queues(queued={'a': 3})
# Start two workers and wait until they start processing.
worker1 = Process(target=external_worker,
kwargs={'max_workers_per_queue': 2})
worker2 = Process(target=external_worker,
kwargs={'max_workers_per_queue': 2})
worker1.start()
worker2.start()
time.sleep(DELAY)
# This worker should fail to get the queue lock and exit immediately
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(active={'a': 2}, queued={'a': 1})
# Wait for external workers
worker1.join()
worker2.join()
def external_worker(n=None, patch_config=None, max_workers_per_queue=None):
"""
Runs a worker. To be used with multiprocessing.Pool.map.
"""
tiger = get_tiger()
if patch_config:
tiger.config.update(patch_config)
worker = Worker(tiger)
if max_workers_per_queue is not None:
worker.max_workers_per_queue = max_workers_per_queue
worker.run(once=True, force_once=True)
def external_worker(n=None, patch_config=None, max_workers_per_queue=None):
"""
Runs a worker. To be used with multiprocessing.Pool.map.
"""
tiger = get_tiger()
if patch_config:
tiger.config.update(patch_config)
worker = Worker(tiger)
if max_workers_per_queue is not None:
worker.max_workers_per_queue = max_workers_per_queue
worker.run(once=True)
def test_queue_system_lock(self):
"""Test queue system lock."""
with FreezeTime(datetime.datetime(2014, 1, 1)):
# Queue three tasks
for i in range(0, 3):
task = Task(self.tiger, long_task_ok, queue='a')
task.delay()
self._ensure_queues(queued={'a': 3})
# Ensure we can process one
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(queued={'a': 2})
# Set system lock so no processing should occur for 10 seconds
self.tiger.set_queue_system_lock('a', 10)
lock_timeout = self.tiger.get_queue_system_lock('a')
assert lock_timeout == time.time() + 10
# Confirm tasks don't get processed within the system lock timeout
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 9)):
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(queued={'a': 2})
# 10 seconds in the future the lock should have expired
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 10)):
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(queued={'a': 1})
def test_queue_system_lock(self):
"""Test queue system lock."""
with FreezeTime(datetime.datetime(2014, 1, 1)):
# Queue three tasks
for i in range(0, 3):
task = Task(self.tiger, long_task_ok, queue='a')
task.delay()
self._ensure_queues(queued={'a': 3})
# Ensure we can process one
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(queued={'a': 2})
# Set system lock so no processing should occur for 10 seconds
self.tiger.set_queue_system_lock('a', 10)
lock_timeout = self.tiger.get_queue_system_lock('a')
assert lock_timeout == time.time() + 10
# Confirm tasks don't get processed within the system lock timeout
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 9)):
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(queued={'a': 2})
# 10 seconds in the future the lock should have expired
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 10)):
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(queued={'a': 1})