def external_worker(n=None):
"""
Runs a worker. To be used with multiprocessing.Pool.map.
"""
tiger = get_tiger()
worker = Worker(tiger)
worker.run(once=True)
def test_periodic_execution_unique_ids_manual_scheduling(self):
"""
Periodic tasks should have the same unique ids when manually scheduled
When a periodic task is scheduled initially as part of worker startup
vs ``.delay``'d manually, the unique id generated should be the same.
If they aren't it could result in duplicate tasks.
"""
# Sleep until the next second
sleep_until_next_second()
# After the first worker run, the periodic task will be queued.
# Note that since periodic tasks register with the Tiger instance, it
# must be the same instance that was used to decorate the task. We
# therefore use `tiger` from the tasks module instead of `self.tiger`.
self._ensure_queues()
Worker(tiger).run(once=True)
self._ensure_queues(scheduled={'periodic': 1})
time.sleep(1)
Worker(tiger).run(once=True)
self._ensure_queues(queued={'periodic': 1})
# schedule the task manually
periodic_task.delay()
# make sure a duplicate wasn't scheduled
self._ensure_queues(queued={'periodic': 1})
def test_file_args_task(self):
# Use a temp file to communicate since we're forking.
tmpfile = tempfile.NamedTemporaryFile()
worker = Worker(self.tiger)
self.tiger.delay(file_args_task, args=(tmpfile.name,))
queues = self._ensure_queues(queued={'default': 1})
task = queues['queued']['default'][0]
assert task['func'] == 'tests.tasks:file_args_task'
worker.run(once=True)
self._ensure_queues(queued={'default': 0})
json_data = tmpfile.read().decode('utf8')
assert json.loads(json_data) == {
'args': [],
'kwargs': {}
}
tmpfile.seek(0)
self.tiger.delay(file_args_task, args=(tmpfile.name, 123, 'args'),
kwargs={'more': [1, 2, 3]})
self._ensure_queues(queued={'default': 1})
worker.run(once=True)
self._ensure_queues(queued={'default': 0})
json_data = tmpfile.read().decode('utf8')
assert json.loads(json_data) == {
'args': [123, 'args'],
'kwargs': {'more': [1, 2, 3]}
}
def test_max_workers(self):
"""Test Single Worker Queue."""
# Queue three tasks
for i in range(0, 3):
task = Task(self.tiger, long_task_killed, queue='a')
task.delay()
self._ensure_queues(queued={'a': 3})
# Start two workers and wait until they start processing.
worker1 = Process(target=external_worker,
kwargs={'max_workers_per_queue': 2})
worker2 = Process(target=external_worker,
kwargs={'max_workers_per_queue': 2})
worker1.start()
worker2.start()
time.sleep(DELAY)
# This worker should fail to get the queue lock and exit immediately
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(active={'a': 2}, queued={'a': 1})
# Wait for external workers
worker1.join()
worker2.join()
def external_worker(n=None):
"""
Runs a worker. To be used with multiprocessing.Pool.map.
"""
tiger = get_tiger()
worker = Worker(tiger)
worker.run(once=True)
def test_locked_task(self):
self.tiger.delay(locked_task, kwargs={'key': '1'})
self.tiger.delay(locked_task, kwargs={'key': '2'})
self.tiger.delay(locked_task, kwargs={'key': '2'})
self._ensure_queues(queued={'default': 3},
scheduled={'default': 0},
error={'default': 0})
Pool(3).map(external_worker, range(3))
# One task with keys 1 and 2 executed, but one is scheduled because
# it hit a lock.
self._ensure_queues(queued={'default': 0},
scheduled={'default': 1},
error={'default': 0})
time.sleep(DELAY)
# Two runs: The first one picks the task up from the "scheduled" queue,
# the second one processes it.
Worker(self.tiger).run(once=True)
self._ensure_queues(queued={'default': 1},
scheduled={'default': 0},
error={'default': 0})
Worker(self.tiger).run(once=True)
self._ensure_queues(queued={'default': 0},
scheduled={'default': 0},
error={'default': 0})
def test_max_workers(self):
"""Test Single Worker Queue."""
# Queue three tasks
for i in range(0, 3):
task = Task(self.tiger, long_task_ok, queue='a')
task.delay()
self._ensure_queues(queued={'a': 3})
# Start two workers and wait until they start processing.
worker1 = Process(target=external_worker,
kwargs={'max_workers_per_queue': 2})
worker2 = Process(target=external_worker,
kwargs={'max_workers_per_queue': 2})
worker1.start()
worker2.start()
# Wait for both tasks to start
wait_for_long_task()
wait_for_long_task()
# Verify they both are active
self._ensure_queues(active={'a': 2}, queued={'a': 1})
# This worker should fail to get the queue lock and exit immediately
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(active={'a': 2}, queued={'a': 1})
# Wait for external workers
worker1.join()
worker2.join()
def test_single_worker_queue(self):
"""
Test Single Worker Queue.
Single worker queues are the same as running with MAX_WORKERS_PER_QUEUE
set to 1.
"""
# Queue two tasks
task = Task(self.tiger, long_task_ok, queue='swq')
task.delay()
task = Task(self.tiger, long_task_ok, queue='swq')
task.delay()
self._ensure_queues(queued={'swq': 2})
# Start a worker and wait until it starts processing.
# It should start processing one task and hold a lock on the queue
worker = Process(target=external_worker)
worker.start()
# Wait for task to start
wait_for_long_task()
# This worker should fail to get the queue lock and exit immediately
Worker(self.tiger).run(once=True, force_once=True)
self._ensure_queues(active={'swq': 1}, queued={'swq': 1})
# Wait for external worker
worker.join()
# Clear out second task
Worker(self.tiger).run(once=True, force_once=True)
self.conn.delete('long_task_ok')
# Retest using a non-single worker queue
# Queue two tasks
task = Task(self.tiger, long_task_ok, queue='not_swq')
task.delay()
task = Task(self.tiger, long_task_ok, queue='not_swq')
task.delay()
self._ensure_queues(queued={'not_swq': 2})
# Start a worker and wait until it starts processing.
# It should start processing one task
worker = Process(target=external_worker)
worker.start()
# Wait for task to start processing
wait_for_long_task()
# This worker should process the second task
Worker(self.tiger).run(once=True, force_once=True)
# Queues should be empty since the first task will have to
# have finished before the second task finishes.
self._ensure_queues()
worker.join()
def test_task_disappears(self):
"""
Ensure that a task object that disappears while the task is processing
is handled properly. This could happen when a worker processes a task,
then hangs for a long time, causing another worker to pick up and finish
the task. Then, when the original worker resumes, the task object will
be gone. Make sure we log a "not found" error and move on.
"""
task = Task(self.tiger, sleep_task, kwargs={'delay': 2 * DELAY})
task.delay()
self._ensure_queues(queued={'default': 1})
# Start a worker and wait until it starts processing.
worker = Process(target=external_worker)
worker.start()
time.sleep(DELAY)
# Remove the task object while the task is processing.
assert self.conn.delete('t:task:{}'.format(task.id)) == 1
# Kill the worker while it's still processing the task.
os.kill(worker.pid, signal.SIGKILL)
# _ensure_queues() breaks here because it can't find the task
assert self.conn.scard('t:queued') == 0
assert self.conn.scard('t:active') == 1
assert self.conn.scard('t:error') == 0
assert self.conn.scard('t:scheduled') == 0
# Capture logger
errors = []
def fake_error(msg):
errors.append(msg)
with Patch(self.tiger.log._logger, 'error', fake_error):
# Since ACTIVE_TASK_UPDATE_TIMEOUT hasn't elapsed yet, re-running
# the worker at this time won't change anything. (run twice to move
# from scheduled to queued)
Worker(self.tiger).run(once=True)
Worker(self.tiger).run(once=True)
assert len(errors) == 0
assert self.conn.scard('t:queued') == 0
assert self.conn.scard('t:active') == 1
assert self.conn.scard('t:error') == 0
assert self.conn.scard('t:scheduled') == 0
# After waiting and re-running the worker, queues will clear.
time.sleep(2 * DELAY)
Worker(self.tiger).run(once=True)
Worker(self.tiger).run(once=True)
self._ensure_queues()
assert len(errors) == 1
assert "not found" in errors[0]
def ensure_run(n):
# Run worker twice (once to move from scheduled to queued, and once
# to execute the task)
Worker(tiger).run(once=True)
self._ensure_queues(queued={'periodic': 1})
Worker(tiger).run(once=True)
self._ensure_queues(scheduled={'periodic': 1})
assert int(self.conn.get('period_count')) == n
# The task is requeued for the next period
self._ensure_queues(scheduled={'periodic': 1})
def test_periodic_execution(self):
"""
Test periodic task execution.
Test periodic_task() runs as expected and periodic_task_ignore()
is not queued.
"""
# Sleep until the next second to ensure we have enough time to start
# the worker and get the periodic task queued before the following
# second starts.
sleep_until_next_second()
# After the first worker run, the periodic task will be queued.
# Note that since periodic tasks register with the Tiger instance, it
# must be the same instance that was used to decorate the task. We
# therefore use `tiger` from the tasks module instead of `self.tiger`.
self._ensure_queues()
Worker(tiger).run(once=True)
# NOTE: When the worker is started just before the second elapses,
# it's possible that the periodic task is in "queued" state instead
# of "scheduled" to ensure immediate execution. We capture this
# condition by running the task, and retry.
try:
self._ensure_queues(scheduled={'periodic': 1})
except AssertionError:
Worker(tiger).run(once=True)
self._ensure_queues(scheduled={'periodic': 1})
assert int(self.conn.get('period_count')) == 1
self.conn.delete('period_count')
def ensure_run(n):
# Run worker twice (once to move from scheduled to queued, and once
# to execute the task)
Worker(tiger).run(once=True)
self._ensure_queues(queued={'periodic': 1})
Worker(tiger).run(once=True)
self._ensure_queues(scheduled={'periodic': 1})
assert int(self.conn.get('period_count')) == n
# The task is requeued for the next period
self._ensure_queues(scheduled={'periodic': 1})
# Sleep until the next second
sleep_until_next_second()
ensure_run(1)
# Within less than a second, the task will be processed again.
time.sleep(1)
ensure_run(2)
def test_single_worker_queue(self):
"""
Test Single Worker Queue.
Single worker queues are the same as running with MAX_WORKERS_PER_QUEUE
set to 1.
"""
# Queue two tasks
task = Task(self.tiger, long_task_ok, queue='swq')
task.delay()
task = Task(self.tiger, long_task_ok, queue='swq')
task.delay()
self._ensure_queues(queued={'swq': 2})
# Start a worker and wait until it starts processing.
# It should start processing one task and hold a lock on the queue
worker = Process(target=external_worker)
worker.start()
# Wait up to 2 seconds for external task to start
result = self.conn.blpop('long_task_ok', 2)
assert result[1] == '1'
# This worker should fail to get the queue lock and exit immediately
Worker(self.tiger).run(once=True, force_once=True)
self._ensure_queues(active={'swq': 1}, queued={'swq': 1})
# Wait for external worker
worker.join()
# Retest using a non-single worker queue
# Queue two tasks
task = Task(self.tiger, long_task_ok, queue='not_swq')
task.delay()
task = Task(self.tiger, long_task_ok, queue='not_swq')
task.delay()
self._ensure_queues(queued={'not_swq': 2})
# Start a worker and wait until it starts processing.
# It should start processing one task
worker = Process(target=external_worker)
worker.start()
time.sleep(DELAY)
# This worker should process the second task
Worker(self.tiger).run(once=True, force_once=True)
# Queues should be empty
self._ensure_queues()
worker.join()
def test_successful_execution_clears_executions_from_retries(self):
"""
Ensure previous executions from retries are cleared after a successful
execution.
"""
sleep_until_next_second()
# Queue the periodic task.
self._ensure_queues()
Worker(tiger).run(once=True)
# Prepare to execute the periodic task (as retriable failure).
tiger.connection.set('fail-periodic-task', 'retriable')
n_total, tasks = Task.tasks_from_queue(tiger, 'periodic', SCHEDULED)
task_id = tasks[0].id
time.sleep(1)
# Queue the periodic task.
self._ensure_queues(scheduled={'periodic': 1})
Worker(tiger).run(once=True)
# Run the failing periodic task.
self._ensure_queues(queued={'periodic': 1})
Worker(tiger).run(once=True)
task = Task.from_id(tiger,
'periodic',
SCHEDULED,
task_id,
load_executions=10)
assert len(task.executions) == 1
tiger.connection.delete('fail-periodic-task')
time.sleep(1)
# Queue the periodic task.
self._ensure_queues(scheduled={'periodic': 1})
Worker(tiger).run(once=True)
# Run the successful periodic task.
self._ensure_queues(queued={'periodic': 1})
Worker(tiger).run(once=True)
# Ensure we cleared any previous executions.
task = Task.from_id(tiger,
'periodic',
SCHEDULED,
task_id,
load_executions=10)
assert len(task.executions) == 0
def test_successful_execution_doesnt_clear_previous_errors(self):
"""
Ensure previous executions are not cleared if we have had non-retriable
errors.
"""
sleep_until_next_second()
# Queue the periodic task.
self._ensure_queues()
Worker(tiger).run(once=True)
# Prepare to execute the periodic task (as permanent failure).
tiger.connection.set('fail-periodic-task', 'permanent')
n_total, tasks = Task.tasks_from_queue(tiger, 'periodic', SCHEDULED)
task_id = tasks[0].id
time.sleep(1)
# Queue the periodic task.
self._ensure_queues(scheduled={'periodic': 1})
Worker(tiger).run(once=True)
# Run the failing periodic task.
self._ensure_queues(queued={'periodic': 1})
Worker(tiger).run(once=True)
task = Task.from_id(tiger,
'periodic',
SCHEDULED,
task_id,
load_executions=10)
assert len(task.executions) == 1
tiger.connection.delete('fail-periodic-task')
time.sleep(1)
# Queue the periodic task.
self._ensure_queues(scheduled={'periodic': 1}, error={'periodic': 1})
Worker(tiger).run(once=True)
# Run the successful periodic task.
self._ensure_queues(queued={'periodic': 1}, error={'periodic': 1})
Worker(tiger).run(once=True)
# Ensure we didn't clear previous executions.
task = Task.from_id(tiger,
'periodic',
SCHEDULED,
task_id,
load_executions=10)
assert len(task.executions) == 1
def external_worker(n=None, patch_config=None, max_workers_per_queue=None):
"""
Runs a worker. To be used with multiprocessing.Pool.map.
"""
tiger = get_tiger()
if patch_config:
tiger.config.update(patch_config)
worker = Worker(tiger)
if max_workers_per_queue is not None:
worker.max_workers_per_queue = max_workers_per_queue
worker.run(once=True, force_once=True)
def external_worker(n=None, patch_config=None, max_workers_per_queue=None):
"""
Runs a worker. To be used with multiprocessing.Pool.map.
"""
tiger = get_tiger()
if patch_config:
tiger.config.update(patch_config)
worker = Worker(tiger)
if max_workers_per_queue is not None:
worker.max_workers_per_queue = max_workers_per_queue
worker.run(once=True)
def test_queue(self):
self.tiger.delay(simple_task, queue='a')
self._ensure_queues(queued={'a': 1, 'b': 0, 'c': 0})
self.tiger.delay(simple_task, queue='b')
self._ensure_queues(queued={'a': 1, 'b': 1, 'c': 0})
self.tiger.delay(simple_task, queue='c')
self._ensure_queues(queued={'a': 1, 'b': 1, 'c': 1})
Worker(self.tiger, queues=['a', 'b']).run(once=True)
self._ensure_queues(queued={'a': 0, 'b': 0, 'c': 1})
Worker(self.tiger).run(once=True)
self._ensure_queues(queued={'a': 0, 'b': 0, 'c': 0})
def test_batch_2(self):
self.tiger.delay(batch_task, args=[1])
self.tiger.delay(non_batch_task, args=[5])
self.tiger.delay(batch_task, args=[2])
self.tiger.delay(batch_task, args=[3])
self.tiger.delay(batch_task, args=[4])
self.tiger.delay(non_batch_task, args=[6])
self.tiger.delay(non_batch_task, args=[7])
self._ensure_queues(queued={'batch': 7})
Worker(self.tiger).run(once=True)
self._ensure_queues(queued={'batch': 0})
data = [json.loads(d) for d in self.conn.lrange('batch_task', 0, -1)]
assert data == [
[{
'args': [1],
'kwargs': {}
}, {
'args': [2],
'kwargs': {}
}],
5,
[{
'args': [3],
'kwargs': {}
}, {
'args': [4],
'kwargs': {}
}],
6,
7,
]
def test_mixed_runner_class_batch_task(self):
"""Ensure all tasks in a batch task must have the same runner class."""
self.tiger.delay(batch_task, args=[1], runner_class=MyRunnerClass)
self.tiger.delay(batch_task, args=[2])
Worker(self.tiger).run(once=True)
assert self.conn.get('task_args') is None
self._ensure_queues(error={'batch': 2})
def test_periodic_execution_unique_ids_self_correct(self):
"""
Test that periodic tasks will self-correct unique ids
"""
# Sleep until the next second
sleep_until_next_second()
# generate the ids
correct_unique_id = gen_unique_id(serialize_func_name(periodic_task),
[], {})
malformed_unique_id = gen_unique_id(serialize_func_name(periodic_task),
None, None)
task = Task(tiger, func=periodic_task)
# patch the id to something slightly wrong
assert task.id == correct_unique_id
task._data['id'] = malformed_unique_id
assert task.id == malformed_unique_id
# schedule the task
task.delay()
self._ensure_queues(queued={'periodic': 1})
# pull task out of the queue by the malformed id
task = Task.from_id(tiger, 'periodic', QUEUED, malformed_unique_id)
assert task is not None
Worker(tiger).run(once=True)
self._ensure_queues(scheduled={'periodic': 1})
# pull task out of the queue by the self-corrected id
task = Task.from_id(tiger, 'periodic', SCHEDULED, correct_unique_id)
assert task is not None
def test_custom_runner_class_batch_task(self):
self.tiger.delay(batch_task, args=[1], runner_class=MyRunnerClass)
self.tiger.delay(batch_task, args=[2], runner_class=MyRunnerClass)
Worker(self.tiger).run(once=True)
assert self.conn.get('task_args') == "1,2"
self.conn.delete('task_args')
self._ensure_queues()
def test_structlog_processor(self):
try:
# Use ReturnLogger for testing
structlog.configure(
processors=[tasktiger_processor],
context_class=dict,
logger_factory=structlog.ReturnLoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
# Run a simple task. Logging output is verified in
# the task.
self.tiger.delay(logging_task)
queues = self._ensure_queues(queued={"default": 1})
task = queues["queued"]["default"][0]
assert task["func"] == "tests.test_logging:logging_task"
Worker(self.tiger).run(once=True)
self._ensure_queues(queued={"default": 0})
assert not self.conn.exists("t:task:%s" % task["id"])
finally:
structlog.configure(
processors=[
structlog.stdlib.add_log_level,
structlog.stdlib.filter_by_level,
structlog.processors.TimeStamper(fmt="iso", utc=True),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
structlog.processors.JSONRenderer(),
],
context_class=dict,
logger_factory=structlog.ReturnLoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
def test_current_tasks(self):
task1 = Task(self.tiger, verify_current_tasks)
task1.delay()
task2 = Task(self.tiger, verify_current_tasks)
task2.delay()
Worker(self.tiger).run(once=True)
assert self.conn.lrange('task_ids', 0, -1) == [task1.id, task2.id]
def _test_expired_task(self, task, expected_state):
"""
Ensure the given task ends up in the expected state if the worker is
killed prematurely. The task needs to run for longer than DELAY for
this test to work.
"""
task.delay()
self._ensure_queues(queued={'default': 1})
# Start a worker and wait until it starts processing.
worker = Process(target=external_worker)
worker.start()
time.sleep(DELAY)
# Kill the worker while it's still processing the task.
os.kill(worker.pid, signal.SIGKILL)
self._ensure_queues(active={'default': 1})
# Wait for (at least) ACTIVE_TASK_UPDATE_TIMEOUT
time.sleep(2 * DELAY)
Worker(self.tiger).run(once=True)
self._ensure_queues(**{expected_state: {'default': 1}})
def test_batch_exception_3(self):
self.tiger.delay(batch_task, args=[1])
self.tiger.delay(non_batch_task, args=[2])
self.tiger.delay(batch_task, args=[10])
self._ensure_queues(queued={'batch': 3})
Worker(self.tiger).run(once=True)
self._ensure_queues(queued={'batch': 0}, error={'batch': 2})
def test_exclude_queues(self):
"""
Test combining ONLY_QUEUES and EXCLUDE_QUEUES, and precedence in case
of subqueues or overlaps.
"""
self.tiger.config['ONLY_QUEUES'] = ['a', 'a.b.c', 'b', 'c']
self.tiger.config['EXCLUDE_QUEUES'] = ['a.b', 'b']
# Queues that should be processed
process_queues = ['a', 'a.a', 'a.b.c', 'c', 'c.a']
# Queues that should be excluded
ignore_queues = ['a.b', 'a.b.d', 'b', 'b.a', 'd', 'd.a']
all_queues = process_queues + ignore_queues
for queue in all_queues:
self.tiger.delay(simple_task, queue=queue)
self._ensure_queues(queued={q: 1 for q in all_queues})
Worker(self.tiger).run(once=True)
self._ensure_queues(queued={q: 1 for q in ignore_queues})
def test_retry_exception_2(self):
task = self.tiger.delay(retry_task_2)
self._ensure_queues(queued={'default': 1})
assert task.n_executions() == 0
Worker(self.tiger).run(once=True)
self._ensure_queues(scheduled={'default': 1})
assert task.n_executions() == 1
time.sleep(DELAY)
Worker(self.tiger).run(once=True)
Worker(self.tiger).run(once=True)
self._ensure_queues()
pytest.raises(TaskNotFound, task.n_executions)
def test_batch_4(self):
self.tiger.delay(batch_task, queue='batch.sub', args=[1])
self.tiger.delay(batch_task, queue='batch.sub', args=[2])
self.tiger.delay(batch_task, queue='batch.sub', args=[3])
self.tiger.delay(batch_task, queue='batch.sub', args=[4])
self._ensure_queues(queued={'batch.sub': 4})
Worker(self.tiger).run(once=True)
self._ensure_queues(queued={'batch.sub': 0})
data = [json.loads(d) for d in self.conn.lrange('batch_task', 0, -1)]
assert data == [
[
{
'args': [1],
'kwargs': {}
},
{
'args': [2],
'kwargs': {}
},
{
'args': [3],
'kwargs': {}
},
],
[
{
'args': [4],
'kwargs': {}
},
],
]
def test_retry_on_2(self):
# Will be retried
self.tiger.delay(exception_task, retry_on=[ValueError, Exception])
Worker(self.tiger).run(once=True)
self._ensure_queues(queued={'default': 0},
scheduled={'default': 1},
error={'default': 0})
def test_retry_on_3(self):
# Make sure we catch superclasses.
self.tiger.delay(exception_task, retry_on=[Exception])
Worker(self.tiger).run(once=True)
self._ensure_queues(queued={'default': 0},
scheduled={'default': 1},
error={'default': 0})
def test_retry_on_1(self):
# Fails immediately
self.tiger.delay(exception_task, retry_on=[ValueError, IndexError])
Worker(self.tiger).run(once=True)
self._ensure_queues(queued={'default': 0},
scheduled={'default': 0},
error={'default': 1})
def test_permanent_error(self):
task = self.tiger.delay(exception_task,
runner_class=MyErrorRunnerClass)
Worker(self.tiger).run(once=True)
assert self.conn.get('task_id') == task.id
self.conn.delete('task_id')
self._ensure_queues(error={'default': 1})
def test_when(self):
self.tiger.delay(simple_task, when=datetime.timedelta(seconds=DELAY))
self._ensure_queues(queued={'default': 0}, scheduled={'default': 1})
Worker(self.tiger).run(once=True)
self._ensure_queues(queued={'default': 0}, scheduled={'default': 1})
time.sleep(DELAY)
# Two runs: The first one picks the task up from the "scheduled" queue,
# the second one processes it.
Worker(self.tiger).run(once=True)
self._ensure_queues(queued={'default': 1}, scheduled={'default': 0})
Worker(self.tiger).run(once=True)
self._ensure_queues(queued={'default': 0}, scheduled={'default': 0})
def test_periodic_execution(self):
"""
Test periodic task execution.
"""
# After the first worker run, the periodic task will be queued.
# Note that since periodic tasks register with the Tiger instance, it
# must be the same instance that was used to decorate the task. We
# therefore use `tiger` from the tasks module instead of `self.tiger`.
self._ensure_queues()
Worker(tiger).run(once=True)
# NOTE: When the worker is started just before the second elapses,
# it's possible that the periodic task is in "queued" state instead
# of "scheduled" to ensure immediate execution. We capture this
# condition by running the task, and retry.
try:
self._ensure_queues(scheduled={'periodic': 1})
except AssertionError:
Worker(tiger).run(once=True)
assert int(self.conn.get('period_count')) == 1
self.conn.delete('period_count')
self._ensure_queues(scheduled={'periodic': 1})
def ensure_run(n):
# Run worker twice (once to move from scheduled to queued, and once
# to execute the task)
Worker(tiger).run(once=True)
self._ensure_queues(queued={'periodic': 1})
Worker(tiger).run(once=True)
self._ensure_queues(scheduled={'periodic': 1})
assert int(self.conn.get('period_count')) == n
# The task is requeued for the next period
self._ensure_queues(scheduled={'periodic': 1})
# Sleep until the next second
now = datetime.datetime.utcnow()
time.sleep(1 - now.microsecond / 10.**6)
ensure_run(1)
# Within less than a second, the task will be processed again.
time.sleep(1)
ensure_run(2)
def test_queue_system_lock(self):
"""Test queue system lock."""
with FreezeTime(datetime.datetime(2014, 1, 1)):
# Queue three tasks
for i in range(0, 3):
task = Task(self.tiger, long_task_ok, queue='a')
task.delay()
self._ensure_queues(queued={'a': 3})
# Ensure we can process one
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(queued={'a': 2})
# Set system lock so no processing should occur for 10 seconds
self.tiger.set_queue_system_lock('a', 10)
lock_timeout = self.tiger.get_queue_system_lock('a')
assert lock_timeout == time.time() + 10
# Confirm tasks don't get processed within the system lock timeout
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 9)):
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(queued={'a': 2})
# 10 seconds in the future the lock should have expired
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 10)):
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(queued={'a': 1})
