def run_schedule_task(_id, request_id, username, **kwargs):
db = Mongo()
params = (_id, request_id, username)
queue_name = get_queue_by_job(_id)
job = db.collection('jobs').find_one({'_id': ObjectId(_id)})
func = run_playbook_task
if job.get('type') == 'adhoc':
func = run_adhoc_task
task = Task(tiger,
func=func,
args=params,
kwargs=kwargs,
queue=queue_name,
unique=False,
lock=True,
lock_key=_id)
task_record = {
'job_id': _id,
'state': QUEUED,
'type': 'schedule',
'ansible': job.get('type'),
'queue': queue_name,
'result': '',
'request_id': request_id,
't_id': task.id,
'created_at': time(),
}
db.collection('tasks').insert_one(task_record)
task.delay()
def test_max_workers(self):
"""Test Single Worker Queue."""
# Queue three tasks
for i in range(0, 3):
task = Task(self.tiger, long_task_ok, queue='a')
task.delay()
self._ensure_queues(queued={'a': 3})
# Start two workers and wait until they start processing.
worker1 = Process(target=external_worker,
kwargs={'max_workers_per_queue': 2})
worker2 = Process(target=external_worker,
kwargs={'max_workers_per_queue': 2})
worker1.start()
worker2.start()
# Wait for both tasks to start
wait_for_long_task()
wait_for_long_task()
# Verify they both are active
self._ensure_queues(active={'a': 2}, queued={'a': 1})
# This worker should fail to get the queue lock and exit immediately
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(active={'a': 2}, queued={'a': 1})
# Wait for external workers
worker1.join()
worker2.join()
def test_queue_system_lock(self):
"""Test queue system lock."""
with FreezeTime(datetime.datetime(2014, 1, 1)):
# Queue three tasks
for i in range(0, 3):
task = Task(self.tiger, long_task_ok, queue='a')
task.delay()
self._ensure_queues(queued={'a': 3})
# Ensure we can process one
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(queued={'a': 2})
# Set system lock so no processing should occur for 10 seconds
self.tiger.set_queue_system_lock('a', 10)
lock_timeout = self.tiger.get_queue_system_lock('a')
assert lock_timeout == time.time() + 10
# Confirm tasks don't get processed within the system lock timeout
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 9)):
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(queued={'a': 2})
# 10 seconds in the future the lock should have expired
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 10)):
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(queued={'a': 1})
def test_current_task_eager(self):
self.tiger.config['ALWAYS_EAGER'] = True
task = Task(self.tiger, verify_current_task)
task.delay()
assert not self.conn.exists('runtime_error')
assert self.conn.get('task_id') == task.id
def test_max_workers(self):
"""Test Single Worker Queue."""
# Queue three tasks
for i in range(0, 3):
task = Task(self.tiger, long_task_killed, queue='a')
task.delay()
self._ensure_queues(queued={'a': 3})
# Start two workers and wait until they start processing.
worker1 = Process(target=external_worker,
kwargs={'max_workers_per_queue': 2})
worker2 = Process(target=external_worker,
kwargs={'max_workers_per_queue': 2})
worker1.start()
worker2.start()
time.sleep(DELAY)
# This worker should fail to get the queue lock and exit immediately
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(active={'a': 2}, queued={'a': 1})
# Wait for external workers
worker1.join()
worker2.join()
def test_current_tasks_eager(self):
self.tiger.config['ALWAYS_EAGER'] = True
task = Task(self.tiger, verify_current_tasks)
task.delay()
assert not self.conn.exists('runtime_error')
assert self.conn.lrange('task_ids', 0, -1) == [task.id]
def test_current_task_eager(self):
self.tiger.config['ALWAYS_EAGER'] = True
task = Task(self.tiger, verify_current_task)
task.delay()
assert not self.conn.exists('runtime_error')
assert self.conn.get('task_id') == task.id
def test_current_tasks_eager(self):
self.tiger.config['ALWAYS_EAGER'] = True
task = Task(self.tiger, verify_current_tasks)
task.delay()
assert not self.conn.exists('runtime_error')
assert self.conn.lrange('task_ids', 0, -1) == [task.id]
def test_queue_system_lock(self):
"""Test queue system lock."""
with FreezeTime(datetime.datetime(2014, 1, 1)):
# Queue three tasks
for i in range(0, 3):
task = Task(self.tiger, long_task_ok, queue='a')
task.delay()
self._ensure_queues(queued={'a': 3})
# Ensure we can process one
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(queued={'a': 2})
# Set system lock so no processing should occur for 10 seconds
self.tiger.set_queue_system_lock('a', 10)
lock_timeout = self.tiger.get_queue_system_lock('a')
assert lock_timeout == time.time() + 10
# Confirm tasks don't get processed within the system lock timeout
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 9)):
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(queued={'a': 2})
# 10 seconds in the future the lock should have expired
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 10)):
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(queued={'a': 1})
def dispatch(payload):
hosts = payload.get('inventory')
tasks = payload.get('tasks')
if not hosts or not tasks:
return None
uid = md5(str(json.dumps(payload)))
username = payload.get('username')
run_id = payload.get('req_id') or str(uuid.uuid4())
params = [run_id, payload]
queue_name = 'book_runtime'
func = run
task = Task(tiger,
func=func,
args=params,
queue=queue_name,
unique=True,
lock=True,
lock_key=uid)
run_record = {
'uid': uid,
'run_id': run_id,
'run_by': username,
'options': payload,
'result': '',
'state': 'pending',
'created_at': time.time(),
'updated_at': time.time(),
}
result = Perform.insert_one(run_record)
task.delay()
return result.inserted_id
def test_delay(self):
task = Task(self.tiger, simple_task)
self._ensure_queues()
task.delay()
self._ensure_queues(queued={'default': 1})
# Canceling only works for scheduled tasks.
pytest.raises(TaskNotFound, task.cancel)
def test_delay(self):
task = Task(self.tiger, simple_task)
self._ensure_queues()
task.delay()
self._ensure_queues(queued={'default': 1})
# Canceling only works for scheduled tasks.
pytest.raises(TaskNotFound, task.cancel)
def run_job(_id, history_id=None, **kwargs):
db = Mongo()
record = Job.find_by_id(_id)
if not record:
return False
request_id = str(current_request_id())
username = None if not login_user else login_user.get('username')
params = (_id, request_id, username, history_id)
queue_name = get_queue_by_job(_id)
extra = record.get('extra')
template = record.get('template')
schedule = extra.get('schedule')
ansible_type = record.get('type')
if template.get('run_type') == 'schedule':
existed = db.collection('scheduler_jobs').find_one(
{'_id': record['_id']})
if existed:
return False
scheduler.add_job(func=run_schedule_task,
trigger='cron',
args=params,
coalesce=True,
kwargs=kwargs,
id=str(record.get('_id')),
max_instances=1,
name=record.get('name'),
**schedule)
return True
else:
func = run_playbook_task if ansible_type != 'adhoc' else run_adhoc_task
task = Task(tiger,
func=func,
args=params,
kwargs=kwargs,
queue=queue_name,
unique=True,
lock=True,
lock_key=_id)
task_record = {
'job_id': _id,
'type': 'trigger',
'ansible': ansible_type,
'state': QUEUED,
'queue': queue_name,
'result': '',
'request_id': request_id,
't_id': task.id,
'created_at': time(),
'kwargs': kwargs,
}
result = db.collection('tasks').insert_one(task_record)
task.delay()
return result.inserted_id
def test_task_disappears(self):
"""
Ensure that a task object that disappears while the task is processing
is handled properly. This could happen when a worker processes a task,
then hangs for a long time, causing another worker to pick up and finish
the task. Then, when the original worker resumes, the task object will
be gone. Make sure we log a "not found" error and move on.
"""
task = Task(self.tiger, sleep_task, kwargs={'delay': 2 * DELAY})
task.delay()
self._ensure_queues(queued={'default': 1})
# Start a worker and wait until it starts processing.
worker = Process(target=external_worker)
worker.start()
time.sleep(DELAY)
# Remove the task object while the task is processing.
assert self.conn.delete('t:task:{}'.format(task.id)) == 1
# Kill the worker while it's still processing the task.
os.kill(worker.pid, signal.SIGKILL)
# _ensure_queues() breaks here because it can't find the task
assert self.conn.scard('t:queued') == 0
assert self.conn.scard('t:active') == 1
assert self.conn.scard('t:error') == 0
assert self.conn.scard('t:scheduled') == 0
# Capture logger
errors = []
def fake_error(msg):
errors.append(msg)
with Patch(self.tiger.log._logger, 'error', fake_error):
# Since ACTIVE_TASK_UPDATE_TIMEOUT hasn't elapsed yet, re-running
# the worker at this time won't change anything. (run twice to move
# from scheduled to queued)
Worker(self.tiger).run(once=True)
Worker(self.tiger).run(once=True)
assert len(errors) == 0
assert self.conn.scard('t:queued') == 0
assert self.conn.scard('t:active') == 1
assert self.conn.scard('t:error') == 0
assert self.conn.scard('t:scheduled') == 0
# After waiting and re-running the worker, queues will clear.
time.sleep(2 * DELAY)
Worker(self.tiger).run(once=True)
Worker(self.tiger).run(once=True)
self._ensure_queues()
assert len(errors) == 1
assert "not found" in errors[0]
def test_task_disappears(self):
"""
Ensure that a task object that disappears while the task is processing
is handled properly. This could happen when a worker processes a task,
then hangs for a long time, causing another worker to pick up and finish
the task. Then, when the original worker resumes, the task object will
be gone. Make sure we log a "not found" error and move on.
"""
task = Task(self.tiger, sleep_task, kwargs={'delay': 2 * DELAY})
task.delay()
self._ensure_queues(queued={'default': 1})
# Start a worker and wait until it starts processing.
worker = Process(target=external_worker)
worker.start()
time.sleep(DELAY)
# Remove the task object while the task is processing.
assert self.conn.delete('t:task:{}'.format(task.id)) == 1
# Kill the worker while it's still processing the task.
os.kill(worker.pid, signal.SIGKILL)
# _ensure_queues() breaks here because it can't find the task
assert self.conn.scard('t:queued') == 0
assert self.conn.scard('t:active') == 1
assert self.conn.scard('t:error') == 0
assert self.conn.scard('t:scheduled') == 0
# Capture logger
errors = []
def fake_error(msg):
errors.append(msg)
with Patch(self.tiger.log._logger, 'error', fake_error):
# Since ACTIVE_TASK_UPDATE_TIMEOUT hasn't elapsed yet, re-running
# the worker at this time won't change anything. (run twice to move
# from scheduled to queued)
Worker(self.tiger).run(once=True)
Worker(self.tiger).run(once=True)
assert len(errors) == 0
assert self.conn.scard('t:queued') == 0
assert self.conn.scard('t:active') == 1
assert self.conn.scard('t:error') == 0
assert self.conn.scard('t:scheduled') == 0
# After waiting and re-running the worker, queues will clear.
time.sleep(2 * DELAY)
Worker(self.tiger).run(once=True)
Worker(self.tiger).run(once=True)
self._ensure_queues()
assert len(errors) == 1
assert "not found" in errors[0]
def test_delay_scheduled(self):
task = Task(self.tiger, simple_task, queue='a')
task.delay(when=datetime.timedelta(minutes=5))
self._ensure_queues(scheduled={'a': 1})
# Test canceling a scheduled task.
task.cancel()
self._ensure_queues()
# Canceling again raises an error
pytest.raises(TaskNotFound, task.cancel)
def test_update_scheduled_time(self):
task = Task(self.tiger, simple_task, unique=True)
task.delay(when=datetime.timedelta(minutes=5))
self._ensure_queues(scheduled={'default': 1})
old_score = self.conn.zscore('t:scheduled:default', task.id)
task.update_scheduled_time(when=datetime.timedelta(minutes=6))
self._ensure_queues(scheduled={'default': 1})
new_score = self.conn.zscore('t:scheduled:default', task.id)
# The difference can be slightly over 60 due to processing time, but
# shouldn't be much higher.
assert 60 <= new_score - old_score < 61
def test_successful_execution_doesnt_clear_previous_errors(self):
"""
Ensure previous executions are not cleared if we have had non-retriable
errors.
"""
sleep_until_next_second()
# Queue the periodic task.
self._ensure_queues()
Worker(tiger).run(once=True)
# Prepare to execute the periodic task (as permanent failure).
tiger.connection.set('fail-periodic-task', 'permanent')
n_total, tasks = Task.tasks_from_queue(tiger, 'periodic', SCHEDULED)
task_id = tasks[0].id
time.sleep(1)
# Queue the periodic task.
self._ensure_queues(scheduled={'periodic': 1})
Worker(tiger).run(once=True)
# Run the failing periodic task.
self._ensure_queues(queued={'periodic': 1})
Worker(tiger).run(once=True)
task = Task.from_id(tiger,
'periodic',
SCHEDULED,
task_id,
load_executions=10)
assert len(task.executions) == 1
tiger.connection.delete('fail-periodic-task')
time.sleep(1)
# Queue the periodic task.
self._ensure_queues(scheduled={'periodic': 1}, error={'periodic': 1})
Worker(tiger).run(once=True)
# Run the successful periodic task.
self._ensure_queues(queued={'periodic': 1}, error={'periodic': 1})
Worker(tiger).run(once=True)
# Ensure we didn't clear previous executions.
task = Task.from_id(tiger,
'periodic',
SCHEDULED,
task_id,
load_executions=10)
assert len(task.executions) == 1
def test_successful_execution_clears_executions_from_retries(self):
"""
Ensure previous executions from retries are cleared after a successful
execution.
"""
sleep_until_next_second()
# Queue the periodic task.
self._ensure_queues()
Worker(tiger).run(once=True)
# Prepare to execute the periodic task (as retriable failure).
tiger.connection.set('fail-periodic-task', 'retriable')
n_total, tasks = Task.tasks_from_queue(tiger, 'periodic', SCHEDULED)
task_id = tasks[0].id
time.sleep(1)
# Queue the periodic task.
self._ensure_queues(scheduled={'periodic': 1})
Worker(tiger).run(once=True)
# Run the failing periodic task.
self._ensure_queues(queued={'periodic': 1})
Worker(tiger).run(once=True)
task = Task.from_id(tiger,
'periodic',
SCHEDULED,
task_id,
load_executions=10)
assert len(task.executions) == 1
tiger.connection.delete('fail-periodic-task')
time.sleep(1)
# Queue the periodic task.
self._ensure_queues(scheduled={'periodic': 1})
Worker(tiger).run(once=True)
# Run the successful periodic task.
self._ensure_queues(queued={'periodic': 1})
Worker(tiger).run(once=True)
# Ensure we cleared any previous executions.
task = Task.from_id(tiger,
'periodic',
SCHEDULED,
task_id,
load_executions=10)
assert len(task.executions) == 0
def clear_queue():
'''
Clears all inactive tasks from queue.
'''
for queue, stats in tiger.get_queue_stats().items():
for state, tasks in stats.items():
if state == 'error':
for task in Task.tasks_from_queue(tiger, queue, state)[-1]:
task.delete()
logging.info('Task %s cleared from queue %s', task, queue)
elif state == 'scheduled':
for task in Task.tasks_from_queue(tiger, queue, state)[-1]:
task.cancel()
logging.info('Task %s cleared from queue %s', task, queue)
def test_current_tasks(self):
task1 = Task(self.tiger, verify_current_tasks)
task1.delay()
task2 = Task(self.tiger, verify_current_tasks)
task2.delay()
Worker(self.tiger).run(once=True)
assert self.conn.lrange('task_ids', 0, -1) == [task1.id, task2.id]
def dispatch(book_id, entry, payload):
print('xxxx', book_id, entry, payload)
book = Book.find_by_id(book_id)
if not book:
return False
username = payload.get('username')
run_id = payload.get('req_id') or str(uuid.uuid4())
params = [book_id, run_id]
options = payload.get('options')
if not entry:
return False
if type(options) == str:
args = options.split(' ')
pb = PlaybookCLI(args)
pb.init_parser()
options, args = pb.parser.parse_args(args[1:])
options, args = pb.post_process_args(options, args)
options = options.__dict__
options['entry'] = args
for i in options['inventory']:
if not os.path.isfile(i):
i = os.path.basename(i)
options['inventory'] = i
break
queue_name = 'book_runtime'
func = run
task = Task(tiger,
func=func,
args=params,
kwargs=options,
queue=queue_name,
unique=True,
lock=True,
lock_key=book_id)
run_record = {
'book_id': book_id,
'run_id': run_id,
'run_by': username,
'options': options,
'result': '',
'state': 'pending',
'created_at': 1,
'updated_at': 2,
}
result = Perform.insert_one(run_record)
task.delay()
return result.inserted_id
def task_detail(self, queue, state, task_id):
LIMIT = 1000
try:
task = Task.from_id(self.tiger,
queue,
state,
task_id,
load_executions=LIMIT)
except TaskNotFound:
abort(404)
executions_dumped = []
for execution in task.executions:
traceback = execution.pop('traceback', None)
executions_dumped.append((json.dumps(execution,
indent=2,
sort_keys=True), traceback))
return self.render(
'tasktiger_admin/tasktiger_task_detail.html',
queue=queue,
state=state,
task=task,
task_dumped=json.dumps(task.data, indent=2, sort_keys=True),
executions_dumped=executions_dumped,
)
def test_requeue_expired_task(self):
"""
Ensure a retriable task ends up in "queued" state if the worker is
killed prematurely.
"""
task = Task(self.tiger, sleep_task, retry_on=[JobTimeoutException])
self._test_expired_task(task, 'queued')
def retry(_id, state):
record = db.collection('tasks').find_one({'_id': ObjectId(_id)})
if not record:
return jsonify({
'message': 'task not found',
'code': 194041
}), 404
task_id = record.get('t_id')
queue = record.get('queue')
try:
task = Task.from_id(tiger, queue, state, task_id)
task.retry()
db.collection('tasks').update_one({'_id': record['_id']}, {'$set': {
'updated_at': datetime.now(),
'state': state,
}})
extra = {
'queue': queue,
'task_id': task_id,
'from_state': state,
'to_state': QUEUED
}
logger.info('retry task', extra=extra)
except TaskNotFound:
return jsonify({
'message': 'invalid task',
'code': 104044
}), 404
return jsonify({
'message': 'ok',
'code': 0,
})
def delete_task(_id, state):
record = db.collection('tasks').find_one({'t_id': _id})
if not record:
return jsonify({
'message': 'task not found',
'code': 194041
}), 404
task_id = record.get('t_id')
queue = record.get('queue')
try:
task = Task.from_id(tiger, queue, state, task_id)
task._move(from_state=state)
db.collection('tasks').update_one({'_id': record['_id']}, {'$set': {
'updated_at': datetime.now(),
'state': 'delete',
}})
extra = {
'queue': queue,
'task_id': task_id,
'from_state': state,
'to_state': None
}
logger.info('cancel task', extra=extra)
except TaskNotFound:
return jsonify({
'message': 'invalid task',
'code': 104044
}), 404
return jsonify({
'message': 'ok',
'code': 0,
})
def get_task_info(_id):
record = db.collection('tasks').find_one({'_id': ObjectId(_id)})
if not record:
return jsonify({
'message': 'record not found',
'code': 104040
}), 404
log = db.collection('logs').find_one({'task_id': str(record.get('_id'))})
# log = db.collection('logs').find_one({'task_id': '5d6d4e0ae3f7e086eaa30321'})
record['log'] = log
job = db.collection('jobs').find_one({'_id': ObjectId(record.get('job_id'))})
record['job'] = job
queue = record.get('queue')
state = record.get('state')
task_id = record.get('t_id')
try:
task = Task.from_id(tiger, queue, state, task_id)
record['queue_info'] = task.data.copy()
except TaskNotFound:
record['queue_info'] = None
return jsonify({
'message': 'ok',
'code': 0,
'data': record,
})
def task_delete(self, queue, state, task_id):
try:
task = Task.from_id(self.tiger, queue, state, task_id)
except TaskNotFound:
abort(404)
task.delete()
return redirect(url_for('.queue_detail', queue=queue, state=state))
def task_delete(self, queue, state, task_id):
try:
task = Task.from_id(self.tiger, queue, state, task_id)
except TaskNotFound:
abort(404)
task.delete()
return redirect(url_for('.queue_detail', queue=queue, state=state))
def test_discard_expired_task(self):
"""
Ensure a non-retriable task ends up in "error" state if the worker is
killed prematurely.
"""
task = Task(self.tiger, sleep_task)
self._test_expired_task(task, 'error')
def test_delay_scheduled_3(self):
task = Task(self.tiger, simple_task, unique=True)
task.delay(when=datetime.timedelta(minutes=5))
self._ensure_queues(scheduled={'default': 1})
# We can look up a unique task by recreating it.
task = Task(self.tiger, simple_task, unique=True)
task.cancel()
self._ensure_queues()
def test_periodic_execution_unique_ids_self_correct(self):
"""
Test that periodic tasks will self-correct unique ids
"""
# Sleep until the next second
sleep_until_next_second()
# generate the ids
correct_unique_id = gen_unique_id(serialize_func_name(periodic_task),
[], {})
malformed_unique_id = gen_unique_id(serialize_func_name(periodic_task),
None, None)
task = Task(tiger, func=periodic_task)
# patch the id to something slightly wrong
assert task.id == correct_unique_id
task._data['id'] = malformed_unique_id
assert task.id == malformed_unique_id
# schedule the task
task.delay()
self._ensure_queues(queued={'periodic': 1})
# pull task out of the queue by the malformed id
task = Task.from_id(tiger, 'periodic', QUEUED, malformed_unique_id)
assert task is not None
Worker(tiger).run(once=True)
self._ensure_queues(scheduled={'periodic': 1})
# pull task out of the queue by the self-corrected id
task = Task.from_id(tiger, 'periodic', SCHEDULED, correct_unique_id)
assert task is not None
def test_periodic_execution_unique_ids(self):
"""
Test that periodic tasks generate the same unique ids
When a periodic task is scheduled initially as part of worker startup
vs re-scheduled from within python the unique id generated should be
the same. If they aren't it could result in duplicate tasks.
"""
# Sleep until the next second
now = datetime.datetime.utcnow()
time.sleep(1 - now.microsecond / 10.0**6)
# After the first worker run, the periodic task will be queued.
# Note that since periodic tasks register with the Tiger instance, it
# must be the same instance that was used to decorate the task. We
# therefore use `tiger` from the tasks module instead of `self.tiger`.
self._ensure_queues()
Worker(tiger).run(once=True)
self._ensure_queues(scheduled={'periodic': 1})
time.sleep(1)
Worker(tiger).run(once=True)
self._ensure_queues(queued={'periodic': 1})
# generate the expected unique id
expected_unique_id = gen_unique_id(serialize_func_name(periodic_task),
[], {})
# pull task out of the queue by id. If found, then the id is correct
task = Task.from_id(tiger, 'periodic', QUEUED, expected_unique_id)
assert task is not None
# execute and reschedule the task
self._ensure_queues(queued={'periodic': 1})
Worker(tiger).run(once=True)
self._ensure_queues(scheduled={'periodic': 1})
# wait for the task to need to be queued
time.sleep(1)
Worker(tiger).run(once=True)
self._ensure_queues(queued={'periodic': 1})
# The unique id shouldn't change between executions. Try finding the
# task by id again
task = Task.from_id(tiger, 'periodic', QUEUED, expected_unique_id)
assert task is not None
def test_task_all_states(self):
"""Test max queue size with tasks in all three states."""
# Active
task = Task(self.tiger, sleep_task, queue='a')
task.delay()
self._ensure_queues(queued={'a': 1})
# Start a worker and wait until it starts processing.
worker = Process(target=external_worker)
worker.start()
time.sleep(DELAY)
# Kill the worker while it's still processing the task.
os.kill(worker.pid, signal.SIGKILL)
self._ensure_queues(active={'a': 1})
# Scheduled
self.tiger.delay(
simple_task,
queue='a',
max_queue_size=3,
when=datetime.timedelta(seconds=10),
)
# Queued
self.tiger.delay(simple_task, queue='a', max_queue_size=3)
self._ensure_queues(active={'a': 1},
queued={'a': 1},
scheduled={'a': 1})
# Should fail to queue task to run immediately
with pytest.raises(QueueFullException):
self.tiger.delay(simple_task, queue='a', max_queue_size=3)
# Should fail to queue task to run in the future
with pytest.raises(QueueFullException):
self.tiger.delay(
simple_task,
queue='a',
max_queue_size=3,
when=datetime.timedelta(seconds=10),
)
def queue_detail(self, queue, state):
n, tasks = Task.tasks_from_queue(self.tiger, queue, state,
load_executions=1)
return self.render('tasktiger_admin/tasktiger_queue_detail.html',
queue=queue,
state=state,
n=n,
tasks=tasks,
)
def test_tasks_from_queue_with_executions(self):
task = self.tiger.delay(exception_task, retry=True)
# Get two executions in task
Worker(self.tiger).run(once=True)
time.sleep(DELAY)
# Second run (run twice to move from scheduled to queued)
Worker(self.tiger).run(once=True)
Worker(self.tiger).run(once=True)
n, tasks = Task.tasks_from_queue(self.tiger, 'default', 'scheduled',
load_executions=1)
assert n == 1
assert len(tasks[0].executions) == 1
n, tasks = Task.tasks_from_queue(self.tiger, 'default', 'scheduled',
load_executions=10)
assert n == 1
assert len(tasks[0].executions) == 2
def test_tasks_from_queue(self):
task0 = Task(self.tiger, simple_task)
task1 = Task(self.tiger, exception_task)
task2 = Task(self.tiger, simple_task, queue='other')
task0.delay()
task1.delay()
task2.delay()
n, tasks = Task.tasks_from_queue(self.tiger, 'default', 'queued')
assert n == 2
assert task0.id == tasks[0].id
assert task0.func == simple_task
assert task0.func == tasks[0].func
assert task0.serialized_func == 'tests.tasks:simple_task'
assert task0.serialized_func == tasks[0].serialized_func
assert task0.state == tasks[0].state
assert task0.state == 'queued'
assert task0.queue == tasks[0].queue
assert task0.queue == 'default'
def test_current_tasks(self):
task1 = Task(self.tiger, verify_current_tasks)
task1.delay()
task2 = Task(self.tiger, verify_current_tasks)
task2.delay()
Worker(self.tiger).run(once=True)
assert self.conn.lrange('task_ids', 0, -1) == [task1.id, task2.id]
def queue_detail(self, queue, state):
n, tasks = Task.tasks_from_queue(self.tiger,
queue,
state,
load_executions=1)
return self.render(
'tasktiger_admin/tasktiger_queue_detail.html',
queue=queue,
state=state,
n=n,
tasks=tasks,
)
def test_delay_scheduled_3(self):
task = Task(self.tiger, simple_task, unique=True)
task.delay(when=datetime.timedelta(minutes=5))
self._ensure_queues(scheduled={'default': 1})
# We can look up a unique task by recreating it.
task = Task(self.tiger, simple_task, unique=True)
task.cancel()
self._ensure_queues()
def test_delay_scheduled_2(self):
task = Task(self.tiger, simple_task, queue='a')
task.delay(when=datetime.timedelta(minutes=5))
self._ensure_queues(scheduled={'a': 1})
task_id = task.id
# We can't look up a non-unique task by recreating it.
task = Task(self.tiger, simple_task, queue='a')
pytest.raises(TaskNotFound, task.cancel)
# We can look up a task by its ID.
fetch_task = lambda: Task.from_id(self.tiger, 'a', 'scheduled', task_id)
task = fetch_task()
task.cancel()
self._ensure_queues()
# Task.from_id raises if it doesn't exist.
pytest.raises(TaskNotFound, fetch_task)
def test_task_all_states(self):
"""Test max queue size with tasks in all three states."""
# Active
task = Task(self.tiger, sleep_task, queue='a')
task.delay()
self._ensure_queues(queued={'a': 1})
# Start a worker and wait until it starts processing.
worker = Process(target=external_worker)
worker.start()
time.sleep(DELAY)
# Kill the worker while it's still processing the task.
os.kill(worker.pid, signal.SIGKILL)
self._ensure_queues(active={'a': 1})
# Scheduled
self.tiger.delay(simple_task, queue='a', max_queue_size=3,
when=datetime.timedelta(seconds=10))
# Queued
self.tiger.delay(simple_task, queue='a', max_queue_size=3)
self._ensure_queues(active={'a': 1},
queued={'a': 1},
scheduled={'a': 1})
# Should fail to queue task to run immediately
with pytest.raises(QueueFullException):
self.tiger.delay(simple_task, queue='a', max_queue_size=3)
# Should fail to queue task to run in the future
with pytest.raises(QueueFullException):
self.tiger.delay(simple_task, queue='a', max_queue_size=3,
when=datetime.timedelta(seconds=10))
def test_delay_scheduled(self):
task = Task(self.tiger, simple_task, queue='a')
task.delay(when=datetime.timedelta(minutes=5))
self._ensure_queues(scheduled={'a': 1})
# Test canceling a scheduled task.
task.cancel()
self._ensure_queues()
# Canceling again raises an error
pytest.raises(TaskNotFound, task.cancel)
def task_detail(self, queue, state, task_id):
LIMIT = 1000
try:
task = Task.from_id(self.tiger,
queue,
state,
task_id,
load_executions=LIMIT)
except TaskNotFound:
abort(404)
executions_dumped = []
for execution in task.executions:
traceback = execution.pop('traceback', None)
execution_integrations = generate_integrations(
self.integration_config.get('EXECUTION_INTEGRATION_LINKS', []),
task,
execution,
)
execution_converted = convert_keys_to_datetime(
execution, ['time_failed', 'time_started'])
executions_dumped.append((
json.dumps(
execution_converted,
indent=2,
sort_keys=True,
default=str,
),
traceback,
execution_integrations,
execution_converted,
))
integrations = generate_integrations(
self.integration_config.get('INTEGRATION_LINKS', []), task, None)
return self.render(
'tasktiger_admin/tasktiger_task_detail.html',
queue=queue,
state=state,
task=task,
task_data=task.data,
task_data_dumped=json.dumps(task.data, indent=2, sort_keys=True),
executions_dumped=reversed(executions_dumped),
integrations=integrations,
)
def task_detail(self, queue, state, task_id):
LIMIT = 1000
try:
task = Task.from_id(self.tiger, queue, state, task_id,
load_executions=LIMIT)
except TaskNotFound:
abort(404)
executions_dumped = []
for execution in task.executions:
traceback = execution.pop('traceback', None)
executions_dumped.append((
json.dumps(execution, indent=2, sort_keys=True),
traceback)
)
return self.render('tasktiger_admin/tasktiger_task_detail.html',
queue=queue,
state=state,
task=task,
task_dumped=json.dumps(task.data, indent=2, sort_keys=True),
executions_dumped=executions_dumped,
)
def task_retry_multiple(self, queue, state):
LIMIT = 50
n, tasks = Task.tasks_from_queue(self.tiger, queue, state, limit=LIMIT)
for task in tasks:
task.retry()
return redirect(url_for('.queue_detail', queue=queue, state=state))
def test_eager(self):
self.tiger.config['ALWAYS_EAGER'] = True
# Ensure task is immediately executed.
task = Task(self.tiger, simple_task)
task.delay()
self._ensure_queues()
# Ensure task is immediately executed.
task = Task(self.tiger, exception_task)
pytest.raises(Exception, task.delay)
self._ensure_queues()
# Even when we specify "when" in the past
task = Task(self.tiger, simple_task)
task.delay(when=datetime.timedelta(seconds=-5))
self._ensure_queues()
# or with a zero timedelta.
task = Task(self.tiger, simple_task)
task.delay(when=datetime.timedelta(seconds=0))
self._ensure_queues()
# Ensure there is an exception if we can't serialize the task.
task = Task(self.tiger, decorated_task,
args=[object()])
pytest.raises(TypeError, task.delay)
self._ensure_queues()
# Ensure task is not executed if it's scheduled in the future.
task = Task(self.tiger, simple_task)
task.delay(when=datetime.timedelta(seconds=5))
self._ensure_queues(scheduled={'default': 1})
def test_current_task(self):
task = Task(self.tiger, verify_current_task)
task.delay()
Worker(self.tiger).run(once=True)
assert not self.conn.exists('runtime_error')
assert self.conn.get('task_id') == task.id
def test_single_worker_queue(self):
"""
Test Single Worker Queue.
Single worker queues are the same as running with MAX_WORKERS_PER_QUEUE
set to 1.
"""
# Queue two tasks
task = Task(self.tiger, long_task_ok, queue='swq')
task.delay()
task = Task(self.tiger, long_task_ok, queue='swq')
task.delay()
self._ensure_queues(queued={'swq': 2})
# Start a worker and wait until it starts processing.
# It should start processing one task and hold a lock on the queue
worker = Process(target=external_worker)
worker.start()
# Wait up to 2 seconds for external task to start
result = self.conn.blpop('long_task_ok', 2)
assert result[1] == '1'
# This worker should fail to get the queue lock and exit immediately
Worker(self.tiger).run(once=True, force_once=True)
self._ensure_queues(active={'swq': 1}, queued={'swq': 1})
# Wait for external worker
worker.join()
# Retest using a non-single worker queue
# Queue two tasks
task = Task(self.tiger, long_task_ok, queue='not_swq')
task.delay()
task = Task(self.tiger, long_task_ok, queue='not_swq')
task.delay()
self._ensure_queues(queued={'not_swq': 2})
# Start a worker and wait until it starts processing.
# It should start processing one task
worker = Process(target=external_worker)
worker.start()
time.sleep(DELAY)
# This worker should process the second task
Worker(self.tiger).run(once=True, force_once=True)
# Queues should be empty
self._ensure_queues()
worker.join()
