def test_empty(self, assert_atomic, connection, mocker):
q = Queue()
t1 = q.enqueue_call()
t2 = q.enqueue_call()
assert connection.exists(t1.key)
with assert_atomic():
q.empty()
assert not connection.exists(q.key)
assert not connection.exists(q.unblock_key)
assert q.name in queue_registry.get_names()
assert not connection.exists(t1.key)
assert not connection.exists(t2.key)
def test_push(assert_atomic):
q = Queue()
q.enqueue_call()
q.enqueue_call()
task = TaskFactory()
with assert_atomic():
q.push(task)
assert q.get_task_ids()[-1] == task.id
task = TaskFactory()
q.push(task, at_front=True)
assert q.get_task_ids()[0] == task.id
def test_get_tasks():
q = Queue()
tasks = [TaskFactory() for i in range(3)]
for task in tasks:
task._save()
q.push(task)
assert q.get_task_ids() == id_list(tasks)
assert id_list(q.get_tasks()) == id_list(tasks)
q.remove_and_delete(tasks[1])
assert q.get_task_ids() == [tasks[0].id, tasks[2].id]
def info(interval, by_queue, queues, **options):
"""Display information about active queues and workers"""
if queues:
queues = [Queue(q) for q in queues]
def display():
show_queues(queues)
click.echo('')
show_workers(queues, by_queue)
click.echo('')
try:
if interval:
while True:
click.clear()
display()
click.echo('Updated: %s' % datetime.datetime.now())
time.sleep(interval)
else:
display()
except ConnectionError as e:
click.echo(e)
sys.exit(1)
except KeyboardInterrupt:
click.echo()
sys.exit(0)
def test_queue_basics(assert_atomic):
worker = WorkerFactory()
q = Queue()
assert q.name == 'default'
assert str(q) == '<Queue default>'
assert repr(q) == "Queue('default')"
assert q.count() == 0
with assert_atomic():
task = q.enqueue_call()
q.enqueue_call()
assert q.count() == 2
assert q.dequeue(worker).id == task.id
assert q.count() == 1
def empty(all, delete, queues, **options):
"""Empty selected queues"""
if all:
queues = Queue.all()
elif queues:
queues = [Queue(q) for q in queues]
else:
raise click.UsageError("No queues specified")
if not queues:
click.echo('Nothing to do')
sys.exit(0)
for queue in queues:
if delete:
queue.delete()
else:
queue.empty()
click.echo(f'Queue f{queue.name} emptied')
def test_empty(cli_run, stub):
queues = [QueueFactory() for i in range(5)]
for q in queues:
q.enqueue_call(stub)
with pytest.raises(click.UsageError):
cli_run('empty')
assert all(q.count() == 1 for q in queues)
cli_run('empty', queues[0].name, queues[1].name)
assert queues[0].count() == 0
assert queues[1].count() == 0
assert queues[2].count() == 1
assert set(queues) == set(Queue.all())
cli_run('empty', '--delete', queues[1].name, queues[2].name)
assert set(queues) - {queues[1], queues[2]} == set(Queue.all())
cli_run('empty', '--all')
assert all(q.count() == 0 for q in queues)
def test_delete_transaction(self, assert_atomic, connection, mocker):
q = Queue()
task1 = q.enqueue_call()
task2 = None
def task_delete(task_ids, **kwargs):
nonlocal task2
if not task2:
# interrupt the transaction with the creation of a new task
task2 = q.enqueue_call()
else:
# assert that the previous transaction attempt was canceled
assert connection.exists(task1.key)
orig_delete_many(task_ids, **kwargs)
orig_delete_many = Task.delete_many
mocker.patch.object(Task, 'delete_many', new=task_delete)
assert connection.exists(task1.key)
q.empty()
assert not connection.exists(task1.key)
assert not connection.exists(task2.key)
def test_delete(self, assert_atomic, connection, mocker):
q = Queue()
q.enqueue_call()
q.enqueue_call()
with assert_atomic():
q.delete()
assert not connection.exists(q.key)
assert not connection.exists(q.unblock_key)
assert q.name not in queue_registry.get_names()
def test_unblocking(connection):
worker = WorkerFactory()
q = Queue()
q.enqueue_call()
assert connection.llen(q.unblock_key)
assert q.dequeue(worker) is not None
assert connection.llen(q.unblock_key)
assert q.dequeue(worker) is None
assert not connection.llen(q.unblock_key)
def show_workers(queues, by_queue):
def state_symbol(state):
return {
WorkerState.BUSY: red('busy'),
WorkerState.IDLE: green('idle')
}.get(state, state)
workers = Worker.all()
all_queues = set(Queue.all()) | {q for w in workers for q in w.queues}
if queues:
workers = [w for w in workers if any(set(w.queues) & set(queues))]
else:
queues = all_queues
if not by_queue:
click.secho("Workers", bold=True)
else:
click.secho("Workers per Queue", bold=True)
print_separator()
if not by_queue:
for worker in sorted(workers, key=attrgetter('description')):
worker_queues = ', '.join(q.name for q in worker.queues)
click.echo(
f'{worker.description} {state_symbol(worker.state)}: {worker_queues}'
)
else:
queue_map = {q: [] for q in all_queues}
for w in workers:
for q in w.queues:
queue_map[q].append(w)
max_qname = max(len(q.name) for q in queues)
for queue in sorted(queues, key=attrgetter('name')):
q_workers = queue_map[queue]
workers_str = ", ".join(
sorted(f'{w.description} {state_symbol(w.state)}'
for w in q_workers))
if not workers_str:
workers_str = '–'
click.echo(f'{queue.name:>{max_qname}}: {workers_str}')
click.echo(f'{len(workers)} worker(s)')
def show_queues(queues):
if not queues:
queues = Queue.all()
counts = {q: q.count() for q in queues}
chart_width = 20
chart_max = max((1, *counts.values()))
# Round up to the next fixed marker or the next power of two times 1000
chart_max = next(
(x for x in [20, 50, 100, 200, 400, 600, 800, 1000] if x >= chart_max),
2**math.ceil(math.log2(chart_max / 1000)) * 1000)
click.secho("Queues", bold=True)
print_separator()
for q in sorted(queues, key=attrgetter('name')):
count = counts[q]
chart = green('|' + '█' * int(count / chart_max * chart_width))
click.echo(f'{q.name:<12} {chart} {count}')
click.echo(f'{len(queues)} queue(s), {sum(counts.values())} task(s) total')
def test_dequeue(connection):
worker = WorkerFactory()
q = Queue()
assert q.dequeue(worker) is None
task1 = q.enqueue_call()
task2 = q.enqueue_call()
assert connection.llen(q.unblock_key) == 2
assert q.dequeue(worker).id == task1.id
assert worker.current_task_id == task1.id
assert connection.lrange(worker.task_key, 0, -1) == [task1.id.encode()]
assert q.dequeue(worker).id == task2.id
assert worker.current_task_id == task2.id
assert connection.lrange(worker.task_key, 0, -1) == [task2.id.encode(), task1.id.encode()]
assert connection.llen(q.unblock_key) == 2
# Clear block_key when queue is empty
assert q.dequeue(worker) is None
assert connection.llen(q.unblock_key) == 0
def QueueFactory():
global queue_sequence
queue_sequence += 1
name = f"queue_{queue_sequence}"
return Queue(name)
import tasks
from redis_tasks.queue import Queue
low_prio_queue = Queue('low_prio_queue')
low_prio_queue.enqueue_call(tasks.count_words_at_url,
['http://www.python.org'])
import tasks from redis_tasks.queue import Queue queue = Queue() queue.enqueue_call(tasks.count_words_at_url, ['http://www.python.org'])
def test_empty_empty_queue(self):
q = Queue()
q.empty()
def test_await_multi():
q1 = Queue('first')
q2 = Queue('second')
blocked = False
def in_thread():
nonlocal blocked
blocked = True
q = Queue.await_multi([q1, q2], 1)
blocked = False
return q
with futures.ThreadPoolExecutor(max_workers=5) as executor:
future = executor.submit(in_thread)
time.sleep(0.005)
assert blocked
q1.push(TaskFactory())
time.sleep(0.005)
assert not blocked
assert future.result().name == q1.name
future = executor.submit(in_thread)
time.sleep(0.005)
q2.push(TaskFactory())
q1.push(TaskFactory())
assert future.result().name == q2.name
assert Queue.await_multi([q1, q2], 1).name == q1.name
def test_queue_all():
q1 = Queue('a')
q2 = Queue('b')
q3 = Queue('c')
q2.enqueue_call()
q3.enqueue_call()
assert [x.name for x in Queue.all()] == ['b', 'c']
q2.delete()
q3.empty()
q1.enqueue_call()
assert [x.name for x in Queue.all()] == ['a', 'c']
def in_thread():
nonlocal blocked
blocked = True
q = Queue.await_multi([q1, q2], 1)
blocked = False
return q
def test_remove_and_delete(assert_atomic, connection):
q = Queue()
q.enqueue_call()
task = q.enqueue_call()
assert q.count() == 2
assert connection.exists(task.key)
with assert_atomic():
q.remove_and_delete(task)
assert not connection.exists(task.key)
assert q.count() == 1
with pytest.raises(TaskDoesNotExist):
q.remove_and_delete(task)
assert q.count() == 1
def test_queue_await_multi_empty():
assert Queue.await_multi([Queue()], 1) is None
