def ping(request):
"""
Ping action
@param request: dict - defined in "on_request"
"""
respond(request, request['data'])
def _delete_queue(request, queue):
"""
Delete queue command
@param request: dict - defined in "on_request"
@param queue: str
"""
if log.level == logging.DEBUG or config['grep']:
verbose('w{}: deleting queue {}'.format(state.worker, queue))
confirm = request['confirm']
request['confirm'] = False # To avoid double confirm.
rebind(request, queue) # Unbind all by default.
for consumer_id, queue_of_consumer in state.queues_by_consumer_ids.items():
if queue_of_consumer == queue:
_delete_consumer(request, consumer_id)
state.queues.pop(queue, None)
state.queues_to_delete_when_unused.pop(queue, None)
queue_used = state.queues_used.pop(queue, None)
if queue_used:
queue_used.set() # Cancel "_wait_used_or_delete_queue" greenlet.
if log.level == logging.DEBUG or config['grep']:
verbose('w{}: deleted queue {}'.format(state.worker, queue))
if confirm:
respond(request)
def _delete_consumer(request, consumer_id):
"""
Delete consumer command
@param request: dict - defined in "on_request"
@param consumer_id: str
"""
confirm = request['confirm']
request['confirm'] = False # To avoid double confirm.
for client, consumer_ids in state.consumer_ids_by_clients.items():
# We should check all clients, not only request['client'],
# because _delete_consumer() may be called from _delete_queue() by another client.
consumer_ids.discard(consumer_id)
if not consumer_ids:
state.consumer_ids_by_clients.pop(client, None)
state.clients_by_consumer_ids.pop(consumer_id, None)
queue = state.queues_by_consumer_ids.pop(consumer_id, None)
if not queue:
return
consumers = state.consumers_by_queues.get(queue)
if consumers:
consumers.pop(consumer_id, None)
if not consumers:
state.consumers_by_queues.pop(queue, None)
_reject(request, queue, consumer_id, '--all')
if queue not in state.queues_by_consumer_ids.itervalues():
queue_used = state.queues_used.get(queue)
if queue_used is not None:
queue_used.clear()
delete_queue_when_unused = state.queues_to_delete_when_unused.get(
queue)
if delete_queue_when_unused is True:
_delete_queue(request, queue)
elif delete_queue_when_unused is not None:
spawn(_wait_used_or_delete_queue,
request['client'],
queue,
seconds=delete_queue_when_unused)
if confirm:
respond(request)
def _eval(request):
"""
Eval action
@param request: dict - defined in "on_request" with (
data: str - "len(state.queues)", "--worker=0 log.setLevel(logging.DEBUG)", etc - see "stats.py"
...
)
"""
code = request['data']
if code.startswith('--worker='): # back-compat
worker, code = code.split(' ', 1)
worker = int(worker.split('=')[1])
assert worker == state.worker, (
worker, state.worker,
'New client should connect directly to --worker requested!')
respond(request, str(eval(code)))
def publish(request):
"""
Publish action
@param request: dict - defined in "on_request"
"""
state.published += 1
event, data = request['data'].split(' ', 1)
msg = '{} event={} {}'.format(request['id'], event, data)
queues = state.queues_by_events.get(event)
if queues:
_put_to_queues(request, queues, msg)
if request['confirm']:
respond(request) # Once.
if config['top_events']:
event_mask = config['top_events_id'].sub('{id}', event)
state.top_events[event_mask] = state.top_events.get(event_mask, 0) + 1
def _consume_loop(request, queue, consumer_id, consumer_ids, manual_ack):
"""
Async loop to consume messages and to send them to clients.
@param request: dict - defined in "on_request"
@param queue: gevent.queue.Queue
@param consumer_id: str
@param consumer_ids: set([str])
@param manual_ack: bool
"""
try:
while consumer_id in consumer_ids:
try:
data = queue.get(timeout=config['block_seconds'])
except Empty:
continue
if consumer_id in consumer_ids: # Consumer could be deleted while being blocked above.
if manual_ack:
wall = gbn('manual_ack')
msg_id, _ = data.split(' ', 1)
state.messages_by_consumer_ids.setdefault(
consumer_id, {})[msg_id] = data
gbn(wall=wall)
respond(request, data)
state.consumed += 1
else:
wall = gbn('consume_back')
queue.put(data)
gbn(wall=wall)
# Don't try to preserve chronological order of messages in this edge case:
# "peek() + get()" does not fit for multiple consumers from the same queue.
# "JoinableQueue().task_done()" is heavier and "reject()" will change order anyway.
time.sleep(0)
except Exception:
on_error('_consume_loop', request)
def _reject(request, queue, consumer_id, msg_id):
"""
Reject command
@param request: dict - defined in "on_request"
@param queue: str
@param consumer_id: str
@param msg_id: str
"""
if msg_id == '--all':
msgs = state.messages_by_consumer_ids.pop(consumer_id, {}).itervalues()
else:
msg = state.messages_by_consumer_ids.get(consumer_id,
{}).pop(msg_id, None)
msgs = () if msg is None else (msg, )
queue = state.queues.get(queue)
if queue:
for msg in msgs:
msg_id, props, data = msg.split(' ', 2)
new_props = []
found_retry = False
for prop in props.split(','):
name, value = prop.split('=', 1)
if name == 'retry':
value = str(int(value) + 1)
found_retry = True
new_props.append((name, value))
if not found_retry:
new_props.append(('retry', '1'))
msg = ' '.join(
(msg_id, ','.join('='.join(prop) for prop in new_props), data))
queue.put(msg)
if request['confirm']:
respond(request)
def ack(request):
"""
Ack action
@param request: dict - defined in "on_request" with (
data: str - "{consumer_id} {msg_id}" or "{consumer_id} --all",
...
)
"""
consumer_id, msg_id = request['data'].split(' ', 1)
queue = state.queues_by_consumer_ids.get(consumer_id)
if not queue:
if log.level == logging.DEBUG or config['grep']:
verbose('w{}: found no queue for request={}'.format(state.worker, request))
return
if msg_id == '--all':
state.messages_by_consumer_ids.pop(consumer_id, {}).clear()
else:
state.messages_by_consumer_ids.get(consumer_id, {}).pop(msg_id, None)
if request['confirm']:
respond(request)
def rebind(request, queue=None, update_consumers=False, dont_update_consumer_id=None, **args):
"""
Rebind action
@param request: dict - defined in "on_request" with (
data: str - "{queue} \
[{event} ... {event}] \
[--remove {event} ... {event}] \
[--remove-mask {event_mask} ... {event_mask}] \
[--add {event} ... {event}]"
)
# When "rebind" is called from other actions:
@param queue: str
@param update_consumers: bool - If there is some other need to update consumers, e.g. changing --delete-queue-when-unused.
@param dont_update_consumer_id: str|None - No need to update consumer that initiated rebind on "consume" without "--add".
@param args: {'replace': [], 'remove': [], 'remove-mask': [], 'add': []} - No args means remove all.
"""
### when called from other actions
if queue:
wall = None
remove_all = not args
### parse
else:
wall = gbn('rebind.parse')
queue, data = request['data'].split(' ', 1)
remove_all = not data
if not remove_all:
args = {'replace': [], 'remove': [], 'remove-mask': [], 'add': []}
arg = args['replace'] # Default.
for part in data.split(' '):
if part.startswith('--'):
arg = args[part[2:]]
elif part:
arg.append(part)
### old_events
wall = gbn('rebind.old_events', wall=wall)
old_events = state.events_by_queues.get(queue) or set()
### remove_all
if remove_all:
wall = gbn('rebind.remove_all', wall=wall)
remove = old_events
add = new_events = ()
else:
### replace
if args.get('replace'):
wall = gbn('rebind.replace', wall=wall)
new_events = set(args['replace'])
else:
new_events = old_events.copy()
### remove
if args.get('remove') and new_events:
wall = gbn('rebind.remove', wall=wall)
new_events.difference_update(args['remove'])
### remove mask
if args.get('remove-mask') and new_events:
wall = gbn('rebind.remove-mask', wall=wall)
key = tuple(args['remove-mask'])
regexp = state.remove_mask_cache.get(key)
if not regexp:
regexp = state.remove_mask_cache[key] = re.compile('|'.join(
'[^.]*'.join(
re.escape(part) for part in mask.split('*')
) + '$' for mask in args['remove-mask']
))
if len(state.remove_mask_cache) > config['remove_mask_cache_limit']:
state.remove_mask_cache.clear()
new_events.difference_update([event for event in new_events if '.' in event and regexp.match(event)])
### add
if args.get('add'):
wall = gbn('rebind.add', wall=wall)
new_events.update(args['add'])
### diff
wall = gbn('rebind.diff', wall=wall)
if not old_events:
remove = ()
add = new_events
elif not new_events:
remove = old_events
add = ()
else: # Cases above are just optimizations of this code.
remove = old_events - new_events
add = new_events - old_events
### update_consumers
if update_consumers or new_events != old_events:
wall = gbn('rebind.update_consumers', wall=wall)
consumers = state.consumers_by_queues.get(queue)
if consumers:
delete_queue_when_unused = state.queues_to_delete_when_unused.get(queue)
response_data = ''.join((
'--update ', queue,
' ' + ' '.join(sorted(new_events)) if new_events else '',
'' if delete_queue_when_unused is None else ' --delete-queue-when-unused' + (
'' if delete_queue_when_unused is True else '={}'.format(delete_queue_when_unused)
),
))
for consumer_id, manual_ack in consumers.iteritems():
if consumer_id != dont_update_consumer_id:
consumer_client = state.clients_by_consumer_ids.get(consumer_id)
if consumer_client:
consumer_request = dict(id=consumer_id, client=consumer_client, worker=state.worker) # Consumer clients are connected to worker of queue, processing rebind.
respond(consumer_request, response_data + (' --manual-ack' if manual_ack else ''))
### send
if remove or add:
wall = gbn('rebind.send', wall=wall)
# Split by unique workers of events:
partial_rebinds = {}
for events in remove, add:
is_add_index = int(events is add) # 0=remove, 1=add
for event in events:
worker = get_worker(event)
if worker == state.worker:
continue # Worker of queue will get full _rebind.
if worker not in partial_rebinds:
partial_rebinds[worker] = ([], []) # Owl with square eyes: partial_remove, partial_add.
partial_rebinds[worker][is_add_index].append(event)
# Send partial _rebind to unique workers of events:
for worker, (partial_remove, partial_add) in partial_rebinds.iteritems():
send_to_worker(worker, '_rebind', request, (queue, ' '.join(partial_remove), ' '.join(partial_add)))
# Send full _rebind to self - worker of queue:
_rebind(request, queue, ' '.join(remove), ' '.join(add))
# This "_rebind" will confirm instead of "rebind".
gbn(wall=wall)
### no-op
else:
wall = gbn('rebind.no-op', wall=wall) # Just to count percent.
gbn(wall=wall)
if request['confirm']:
respond(request)