def send_to_worker(worker, func_name, request, args=()):
"""
Send command to other worker or execute command locally.
@param worker: int
@param func_name: str
@param request: dict - defined in "on_request"
@param args: tuple(str)
"""
if log.level == logging.DEBUG or config['grep']:
verbose('w{} > w{}: {}{} for request={}'.format(
state.worker, worker, func_name, args, request))
if worker == state.worker:
execute(func_name, request, args)
else:
wall = gbn('send_to_worker.other')
# "command protocol" encodes 40x faster than default "pickle.dumps" and produces 9x smaller result:
command = '\t'.join((func_name, request['id'],
request.get('client', '-'),
str(request.get('worker', -1)),
str(int(request.get('confirm', False)))) + args)
if len(command) >= config['warn_command_bytes']:
crit('WARNING! Too big: "{}" == {} >= {} bytes'.format(
command, len(command), config['warn_command_bytes']))
state.commands_to_workers[worker].put(command)
state.commands_put += 1
gbn(wall=wall)
def _delete_queue(request, queue):
"""
Delete queue command
@param request: dict - defined in "on_request"
@param queue: str
"""
if log.level == logging.DEBUG or config['grep']:
verbose('w{}: deleting queue {}'.format(state.worker, queue))
confirm = request['confirm']
request['confirm'] = False # To avoid double confirm.
rebind(request, queue) # Unbind all by default.
for consumer_id, queue_of_consumer in state.queues_by_consumer_ids.items():
if queue_of_consumer == queue:
_delete_consumer(request, consumer_id)
state.queues.pop(queue, None)
state.queues_to_delete_when_unused.pop(queue, None)
queue_used = state.queues_used.pop(queue, None)
if queue_used:
queue_used.set() # Cancel "_wait_used_or_delete_queue" greenlet.
if log.level == logging.DEBUG or config['grep']:
verbose('w{}: deleted queue {}'.format(state.worker, queue))
if confirm:
respond(request)
def delete_consumers(client):
"""
Delete consumers on disconnect of client.
@param client: str
"""
wall = gbn('delete_consumers')
for consumer_id in list(state.consumer_ids_by_clients.get(client, ())):
# get() is used instead of pop() because
# "_delete_consumer" will discard "consumer_id" from "consumer_ids" to stop "_consume_loop",
# and once client has no consumers - it will pop().
# list(set()) is used to avoid "Set changed size during iteration".
if log.level == logging.DEBUG or config['grep']:
verbose(
'w{}: deleting consumer {} on disconnect of client {}'.format(
state.worker, consumer_id, client))
request = dict(id='disconnect',
client=client,
worker=state.worker,
confirm=False)
_delete_consumer(request, consumer_id)
gbn(wall=wall)
def on_request(request):
"""
Handler of request from client.
@param request: dict(
client: str,
worker: int,
body: str,
)
"action()" gets "request" without "body" but with (
id: str,
action: str,
data: str - should be parsed inside "action()",
confirm: bool,
)
Command in another worker gets "request" without (
action: str - unused,
data: str - unused and may be very big
)
"""
wall = gbn('on_request')
request['id'] = None
try:
body = request['body'].rstrip()
request['id'], request['action'], request['data'] = body.split(' ', 2)
del request[
'body'] # Less data to pass between workers. On error "request" with "id, action, data" will be logged.
if log.level == logging.DEBUG or config['grep']:
verbose('w{}: {}#{} > {} {}'.format(state.worker,
request['client'],
request['id'],
request['action'],
request['data']))
request['confirm'] = request['data'].startswith('--confirm ')
if request['confirm']:
request['data'] = request['data'][10:]
action = state.actions[request['action']]
wall = gbn(request['action'], wall=wall)
action(request)
gbn(wall=wall)
except Exception:
request['error_id'] = dtid(config['id_length'])
crit(also=request)
try:
respond(request)
except Exception:
crit(also=request)
gbn(wall=wall)
def responder(client):
"""
Sends queued responses to socket of client.
See also "mqks.server.lib.workers.respond()" that enqueues response to "state.responses_by_clients[client]".
@param client: str
"""
response = None
try:
responses = state.responses_by_clients.get(client)
sock = state.socks_by_clients.get(client)
if not responses or not sock:
return
while client in state.responses_by_clients:
try:
response = responses.get(timeout=config['block_seconds'])
except Empty:
continue
wall = gbn('responder')
try:
request, data = response
error_id = request.get('error_id')
response = '{} {}'.format('error' if error_id else 'ok',
error_id or data)
if log.level == logging.DEBUG or config['grep']:
verbose('w{}: {}#{} < {}'.format(state.worker, client,
request['id'], response))
response = '{} {}\n'.format(request['id'], response)
except Exception:
gbn(wall=wall)
crit(also=dict(response=response))
continue
try:
sock.sendall(response)
# Disconnect on socket error.
finally:
gbn(wall=wall)
time.sleep(0)
except Exception as e:
if not is_disconnect(e):
crit(also=dict(client=client, response=response))
def reject(request):
"""
Reject action
@param request: dict - defined in "on_request" with (
data: str - "{consumer_id} {msg_id}",
...
)
"""
consumer_id, msg_id = request['data'].split(' ', 1)
queue = state.queues_by_consumer_ids.get(consumer_id)
if queue:
_reject(request, queue, consumer_id, msg_id)
elif log.level == logging.DEBUG or config['grep']:
verbose('w{}: found no queue for request={}'.format(
state.worker, request))
def execute(func_name, request, args):
"""
Execute the command and handle errors.
@param func_name: str
@param request: dict - defined in "on_request"
@param args: sequence(str)
"""
try:
wall = gbn(func_name)
if log.level == logging.DEBUG or config['grep']:
verbose('w{} < {}{} for request={}'.format(state.worker, func_name,
args, request))
func = state.funcs[func_name]
func(request, *args)
gbn(wall=wall)
except Exception:
on_error((func_name, request, args), request)
def ack(request):
"""
Ack action
@param request: dict - defined in "on_request" with (
data: str - "{consumer_id} {msg_id}" or "{consumer_id} --all",
...
)
"""
consumer_id, msg_id = request['data'].split(' ', 1)
queue = state.queues_by_consumer_ids.get(consumer_id)
if not queue:
if log.level == logging.DEBUG or config['grep']:
verbose('w{}: found no queue for request={}'.format(state.worker, request))
return
if msg_id == '--all':
state.messages_by_consumer_ids.pop(consumer_id, {}).clear()
else:
state.messages_by_consumer_ids.get(consumer_id, {}).pop(msg_id, None)
if request['confirm']:
respond(request)