def pack_binary_command(cmd_type, cmd_args, is_response=False):
"""Packs the given command using the parameter ordering specified in GEARMAN_PARAMS_FOR_COMMAND.
*NOTE* Expects that all arguments in cmd_args are already str's.
"""
expected_cmd_params = GEARMAN_PARAMS_FOR_COMMAND.get(cmd_type, None)
if expected_cmd_params is None or cmd_type == GEARMAN_COMMAND_TEXT_COMMAND:
raise ProtocolError('Received unknown binary command: %s' % get_command_name(cmd_type))
expected_parameter_set = set(expected_cmd_params)
received_parameter_set = set(cmd_args.keys())
if expected_parameter_set != received_parameter_set:
raise ProtocolError('Received arguments did not match expected arguments: %r != %r' % (expected_parameter_set, received_parameter_set))
# Select the right expected magic
if is_response:
magic = MAGIC_RES_STRING
else:
magic = MAGIC_REQ_STRING
# !NOTE! str should be replaced with bytes in Python 3.x
# We will iterate in ORDER and str all our command arguments
if compat.any(type(param_value) != str for param_value in cmd_args.values()):
raise ProtocolError('Received non-binary arguments: %r' % cmd_args)
data_items = [cmd_args[param] for param in expected_cmd_params]
binary_payload = NULL_CHAR.join(data_items)
# Pack the header in the !4sII format then append the binary payload
payload_size = len(binary_payload)
packing_format = '!4sII%ds' % payload_size
return struct.pack(packing_format, magic.encode(), cmd_type, payload_size, binary_payload.encode())
def recv_server_workers(self, raw_text):
"""Slowly assemble a server workers message line by line"""
# If we received a '.', we've finished parsing this workers message
# Pack up our output and reset our response queue
if raw_text == '.':
output_response = tuple(self._workers_response)
self._recv_responses.append(output_response)
self._workers_response = []
return False
split_tokens = raw_text.split(' ')
if len(split_tokens) < self.WORKERS_FIELDS:
raise ProtocolError(
'Received %d tokens, expected >= 4 tokens: %r' %
(len(split_tokens), split_tokens))
if split_tokens[3] != ':':
raise ProtocolError('Malformed worker response: %r' %
(split_tokens, ))
# Label our fields and make the results Python friendly
worker_dict = {}
worker_dict['file_descriptor'] = split_tokens[0]
worker_dict['ip'] = split_tokens[1]
worker_dict['client_id'] = split_tokens[2]
worker_dict['tasks'] = tuple(split_tokens[4:])
self._workers_response.append(worker_dict)
return True
def parse_binary_command(in_buffer, is_response=True):
"""Parse data and return (command type, command arguments dict, command size)
or (None, None, data) if there's not enough data for a complete command.
"""
in_buffer_size = len(in_buffer)
magic = None
cmd_type = None
cmd_args = None
cmd_len = 0
expected_packet_size = None
# If we don't have enough data to parse, error early
if in_buffer_size < COMMAND_HEADER_SIZE:
return cmd_type, cmd_args, cmd_len
# By default, we'll assume we're dealing with a gearman command
magic, cmd_type, cmd_len = struct.unpack('!4sII',
in_buffer[:COMMAND_HEADER_SIZE])
magic = magic.decode('utf-8')
received_bad_response = is_response and bool(magic != MAGIC_RES_STRING)
received_bad_request = not is_response and bool(magic != MAGIC_REQ_STRING)
if received_bad_response or received_bad_request:
raise ProtocolError('Malformed Magic')
expected_cmd_params = GEARMAN_PARAMS_FOR_COMMAND.get(cmd_type, None)
# GEARMAN_COMMAND_TEXT_COMMAND is a faked command that we use to support server text-based commands
if expected_cmd_params is None or cmd_type == GEARMAN_COMMAND_TEXT_COMMAND:
raise ProtocolError('Received unknown binary command: %s' % cmd_type)
# If everything indicates this is a valid command, we should check to see if we have enough stuff to read in our buffer
expected_packet_size = COMMAND_HEADER_SIZE + cmd_len
if in_buffer_size < expected_packet_size:
return None, None, 0
binary_payload = in_buffer[COMMAND_HEADER_SIZE:expected_packet_size]
binary_payload = binary_payload.decode('utf-8')
split_arguments = []
if len(expected_cmd_params) > 0:
split_arguments = binary_payload.split(NULL_CHAR,
len(expected_cmd_params) - 1)
elif binary_payload:
raise ProtocolError('Expected no binary payload: %s' %
get_command_name(cmd_type))
# This is a sanity check on the binary_payload.split() phase
# We should never be able to get here with any VALID gearman data
if len(split_arguments) != len(expected_cmd_params):
raise ProtocolError(
'Received %d argument(s), expecting %d argument(s): %s' %
(len(split_arguments), len(expected_cmd_params),
get_command_name(cmd_type)))
# Iterate through the split arguments and assign them labels based on their order
cmd_args = dict((param_label, param_value)
for param_label, param_value in zip(
expected_cmd_params, split_arguments))
return cmd_type, cmd_args, expected_packet_size
def pack_text_command(cmd_type, cmd_args):
"""Parse a text command and return a single line at a time"""
if cmd_type != GEARMAN_COMMAND_TEXT_COMMAND:
raise ProtocolError('Unknown cmd_type: Received %s, expecting %s' % (get_command_name(cmd_type), get_command_name(GEARMAN_COMMAND_TEXT_COMMAND)))
cmd_line = cmd_args.get('raw_text')
if cmd_line is None:
raise ProtocolError('Did not receive arguments any valid arguments: %s' % cmd_args)
return str(cmd_line)
def recv_server_status(self, raw_text):
"""Slowly assemble a server status message line by line"""
# If we received a '.', we've finished parsing this status message
# Pack up our output and reset our response queue
if raw_text == '.':
output_response = tuple(self._status_response)
self._recv_responses.append(output_response)
self._status_response = []
return False
# If we didn't get a final response, split our line and interpret all the data
split_tokens = raw_text.split('\t')
if len(split_tokens) != self.STATUS_FIELDS:
raise ProtocolError(
'Received %d tokens, expected %d tokens: %r' %
(len(split_tokens), self.STATUS_FIELDS, split_tokens))
# Label our fields and make the results Python friendly
task, queued_count, running_count, worker_count = split_tokens
status_dict = {}
status_dict['task'] = task
status_dict['queued'] = int(queued_count)
status_dict['running'] = int(running_count)
status_dict['workers'] = int(worker_count)
self._status_response.append(status_dict)
return True
def recv_server_maxqueue(self, raw_text):
"""Maxqueue response is a simple passthrough"""
if raw_text != 'OK':
raise ProtocolError("Expected 'OK', received: %s" % raw_text)
self._recv_responses.append(raw_text)
return False
def recv_server_show_jobs(self, raw_text):
"""Slowly assemble a show jobs message line by line"""
# If we received a '.', we've finished parsing this status message
# Pack up our output and reset our response queue
if raw_text == '.':
output_response = tuple(self._status_response)
self._recv_responses.append(output_response)
self._status_response = []
return False
# If we didn't get a final response, split our line and interpret all the data
split_tokens = raw_text.split('\t')
if len(split_tokens) != self.JOB_FIELDS:
raise ProtocolError(
'Received %d tokens, expected %d tokens: %r' %
(len(split_tokens), self.JOB_FIELDS, split_tokens))
# Label our fields and make the results Python friendly
handle, queued_count, canceled_count, enabled_count = split_tokens
job_dict = {
"handle": handle,
"queued": int(queued_count),
"canceled": int(canceled_count),
"enabled": int(enabled_count),
}
self._status_response.append(job_dict)
return True
def pack_binary_command(cmd_type, cmd_args, is_response=False):
"""Packs the given command using the parameter ordering specified in GEARMAN_PARAMS_FOR_COMMAND.
*NOTE* Expects that all arguments in cmd_args are already str's.
"""
expected_cmd_params = GEARMAN_PARAMS_FOR_COMMAND.get(cmd_type, None)
if expected_cmd_params is None or cmd_type == GEARMAN_COMMAND_TEXT_COMMAND:
raise ProtocolError('Received unknown binary command: %s' %
get_command_name(cmd_type))
expected_parameter_set = set(expected_cmd_params)
received_parameter_set = set(cmd_args.keys())
if expected_parameter_set != received_parameter_set:
raise ProtocolError(
'Received arguments did not match expected arguments: %r != %r' %
(expected_parameter_set, received_parameter_set))
# Select the right expected magic
if is_response:
magic = MAGIC_RES_STRING
else:
magic = MAGIC_REQ_STRING
for key, value in cmd_args.items():
if not isinstance(value, compat.binary_type):
if isinstance(value, str) or isinstance(value,
compat.unicode_type):
# Postel's: Provide Python 2 => Python 3 compatibility.
cmd_args[key] = value.encode()
else:
raise ProtocolError('Received non-binary arguments: %r' %
cmd_args)
data_items = [cmd_args[param] for param in expected_cmd_params]
# Now check that all but the last argument are free of \0 as per the protocol spec.
if any(b'\0' in argument for argument in data_items[:-1]):
raise ProtocolError(
'Received arguments with NULL byte in non-final argument')
binary_payload = NULL_CHAR.join(data_items)
# Pack the header in the !4sII format then append the binary payload
payload_size = len(binary_payload)
packing_format = '!4sII%ds' % payload_size
return struct.pack(packing_format, magic, cmd_type, payload_size,
binary_payload)
def pack_binary_command(cmd_type, cmd_args, is_response=False):
"""Packs the given command using the parameter ordering specified in GEARMAN_PARAMS_FOR_COMMAND.
*NOTE* Expects that all arguments in cmd_args are already str's.
"""
expected_cmd_params = GEARMAN_PARAMS_FOR_COMMAND.get(cmd_type, None)
if expected_cmd_params is None or cmd_type == GEARMAN_COMMAND_TEXT_COMMAND:
raise ProtocolError('Received unknown binary command: %s' %
get_command_name(cmd_type))
expected_parameter_set = set(expected_cmd_params)
received_parameter_set = set(cmd_args.keys())
if expected_parameter_set != received_parameter_set:
raise ProtocolError(
'Received arguments did not match expected arguments: %r != %r' %
(expected_parameter_set, received_parameter_set))
# Select the right expected magic
if is_response:
magic = MAGIC_RES_STRING
else:
magic = MAGIC_REQ_STRING
# !NOTE! str should be replaced with bytes in Python 3.x
# The binary protocol is null byte delimited, so let's make sure we don't
# have null bytes in our values and we're dealing with strings we can probably encode.
if compat.any(
not isinstance(param_value, basestring) or '\0' in param_value
for param_value in cmd_args.itervalues()):
raise ProtocolError('Received un-encodable arguments: %r' % cmd_args)
data_items = [
cmd_args[param].encode('ascii') for param in expected_cmd_params
]
binary_payload = NULL_CHAR.join(data_items)
# Pack the header in the !4sII format then append the binary payload
payload_size = len(binary_payload)
packing_format = '!4sII%ds' % payload_size
return struct.pack(packing_format, magic, cmd_type, payload_size,
binary_payload)
def _pack_command(self, cmd_type, cmd_args):
"""Converts a command to its raw binary format"""
if cmd_type not in GEARMAN_PARAMS_FOR_COMMAND:
raise ProtocolError('Unknown command: %r' % get_command_name(cmd_type))
if _DEBUG_MODE_:
gearman_logger.debug('%s - Send - %s - %r', hex(id(self)), get_command_name(cmd_type), cmd_args)
if cmd_type == GEARMAN_COMMAND_TEXT_COMMAND:
return pack_text_command(cmd_type, cmd_args)
else:
# We'll be sending a response if we know we're a server side command
is_response = bool(self._is_server_side)
return pack_binary_command(cmd_type, cmd_args, is_response)
def send_text_command(self, command_line):
"""Send our administrative text command"""
expected_server_command = None
for server_command in EXPECTED_GEARMAN_SERVER_COMMANDS:
if command_line.startswith(server_command):
expected_server_command = server_command
break
if not expected_server_command:
raise ProtocolError(
'Attempted to send an unknown server command: %r' %
command_line)
self._sent_commands.append(expected_server_command)
output_text = '%s\n' % command_line
self.send_command(GEARMAN_COMMAND_TEXT_COMMAND, raw_text=output_text)
def parse_text_command(in_buffer):
"""Parse a text command and return a single line at a time"""
cmd_type = None
cmd_args = None
cmd_len = 0
if '\n' not in in_buffer:
return cmd_type, cmd_args, cmd_len
text_command, in_buffer = in_buffer.split('\n', 1)
if NULL_CHAR in text_command:
raise ProtocolError('Received unexpected character: %s' % text_command)
# Fake gearman command "TEXT_COMMAND" used to process server admin client responses
cmd_type = GEARMAN_COMMAND_TEXT_COMMAND
cmd_args = dict(raw_text=text_command)
cmd_len = len(text_command) + 1
return cmd_type, cmd_args, cmd_len
def recv_server_show_unique_jobs(self, raw_text):
"""Slowly assemble a server show unique jobs message line by line"""
# If we received a '.', we've finished parsing this status message
# Pack up our output and reset our response queue
if raw_text == '.':
output_response = tuple(self._status_response)
self._recv_responses.append(output_response)
self._status_response = []
return False
# If we didn't get a final response, split our line and interpret all the data
split_tokens = raw_text.split('\t')
if len(split_tokens) != self.UNIQUE_JOB_FIELDS:
raise ProtocolError(
'Received %d tokens, expected %d tokens: %r' %
(len(split_tokens), self.UNIQUE_JOB_FIELDS, split_tokens))
# Label our fields and make the results Python friendly
unique = split_tokens
job_dict = {}
job_dict['unique'] = unique
self._status_response.append(job_dict)
return True
