def recv_server_workers(self, raw_text): """Slowly assemble a server workers message line by line""" # If we received a '.', we've finished parsing this workers message # Pack up our output and reset our response queue if raw_text == '.': output_response = tuple(self._workers_response) self._recv_responses.append(output_response) self._workers_response = [] return False split_tokens = raw_text.split(' ') if len(split_tokens) < self.WORKERS_FIELDS: raise ProtocolError( 'Received %d tokens, expected >= 4 tokens: %r' % (len(split_tokens), split_tokens)) if split_tokens[3] != ':': raise ProtocolError('Malformed worker response: %r' % (split_tokens, )) # Label our fields and make the results Python friendly worker_dict = {} worker_dict['file_descriptor'] = split_tokens[0] worker_dict['ip'] = split_tokens[1] worker_dict['client_id'] = split_tokens[2] worker_dict['tasks'] = tuple(split_tokens[4:]) self._workers_response.append(worker_dict) return True
def pack_text_command(cmd_type, cmd_args): """Parse a text command and return a single line at a time""" if cmd_type != GEARMAN_COMMAND_TEXT_COMMAND: raise ProtocolError('Unknown cmd_type: Received %s, expecting %s' % (get_command_name(cmd_type), get_command_name(GEARMAN_COMMAND_TEXT_COMMAND))) cmd_line = cmd_args.get('raw_text') if cmd_line is None: raise ProtocolError( 'Did not receive arguments any valid arguments: %s' % cmd_args) return str(cmd_line)
def recv_server_maxqueue(self, raw_text): """Maxqueue response is a simple passthrough""" if raw_text != 'OK': raise ProtocolError("Expected 'OK', received: %s" % raw_text) self._recv_responses.append(raw_text) return False
def recv_server_status(self, raw_text): """Slowly assemble a server status message line by line""" # If we received a '.', we've finished parsing this status message # Pack up our output and reset our response queue if raw_text == '.': output_response = tuple(self._status_response) self._recv_responses.append(output_response) self._status_response = [] return False # If we didn't get a final response, split our line and interpret all # the data split_tokens = raw_text.split('\t') if len(split_tokens) != self.STATUS_FIELDS: raise ProtocolError( 'Received %d tokens, expected %d tokens: %r' % (len(split_tokens), self.STATUS_FIELDS, split_tokens)) # Label our fields and make the results Python friendly task, queued_count, running_count, worker_count = split_tokens status_dict = {} status_dict['task'] = task status_dict['queued'] = int(queued_count) status_dict['running'] = int(running_count) status_dict['workers'] = int(worker_count) self._status_response.append(status_dict) return True
def pack_binary_command(cmd_type, cmd_args, is_response=False): """Packs the given command using the parameter ordering specified in GEARMAN_PARAMS_FOR_COMMAND. *NOTE* Expects that all arguments in cmd_args are already str's. """ expected_cmd_params = GEARMAN_PARAMS_FOR_COMMAND.get(cmd_type, None) if expected_cmd_params is None or cmd_type == GEARMAN_COMMAND_TEXT_COMMAND: raise ProtocolError( 'Received unknown binary command: %s' % get_command_name(cmd_type)) expected_parameter_set = set(expected_cmd_params) received_parameter_set = set(cmd_args.keys()) if expected_parameter_set != received_parameter_set: raise ProtocolError( 'Received arguments did not match expected arguments: %r != %r' % (expected_parameter_set, received_parameter_set)) # Select the right expected magic if is_response: magic = MAGIC_RES_STRING else: magic = MAGIC_REQ_STRING # !NOTE! str should be replaced with bytes in Python 3.x # We will iterate in ORDER and str all our command arguments if compat.any(not isinstance(param_value, str) for param_value in cmd_args.values()): raise ProtocolError('Received non-binary arguments: %r' % cmd_args) data_items = [cmd_args[param] for param in expected_cmd_params] binary_payload = NULL_CHAR.join(data_items).encode() # Pack the header in the !4sII format then append the binary payload payload_size = len(binary_payload) packing_format = '!4sII%ds' % payload_size return struct.pack( packing_format, magic.encode(), cmd_type, payload_size, binary_payload)
def send_text_command(self, command_line): """Send our administrative text command""" expected_server_command = None for server_command in EXPECTED_GEARMAN_SERVER_COMMANDS: if command_line.startswith(server_command): expected_server_command = server_command break if not expected_server_command: raise ProtocolError( 'Attempted to send an unknown server command: %r' % command_line) self._sent_commands.append(expected_server_command) output_text = '%s\n' % command_line self.send_command(GEARMAN_COMMAND_TEXT_COMMAND, raw_text=output_text)
def _pack_command(self, cmd_type, cmd_args): """Converts a command to its raw binary format""" if cmd_type not in GEARMAN_PARAMS_FOR_COMMAND: raise ProtocolError('Unknown command: %r' % get_command_name(cmd_type)) if _DEBUG_MODE_: gearman_logger.debug('%s - Send - %s - %r', hex(id(self)), get_command_name(cmd_type), cmd_args) if cmd_type == GEARMAN_COMMAND_TEXT_COMMAND: return pack_text_command(cmd_type, cmd_args) else: # We'll be sending a response if we know we're a server side # command is_response = bool(self._is_server_side) return pack_binary_command(cmd_type, cmd_args, is_response)
def parse_text_command(in_buffer): """Parse a text command and return a single line at a time""" cmd_type = None cmd_args = None cmd_len = 0 in_buffer = in_buffer.decode() if '\n' not in in_buffer: return cmd_type, cmd_args, cmd_len text_command, in_buffer = in_buffer.split('\n', 1) if NULL_CHAR in text_command: raise ProtocolError('Received unexpected character: %s' % text_command) # Fake gearman command "TEXT_COMMAND" used to process server admin client # responses cmd_type = GEARMAN_COMMAND_TEXT_COMMAND cmd_args = dict(raw_text=text_command) cmd_len = len(text_command) + 1 return cmd_type, cmd_args, cmd_len
def parse_binary_command(in_buffer, is_response=True): """Parse data and return (command type, command arguments dict, command size) or (None, None, data) if there's not enough data for a complete command. """ in_buffer_size = len(in_buffer) magic = None cmd_type = None cmd_args = None cmd_len = 0 expected_packet_size = None # If we don't have enough data to parse, error early if in_buffer_size < COMMAND_HEADER_SIZE: return cmd_type, cmd_args, cmd_len # By default, we'll assume we're dealing with a gearman command magic, cmd_type, cmd_len = struct.unpack( '!4sII', in_buffer[:COMMAND_HEADER_SIZE]) magic = magic.decode('utf-8') received_bad_response = is_response and bool(magic != MAGIC_RES_STRING) received_bad_request = not is_response and bool(magic != MAGIC_REQ_STRING) if received_bad_response or received_bad_request: raise ProtocolError('Malformed Magic') expected_cmd_params = GEARMAN_PARAMS_FOR_COMMAND.get(cmd_type, None) # GEARMAN_COMMAND_TEXT_COMMAND is a faked command that we use to support # server text-based commands if expected_cmd_params is None or cmd_type == GEARMAN_COMMAND_TEXT_COMMAND: raise ProtocolError('Received unknown binary command: %s' % cmd_type) # If everything indicates this is a valid command, we should check to see # if we have enough stuff to read in our buffer expected_packet_size = COMMAND_HEADER_SIZE + cmd_len if in_buffer_size < expected_packet_size: return None, None, 0 binary_payload = in_buffer[COMMAND_HEADER_SIZE:expected_packet_size] binary_payload = binary_payload.decode('utf-8') split_arguments = [] if len(expected_cmd_params) > 0: split_arguments = binary_payload.split( NULL_CHAR, len(expected_cmd_params) - 1) elif binary_payload: raise ProtocolError( 'Expected no binary payload: %s' % get_command_name(cmd_type)) # This is a sanity check on the binary_payload.split() phase # We should never be able to get here with any VALID gearman data if len(split_arguments) != len(expected_cmd_params): raise ProtocolError( 'Received %d argument(s), expecting %d argument(s): %s' % ( len(split_arguments), len(expected_cmd_params), get_command_name(cmd_type) ) ) # Iterate through the split arguments and assign them labels based on # their order cmd_args = dict((param_label, param_value) for param_label, param_value in zip(expected_cmd_params, split_arguments)) return cmd_type, cmd_args, expected_packet_size