def poll_connections_until_stopped(self, submitted_connections, callback_fxn, timeout=None):
"""Continue to poll our connections until we receive a stopping condition"""
stopwatch = gearman.util.Stopwatch(timeout)
any_activity = False
callback_ok = callback_fxn(any_activity)
connection_ok = compat.any(current_connection.connected for current_connection in submitted_connections)
while connection_ok and callback_ok:
time_remaining = stopwatch.get_time_remaining()
if time_remaining == 0.0:
break
# Do a single robust select and handle all connection activity
read_connections, write_connections, dead_connections = self.poll_connections_once(submitted_connections, timeout=time_remaining)
self.handle_connection_activity(read_connections, write_connections, dead_connections)
any_activity = compat.any([read_connections, write_connections, dead_connections])
callback_ok = callback_fxn(any_activity)
connection_ok = compat.any(current_connection.connected for current_connection in submitted_connections)
# We should raise here if we have no alive connections (don't go into a select polling loop with no connections)
if not connection_ok:
raise ServerUnavailable('Found no valid connections in list: %r' % self.connection_list)
return bool(connection_ok and callback_ok)
def pack_binary_command(cmd_type, cmd_args, is_response=False):
"""Packs the given command using the parameter ordering specified in GEARMAN_PARAMS_FOR_COMMAND.
*NOTE* Expects that all arguments in cmd_args are already str's.
"""
expected_cmd_params = GEARMAN_PARAMS_FOR_COMMAND.get(cmd_type, None)
if expected_cmd_params is None or cmd_type == GEARMAN_COMMAND_TEXT_COMMAND:
raise ProtocolError('Received unknown binary command: %s' % get_command_name(cmd_type))
expected_parameter_set = set(expected_cmd_params)
received_parameter_set = set(cmd_args.keys())
if expected_parameter_set != received_parameter_set:
raise ProtocolError('Received arguments did not match expected arguments: %r != %r' % (expected_parameter_set, received_parameter_set))
# Select the right expected magic
if is_response:
magic = MAGIC_RES_STRING
else:
magic = MAGIC_REQ_STRING
# We will iterate in ORDER and check all our command arguments are bytes
if compat.any(type(param_value) != compat.bytes_type for param_value in cmd_args.values()):
raise ProtocolError('Received non-binary arguments: %r' % cmd_args)
data_items = [cmd_args[param] for param in expected_cmd_params]
# Now check that all but the last argument are free of \0 as per the protocol spec.
if compat.any(b'\0' in argument for argument in data_items[:-1]):
raise ProtocolError('Received arguments with NULL byte in non-final argument')
binary_payload = NULL_BYTE.join(data_items)
# Pack the header in the !4sII format then append the binary payload
payload_size = len(binary_payload)
packing_format = '!4sII%ds' % payload_size
return struct.pack(packing_format, magic, cmd_type, payload_size, binary_payload)
def poll_connections_until_stopped(self,
submitted_connections,
callback_fxn,
timeout=None,
prehandle=None):
"""Continue to poll our connections until we receive a stopping condition"""
stopwatch = gearman.util.Stopwatch(timeout)
submitted_connections = set(submitted_connections)
connection_map = {}
any_activity = False
callback_ok = callback_fxn(any_activity)
connection_ok = compat.any(
current_connection.connected
for current_connection in submitted_connections)
poller = gearman.io.get_connection_poller()
if connection_ok:
self._register_connections_with_poller(submitted_connections,
poller)
connection_map = dict([(c.fileno(), c)
for c in submitted_connections
if c.connected])
while connection_ok and callback_ok:
time_remaining = stopwatch.get_time_remaining()
if time_remaining == 0.0:
break
# Do a single robust select and handle all connection activity
read_connections, write_connections, dead_connections = self.poll_connections_once(
poller, connection_map, timeout=time_remaining)
if prehandle:
prehandle(read_connections, write_connections,
dead_connections)
# Handle reads and writes and close all of the dead connections
read_connections, write_connections, dead_connections = self.handle_connection_activity(
read_connections, write_connections, dead_connections)
any_activity = compat.any(
[read_connections, write_connections, dead_connections])
# Do not retry dead connections on the next iteration of the loop, as we closed them in handle_error
submitted_connections -= dead_connections
callback_ok = callback_fxn(any_activity)
connection_ok = compat.any(
current_connection.connected
for current_connection in submitted_connections)
poller.close()
# We should raise here if we have no alive connections (don't go into a select polling loop with no connections)
if not connection_ok:
raise ServerUnavailable('Found no valid connections in list: %r' %
self.connection_list)
return bool(connection_ok and callback_ok)
def poll_connections_until_stopped(self, connections, callback_fxn,
timeout=None):
"""Continue to poll our connections until we receive a stopping
condition"""
stopwatch = gearman.util.Stopwatch(timeout)
connections = set(connections)
any_activity = False
callback_ok = callback_fxn(any_activity)
connection_ok = compat.any(conn.connected for conn in connections)
poller = gearman.io.get_connection_poller()
while connection_ok and callback_ok:
time_remaining = stopwatch.get_time_remaining()
if time_remaining == 0.0:
break
self._register_connections_with_poller(connections, poller)
# Do a single robust select and handle all connection activity
readable, writable, death = self.poll_connections_once(
poller, dict([(conn.fileno(), conn)
for conn in connections if conn.connected]),
timeout=time_remaining)
# Handle reads and writes and close all of the dead connections
readable, writable, death = self.handle_connection_activity(
readable, writable, death)
any_activity = compat.any([readable, writable, death])
for conn in connections:
poller.unregister(conn)
# Do not retry dead connections on the next iteration of the loop,
# as we closed them in handle_error
connections -= death
callback_ok = callback_fxn(any_activity)
connection_ok = compat.any(conn.connected for conn in connections)
poller.close()
# We should raise here if we have no alive connections (don't go into a
# select polling loop with no connections)
if not connection_ok:
raise ServerUnavailable('Found no valid connections in list: %r' % self.connection_list)
return bool(connection_ok and callback_ok)
def has_pending_jobs():
for request in request_ls:
if request.state == JOB_UNKNOWN:
self.send_job_request(request)
return compat.any(request.state == JOB_PENDING for
request in request_ls)
def pack_binary_command(cmd_type, cmd_args, is_response=False):
"""Packs the given command using the parameter ordering specified in GEARMAN_PARAMS_FOR_COMMAND.
*NOTE* Expects that all arguments in cmd_args are already str's.
"""
expected_cmd_params = GEARMAN_PARAMS_FOR_COMMAND.get(cmd_type, None)
if expected_cmd_params is None or cmd_type == GEARMAN_COMMAND_TEXT_COMMAND:
raise ProtocolError('Received unknown binary command: %s' % get_command_name(cmd_type))
expected_parameter_set = set(expected_cmd_params)
received_parameter_set = set(cmd_args.keys())
if expected_parameter_set != received_parameter_set:
raise ProtocolError('Received arguments did not match expected arguments: %r != %r' % (expected_parameter_set, received_parameter_set))
# Select the right expected magic
if is_response:
magic = MAGIC_RES_STRING
else:
magic = MAGIC_REQ_STRING
# !NOTE! str should be replaced with bytes in Python 3.x
# We will iterate in ORDER and str all our command arguments
if compat.any(type(param_value) != str for param_value in cmd_args.values()):
raise ProtocolError('Received non-binary arguments: %r' % cmd_args)
data_items = [cmd_args[param] for param in expected_cmd_params]
binary_payload = NULL_CHAR.join(data_items)
# Pack the header in the !4sII format then append the binary payload
payload_size = len(binary_payload)
packing_format = '!4sII%ds' % payload_size
return struct.pack(packing_format, magic.encode(), cmd_type, payload_size, binary_payload.encode())
def pack_binary_command(cmd_type, cmd_args, is_response=False):
"""Packs the given command using the parameter ordering specified in GEARMAN_PARAMS_FOR_COMMAND.
*NOTE* Expects that all arguments in cmd_args are already str's.
"""
expected_cmd_params = GEARMAN_PARAMS_FOR_COMMAND.get(cmd_type, None)
if expected_cmd_params is None or cmd_type == GEARMAN_COMMAND_TEXT_COMMAND:
raise ProtocolError('Received unknown binary command: %s' % get_command_name(cmd_type))
expected_parameter_set = set(expected_cmd_params)
received_parameter_set = set(cmd_args.keys())
if expected_parameter_set != received_parameter_set:
raise ProtocolError('Received arguments did not match expected arguments: %r != %r' % (expected_parameter_set, received_parameter_set))
# Select the right expected magic
if is_response:
magic = MAGIC_RES_STRING
else:
magic = MAGIC_REQ_STRING
# !NOTE! str should be replaced with bytes in Python 3.x
# We will iterate in ORDER and str all our command arguments
if compat.any(type(param_value) != str for param_value in cmd_args.itervalues()):
raise ProtocolError('Received non-binary arguments: %r' % cmd_args)
data_items = [cmd_args[param] for param in expected_cmd_params]
binary_payload = NULL_CHAR.join(data_items)
# Pack the header in the !4sII format then append the binary payload
payload_size = len(binary_payload)
packing_format = '!4sII%ds' % payload_size
return struct.pack(packing_format, magic, cmd_type, payload_size, binary_payload)
def pack_binary_command(cmd_type, cmd_args, is_response=False):
"""Packs the given command using the parameter ordering specified in GEARMAN_PARAMS_FOR_COMMAND.
*NOTE* Expects that all arguments in cmd_args are already str's.
"""
expected_cmd_params = GEARMAN_PARAMS_FOR_COMMAND.get(cmd_type, None)
if expected_cmd_params is None or cmd_type == GEARMAN_COMMAND_TEXT_COMMAND:
raise ProtocolError('Received unknown binary command: %s' % get_command_name(cmd_type))
expected_parameter_set = set(expected_cmd_params)
received_parameter_set = set(cmd_args.keys())
if expected_parameter_set != received_parameter_set:
raise ProtocolError('Received arguments did not match expected arguments: %r != %r' % (expected_parameter_set, received_parameter_set))
# Select the right expected magic
if is_response:
magic = MAGIC_RES_STRING
else:
magic = MAGIC_REQ_STRING
# !NOTE! str should be replaced with bytes in Python 3.x
# The binary protocol is null byte delimited, so let's make sure we don't
# have null bytes in our values and we're dealing with strings we can probably encode.
if compat.any(not isinstance(param_value, basestring) or '\0' in param_value for param_value in cmd_args.itervalues()):
raise ProtocolError('Received un-encodable arguments: %r' % cmd_args)
data_items = [cmd_args[param].encode('ascii') for param in expected_cmd_params]
binary_payload = NULL_CHAR.join(data_items)
# Pack the header in the !4sII format then append the binary payload
payload_size = len(binary_payload)
packing_format = '!4sII%ds' % payload_size
return struct.pack(packing_format, magic, cmd_type, payload_size, binary_payload)
def poll_connections_until_stopped(self, submitted_connections, callback_fxn, timeout=None, prehandle=None):
"""Continue to poll our connections until we receive a stopping condition"""
stopwatch = gearman.util.Stopwatch(timeout)
submitted_connections = set(submitted_connections)
connection_map = {}
any_activity = False
callback_ok = callback_fxn(any_activity)
connection_ok = compat.any(current_connection.connected for current_connection in submitted_connections)
poller = gearman.io.get_connection_poller()
if connection_ok:
self._register_connections_with_poller(submitted_connections, poller)
connection_map = dict([(c.fileno(), c) for c in submitted_connections if c.connected])
while connection_ok and callback_ok:
time_remaining = stopwatch.get_time_remaining()
if time_remaining == 0.0:
break
# Do a single robust select and handle all connection activity
read_connections, write_connections, dead_connections = self.poll_connections_once(
poller, connection_map, timeout=time_remaining
)
if prehandle:
prehandle(read_connections, write_connections, dead_connections)
# Handle reads and writes and close all of the dead connections
read_connections, write_connections, dead_connections = self.handle_connection_activity(
read_connections, write_connections, dead_connections
)
any_activity = compat.any([read_connections, write_connections, dead_connections])
# Do not retry dead connections on the next iteration of the loop, as we closed them in handle_error
submitted_connections -= dead_connections
callback_ok = callback_fxn(any_activity)
connection_ok = compat.any(current_connection.connected for current_connection in submitted_connections)
poller.close()
# We should raise here if we have no alive connections (don't go into a select polling loop with no connections)
if not connection_ok:
raise ServerUnavailable("Found no valid connections in list: %r" % self.connection_list)
return bool(connection_ok and callback_ok)
def continue_while_jobs_pending(any_activity):
for current_request in job_requests:
if current_request.state == JOB_UNKNOWN:
self.send_job_request(current_request)
return compat.any(
is_request_pending(current_request)
for current_request in job_requests)
def poll(self, connections, before_poll, after_poll, timeout=None):
"""Continue to poll our connections until we receive a stopping
condition"""
stopwatch = gearman.util.Stopwatch(timeout)
connections = set(connections)
workable = before_poll()
pollable = compat.any(not c.internal and c.connected for c in connections)
internal = compat.any(c.internal and c.connected for c in connections) \
if self.has_internal_connection else True
while workable and pollable and internal:
time_remaining = stopwatch.get_time_remaining()
if time_remaining == 0.0:
break
self._register(connections)
readable, writable, exceptional = self._poll_once(
connections, timeout=time_remaining)
readable, writable, exceptional = self._process(
readable, writable, exceptional)
any_activity = compat.any([readable, writable, exceptional])
for connection in connections:
self._poller.unregister(connection)
# Do not retry dead connections on the next iteration of the loop,
# as we closed them in handle_error
connections -= exceptional
workable = after_poll()
pollable = compat.any(not c.internal for c in connections)
internal = compat.any(c.internal for c in connections) \
if self.has_internal_connection else True
return workable
def pack_binary_command(cmd_type, cmd_args, is_response=False):
"""Packs the given command using the parameter ordering specified in GEARMAN_PARAMS_FOR_COMMAND.
*NOTE* Expects that all arguments in cmd_args are already str's.
"""
expected_cmd_params = GEARMAN_PARAMS_FOR_COMMAND.get(cmd_type, None)
if expected_cmd_params is None or cmd_type == GEARMAN_COMMAND_TEXT_COMMAND:
raise ProtocolError('Received unknown binary command: %s' %
get_command_name(cmd_type))
expected_parameter_set = set(expected_cmd_params)
received_parameter_set = set(cmd_args.keys())
if expected_parameter_set != received_parameter_set:
raise ProtocolError(
'Received arguments did not match expected arguments: %r != %r' %
(expected_parameter_set, received_parameter_set))
# Select the right expected magic
if is_response:
magic = MAGIC_RES_STRING
else:
magic = MAGIC_REQ_STRING
# !NOTE! str should be replaced with bytes in Python 3.x
# The binary protocol is null byte delimited, so let's make sure we don't
# have null bytes in our values and we're dealing with strings we can probably encode.
if compat.any(
not isinstance(param_value, basestring) or '\0' in param_value
for param_value in cmd_args.itervalues()):
raise ProtocolError('Received un-encodable arguments: %r' % cmd_args)
data_items = [
cmd_args[param].encode('ascii') for param in expected_cmd_params
]
binary_payload = NULL_CHAR.join(data_items)
# Pack the header in the !4sII format then append the binary payload
payload_size = len(binary_payload)
packing_format = '!4sII%ds' % payload_size
return struct.pack(packing_format, magic, cmd_type, payload_size,
binary_payload)
def continue_while_updates_pending(any_activity):
return compat.any(current_connection.writable()
for current_connection in connection_set)
def continue_while_updates_pending(any_activity):
return compat.any(current_connection.writable() for current_connection in connection_set)
def continue_while_jobs_pending(any_activity):
for current_request in job_requests:
if current_request.state == JOB_UNKNOWN:
self.send_job_request(current_request)
return compat.any(is_request_pending(current_request) for current_request in job_requests)