def _handle_error(self, error):
"""
Internal error handling method. Here we expect a socket.error coming in
and will handle different socket errors differently.
"""
# Handle version differences in Python
if hasattr(error, 'errno'): # Python >= 2.6
error_code = error.errno
else:
error_code = error[0] # Python <= 2.5
# Ok errors, just continue what we were doing before
if error_code in (errno.EWOULDBLOCK, errno.EAGAIN, errno.EINTR):
return
# Socket is closed, so lets just go to our handle_close method
elif error_code == errno.EBADF:
log.error("%s: Socket is closed", self.__class__.__name__)
else:
# Haven't run into this one yet, log it.
log.error("%s: Socket Error on %d: %s",
self.__class__.__name__,
self.socket.fileno(),
error_code)
# Disconnect from our IOLoop and let Connection know what's up
self._handle_disconnect()
def _flush_outbound(self):
try:
self._handle_write()
self._socket_timeouts = 0
except socket.timeout:
self._socket_timeouts += 1
if self._socket_timeouts > SOCKET_TIMEOUT_THRESHOLD:
log.error(SOCKET_TIMEOUT_MESSAGE)
self._handle_disconnect()
def _flush_outbound(self):
try:
self._handle_write()
self._socket_timeouts = 0
except socket.timeout:
self._socket_timeouts += 1
if self._socket_timeouts > SOCKET_TIMEOUT_THRESHOLD:
log.error(SOCKET_TIMEOUT_MESSAGE)
self._handle_disconnect()
def _on_basic_get_ok(self, method_frame, header_frame, body):
if self._on_get_ok_callback:
self._on_get_ok_callback(self,
method_frame.method,
header_frame.properties,
body)
self._basic_get_ok_callback = None
else:
log.error("%s._on_basic_get: No callback defined.",
self.__class__.__name__)
def process_data_events(self):
if not self.is_open:
raise AMQPConnectionError
self._flush_outbound()
try:
self._handle_read()
self._socket_timeouts = 0
except socket.timeout:
self._socket_timeouts += 1
if self._socket_timeouts > SOCKET_TIMEOUT_THRESHOLD:
log.error(SOCKET_TIMEOUT_MESSAGE)
self._handle_disconnect()
def _on_channel_flow_ok(self, frame):
"""
Called in response to us asking the server to toggle on Channel.Flow
"""
# Update the channel flow_active state
self.transport.flow_active = frame.method.active
# If we have a callback defined, process it
if self._on_flow_ok_callback:
self._on_flow_ok_callback(frame.method.active)
self._on_flow_ok_callback = None
else:
log.error("%s._on_flow_ok: No callback defined.",
self.__class__.__name__)
def process_data_events(self):
# Make sure we're open, if not raise the exception
if not self.is_open and not self.is_closing:
raise AMQPConnectionError
# Read data
try:
self._handle_read()
self._socket_timeouts = 0
except socket.timeout:
self._socket_timeouts += 1
if self._socket_timeouts > SOCKET_TIMEOUT_THRESHOLD:
log.error(SOCKET_TIMEOUT_MESSAGE)
self._handle_disconnect()
# Process our timeout events
self.process_timeouts()
# Write our data
self._flush_outbound()
def process_data_events(self):
# Make sure we're open, if not raise the exception
if not self.is_open and not self.is_closing:
raise AMQPConnectionError
# Read data
try:
self._handle_read()
self._socket_timeouts = 0
except socket.timeout:
self._socket_timeouts += 1
if self._socket_timeouts > SOCKET_TIMEOUT_THRESHOLD:
log.error(SOCKET_TIMEOUT_MESSAGE)
self._handle_disconnect()
# Process our timeout events
self.process_timeouts()
# Write our data
self._flush_outbound()
def _handle_events(self, fd, events, error=None):
"""
Our IO/Event loop have called us with events, so process them
"""
if not self.socket:
log.error("%s: Got events for closed stream %d", self.__class__.__name__, self.socket.fileno())
return
if events & READ:
self._handle_read()
if events & ERROR:
self._handle_error(error)
if events & WRITE:
self._handle_write()
# Call our event state manager who will decide if we reset our
# event state due to having an empty outbound buffer
self._manage_event_state()
def _handle_events(self, fd, events, error=None):
"""
Our IO/Event loop have called us with events, so process them
"""
if not self.socket:
log.error("%s: Got events for closed stream %d",
self.__class__.__name__, self.socket.fileno())
return
if events & READ:
self._handle_read()
if events & ERROR:
self._handle_error(error)
if events & WRITE:
self._handle_write()
# Call our event state manager who will decide if we reset our
# event state due to having an empty outbound buffer
self._manage_event_state()
def _handle_error(self, error):
"""
Internal error handling method. Here we expect a socket.error coming in
and will handle different socket errors differently.
"""
# Handle version differences in Python
if hasattr(error, 'errno'): # Python >= 2.6
error_code = errno.errorcode[error.errno]
elif error:
error_code = error[0] # Python <= 2.5
# Ok errors, just continue what we were doing before
if error_code in ERRORS_TO_IGNORE:
log.debug("Ignoring %s", error_code)
return None
# Socket is closed, so lets just go to our handle_close method
elif error_code == errno.EBADF:
log.error("%s: Socket is closed", self.__class__.__name__)
self._handle_disconnect()
elif self.parameters.ssl and isinstance(error, ssl.SSLError):
log.error(repr(error))
else:
# Haven't run into this one yet, log it.
log.error("%s: Socket Error on fd %d: %s",
self.__class__.__name__,
self.socket.fileno(),
error_code)
log.debug("Not handled?")
# Disconnect from our IOLoop and let Connection know what's up
self._handle_disconnect()
def process_cluster( self, cluster, hosts,
ts_stale_limit=graphite_min_cycle, ts=None ):
ts_now = ts or time()
if abs(int(cluster['LOCALTIME']) - ts_now) > ts_stale_limit:
log.warn(( 'Localtime for cluster {0[NAME]} ({0}) is way off'
' the local timestamp (limit: {1})' ).format(cluster, ts_stale_limit))
for host, metrics in hosts:
tn, ts_host = it.imap(int, op.itemgetter('TN', 'REPORTED')(host))
ts_host += tn
if tn > ts_stale_limit:
log.error(( 'Data for host {0[NAME]} ({0}) is too'
' stale (limit: {1}), skipping metrics for host' ).format(host, ts_stale_limit))
elif abs(ts_now - ts_host) > ts_stale_limit:
log.error(( '"Reported" timestamp for host {0[NAME]}'
' ({0}) is way off the local timestamp (limit: {1}),'
' skipping metrics for host' ).format(host, ts_stale_limit))
else:
process_name = ft.partial( self.process_name,
*it.imap(op.itemgetter('NAME'), [cluster, host]), ts=ts )
for metric in metrics:
name = process_name(metric['NAME'])
try: yield (name,) + self.process_metric(name, host, metric, ts=ts_host)
except self.IgnoreValue: pass
def _handle_error(self, error):
"""
Internal error handling method. Here we expect a socket.error coming in
and will handle different socket errors differently.
"""
# Handle version differences in Python
if hasattr(error, 'errno'): # Python >= 2.6
error_code = error.errno
elif error is not None:
error_code = error[0] # Python <= 2.5
else:
# This shouldn't happen, but log it in case it does
log.error("%s: Tried to handle an error where no error existed",
self.__class__.__name__)
# Cannot continue as error_code is not set
return
# Ok errors, just continue what we were doing before
if error_code in ERRORS_TO_IGNORE:
log.debug("Ignoring %s", error_code)
return None
# Socket is closed, so lets just go to our handle_close method
elif error_code in (errno.EBADF, errno.ECONNABORTED):
log.error("%s: Socket is closed",
self.__class__.__name__)
elif self.parameters.ssl and isinstance(error, ssl.SSLError):
# SSL socket operation needs to be retried
if error_code in (ssl.SSL_ERROR_WANT_READ,
ssl.SSL_ERROR_WANT_WRITE):
return None
else:
log.error("%s: SSL Socket error on fd %d: %s",
self.__class__.__name__,
self.socket.fileno(),
repr(error))
else:
# Haven't run into this one yet, log it.
log.error("%s: Socket Error on fd %d: %s",
self.__class__.__name__,
self.socket.fileno(),
error_code)
# Disconnect from our IOLoop and let Connection know what's up
self._handle_disconnect()
return None
def process_data_events(self, write_only=False):
# Make sure we're open, if not raise the exception
if not self.is_open and not self.is_closing:
raise AMQPConnectionError
# PATCH #0045: Fixes problem with recursing into a read callback
# while doing a write (in case you put a message back on your
# own queue)
if not write_only:
# Read data
try:
self._handle_read()
self._socket_timeouts = 0
except socket.timeout:
self._socket_timeouts += 1
if self._socket_timeouts > SOCKET_TIMEOUT_THRESHOLD:
log.error(SOCKET_TIMEOUT_MESSAGE)
self._handle_disconnect()
# Process our timeout events
self.process_timeouts()
# Write our data
self._flush_outbound()
def _handle_error(self, error):
"""
Internal error handling method. Here we expect a socket.error coming in
and will handle different socket errors differently.
"""
# Handle version differences in Python
if hasattr(error, 'errno'): # Python >= 2.6
error_code = error.errno
elif error is not None:
error_code = error[0] # Python <= 2.5
else:
# This shouldn't happen, but log it in case it does
log.error("%s: Tried to handle an error where no error existed",
self.__class__.__name__)
# Ok errors, just continue what we were doing before
if error_code in ERRORS_TO_IGNORE:
log.debug("Ignoring %s", error_code)
return None
# Socket is closed, so lets just go to our handle_close method
elif error_code in (errno.EBADF, errno.ECONNABORTED):
log.error("%s: Socket is closed",
self.__class__.__name__)
elif self.parameters.ssl and isinstance(error, ssl.SSLError):
# SSL socket operation needs to be retried
if error_code in (ssl.SSL_ERROR_WANT_READ,
ssl.SSL_ERROR_WANT_WRITE):
return None
else:
log.error("%s: SSL Socket error on fd %d: %s",
self.__class__.__name__,
self.socket.fileno(),
repr(error))
else:
# Haven't run into this one yet, log it.
log.error("%s: Socket Error on fd %d: %s",
self.__class__.__name__,
self.socket.fileno(),
error_code)
# Disconnect from our IOLoop and let Connection know what's up
self._handle_disconnect()
return None
class AsyncoreDispatcher(asyncore.dispatcher):
"""
We extend asyncore.dispatcher here and throw in everything we need to
handle both asyncore's needs and pika's. In the async adapter structure
we expect a ioloop behavior which includes timeouts and a start and stop
function.
"""
def __init__(self, parameters):
"""
Initialize the dispatcher, socket and our defaults. We turn of nageling
in the socket to allow for faster throughput.
"""
asyncore.dispatcher.__init__(self)
# Carry the parameters for this as well
self.parameters = parameters
# Setup defaults
self.connecting = True
self.connection = None
self._timeouts = dict()
self.writable_ = False
self.map = None
# Set our remaining attempts to the value or True if it's none
remaining_attempts = self.parameters.connection_attempts or True
# Loop while we have remaining attempts
while remaining_attempts:
try:
return self._socket_connect()
except socket.error, err:
remaining_attempts -= 1
if not remaining_attempts:
break
log.warning(
"Could not connect: %s. Retrying in %i seconds \
with %i retry(s) left", err[-1], self.parameters.retry_delay,
remaining_attempts)
self.socket.close()
sleep(self.parameters.retry_delay)
# Log the errors and raise the exception
log.error("Could not connect: %s", err[-1])
raise AMQPConnectionError(err[-1])
def _check_state_on_disconnect(self):
"""
Checks to see if we were in opening a connection with RabbitMQ when
we were disconnected and raises exceptions for the anticipated
exception types.
"""
if self.connection_state == CONNECTION_PROTOCOL:
log.error("Incompatible Protocol Versions")
raise IncompatibleProtocolError
elif self.connection_state == CONNECTION_START:
log.error("Socket closed while authenticating indicating a \
probable authentication error")
raise ProbableAuthenticationError
elif self.connection_state == CONNECTION_TUNE:
log.error("Socket closed while tuning the connection indicating a \
probable permission error when accessing a virtual host")
raise ProbableAccessDeniedError
def _check_state_on_disconnect(self):
"""
Checks to see if we were in opening a connection with RabbitMQ when
we were disconnected and raises exceptions for the anticipated
exception types.
"""
if self.connection_state == CONNECTION_PROTOCOL:
log.error("Incompatible Protocol Versions")
raise IncompatibleProtocolError
elif self.connection_state == CONNECTION_START:
log.error("Socket closed while authenticating indicating a \
probable authentication error")
raise ProbableAuthenticationError
elif self.connection_state == CONNECTION_TUNE:
log.error("Socket closed while tuning the connection indicating a \
probable permission error when accessing a virtual host")
raise ProbableAccessDeniedError
except socket.error, err:
reason = err[-1]
self.socket.close()
retry = ''
if remaining_attempts:
retry = "Retrying in %i seconds with %i retry(s) left" % \
(self.parameters.retry_delay, remaining_attempts)
log.warning("Could not connect: %s. %s", reason, retry)
if remaining_attempts:
time.sleep(self.parameters.retry_delay)
# Log the errors and raise the exception
log.error("Could not connect: %s", reason)
raise AMQPConnectionError(reason)
def add_timeout(self, deadline, callback):
return self.ioloop.add_timeout(deadline, callback)
def remove_timeout(self, timeout_id):
self.ioloop.remove_timeout(timeout_id)
def _erase_credentials(self):
pass
def _flush_outbound(self):
"""
Call the state manager who will figure out that we need to write.
"""
class BaseConnection(Connection):
def __init__(self, parameters=None,
on_open_callback=None,
reconnection_strategy=None):
# Let the developer know we could not import SSL
if parameters.ssl and not SSL:
raise Exception("SSL specified but it is not available")
# Set our defaults
self.fd = None
self.ioloop = None
self.socket = None
# Event states (base and current)
self.base_events = READ | ERROR
self.event_state = self.base_events
# Call our parent's __init__
Connection.__init__(self, parameters, on_open_callback,
reconnection_strategy)
def _socket_connect(self):
"""Create socket and connect to it, using SSL if enabled"""
# Create our socket and set our socket options
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
self.socket.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
# Wrap the SSL socket if we SSL turned on
ssl_text = ""
if self.parameters.ssl:
ssl_text = " with SSL"
if self.parameters.ssl_options:
self.socket = ssl.wrap_socket(self.socket,
**self.parameters.ssl_options)
else:
self.socket = ssl.wrap_socket(self.socket)
# Try and connect
log.info("Connecting to %s:%i%s", self.parameters.host,
self.parameters.port, ssl_text)
self.socket.connect((self.parameters.host, self.parameters.port))
# Set the socket to non-blocking
self.socket.setblocking(0)
def _adapter_connect(self):
"""
Base connection function to be extended as needed
"""
# Set our remaining attempts to the value or True if it's none
remaining_attempts = self.parameters.connection_attempts or True
# Loop while we have remaining attempts
while remaining_attempts:
try:
return self._socket_connect()
except socket.error, err:
remaining_attempts -= 1
if not remaining_attempts:
break
log.warning("Could not connect: %s. Retrying in %i seconds \
with %i retry(s) left",
err[-1], self.parameters.retry_delay,
remaining_attempts)
self.socket.close()
sleep(self.parameters.retry_delay)
# Log the errors and raise the exception
log.error("Could not connect: %s", err[-1])
raise AMQPConnectionError(err[-1])
except socket.error, err:
reason = err[-1]
self.socket.close()
retry = ''
if remaining_attempts:
retry = "Retrying in %i seconds with %i \
retry(s) left" % (self.parameters.retry_delay, remaining_attempts)
log.warning("Could not connect: %s. %s", reason, retry)
if remaining_attempts:
sleep(self.parameters.retry_delay)
# Log the errors and raise the exception
log.error("Could not connect: %s", err[-1])
raise AMQPConnectionError(err[-1])
def add_timeout(self, delay_sec, callback):
deadline = time() + delay_sec
return self.ioloop.add_timeout(deadline, callback)
def remove_timeout(self, timeout_id):
self.ioloop.remove_timeout(timeout_id)
def _erase_credentials(self):
pass
def _flush_outbound(self):
"""
Call the state manager who will figure out that we need to write.
except socket.error, err:
reason = err[-1]
self.socket.close()
retry = ''
if remaining_attempts:
retry = "Retrying in %i seconds with %i retry(s) left" % \
(self.parameters.retry_delay, remaining_attempts)
log.warning("Could not connect: %s. %s", reason, retry)
if remaining_attempts:
time.sleep(self.parameters.retry_delay)
# Log the errors and raise the exception
log.error("Could not connect: %s", reason)
raise AMQPConnectionError(reason)
def add_timeout(self, deadline, callback):
return self.ioloop.add_timeout(deadline, callback)
def remove_timeout(self, timeout_id):
self.ioloop.remove_timeout(timeout_id)
def _erase_credentials(self):
pass
def _flush_outbound(self):
"""
Call the state manager who will figure out that we need to write.
"""