def start(self):
"""Start worker."""
connected = False
# Define connection
while (not connected):
try:
self._creds = self._amqp_url.split('amqp://')[1].split("@")[0]
parameters = URLParameters(self._amqp_url)
self._host = parameters.host
self._port = "15672"
self._vhost = parameters.virtual_host
self._blocking_connection = BlockingConnection(parameters)
self._connection = SelectConnection(
URLParameters(self._amqp_url),
on_open_callback=self.on_open_connection)
self._connection.ioloop.start()
connected = True
# Catch a Keyboard Interrupt to make sure that the connection is closed cleanly
except KeyboardInterrupt:
# Gracefully close the connection
self._connection.close()
# Start the IOLoop again so Pika can communicate, it will stop on its own when the connection is closed
self._connection.ioloop.start()
except:
if (self._debug):
print(
" [!] RabbitMQ Host Unrecheable. Reconnecting in {} seconds..."
.format(self._reconection_time))
time.sleep(self._reconection_time)
class RabbitMQConsumer():
def __init__(self, options={}):
self.options = options
self.connection = SelectConnection(on_open_callback=self.on_connected)
def on_connected(self, connection):
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, new_channel):
self.channel = new_channel
self.channel.queue_declare(queue=self.options['queue'],
durable=self.options['durable'],
exclusive=self.options['exclusive'],
auto_delete=self.options['auto_delete'],
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
self.channel.basic_consume(self.handle_delivery,
queue=self.options['queue'])
def handle_delivery(self, channel, method, header, body):
self.options['on_receive'](body)
self.channel.basic_ack(method.delivery_tag)
def start(self):
self.connection.ioloop.start()
def stop(self):
self.connection.close()
def on_connected(self, connection: SelectConnection):
"""Called when the connection to the message broker is completed
Args:
connection: The SelectConnection instance, representing the achieved connection with the
broker.
"""
connection.channel(on_open_callback=self.on_channel_open)
def consume(self):
try:
self.connection = SelectConnection(self.parameters,
self._on_connected)
self.connection.ioloop.start()
except Exception as e:
self.logger.error('{} {}'.format(self.consumer_id, str(e)))
self.connection.close()
self.connection.ioloop.start()
def channel(self):
x = []
def on_open(ch):
x.append(ch)
SelectConnection.channel(self, on_open)
while not x:
self.ioloop.poller.poll()
return x[0]
def channel(self):
x = []
def on_open(ch):
x.append(ch)
SelectConnection.channel(self, on_open)
while not x:
self.ioloop.poller.poll()
return x[0]
def __init__(self, *args, **nargs):
x = []
def on_open(connection):
x.append(0)
nargs['on_open_callback'] = on_open
nargs['on_close_callback'] = self._my_on_close
SelectConnection.__init__(self, *args, **nargs)
while not x:
self.ioloop.poller.poll()
def __init__(self, *args, **nargs):
x = []
def on_open(connection):
x.append(0)
nargs['on_open_callback'] = on_open
nargs['on_close_callback'] = self._my_on_close
SelectConnection.__init__(self, *args, **nargs)
while not x:
self.ioloop.poller.poll()
def __init__(self, host, port, virtual_host, username, password,
exchange, routing_key, message):
self.channel = None
self.exchange = exchange
self.routing_key = routing_key
self.message = message
credentials = PlainCredentials(
username, password, erase_on_connect=False)
parameters = ConnectionParameters(
host, port, virtual_host, credentials=credentials)
self.connection = SelectConnection(
parameters=parameters, on_open_callback=self.on_connect)
class BasicPublish(object):
"""See also: http://pika.github.com/examples.html"""
def __init__(self, host, port, virtual_host, username, password,
exchange, routing_key, message):
self.channel = None
self.exchange = exchange
self.routing_key = routing_key
self.message = message
credentials = PlainCredentials(
username, password, erase_on_connect=False)
parameters = ConnectionParameters(
host, port, virtual_host, credentials=credentials)
self.connection = SelectConnection(
parameters=parameters, on_open_callback=self.on_connect)
def on_connect(self, connection):
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
self.channel = channel
self.channel.exchange_declare(exchange=self.exchange,
type='direct', durable=True,
auto_delete=False,
callback=self.on_exchange_declared)
def on_exchange_declared(self, frame):
self.channel.queue_declare(queue=self.routing_key, durable=True,
exclusive=False, auto_delete=False,
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
self.channel.queue_bind(exchange=self.exchange,
queue=self.routing_key,
routing_key=self.routing_key,
callback=self.on_queue_bound)
def on_queue_bound(self, frame):
properties = BasicProperties(content_type='text/plain',
delivery_mode=1)
self.channel.basic_publish(exchange=self.exchange,
routing_key=self.routing_key,
body=self.message,
properties=properties)
self.connection.close()
def __init__(self, config, callback):
self.reconnection_delay = 1.0
self.caller_callback = callback
self.config = ConnectionConfig()
self.config.read(config)
self.callbacks = self.set_callbacks()
credentials = PlainCredentials(**self.config.credentials)
broker_config = self.config.broker_config
broker_config['credentials'] = credentials
parameters = ConnectionParameters(**broker_config)
try:
parameters.host = self.config.host
except NoOptionError:
pass
try:
parameters.heartbeat = int(self.config.heartbeat)
except NoOptionError:
pass
try:
parameters.heartbeat = int(self.config.heartbeat)
except NoOptionError:
pass
self.connection = SelectConnection(parameters, self.on_connected)
def connect(self):
"""This method connects to RabbitMQ, returning the connection handle.
When the connection is established, the on_connection_open method
will be invoked by pika.
:rtype: pika.SelectConnection
"""
LOGGER.info("Connecting to %s", self._server_details.host)
credentials = PlainCredentials(self._server_details.username,
self._server_details.password)
connection_params = ConnectionParameters(
host=self._server_details.host,
port=self._server_details.port,
virtual_host=self._server_details.vhost,
credentials=credentials,
)
if self._server_details.uses_ssl:
cafile = os.getenv("REQUESTS_CA_BUNDLE")
ssl_context = ssl.create_default_context(cafile=cafile)
connection_params.ssl_options = SSLOptions(ssl_context)
return SelectConnection(
parameters=connection_params,
on_open_callback=self.on_connection_open,
on_open_error_callback=self.on_connection_open_error,
on_close_callback=self.on_connection_closed,
)
def run(self):
credentials = pika.PlainCredentials(self.username, self.password)
ssl_options = None
port = self.port
if self.use_ssl:
port = self.ssl_port
if self.use_ssl and self.ssl_auth:
ssl_options = {
"ca_certs": self.cacertfile,
"certfile": self.certfile,
"keyfile": self.keyfile,
"cert_reqs": CERT_REQUIRED
}
credentials = ExternalCredentials()
try:
parameters = pika.ConnectionParameters(host=self.endpoint,
port=port,
virtual_host=self.vhost,
credentials=credentials,
ssl=self.use_ssl,
ssl_options=ssl_options)
except TypeError:
parameters = pika.ConnectionParameters(host=self.endpoint,
port=port,
virtual_host=self.vhost,
credentials=credentials)
self.logger.debug("Connecting to %s@%s:%s on vhost %s" %
(self.username, self.endpoint, port, self.vhost))
try:
SelectPoller.TIMEOUT = .1
connection = SelectConnection(parameters)
connection.add_on_close_callback(self._on_closed())
connection.callbacks.add(0,
'_on_connection_open',
self._on_connected(),
one_shot=True)
self.connection = connection
connection.ioloop.start()
except IOError, err:
if not self.manual_close:
self.logger.error("Connection error: {0}".format(err))
def connect(self):
"""
:param parameters:
:return:
"""
self.connection = SelectConnection(self.parameters, self.on_connected,
stop_ioloop_on_close=False)
def run(self):
credentials = pika.PlainCredentials(self.username, self.password)
ssl_options = None
port = self.port
if self.use_ssl:
port = self.ssl_port
if self.use_ssl and self.ssl_auth:
ssl_options = {
"ca_certs": self.cacertfile,
"certfile": self.certfile,
"keyfile": self.keyfile,
"cert_reqs": CERT_REQUIRED
}
credentials = ExternalCredentials()
try:
parameters = pika.ConnectionParameters(host=self.endpoint,
port=port,
virtual_host=self.vhost,
credentials=credentials,
ssl=self.use_ssl,
ssl_options=ssl_options)
except TypeError:
parameters = pika.ConnectionParameters(host=self.endpoint,
port=port,
virtual_host=self.vhost,
credentials=credentials)
self.logger.debug("Connecting to %s@%s:%s on vhost %s" % (
self.username, self.endpoint, port, self.vhost))
try:
SelectPoller.TIMEOUT = .1
connection = SelectConnection(parameters)
connection.add_on_close_callback(self._on_closed())
connection.callbacks.add(0, '_on_connection_open',
self._on_connected(), one_shot=True)
self.connection = connection
connection.ioloop.start()
except IOError, err:
if not self.manual_close:
self.logger.error("Connection error: {0}".format(err))
def connect(self):
""" Creating connection object """
url = self.get_url()
logger.info('Connecting to %s', url)
return SelectConnection(
URLParameters(url),
self.on_connection_open,
stop_ioloop_on_close=False
)
def _connect(self):
"""
Assigns a new pika.SelectConnection to the connection and starts its
ioloop to begin connecting.
"""
if self.connection_parameters is not None:
LOGGER.info(f"starting connection towards: "
f"{self.connection_parameters.host}:"
f"{self.connection_parameters.port}")
else:
LOGGER.info("starting connection towards: 127.0.0.1:5672")
self._connection = SelectConnection(
parameters=self.connection_parameters,
on_open_callback=self.on_connection_open,
on_open_error_callback=self.on_connection_open_error,
on_close_callback=self.on_connection_closed)
self._connection.ioloop.start()
def start(self):
"""Start worker."""
# Define connection
while True:
try:
self._connection = SelectConnection(
URLParameters(self._amqp_url),
on_open_callback=self.on_open_connection)
self._connection.ioloop.start()
# Catch a Keyboard Interrupt to make sure that the connection is closed cleanly
except KeyboardInterrupt:
# Gracefully close the connection
self._connection.close()
# Start the IOLoop again so Pika can communicate, it will stop on its own when the connection is closed
self._connection.ioloop.start()
except:
if (self._debug):
print(
" [!] RabbitMQ Host Unreachable. Reconnecting in {} seconds..."
.format(self._reconnection_time))
sleep(self._reconnection_time)
def _init_connection(self, connection_parameters: ConnectionParameters) -> BaseConnection:
"""Initializes a BaseConnection to be used by the listener.
Args:
connection_parameters: A set up ConnectionParameters instance.
Returns:
The set up BaseConnection instance.
"""
connection = SelectConnection(connection_parameters, on_open_callback=self.on_connected)
return connection
def connect(self):
"""This method connects to RabbitMQ, returning the connection handle.
When the connection is established, the on_connection_open method
will be invoked by pika.
:rtype: pika.SelectConnection
"""
url = self.get_url()
logger.info('Connecting to %s', url)
return SelectConnection(
parameters=URLParameters(url),
on_open_callback=self.on_connection_open,
on_open_error_callback=self.on_connection_open_error,
on_close_callback=self.on_connection_closed)
def _close_connection(self, connection: SelectConnection):
"""Closes a given `SelectConnection` instance.
It will call the connection's `close` method, and then `ioloop.start`, to try keep looping
until it is fully closed, as suggested by pika's documentation:
https://pika.readthedocs.io/en/stable/intro.html#io-and-event-looping
Args:
connection: The connection to close.
"""
if not connection:
return
try:
# Gracefully close the connection
connection.close()
# Loop until we're fully closed, will stop on its own
connection.ioloop.start()
except AMQPError as error:
self.logger.error(
"Error while closing connection on %s: %s.", self._queue_name, str(error),
)
def connect(self, host=None):
"""
Connection with Rabbit.
"""
if not host:
_, host = list(json.loads(sd_rabbit.rabbit_nodes).items())[0]
return SelectConnection(ConnectionParameters(
host=host,
port=5672,
virtual_host=VIRTUAL_HOST,
credentials=sd_rabbit.rabbit_credential),
on_open_callback=self.on_connected,
on_close_callback=self.on_closed)
def open_connection(self):
"""Opens a connection to RabbitMQ
This method immediately returns the connection object. However, whether the
connection was successful is not know until a callback is invoked (either
on_open_callback or on_open_error_callback).
Returns:
The SelectConnection object
"""
return SelectConnection(
parameters=self._connection_parameters,
on_open_callback=self.on_connection_open,
on_close_callback=self.on_connection_closed,
on_open_error_callback=self.on_connection_closed,
)
def test_publish(publisher, monkeypatch):
with monkeypatch.context() as m:
publisher._channel = Channel(SelectConnection(), 1, lambda: 1)
m.setattr(publisher._channel, '_state', Channel.OPEN)
mock_publish = Mock()
m.setattr(publisher._channel, 'basic_publish', mock_publish)
item = {'url': 'https://www.example.com/page'}
publisher._publish(item)
mock_publish.assert_called_once_with(
exchange='exchange',
routing_key='exchange.queue',
body=json.dumps(item, ensure_ascii=False),
properties=BasicProperties(app_id='app',
content_type='application/json',
delivery_mode=1),
mandatory=True)
def _connect(self):
# type: (Consumer) -> SelectConnection
""" Connect to RabbitMQ
:return pika.adapters.SelectConnection
"""
param = pika.URLParameters(self.amqp_url)
logger.info('Connecting to: {host}:{port}/{vhost}; SSL: {ssl}'.format(
host=param.host,
port=param.port,
vhost=param.virtual_host,
ssl=param.ssl))
return SelectConnection(
parameters=param,
on_open_callback=self._on_connection_open,
on_open_error_callback=self._on_open_connection_error,
on_close_callback=self._on_connection_closed,
stop_ioloop_on_close=False)
def get(self):
queue_name = "s_connection"
def on_channel(connection):
connection.channel(on_declare)
def on_declare(channel):
channel.queue_declare(publisher(channel), queue_name)
def publisher(channel):
channel.basic_publish(exchange='',
routing_key=queue_name,
body='[%s]It is asyn publish')
connection = SelectConnection(ConnectionParameters('localhost'), on_channel)
# connection = BlockingConnection(ConnectionParameters())
# channel = connection.channel()
# channel.queue_declare(queue=queue_name)
# channel.basic_publish(exchange='', routing_key=queue_name, body='sss')
self.write("[Tornado asyn produce] OK")
class BrokerConnection(object):
"""Connection class which provides connection to AMQP broker."""
def __init__(self, config, callback):
self.reconnection_delay = 1.0
self.caller_callback = callback
self.config = ConnectionConfig()
self.config.read(config)
self.callbacks = self.set_callbacks()
credentials = PlainCredentials(**self.config.credentials)
broker_config = self.config.broker_config
broker_config['credentials'] = credentials
parameters = ConnectionParameters(**broker_config)
try:
parameters.host = self.config.host
except NoOptionError:
pass
try:
parameters.heartbeat = int(self.config.heartbeat)
except NoOptionError:
pass
try:
parameters.heartbeat = int(self.config.heartbeat)
except NoOptionError:
pass
self.connection = SelectConnection(parameters, self.on_connected)
def on_connected(self, connection):
"""Called when we're connected to AMQP broker."""
logger.info("Connected to AMQP-broker.")
connection.channel(self.on_channel_open)
def on_channel_open(self, new_channel):
"""Called when channel has opened"""
logger.info("Channel to AMQP-broker opened.")
self.channel = new_channel
self.channel.add_on_close_callback(self.on_channel_closed)
self.generic_callback()
def set_callbacks(self):
"""Set callbacks for queue factory."""
self.exchange_callbacks = []
self.queue_callbacks = []
self.binding_callbacks = []
for exchange in self.config.exchanges:
tmp = {}
tmp['exchange'] = exchange
tmp.update(self.config.exchanges[exchange])
self.exchange_callbacks.append(tmp)
for queue in self.config.queues:
tmp = {}
tmp['queue'] = queue
tmp.update(self.config.queues[queue])
self.queue_callbacks.append(tmp)
for binding in self.config.bindings:
tmp = {}
tmp['binding'] = binding
tmp.update(self.config.bindings[binding])
self.binding_callbacks.append(tmp)
def generic_callback(self, channel=None, frame=None):
"""Create exchanges, queues and bindings."""
if channel and frame:
# You could do error handling here
# or add code for anonymous queue handling
pass
if self.exchange_callbacks:
config = self.exchange_callbacks.pop()
config['exchange'] = config['exchange'].split(':', 1)[-1]
logger.info("Creating exchange %s." % config['exchange'])
config.update({"callback": self.generic_callback})
self.channel.exchange_declare(**config)
elif self.queue_callbacks:
config = self.queue_callbacks.pop()
config['queue'] = config['queue'].split(':', 1)[-1]
logger.info("Creating queue %s." % config['queue'])
config.update({"callback": self.generic_callback})
self.channel.queue_declare(**config)
elif self.binding_callbacks:
config = self.binding_callbacks.pop()
binding = config['binding'].split(':')
del config['binding']
config['exchange'] = binding[-1]
config['queue'] = binding[1]
logger.info("Creating binding (%s -> %s) with routing key %s" % (
config['exchange'], config['queue'],
config['routing_key']))
config.update({"callback": self.generic_callback})
self.channel.queue_bind(**config)
else:
logger.info("RabbitMQ exchanges, queues and bindings are now "
"configured.")
self.caller_callback()
def reset_reconnection_delay(self, *args):
self.reconnection_delay = 1.0
def loop(self):
"""Main loop"""
while True:
try:
logger.info("Starting main loop")
self.connection.add_on_open_callback(
self.reset_reconnection_delay)
self.connection.ioloop.start()
except socket.error as e:
logger.error("Connection failed or closed unexpectedly: %s", e)
except TypeError as e:
logger.error("Connection failed or closed unexpectedly: %s", e)
except KeyboardInterrupt:
self.connection.close()
self.connection.ioloop.start()
break
finally:
self.reconnection_delay *= (random.random() * 0.5) + 1
self.reconnection_delay = min(self.reconnection_delay, 60.0)
logger.info("Trying reconnection in %s seconds",
self.reconnection_delay)
time.sleep(self.reconnection_delay)
def on_channel_closed(self, channel, code, text):
logger.warning("Channel closed with reason '%s %s'",
code, text)
self.connection.close(code, text)
# Delete the row
elif message['operation'] == 'delete':
data = {'row_id': message['data']['row_id']}
self.cursor.execute(DELETE_QUERY, data)
# Ack the message
channel.basic_ack(delivery_tag=method_frame.delivery_tag)
if __name__ == '__main__':
consumer = ExampleConsumer()
# Connect to RabbitMQ
host = (len(sys.argv) > 1) and sys.argv[1] or '127.0.0.1'
connection = SelectConnection(ConnectionParameters(host),
consumer._on_connected)
# Loop until CTRL-C
try:
# Start our blocking loop
connection.ioloop.start()
except KeyboardInterrupt:
# Close the connection
connection.close()
# Loop until the conneciton is closed
connection.ioloop.start()
class Updater(object):
def __init__(self, debug=True, prefetch_count=1):
self._path = environ['DIR_TILES']
self._debug = debug
self._res_queue = environ['RES_QUEUE']
self._res_xchg = environ['RES_XCHG']
self._amqp_url = environ['AMQP_URL']
self._prefetch_count = prefetch_count
self._reconnection_time = int(environ['TIMEOUT'])
def base64_to_img(self, buff, fpath):
with open(fpath, 'wb') as f:
b64data = buff.split(",")[1]
data = b64decode(b64data.encode("utf-8", "ignore"))
f.write(data)
def callback_res_queue(self, _unused_channel, delivery, properties, body):
"""
Callback triggered when message is received.
"""
if (self._debug):
print(" [x] Received - Body: {}".format(body[:140]))
start_time = time()
data = loads(body)
path = data['earthImage']['path']
if ('filteredImages' in data):
if (self._debug):
print(" [x] Saving Images...")
for image in data['filteredImages']:
new_path = Path(path.replace('RAW', image['filterName']))
dirs = new_path.parts[:-1]
# create dirs
Path("/".join(dirs)).mkdir(parents=True, exist_ok=True)
self.base64_to_img(image['vectorImage'], new_path)
self._channel.basic_ack(delivery_tag=delivery.delivery_tag)
def on_open_connection(self, _unused_frame):
if (self._debug):
print(" [x] Connected - Connection state: OPEN. ")
self._connection.channel(on_open_callback=self.on_channel_open)
def on_channel_open(self, channel):
"""Callback when we have successfully declared the channel."""
if (self._debug):
print(" [x] Channel - Channel state: OPEN. ")
self._channel = channel
self._channel.exchange_declare(exchange=self._res_xchg,
exchange_type='fanout')
cb = partial(self.on_queue_declareok, userdata=self._res_queue)
self._channel.queue_declare(queue=self._res_queue,
durable=True,
callback=cb)
def on_queue_declareok(self, _unused_frame, userdata):
"""Callback when we have successfully declared the queue.
This call tells the server to send us self._prefetch_count message in advance.
This helps overall throughput, but it does require us to deal with the messages
we have promptly.
"""
queue_name = userdata
if (self._debug):
print(" [x] Queue declared - Queue: {}".format(queue_name))
self._channel.basic_qos(prefetch_count=self._prefetch_count,
callback=self.on_qos)
self._channel.queue_bind(exchange=self._res_xchg, queue=queue_name)
def on_qos(self, _unused_frame):
"""Callback when Basic.QOS has completed."""
if (self._debug):
print(" [x] QOS set to {}".format(self._prefetch_count))
self.start_consuming()
def start_consuming(self):
"""Start consuming from task queue."""
if (self._debug):
print(" [x] Waiting for messages...")
self._channel.basic_consume(self._res_queue, self.callback_res_queue)
def start(self):
"""Start worker."""
# Define connection
while True:
try:
self._connection = SelectConnection(
URLParameters(self._amqp_url),
on_open_callback=self.on_open_connection)
self._connection.ioloop.start()
# Catch a Keyboard Interrupt to make sure that the connection is closed cleanly
except KeyboardInterrupt:
# Gracefully close the connection
self._connection.close()
# Start the IOLoop again so Pika can communicate, it will stop on its own when the connection is closed
self._connection.ioloop.start()
except:
if (self._debug):
print(
" [!] RabbitMQ Host Unreachable. Reconnecting in {} seconds..."
.format(self._reconnection_time))
sleep(self._reconnection_time)
class BrokerConnection(object):
"""Connection class which provides connection to AMQP broker."""
exchange_callbacks = []
queue_callbacks = []
binding_callbacks = []
channel = None
def __init__(self, config, callback):
self.closing = False
self.channel = None
self.connection = None
self.reconnection_delay = 1.0
self.caller_callback = callback
if isinstance(config, ConnectionConfig):
self.config = config
else:
self.config = ConnectionConfig()
self.config.read(config)
self.set_callbacks()
credentials = PlainCredentials(**self.config.credentials)
broker_config = self.config.broker_config
broker_config['credentials'] = credentials
self.parameters = ConnectionParameters(**broker_config)
try:
self.parameters.host = self.config.host
except NoOptionError:
pass
try:
self.parameters.heartbeat_interval = int(
self.config.heartbeat_interval)
except NoOptionError:
pass
self.connect()
def connect(self):
"""
:param parameters:
:return:
"""
self.connection = SelectConnection(self.parameters, self.on_connected,
stop_ioloop_on_close=False)
def on_connected(self, connection):
"""Called when we're connected to AMQP broker."""
print("Connected to AMQP-broker.")
self.add_on_connection_close_callback()
connection.channel(self.on_channel_open)
def add_on_connection_close_callback(self):
"""This method adds an on close callback that will be invoked by pika
when RabbitMQ closes the connection to the publisher unexpectedly.
"""
print('Adding connection close callback')
self.connection.add_on_close_callback(self.on_connection_closed)
def on_connection_closed(self, connection, reply_code, reply_text):
"""This method is invoked by pika when the connection to RabbitMQ is
closed unexpectedly. Since it is unexpected, we will reconnect to
RabbitMQ if it disconnects.
:param pika.connection.Connection connection: The closed connection obj
:param int reply_code: The server provided reply_code if given
:param str reply_text: The server provided reply_text if given
"""
self.channel = None
if self.closing:
self.connection.ioloop.stop()
else:
print('Connection closed, reopening in 5 seconds: (%s) %s',
reply_code, reply_text)
self.connection.add_timeout(5, self.reconnect)
def on_channel_open(self, new_channel):
"""Called when channel has opened"""
logger.info("Channel to AMQP-broker opened.")
self.channel = new_channel
self.channel.add_on_close_callback(self.on_channel_closed)
self.generic_callback()
def set_callbacks(self):
"""Set callbacks for queue factory."""
for exchange in self.config.exchanges:
tmp = {'exchange': exchange}
tmp.update(self.config.exchanges[exchange])
self.exchange_callbacks.append(tmp)
for queue in self.config.queues:
tmp = {'queue': queue}
tmp.update(self.config.queues[queue])
self.queue_callbacks.append(tmp)
for binding in self.config.bindings:
tmp = {'binding': binding}
tmp.update(self.config.bindings[binding])
self.binding_callbacks.append(tmp)
def generic_callback(self, channel=None, frame=None):
"""Create exchanges, queues and bindings."""
if channel and frame:
# You could do error handling here
# or add code for anonymous queue handling
pass
if self.exchange_callbacks:
config = self.exchange_callbacks.pop()
config['exchange'] = config['exchange'].split(':', 1)[-1]
logger.info("Creating exchange %s." % config['exchange'])
config.update({"callback": self.generic_callback})
self.channel.exchange_declare(**config)
elif self.queue_callbacks:
config = self.queue_callbacks.pop()
config['queue'] = config['queue'].split(':', 1)[-1]
logger.info("Creating queue %s." % config['queue'])
config.update({"callback": self.generic_callback})
self.channel.queue_declare(**config)
elif self.binding_callbacks:
config = self.binding_callbacks.pop()
binding = config['binding'].split(':')
del config['binding']
config['exchange'] = binding[-1]
config['queue'] = binding[1]
logger.info("Creating binding (%s -> %s) with routing key %s" % (
config['exchange'], config['queue'],
config['routing_key']))
config.update({"callback": self.generic_callback})
self.channel.queue_bind(**config)
else:
logger.info("RabbitMQ exchanges, queues and bindings are now "
"configured.")
self.caller_callback()
def reset_reconnection_delay(self, *args):
self.reconnection_delay = 1.0
def loop(self):
"""Main loop"""
while True:
try:
logger.info("Starting main loop")
self.connection.add_on_open_callback(
self.reset_reconnection_delay)
self.connection.ioloop.start()
except socket.error as e:
logger.error("Connection failed or closed unexpectedly: %s", e,
exc_info=True)
except TypeError as e:
logger.error("Connection failed or closed unexpectedly: %s", e,
exc_info=True)
except KeyboardInterrupt:
self.connection.close()
self.connection.ioloop.start()
break
finally:
self.reconnection_delay *= (random.random() * 0.5) + 1
self.reconnection_delay = min(self.reconnection_delay, 60.0)
logger.info("Trying reconnection in %s seconds",
self.reconnection_delay)
time.sleep(self.reconnection_delay)
def on_channel_closed(self, channel, code, text):
logger.warning("Channel closed with reason '%s %s'",
code, text)
self.connection.close(code, text)
def reconnect(self):
"""Will be invoked by the IOLoop timer if the connection is
closed. See the on_connection_closed method.
"""
# This is the old connection IOLoop instance, stop its ioloop
self.connection.ioloop.stop()
if not self.closing:
# Create a new connection
self.connect()
# There is now a new connection, needs a new ioloop to run
self.connection.ioloop.start()
def add_on_cancel_callback(self):
"""Add a callback that will be invoked if RabbitMQ cancels the consumer
for some reason. If RabbitMQ does cancel the consumer,
on_consumer_cancelled will be invoked by pika.
"""
print('Adding consumer cancellation callback')
self.channel.add_on_cancel_callback(self.on_consumer_cancelled)
def on_consumer_cancelled(self, method_frame):
"""
Invoked by pika when RabbitMQ sends a Basic.Cancel for a consumer
receiving messages.
:param pika.frame.Method method_frame: The Basic.Cancel frame
"""
print('Consumer was cancelled remotely, shutting down: %r' %
method_frame)
if self.channel:
self.channel.close()
def __init__(self, queue: str, connection: pika.SelectConnection):
assert queue is not None and connection is not None
self.channel = connection.channel()
self.queue = queue
QUEUE = 'xxx'
def on_connection_open(connection):
connection.channel(on_channel_open)
def on_channel_open(channel):
global CHANNEL
CHANNEL = channel
CHANNEL.queue_declare(queue=QUEUE, callback=on_queue_declared)
def on_queue_declared(frame):
count = 0
while True:
body = str(count)
CHANNEL.basic_publish(exchange='', routing_key=QUEUE, body=body)
print 'Sent %s' % body
sleep(3)
count += 1
parameters = ConnectionParameters()
connection = SelectConnection(parameters, on_open_callback=on_connection_open)
try:
connection.ioloop.start()
except KeyboardInterrupt:
connection.close()
connection.ioloop.start()
def __init__(self, options={}):
self.options = options
self.connection = SelectConnection(on_open_callback=self.on_connected)
class QueueConsumer(object):
"""The consumer class to manage connections to the AMQP server/queue"""
def __init__(self, queue, logger, parameters, thread_id=0):
self.channel = None
self.connection = None
self.queue_name = queue
self.logger = logger
self.consumer_id = 'Thread: %d' % (thread_id, )
self.parameters = pika.ConnectionParameters(**parameters)
def _on_queue_declared(self, frame):
self.logger.debug('{} ... declaring queue'.format(self.consumer_id))
self.channel.basic_qos(prefetch_count=1)
try:
self.channel.basic_consume(self.queue_name,
self.handle_delivery,
auto_ack=True)
self.logger.info("{} Declared queue...".format(self.consumer_id))
except Exception as e:
self.logger.error('{} crashing:--> {}'.format(
self.consumer_id, str(e)))
def _on_channel_open(self, channel):
self.channel = channel
try:
self.channel.queue_declare(queue=self.queue_name,
exclusive=False,
durable=True,
auto_delete=False,
callback=self._on_queue_declared)
self.logger.info("{} Opened Channel....".format(self.consumer_id))
except Exception as e:
self.logger.error('{} {}'.format(self.consumer_id, str(e)))
def _on_connected(self, connection):
connection.channel(on_open_callback=self._on_channel_open)
def consume(self):
try:
self.connection = SelectConnection(self.parameters,
self._on_connected)
self.connection.ioloop.start()
except Exception as e:
self.logger.error('{} {}'.format(self.consumer_id, str(e)))
self.connection.close()
self.connection.ioloop.start()
def decode(self, body):
try:
_body = body.decode('utf-8')
except AttributeError:
_body = body
return _body
def handle_delivery(self, channel, method, header, body):
try:
start_time = datetime.datetime.now()
_logger.info("[x] Received Frame")
req = json.loads(self.decode(body))
# Process the frame
process_frame(_logger, req)
time_taken = datetime.datetime.now() - start_time
_logger.info("[x] [Consumer {}] Time Taken: {}.{}".format(
self.consumer_id, time_taken.seconds, time_taken.microseconds))
except Exception as err:
_logger.exception(err)
class RMQConnection(ABC):
"""
Abstract base class to be implemented.
Wraps a pika.SelectConnection and channel object to provide easy access to
consumer/producer implementers. Allows for a quick start to writing
consumers and producers, without having to re-write the connection logic.
RMQConnection signals implementers with the on_ready and on_close methods
when the connection/channel state changes.
"""
def __init__(self, connection_parameters=None):
"""
:param connection_parameters: pika.ConnectionParameters
"""
# Kept public to allow restart with different parameters.
self.connection_parameters: pika.ConnectionParameters = \
connection_parameters
self._connection_thread = Thread(target=self._connect, daemon=True)
self._connection = None
self._channel = None
self._reconnect_attempts = 0
self._closing = False
self._restarting = False
@abstractmethod
def on_ready(self):
"""
Implementers can handle what should happen when the RMQConnection is
ready for work.
"""
pass
@abstractmethod
def on_close(self, permanent=False):
"""
Implementers can handle what should happen when the RMQConnection is
suddenly closed. An on_close call will normally be followed by an
on_ready call from the RMQConnection unless the connection is
permanently down. It is possible that the RMQConnection will signal
multiple on_close calls for a single connection failure, therefore the
implementation of on_close must be idempotent.
:param permanent: bool
"""
pass
@abstractmethod
def on_error(self):
"""
Implementers can handle what should happen when the RMQConnection has
run into an error, for example an Exchange/Queue declaration failure.
"""
pass
def start(self):
"""
Starts the connection, opens a connection and a channel by running the
underlying connection thread. When ready, callbacks will be invoked on
each step of the way towards establishing the connection and channel,
starting with on_connection_open if all went well.
NOTE! Starting the connection is not a synchronous operation! It is
this way since starting may or may not be successful, so a return
value would be necessary to indicate success/failure. But, since
retries will be made, an initial failure may in fact be successful the
next second, meaning the user is given faulty information. Start
success is therefore indicated via the 'on_ready' hook instead.
"""
if not self._connection_thread.is_alive():
self._connection_thread.start()
def restart(self):
"""
Re-initiates the connection and channel.
"""
LOGGER.info("restarting connection")
try:
self._connection.close()
self._restarting = True # Only if connection could be closed.
except ConnectionWrongStateError:
LOGGER.info("connection closed and could not be restarted")
def stop(self):
"""
Starts a closing procedure for the connection and channel.
"""
LOGGER.info("stopping connection")
if not self._closing:
self._closing = True
try:
self._connection.close()
self._connection_thread.join()
except ConnectionWrongStateError:
LOGGER.info("connection already closed")
def declare_queue(self, queue_params, callback=None):
"""
:param queue_params: rabbitmq_client.QueueParams
:param callback: callable
"""
LOGGER.info(f"declaring queue: {queue_params.queue}")
self._channel.queue_declare(queue_params.queue,
durable=queue_params.durable,
exclusive=queue_params.exclusive,
auto_delete=queue_params.auto_delete,
arguments=queue_params.arguments,
callback=callback)
def declare_exchange(self, exchange_params, callback=None):
"""
:param exchange_params: rabbitmq_client.ExchangeParams
:param callback: callable
"""
LOGGER.info(f"declaring exchange: {exchange_params.exchange}")
self._channel.exchange_declare(
exchange_params.exchange,
exchange_type=exchange_params.exchange_type,
durable=exchange_params.durable,
auto_delete=exchange_params.auto_delete,
internal=exchange_params.internal,
arguments=exchange_params.arguments,
callback=callback)
def bind_queue(self, queue_bind_params, callback=None):
"""
:param queue_bind_params: rabbitmq_client.QueueBindParams
:param callback: callable
"""
LOGGER.info(f"binding queue {queue_bind_params.queue} to exchange "
f"{queue_bind_params.exchange} with routing key: "
f"{queue_bind_params.routing_key}")
self._channel.queue_bind(queue_bind_params.queue,
queue_bind_params.exchange,
routing_key=queue_bind_params.routing_key,
arguments=queue_bind_params.arguments,
callback=callback)
def basic_consume(self,
consume_params,
on_message_callback_override=None,
callback=None):
"""
:param consume_params: rabbitmq_client.ConsumeParams
:param on_message_callback_override: callable
:param callback: callable
"""
LOGGER.info(f"basic consumer starting for queue: "
f"{consume_params.queue}")
self._channel.basic_consume(
consume_params.queue,
(on_message_callback_override if on_message_callback_override else
consume_params.on_message_callback),
auto_ack=consume_params.auto_ack,
exclusive=consume_params.exclusive,
consumer_tag=consume_params.consumer_tag,
arguments=consume_params.arguments,
callback=callback)
def basic_publish(self,
body,
exchange=DEFAULT_EXCHANGE,
routing_key="",
publish_params=None):
"""
:param body: bytes
:param exchange: str
:param routing_key: str
:param publish_params: rabbitmq_client.PublishParams
"""
LOGGER.debug(f"publishing to exchange: {exchange} and routing key: "
f"{routing_key}")
if not publish_params:
publish_params = PublishParams() # Create defaults
self._channel.basic_publish(exchange,
routing_key,
body,
properties=publish_params.properties,
mandatory=publish_params.mandatory)
def confirm_delivery(self, on_delivery_confirmed, callback=None):
"""
:param on_delivery_confirmed: callable
:param callback: callable
"""
LOGGER.info("activating confirm delivery mode")
self._channel.confirm_delivery(on_delivery_confirmed,
callback=callback)
def _connect(self):
"""
Assigns a new pika.SelectConnection to the connection and starts its
ioloop to begin connecting.
"""
if self.connection_parameters is not None:
LOGGER.info(f"starting connection towards: "
f"{self.connection_parameters.host}:"
f"{self.connection_parameters.port}")
else:
LOGGER.info("starting connection towards: 127.0.0.1:5672")
self._connection = SelectConnection(
parameters=self.connection_parameters,
on_open_callback=self.on_connection_open,
on_open_error_callback=self.on_connection_open_error,
on_close_callback=self.on_connection_closed)
self._connection.ioloop.start()
def on_connection_open(self, _connection):
"""
Called when a new connection has been established. Opens a new channel.
:param _connection: pika.SelectConnection
"""
LOGGER.debug("connection opened")
# Do NOT reset reconnect counter here, do it after channel opened.
self._connection.channel(on_open_callback=self.on_channel_open)
def on_connection_open_error(self, _connection, error):
"""
Called when a connection could not be established towards RabbitMQ.
This will lead to attempts to reconnect with an increasing delay
between attempts.
:param _connection: pika.SelectConnection
:param error: pika.exceptions.?
"""
LOGGER.warning(f"error establishing connection, error: {error}")
# This should ensure the current thread runs to completion after
# reconnect handling is done.
self._connection.ioloop.stop()
# No fancy handling, just retry even if credentials are bad.
self._reconnect()
def on_connection_closed(self, _connection, reason):
"""
Called when the connection is closed, either unexpectedly or on
purpose. Reconnection is attempted if the disconnect reason is
something that is possible to recover, like an unexpected broker
shutdown.
:param _connection: pika.SelectConnection
:param reason: pika.exceptions.?
"""
permanent = False
# This should ensure the current thread runs to completion after
# on_connection_closed handling is done.
self._connection.ioloop.stop()
if not self._closing and type(reason) in RECONNECT_REASONS:
LOGGER.debug(f"connection closed: {reason}, attempting reconnect")
self._reconnect()
elif self._restarting:
LOGGER.debug("connection closed, restarting now")
# Reset to prevent reconnect for any reason
self._restarting = False
self._connection_thread = Thread(target=self._connect, daemon=True)
self._connection_thread.start()
else:
permanent = True
LOGGER.warning(f"connection closed: {reason}, will not reconnect")
# Signal subclass that connection is down.
self.on_close(permanent=permanent)
@property
def _reconnect_delay(self):
return self._reconnect_attempts if self._reconnect_attempts < 9 else 30
def _reconnect(self):
"""
Starts up the connection again after a gradually increasing delay.
"""
LOGGER.debug(f"reconnect, attempt no. {self._reconnect_attempts + 1}")
# Reconnect attempt may have been queued up before 'stop' was called.
if self._closing:
LOGGER.debug("skipping reconnect, connection stopped")
return
# _connect will assign a new connection object
self._connection_thread = Thread(target=self._connect, daemon=True)
if self._reconnect_delay == 0: # retry immediately the first time
self._connection_thread.start()
else: # calculate backoff timer
timer = Timer(self._reconnect_delay, self._connection_thread.start)
# Timer daemon exits if program exits, to avoid eternal retries.
timer.daemon = True
timer.start()
self._reconnect_attempts += 1
def on_channel_open(self, channel):
"""
Called when a new channel has been opened, the consumer is now ready
for work.
:param channel: pika.channel.Channel
"""
LOGGER.debug("channel opened")
self._reconnect_attempts = 0
self._channel = channel
self._channel.add_on_close_callback(self.on_channel_closed)
self.on_ready() # Signal subclass that connection is ready.
def on_channel_closed(self, _channel, reason):
"""
Called when a channel is closed by RabbitMQ. Channels are not
attempted to be re-opened since they are either closed from going down
together with the connection, or because input parameters for some
queue/exchange/other establishment was given faulty parameters.
Retrying such an operation will just result in a failure loop.
:param _channel: pika.channel.Channel
:param reason: pika.exceptions.?
"""
LOGGER.warning("channel closed")
permanent = False
try:
# Reply code 406 = PRECONDITION FAILED
permanent = True if reason.reply_code == 406 else False
except AttributeError: # Not all reasons have a reply code
pass
# Signal subclass that connection is down.
self.on_close(permanent=permanent)
if permanent:
LOGGER.critical(f"connection stopping due to permanent channel "
f"closure: {reason}")
self.stop()
from pika import SelectConnection, ConnectionParameters
from time import sleep
CHANNEL = None
QUEUE = 'xxx'
def on_connection_open(connection):
connection.channel(on_channel_open)
def on_channel_open(channel):
global CHANNEL
CHANNEL = channel
CHANNEL.queue_declare(queue=QUEUE, callback=on_queue_declared)
def on_queue_declared(frame):
count = 0
while True:
body = str(count)
CHANNEL.basic_publish(exchange='', routing_key=QUEUE, body=body)
print 'Sent %s' % body
sleep(3)
count += 1
parameters = ConnectionParameters()
connection = SelectConnection(parameters, on_open_callback=on_connection_open)
try:
connection.ioloop.start()
except KeyboardInterrupt:
connection.close()
connection.ioloop.start()
class RabbitMQClient(object):
def __init__(self,
client,
amqp_url,
queues={},
reconection_time=10,
prefetch_count=1,
debug=True):
self._amqp_url = amqp_url
self._queues = queues
self._reconection_time = reconection_time # 10 seconds
self._last_timestamps = dict()
self._prefetch_count = prefetch_count
self._connection = None
self._channel = None
self._debug = debug
self._client = client
self._blocking_connection = None
def message_count(self, queue_name):
try:
url = f"http://{self._host}:{self._port}/api/queues/{self._vhost}/{queue_name}?columns=message_stats.publish_details.rate"
r = requests.get(url,
auth=(self._creds.split(":")[0],
self._creds.split(":")[1]))
return r.json()["message_stats"]["publish_details"]["rate"]
except requests.exceptions.RequestException as e:
return None
except:
return 0
def on_open_connection(self, _unused_frame):
if (self._debug):
print(" [x] Connected - Connection state: OPEN. ")
self._connection.channel(on_open_callback=self.on_channel_open)
def on_channel_open(self, channel):
"""Callback when we have successfully declared the channel."""
if (self._debug):
print(" [x] Channel - Channel state: OPEN. ")
self._channel = channel
for key, queue in self._queues.items():
cb = functools.partial(self.on_queue_declareok, userdata=key)
self._channel.queue_declare(queue=key,
durable=queue['durable'],
exclusive=queue['exclusive'],
auto_delete=queue['auto_delete'],
callback=cb)
def on_queue_declareok(self, _unused_frame, userdata):
"""Callback when we have successfully declared the queue.
This call tells the server to send us self._prefetch_count message in advance.
This helps overall throughput, but it does require us to deal with the messages
we have promptly.
"""
queue_name = userdata
if (self._debug):
print(" [x] Queue declared - Queue: {}".format(queue_name))
cb = functools.partial(self.on_qos, userdata=queue_name)
self._channel.basic_qos(prefetch_count=self._prefetch_count,
callback=cb)
def on_qos(self, _unused_frame, userdata):
"""Callback when Basic.QOS has completed."""
if (self._debug):
print(" [x] QOS set to {}".format(self._prefetch_count))
queue_name = userdata
self.start_consuming(queue_name)
def on_message(self, channel, method, properties, body):
queue_name = method.routing_key
if (self._queues[queue_name]['callback']):
callback = getattr(self._client,
self._queues[queue_name]['callback'])
callback(channel, method, properties, body)
def start_consuming(self, queue_name):
"""Start consuming from task queue."""
if (self._debug):
print(" [x] Waiting for messages...")
self._channel.basic_consume(
queue_name,
self.on_message,
auto_ack=self._queues[queue_name]['auto_ack'])
def start(self):
"""Start worker."""
connected = False
# Define connection
while (not connected):
try:
self._creds = self._amqp_url.split('amqp://')[1].split("@")[0]
parameters = URLParameters(self._amqp_url)
self._host = parameters.host
self._port = "15672"
self._vhost = parameters.virtual_host
self._blocking_connection = BlockingConnection(parameters)
self._connection = SelectConnection(
URLParameters(self._amqp_url),
on_open_callback=self.on_open_connection)
self._connection.ioloop.start()
connected = True
# Catch a Keyboard Interrupt to make sure that the connection is closed cleanly
except KeyboardInterrupt:
# Gracefully close the connection
self._connection.close()
# Start the IOLoop again so Pika can communicate, it will stop on its own when the connection is closed
self._connection.ioloop.start()
except:
if (self._debug):
print(
" [!] RabbitMQ Host Unrecheable. Reconnecting in {} seconds..."
.format(self._reconection_time))
time.sleep(self._reconection_time)
