class RabbitMQConsumer():
def __init__(self, options={}):
self.options = options
self.connection = SelectConnection(on_open_callback=self.on_connected)
def on_connected(self, connection):
self.connection = connection
self.connection.channel(self.on_channel_open)
def on_channel_open(self, new_channel):
self.channel = new_channel
self.channel.queue_declare(queue=self.options['queue'],
durable=self.options['durable'],
exclusive=self.options['exclusive'],
auto_delete=self.options['auto_delete'],
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
self.channel.basic_consume(self.handle_delivery,
queue=self.options['queue'])
def handle_delivery(self, channel, method, header, body):
self.options['on_receive'](body)
self.channel.basic_ack(method.delivery_tag)
def start(self):
self.connection.ioloop.start()
def stop(self):
self.connection.close()
class BasicPublish(object):
"""See also: http://pika.github.com/examples.html"""
def __init__(self, host, port, virtual_host, username, password,
exchange, routing_key, message):
self.channel = None
self.exchange = exchange
self.routing_key = routing_key
self.message = message
credentials = PlainCredentials(
username, password, erase_on_connect=False)
parameters = ConnectionParameters(
host, port, virtual_host, credentials=credentials)
self.connection = SelectConnection(
parameters=parameters, on_open_callback=self.on_connect)
def on_connect(self, connection):
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
self.channel = channel
self.channel.exchange_declare(exchange=self.exchange,
type='direct', durable=True,
auto_delete=False,
callback=self.on_exchange_declared)
def on_exchange_declared(self, frame):
self.channel.queue_declare(queue=self.routing_key, durable=True,
exclusive=False, auto_delete=False,
callback=self.on_queue_declared)
def on_queue_declared(self, frame):
self.channel.queue_bind(exchange=self.exchange,
queue=self.routing_key,
routing_key=self.routing_key,
callback=self.on_queue_bound)
def on_queue_bound(self, frame):
properties = BasicProperties(content_type='text/plain',
delivery_mode=1)
self.channel.basic_publish(exchange=self.exchange,
routing_key=self.routing_key,
body=self.message,
properties=properties)
self.connection.close()
def _close_connection(self, connection: SelectConnection):
"""Closes a given `SelectConnection` instance.
It will call the connection's `close` method, and then `ioloop.start`, to try keep looping
until it is fully closed, as suggested by pika's documentation:
https://pika.readthedocs.io/en/stable/intro.html#io-and-event-looping
Args:
connection: The connection to close.
"""
if not connection:
return
try:
# Gracefully close the connection
connection.close()
# Loop until we're fully closed, will stop on its own
connection.ioloop.start()
except AMQPError as error:
self.logger.error(
"Error while closing connection on %s: %s.", self._queue_name, str(error),
)
from pika import SelectConnection, ConnectionParameters
from time import sleep
CHANNEL = None
QUEUE = 'xxx'
def on_connection_open(connection):
connection.channel(on_channel_open)
def on_channel_open(channel):
global CHANNEL
CHANNEL = channel
CHANNEL.queue_declare(queue=QUEUE, callback=on_queue_declared)
def on_queue_declared(frame):
count = 0
while True:
body = str(count)
CHANNEL.basic_publish(exchange='', routing_key=QUEUE, body=body)
print 'Sent %s' % body
sleep(3)
count += 1
parameters = ConnectionParameters()
connection = SelectConnection(parameters, on_open_callback=on_connection_open)
try:
connection.ioloop.start()
except KeyboardInterrupt:
connection.close()
connection.ioloop.start()
class QueueConsumer(object):
"""The consumer class to manage connections to the AMQP server/queue"""
def __init__(self, queue, logger, parameters, thread_id=0):
self.channel = None
self.connection = None
self.queue_name = queue
self.logger = logger
self.consumer_id = 'Thread: %d' % (thread_id, )
self.parameters = pika.ConnectionParameters(**parameters)
def _on_queue_declared(self, frame):
self.logger.debug('{} ... declaring queue'.format(self.consumer_id))
self.channel.basic_qos(prefetch_count=1)
try:
self.channel.basic_consume(self.queue_name,
self.handle_delivery,
auto_ack=True)
self.logger.info("{} Declared queue...".format(self.consumer_id))
except Exception as e:
self.logger.error('{} crashing:--> {}'.format(
self.consumer_id, str(e)))
def _on_channel_open(self, channel):
self.channel = channel
try:
self.channel.queue_declare(queue=self.queue_name,
exclusive=False,
durable=True,
auto_delete=False,
callback=self._on_queue_declared)
self.logger.info("{} Opened Channel....".format(self.consumer_id))
except Exception as e:
self.logger.error('{} {}'.format(self.consumer_id, str(e)))
def _on_connected(self, connection):
connection.channel(on_open_callback=self._on_channel_open)
def consume(self):
try:
self.connection = SelectConnection(self.parameters,
self._on_connected)
self.connection.ioloop.start()
except Exception as e:
self.logger.error('{} {}'.format(self.consumer_id, str(e)))
self.connection.close()
self.connection.ioloop.start()
def decode(self, body):
try:
_body = body.decode('utf-8')
except AttributeError:
_body = body
return _body
def handle_delivery(self, channel, method, header, body):
try:
start_time = datetime.datetime.now()
_logger.info("[x] Received Frame")
req = json.loads(self.decode(body))
# Process the frame
process_frame(_logger, req)
time_taken = datetime.datetime.now() - start_time
_logger.info("[x] [Consumer {}] Time Taken: {}.{}".format(
self.consumer_id, time_taken.seconds, time_taken.microseconds))
except Exception as err:
_logger.exception(err)
# Delete the row
elif message['operation'] == 'delete':
data = {'row_id': message['data']['row_id']}
self.cursor.execute(DELETE_QUERY, data)
# Ack the message
channel.basic_ack(delivery_tag=method_frame.delivery_tag)
if __name__ == '__main__':
consumer = ExampleConsumer()
# Connect to RabbitMQ
host = (len(sys.argv) > 1) and sys.argv[1] or '127.0.0.1'
connection = SelectConnection(ConnectionParameters(host),
consumer._on_connected)
# Loop until CTRL-C
try:
# Start our blocking loop
connection.ioloop.start()
except KeyboardInterrupt:
# Close the connection
connection.close()
# Loop until the conneciton is closed
connection.ioloop.start()
class BrokerConnection(object):
"""Connection class which provides connection to AMQP broker."""
def __init__(self, config, callback):
self.reconnection_delay = 1.0
self.caller_callback = callback
self.config = ConnectionConfig()
self.config.read(config)
self.callbacks = self.set_callbacks()
credentials = PlainCredentials(**self.config.credentials)
broker_config = self.config.broker_config
broker_config['credentials'] = credentials
parameters = ConnectionParameters(**broker_config)
try:
parameters.host = self.config.host
except NoOptionError:
pass
try:
parameters.heartbeat = int(self.config.heartbeat)
except NoOptionError:
pass
try:
parameters.heartbeat = int(self.config.heartbeat)
except NoOptionError:
pass
self.connection = SelectConnection(parameters, self.on_connected)
def on_connected(self, connection):
"""Called when we're connected to AMQP broker."""
logger.info("Connected to AMQP-broker.")
connection.channel(self.on_channel_open)
def on_channel_open(self, new_channel):
"""Called when channel has opened"""
logger.info("Channel to AMQP-broker opened.")
self.channel = new_channel
self.channel.add_on_close_callback(self.on_channel_closed)
self.generic_callback()
def set_callbacks(self):
"""Set callbacks for queue factory."""
self.exchange_callbacks = []
self.queue_callbacks = []
self.binding_callbacks = []
for exchange in self.config.exchanges:
tmp = {}
tmp['exchange'] = exchange
tmp.update(self.config.exchanges[exchange])
self.exchange_callbacks.append(tmp)
for queue in self.config.queues:
tmp = {}
tmp['queue'] = queue
tmp.update(self.config.queues[queue])
self.queue_callbacks.append(tmp)
for binding in self.config.bindings:
tmp = {}
tmp['binding'] = binding
tmp.update(self.config.bindings[binding])
self.binding_callbacks.append(tmp)
def generic_callback(self, channel=None, frame=None):
"""Create exchanges, queues and bindings."""
if channel and frame:
# You could do error handling here
# or add code for anonymous queue handling
pass
if self.exchange_callbacks:
config = self.exchange_callbacks.pop()
config['exchange'] = config['exchange'].split(':', 1)[-1]
logger.info("Creating exchange %s." % config['exchange'])
config.update({"callback": self.generic_callback})
self.channel.exchange_declare(**config)
elif self.queue_callbacks:
config = self.queue_callbacks.pop()
config['queue'] = config['queue'].split(':', 1)[-1]
logger.info("Creating queue %s." % config['queue'])
config.update({"callback": self.generic_callback})
self.channel.queue_declare(**config)
elif self.binding_callbacks:
config = self.binding_callbacks.pop()
binding = config['binding'].split(':')
del config['binding']
config['exchange'] = binding[-1]
config['queue'] = binding[1]
logger.info("Creating binding (%s -> %s) with routing key %s" % (
config['exchange'], config['queue'],
config['routing_key']))
config.update({"callback": self.generic_callback})
self.channel.queue_bind(**config)
else:
logger.info("RabbitMQ exchanges, queues and bindings are now "
"configured.")
self.caller_callback()
def reset_reconnection_delay(self, *args):
self.reconnection_delay = 1.0
def loop(self):
"""Main loop"""
while True:
try:
logger.info("Starting main loop")
self.connection.add_on_open_callback(
self.reset_reconnection_delay)
self.connection.ioloop.start()
except socket.error as e:
logger.error("Connection failed or closed unexpectedly: %s", e)
except TypeError as e:
logger.error("Connection failed or closed unexpectedly: %s", e)
except KeyboardInterrupt:
self.connection.close()
self.connection.ioloop.start()
break
finally:
self.reconnection_delay *= (random.random() * 0.5) + 1
self.reconnection_delay = min(self.reconnection_delay, 60.0)
logger.info("Trying reconnection in %s seconds",
self.reconnection_delay)
time.sleep(self.reconnection_delay)
def on_channel_closed(self, channel, code, text):
logger.warning("Channel closed with reason '%s %s'",
code, text)
self.connection.close(code, text)
class Updater(object):
def __init__(self, debug=True, prefetch_count=1):
self._path = environ['DIR_TILES']
self._debug = debug
self._res_queue = environ['RES_QUEUE']
self._res_xchg = environ['RES_XCHG']
self._amqp_url = environ['AMQP_URL']
self._prefetch_count = prefetch_count
self._reconnection_time = int(environ['TIMEOUT'])
def base64_to_img(self, buff, fpath):
with open(fpath, 'wb') as f:
b64data = buff.split(",")[1]
data = b64decode(b64data.encode("utf-8", "ignore"))
f.write(data)
def callback_res_queue(self, _unused_channel, delivery, properties, body):
"""
Callback triggered when message is received.
"""
if (self._debug):
print(" [x] Received - Body: {}".format(body[:140]))
start_time = time()
data = loads(body)
path = data['earthImage']['path']
if ('filteredImages' in data):
if (self._debug):
print(" [x] Saving Images...")
for image in data['filteredImages']:
new_path = Path(path.replace('RAW', image['filterName']))
dirs = new_path.parts[:-1]
# create dirs
Path("/".join(dirs)).mkdir(parents=True, exist_ok=True)
self.base64_to_img(image['vectorImage'], new_path)
self._channel.basic_ack(delivery_tag=delivery.delivery_tag)
def on_open_connection(self, _unused_frame):
if (self._debug):
print(" [x] Connected - Connection state: OPEN. ")
self._connection.channel(on_open_callback=self.on_channel_open)
def on_channel_open(self, channel):
"""Callback when we have successfully declared the channel."""
if (self._debug):
print(" [x] Channel - Channel state: OPEN. ")
self._channel = channel
self._channel.exchange_declare(exchange=self._res_xchg,
exchange_type='fanout')
cb = partial(self.on_queue_declareok, userdata=self._res_queue)
self._channel.queue_declare(queue=self._res_queue,
durable=True,
callback=cb)
def on_queue_declareok(self, _unused_frame, userdata):
"""Callback when we have successfully declared the queue.
This call tells the server to send us self._prefetch_count message in advance.
This helps overall throughput, but it does require us to deal with the messages
we have promptly.
"""
queue_name = userdata
if (self._debug):
print(" [x] Queue declared - Queue: {}".format(queue_name))
self._channel.basic_qos(prefetch_count=self._prefetch_count,
callback=self.on_qos)
self._channel.queue_bind(exchange=self._res_xchg, queue=queue_name)
def on_qos(self, _unused_frame):
"""Callback when Basic.QOS has completed."""
if (self._debug):
print(" [x] QOS set to {}".format(self._prefetch_count))
self.start_consuming()
def start_consuming(self):
"""Start consuming from task queue."""
if (self._debug):
print(" [x] Waiting for messages...")
self._channel.basic_consume(self._res_queue, self.callback_res_queue)
def start(self):
"""Start worker."""
# Define connection
while True:
try:
self._connection = SelectConnection(
URLParameters(self._amqp_url),
on_open_callback=self.on_open_connection)
self._connection.ioloop.start()
# Catch a Keyboard Interrupt to make sure that the connection is closed cleanly
except KeyboardInterrupt:
# Gracefully close the connection
self._connection.close()
# Start the IOLoop again so Pika can communicate, it will stop on its own when the connection is closed
self._connection.ioloop.start()
except:
if (self._debug):
print(
" [!] RabbitMQ Host Unreachable. Reconnecting in {} seconds..."
.format(self._reconnection_time))
sleep(self._reconnection_time)
class BrokerConnection(object):
"""Connection class which provides connection to AMQP broker."""
exchange_callbacks = []
queue_callbacks = []
binding_callbacks = []
channel = None
def __init__(self, config, callback):
self.closing = False
self.channel = None
self.connection = None
self.reconnection_delay = 1.0
self.caller_callback = callback
if isinstance(config, ConnectionConfig):
self.config = config
else:
self.config = ConnectionConfig()
self.config.read(config)
self.set_callbacks()
credentials = PlainCredentials(**self.config.credentials)
broker_config = self.config.broker_config
broker_config['credentials'] = credentials
self.parameters = ConnectionParameters(**broker_config)
try:
self.parameters.host = self.config.host
except NoOptionError:
pass
try:
self.parameters.heartbeat_interval = int(
self.config.heartbeat_interval)
except NoOptionError:
pass
self.connect()
def connect(self):
"""
:param parameters:
:return:
"""
self.connection = SelectConnection(self.parameters, self.on_connected,
stop_ioloop_on_close=False)
def on_connected(self, connection):
"""Called when we're connected to AMQP broker."""
print("Connected to AMQP-broker.")
self.add_on_connection_close_callback()
connection.channel(self.on_channel_open)
def add_on_connection_close_callback(self):
"""This method adds an on close callback that will be invoked by pika
when RabbitMQ closes the connection to the publisher unexpectedly.
"""
print('Adding connection close callback')
self.connection.add_on_close_callback(self.on_connection_closed)
def on_connection_closed(self, connection, reply_code, reply_text):
"""This method is invoked by pika when the connection to RabbitMQ is
closed unexpectedly. Since it is unexpected, we will reconnect to
RabbitMQ if it disconnects.
:param pika.connection.Connection connection: The closed connection obj
:param int reply_code: The server provided reply_code if given
:param str reply_text: The server provided reply_text if given
"""
self.channel = None
if self.closing:
self.connection.ioloop.stop()
else:
print('Connection closed, reopening in 5 seconds: (%s) %s',
reply_code, reply_text)
self.connection.add_timeout(5, self.reconnect)
def on_channel_open(self, new_channel):
"""Called when channel has opened"""
logger.info("Channel to AMQP-broker opened.")
self.channel = new_channel
self.channel.add_on_close_callback(self.on_channel_closed)
self.generic_callback()
def set_callbacks(self):
"""Set callbacks for queue factory."""
for exchange in self.config.exchanges:
tmp = {'exchange': exchange}
tmp.update(self.config.exchanges[exchange])
self.exchange_callbacks.append(tmp)
for queue in self.config.queues:
tmp = {'queue': queue}
tmp.update(self.config.queues[queue])
self.queue_callbacks.append(tmp)
for binding in self.config.bindings:
tmp = {'binding': binding}
tmp.update(self.config.bindings[binding])
self.binding_callbacks.append(tmp)
def generic_callback(self, channel=None, frame=None):
"""Create exchanges, queues and bindings."""
if channel and frame:
# You could do error handling here
# or add code for anonymous queue handling
pass
if self.exchange_callbacks:
config = self.exchange_callbacks.pop()
config['exchange'] = config['exchange'].split(':', 1)[-1]
logger.info("Creating exchange %s." % config['exchange'])
config.update({"callback": self.generic_callback})
self.channel.exchange_declare(**config)
elif self.queue_callbacks:
config = self.queue_callbacks.pop()
config['queue'] = config['queue'].split(':', 1)[-1]
logger.info("Creating queue %s." % config['queue'])
config.update({"callback": self.generic_callback})
self.channel.queue_declare(**config)
elif self.binding_callbacks:
config = self.binding_callbacks.pop()
binding = config['binding'].split(':')
del config['binding']
config['exchange'] = binding[-1]
config['queue'] = binding[1]
logger.info("Creating binding (%s -> %s) with routing key %s" % (
config['exchange'], config['queue'],
config['routing_key']))
config.update({"callback": self.generic_callback})
self.channel.queue_bind(**config)
else:
logger.info("RabbitMQ exchanges, queues and bindings are now "
"configured.")
self.caller_callback()
def reset_reconnection_delay(self, *args):
self.reconnection_delay = 1.0
def loop(self):
"""Main loop"""
while True:
try:
logger.info("Starting main loop")
self.connection.add_on_open_callback(
self.reset_reconnection_delay)
self.connection.ioloop.start()
except socket.error as e:
logger.error("Connection failed or closed unexpectedly: %s", e,
exc_info=True)
except TypeError as e:
logger.error("Connection failed or closed unexpectedly: %s", e,
exc_info=True)
except KeyboardInterrupt:
self.connection.close()
self.connection.ioloop.start()
break
finally:
self.reconnection_delay *= (random.random() * 0.5) + 1
self.reconnection_delay = min(self.reconnection_delay, 60.0)
logger.info("Trying reconnection in %s seconds",
self.reconnection_delay)
time.sleep(self.reconnection_delay)
def on_channel_closed(self, channel, code, text):
logger.warning("Channel closed with reason '%s %s'",
code, text)
self.connection.close(code, text)
def reconnect(self):
"""Will be invoked by the IOLoop timer if the connection is
closed. See the on_connection_closed method.
"""
# This is the old connection IOLoop instance, stop its ioloop
self.connection.ioloop.stop()
if not self.closing:
# Create a new connection
self.connect()
# There is now a new connection, needs a new ioloop to run
self.connection.ioloop.start()
def add_on_cancel_callback(self):
"""Add a callback that will be invoked if RabbitMQ cancels the consumer
for some reason. If RabbitMQ does cancel the consumer,
on_consumer_cancelled will be invoked by pika.
"""
print('Adding consumer cancellation callback')
self.channel.add_on_cancel_callback(self.on_consumer_cancelled)
def on_consumer_cancelled(self, method_frame):
"""
Invoked by pika when RabbitMQ sends a Basic.Cancel for a consumer
receiving messages.
:param pika.frame.Method method_frame: The Basic.Cancel frame
"""
print('Consumer was cancelled remotely, shutting down: %r' %
method_frame)
if self.channel:
self.channel.close()
class RabbitMQClient(object):
def __init__(self,
client,
amqp_url,
queues={},
reconection_time=10,
prefetch_count=1,
debug=True):
self._amqp_url = amqp_url
self._queues = queues
self._reconection_time = reconection_time # 10 seconds
self._last_timestamps = dict()
self._prefetch_count = prefetch_count
self._connection = None
self._channel = None
self._debug = debug
self._client = client
self._blocking_connection = None
def message_count(self, queue_name):
try:
url = f"http://{self._host}:{self._port}/api/queues/{self._vhost}/{queue_name}?columns=message_stats.publish_details.rate"
r = requests.get(url,
auth=(self._creds.split(":")[0],
self._creds.split(":")[1]))
return r.json()["message_stats"]["publish_details"]["rate"]
except requests.exceptions.RequestException as e:
return None
except:
return 0
def on_open_connection(self, _unused_frame):
if (self._debug):
print(" [x] Connected - Connection state: OPEN. ")
self._connection.channel(on_open_callback=self.on_channel_open)
def on_channel_open(self, channel):
"""Callback when we have successfully declared the channel."""
if (self._debug):
print(" [x] Channel - Channel state: OPEN. ")
self._channel = channel
for key, queue in self._queues.items():
cb = functools.partial(self.on_queue_declareok, userdata=key)
self._channel.queue_declare(queue=key,
durable=queue['durable'],
exclusive=queue['exclusive'],
auto_delete=queue['auto_delete'],
callback=cb)
def on_queue_declareok(self, _unused_frame, userdata):
"""Callback when we have successfully declared the queue.
This call tells the server to send us self._prefetch_count message in advance.
This helps overall throughput, but it does require us to deal with the messages
we have promptly.
"""
queue_name = userdata
if (self._debug):
print(" [x] Queue declared - Queue: {}".format(queue_name))
cb = functools.partial(self.on_qos, userdata=queue_name)
self._channel.basic_qos(prefetch_count=self._prefetch_count,
callback=cb)
def on_qos(self, _unused_frame, userdata):
"""Callback when Basic.QOS has completed."""
if (self._debug):
print(" [x] QOS set to {}".format(self._prefetch_count))
queue_name = userdata
self.start_consuming(queue_name)
def on_message(self, channel, method, properties, body):
queue_name = method.routing_key
if (self._queues[queue_name]['callback']):
callback = getattr(self._client,
self._queues[queue_name]['callback'])
callback(channel, method, properties, body)
def start_consuming(self, queue_name):
"""Start consuming from task queue."""
if (self._debug):
print(" [x] Waiting for messages...")
self._channel.basic_consume(
queue_name,
self.on_message,
auto_ack=self._queues[queue_name]['auto_ack'])
def start(self):
"""Start worker."""
connected = False
# Define connection
while (not connected):
try:
self._creds = self._amqp_url.split('amqp://')[1].split("@")[0]
parameters = URLParameters(self._amqp_url)
self._host = parameters.host
self._port = "15672"
self._vhost = parameters.virtual_host
self._blocking_connection = BlockingConnection(parameters)
self._connection = SelectConnection(
URLParameters(self._amqp_url),
on_open_callback=self.on_open_connection)
self._connection.ioloop.start()
connected = True
# Catch a Keyboard Interrupt to make sure that the connection is closed cleanly
except KeyboardInterrupt:
# Gracefully close the connection
self._connection.close()
# Start the IOLoop again so Pika can communicate, it will stop on its own when the connection is closed
self._connection.ioloop.start()
except:
if (self._debug):
print(
" [!] RabbitMQ Host Unrecheable. Reconnecting in {} seconds..."
.format(self._reconection_time))
time.sleep(self._reconection_time)