def get_consumers(self, Consumer, channel):
assert self.mq_config
# Declaring ourselves rather than use auto-declare.
log.debug("Declaring %s exchange", self.mq_config.exchange)
self.exchange(channel).declare()
queues = []
for queue_name, routing_keys in self.mq_config.queues.items():
queue = Queue(name=queue_name,
exchange=self.exchange,
channel=channel,
durable=True)
log.debug("Declaring queue %s", queue_name)
queue.declare()
for routing_key in routing_keys:
log.debug("Binding queue %s to %s", queue_name, routing_key)
queue.bind_to(exchange=self.exchange, routing_key=routing_key)
queues.append(queue)
consumer = Consumer(queues=queues,
callbacks=[self._callback],
auto_declare=False)
consumer.qos(prefetch_count=1, apply_global=True)
return [consumer]
def declare_exchange(self, name, type='direct', queues=None, **options):
"""Create or update exchange
:param name: name of exchange
:type name: str
:param type: type of exchange - direct, fanout, topic, match
:type type: str
:param queues: list of queues with routing keys: [[queue_name, routing_key], [queue_name, routing_key], ...]
:type queues: list, None or tuple
:param options: additional options for Exchange creation
"""
if queues is None:
queues = [] # pragma: no cover
with self.connections[self.connection].acquire() as conn:
exchange = Exchange(name, type=type, channel=conn, **options)
exchange.declare()
self.exchanges[name] = exchange
for q_name, routing_key in queues:
queue = Queue(name=q_name, channel=conn)
queue.declare()
queue.bind_to(exchange=name, routing_key=routing_key)
self.logger.debug(
'Queue "%s" with routing_key "%s" was bond to exchange "%s"',
q_name, routing_key if routing_key else q_name, name)
def get_consumers(self, Consumer, channel):
assert self.mq_config
# Declaring ourselves rather than use auto-declare.
log.debug("Declaring %s exchange", self.mq_config.exchange)
self.exchange(channel).declare()
queues = []
for queue_name, routing_keys in self.mq_config.queues.items():
queue = Queue(name=queue_name,
exchange=self.exchange,
channel=channel,
durable=True)
log.debug("Declaring queue %s", queue_name)
queue.declare()
for routing_key in routing_keys:
log.debug("Binding queue %s to %s", queue_name, routing_key)
queue.bind_to(exchange=self.exchange,
routing_key=routing_key)
queues.append(queue)
consumer = Consumer(queues=queues,
callbacks=[self._callback],
auto_declare=False)
consumer.qos(prefetch_count=1, apply_global=True)
return [consumer]
class JobConsumer(ConsumerMixin):
"""
Consume jobs from Pulse exchanges
"""
def __init__(self, connection):
self.connection = connection
self.consumers = []
self.queue = None
config = settings.PULSE_DATA_INGESTION_CONFIG
if not config:
raise ValueError("PULSE_DATA_INGESTION_CONFIG is required for the "
"JobConsumer class.")
self.queue_name = "queue/{}/jobs".format(config.username)
def get_consumers(self, Consumer, channel):
return [
Consumer(**c) for c in self.consumers
]
def bind_to(self, exchange, routing_key):
if not self.queue:
self.queue = Queue(
name=self.queue_name,
channel=self.connection.channel(),
exchange=exchange,
routing_key=routing_key,
durable=settings.PULSE_DATA_INGESTION_QUEUES_DURABLE,
auto_delete=settings.PULSE_DATA_INGESTION_QUEUES_AUTO_DELETE
)
self.consumers.append(dict(queues=self.queue,
callbacks=[self.on_message]))
# just in case the queue does not already exist on Pulse
self.queue.declare()
else:
self.queue.bind_to(exchange=exchange, routing_key=routing_key)
def unbind_from(self, exchange, routing_key):
self.queue.unbind_from(exchange, routing_key)
def on_message(self, body, message):
store_pulse_jobs.apply_async(
args=[body,
message.delivery_info["exchange"],
message.delivery_info["routing_key"]],
routing_key='store_pulse_jobs'
)
message.ack()
def close(self):
self.connection.release()
def init(self):
try:
self.channel = channel = self.connection.channel()
exchange = Exchange(name='policymq.direct', type='direct', durable=False)
exchange.declare(channel=channel)
queue = Queue(name=self.queueName, durable=False)
queue.queue_declare(channel=channel)
queue.bind_to(exchange=exchange, routing_key='ALL_', channel=channel)
queue.bind_to(exchange=exchange, routing_key=self.queueName, channel=channel)
consumer = Consumer(channel=channel, queues=[queue], callbacks=[self._consume])
consumer.consume()
return True
except Exception as e:
logger.exception(e)
return False
def declare_queues(self, queues_names):
with self.connection as _conn:
_conn.connect()
channel = _conn.channel()
for queue_name in queues_names:
topic = Exchange(
name=f"TOPIC/{queue_name}", type="topic", channel=channel
)
topic.declare()
queue = Queue(name=queue_name, channel=channel)
queue.declare()
queue.bind_to(exchange=f"TOPIC/{queue_name}")
class JobConsumer(ConsumerMixin):
"""
Consume jobs from Pulse exchanges
"""
def __init__(self, connection):
self.connection = connection
self.consumers = []
self.queue = None
config = settings.PULSE_DATA_INGESTION_CONFIG
if not config:
raise ValueError("PULSE_DATA_INGESTION_CONFIG is required for the "
"JobConsumer class.")
self.queue_name = "queue/{}/jobs".format(config.username)
def get_consumers(self, Consumer, channel):
return [Consumer(**c) for c in self.consumers]
def bind_to(self, exchange, routing_key):
if not self.queue:
self.queue = Queue(
name=self.queue_name,
channel=self.connection.channel(),
exchange=exchange,
routing_key=routing_key,
durable=settings.PULSE_DATA_INGESTION_QUEUES_DURABLE,
auto_delete=settings.PULSE_DATA_INGESTION_QUEUES_AUTO_DELETE)
self.consumers.append(
dict(queues=self.queue, callbacks=[self.on_message]))
# just in case the queue does not already exist on Pulse
self.queue.declare()
else:
self.queue.bind_to(exchange=exchange, routing_key=routing_key)
def unbind_from(self, exchange, routing_key):
self.queue.unbind_from(exchange, routing_key)
def on_message(self, body, message):
store_pulse_jobs.apply_async(args=[
body, message.delivery_info["exchange"],
message.delivery_info["routing_key"]
],
routing_key='store_pulse_jobs')
message.ack()
def close(self):
self.connection.release()
def append_to_topic(self, topic_name, queue_name, routing_key=None):
with self.connection as _conn:
_conn.connect()
channel = _conn.channel()
if not topic_name.startswith(self.config.get_event_name_prefix()):
topic_name = f"{self.config.get_event_name_prefix()}{topic_name}"
topic = Exchange(name=f"{topic_name}",
type="topic",
channel=channel)
topic.declare()
queue = Queue(name=queue_name,
channel=channel,
routing_key=routing_key)
queue.declare()
queue.bind_to(exchange=f"{topic_name}", routing_key=routing_key)
def push_sync(self,upload=True,delay=0,dryrun=False,timeout=None): # pragma: no cover
"wait for push messages"
from kombu import Connection, Exchange, Queue, Consumer
import socket, ssl
url, opts, exchange, queue = self.get_broker()
def callback(body, message):
self.process_update(body)
message.ack()
with Connection(url,**opts) as conn:
self.connection = conn
queue = Queue(queue, channel=conn)
queue.queue_declare()
queue.bind_to(exchange)
try: excpt = (socket.error, ssl.SSLZeroReturnError)
except AttributeError: excpt = socket.error
with conn.Consumer(queue, accept=['json'], callbacks=[callback]) as consumer:
while True:
try: conn.drain_events(timeout=timeout)
except socket.timeout: pass
except excpt: break
def start(self):
"""
Start subscriber.
"""
with Connection(self.config.broker_url) as conn:
data_pipeline_exchange = Exchange(self.config.exchange, 'topic')
queue = Queue(self.config.queue_name)
queue.maybe_bind(conn)
queue.declare()
for routing_key in set(
[i['routing_key'] for i in self.config.task_mapping]):
queue.bind_to(data_pipeline_exchange, routing_key)
worker = Worker(self.config, conn, [queue])
logger.info('Starting subscriber...')
worker.run()
def declare_exchange(self, name, type='direct', queues=None, **options):
"""Create or update exchange
:param name: name of exchange
:type name: str
:param type: type of exchange - direct, fanout, topic, match
:type type: str
:param queues: list of queues with routing keys: [[queue_name, routing_key], [queue_name, routing_key], ...]
:type queues: list, None or tuple
:param options: additional options for Exchange creation
"""
if queues is None:
queues = []
with connections[self.connection].acquire() as conn:
exchange = Exchange(name, type=type, channel=conn, **options)
exchange.declare()
self.exchanges[name] = exchange
for q_name, routing_key in queues:
queue = Queue(name=q_name, channel=conn)
queue.declare()
queue.bind_to(exchange=name, routing_key=routing_key)
self.logger.debug('Queue "%s" with routing_key "%s" was bond to exchange "%s"', q_name,
routing_key if routing_key else q_name, name)
class PulseConsumer(ConsumerMixin):
"""
Consume jobs from Pulse exchanges
"""
def __init__(self, connection, queue_suffix):
self.connection = connection
self.consumers = []
self.queue = None
config = settings.PULSE_DATA_INGESTION_CONFIG
if not config:
raise ValueError("PULSE_DATA_INGESTION_CONFIG is required for the "
"JobConsumer class.")
self.queue_name = "queue/{}/{}".format(config.username, queue_suffix)
def get_consumers(self, Consumer, channel):
return [
Consumer(**c) for c in self.consumers
]
def bind_to(self, exchange, routing_key):
if not self.queue:
self.queue = Queue(
name=self.queue_name,
channel=self.connection.channel(),
exchange=exchange,
routing_key=routing_key,
durable=settings.PULSE_DATA_INGESTION_QUEUES_DURABLE,
auto_delete=settings.PULSE_DATA_INGESTION_QUEUES_AUTO_DELETE
)
self.consumers.append(dict(queues=self.queue,
callbacks=[self.on_message]))
# just in case the queue does not already exist on Pulse
self.queue.declare()
else:
self.queue.bind_to(exchange=exchange, routing_key=routing_key)
def unbind_from(self, exchange, routing_key):
self.queue.unbind_from(exchange, routing_key)
def close(self):
self.connection.release()
def prune_bindings(self, new_bindings):
# get the existing bindings for the queue
bindings = []
try:
bindings = self.get_bindings(self.queue_name)["bindings"]
except Exception:
logger.error("Unable to fetch existing bindings for {}".format(
self.queue_name))
logger.error("Data ingestion may proceed, "
"but no bindings will be pruned")
# Now prune any bindings from the queue that were not
# established above.
# This indicates that they are no longer in the config, and should
# therefore be removed from the durable queue bindings list.
for binding in bindings:
if binding["source"]:
binding_str = self.get_binding_str(binding["source"],
binding["routing_key"])
if binding_str not in new_bindings:
self.unbind_from(Exchange(binding["source"]),
binding["routing_key"])
logger.info("Unbound from: {}".format(binding_str))
def get_binding_str(self, exchange, routing_key):
"""Use consistent string format for binding comparisons"""
return "{} {}".format(exchange, routing_key)
def get_bindings(self, queue_name):
"""Get list of bindings from the pulse API"""
return fetch_json("{}queue/{}/bindings".format(
settings.PULSE_GUARDIAN_URL, queue_name))
class RabbitMQConnection:
"""
Class handling receiving and publishing message on the RabbitMQ messages bus
"""
def __init__(self, message_callback):
self.message_callback = message_callback
self.exchange = Exchange(CFG.exchange)
self.connection = Connection(transport='amqp',
hostname=CFG.host,
port=CFG.port,
userid=CFG.username,
password=CFG.password,
virtual_host=CFG.vhost,
ssl=True)
self.connection.connect()
self.producer = self.connection.Producer(serializer='json',
auto_declare=True)
self.queue = Queue(channel=self.connection.channel(),
name=CFG.queue,
routing_key=CFG.routing_key)
self.queue.declare()
self.queue.bind_to(exchange=Exchange(CFG.exchange),
routing_key=CFG.routing_key)
self.consumer = self.connection.\
Consumer(
queues=self.queue,
callbacks=[self._handle_message],
prefetch_count=
CFG.prefetch_count)
self.consuming = True
def _handle_message(self, body, message):
"""
Callback called by consumer.
:param body:
:param message:
:return:
"""
# body is sometimes dict and sometimes str
# make sure it's a json dict before passing it on
json_body = dict()
if isinstance(body, dict):
json_body = body
elif isinstance(body, str):
json_body = json.loads(body)
self.message_callback(json_body)
message.ack()
def publish_message(self, message):
"""
Publishes passed message on the RabbitMQ message bus
:param message:
:return:
"""
self.producer.publish(message,
retry=True,
retry_policy={
'interval_start': 0,
'interval_step': 2,
'interval_max': 30,
'max_retries': 30,
},
exchange=self.exchange,
routing_key=CFG.routing_key)
def read_messages(self):
"""
Method reading messages from the queue in a while-true loop.
Callback is defined in __init__
:return:
"""
with self.consumer:
while self.consuming:
self.connection.drain_events()
def close_connection(self):
"""
Closes the channels/connections.
:return:
"""
# for now called when you press Ctrl-C
self.consuming = False
self.producer.release()
self.connection.release()
class PulseConsumer(ConsumerMixin):
"""
Consume jobs from Pulse exchanges
"""
def __init__(self, source, build_routing_key):
self.connection = Connection(source['pulse_url'])
self.consumers = []
self.queue = None
self.queue_name = "queue/{}/{}".format(self.connection.userid,
self.queue_suffix)
self.root_url = source['root_url']
self.source = source
self.build_routing_key = build_routing_key
def get_consumers(self, Consumer, channel):
return [Consumer(**c) for c in self.consumers]
def bindings(self):
"""Get the bindings for this consumer, each of the form `<exchange>.<routing_keys>`,
with `<routing_keys>` being `:`-separated."""
return []
def prepare(self):
bindings = []
for binding in self.bindings():
# split source string into exchange and routing key sections
exchange, _, routing_keys = binding.partition('.')
# built an exchange object with our connection and exchange name
exchange = get_exchange(self.connection, exchange)
# split the routing keys up using the delimiter
for routing_key in routing_keys.split(':'):
if self.build_routing_key is not None: # build routing key
routing_key = self.build_routing_key(routing_key)
binding = self.bind_to(exchange, routing_key)
bindings.append(binding)
# prune stale queues using the binding strings
self.prune_bindings(bindings)
def bind_to(self, exchange, routing_key):
if not self.queue:
self.queue = Queue(
name=self.queue_name,
channel=self.connection.channel(),
exchange=exchange,
routing_key=routing_key,
durable=True,
auto_delete=False,
)
self.consumers.append(
dict(queues=self.queue, callbacks=[self.on_message]))
# just in case the queue does not already exist on Pulse
self.queue.declare()
else:
self.queue.bind_to(exchange=exchange, routing_key=routing_key)
# get the binding key for this consumer
binding = self.get_binding_str(exchange.name, routing_key)
logger.info("Pulse queue {} bound to: {}".format(
self.queue_name, binding))
return binding
def unbind_from(self, exchange, routing_key):
self.queue.unbind_from(exchange, routing_key)
def close(self):
self.connection.release()
def prune_bindings(self, new_bindings):
# get the existing bindings for the queue
bindings = []
try:
bindings = self.get_bindings(self.queue_name)["bindings"]
except Exception:
logger.error(
"Unable to fetch existing bindings for %s. Data ingestion may proceed, "
"but no bindings will be pruned", self.queue_name)
# Now prune any bindings from the queue that were not
# established above.
# This indicates that they are no longer in the config, and should
# therefore be removed from the durable queue bindings list.
for binding in bindings:
if binding["source"]:
binding_str = self.get_binding_str(binding["source"],
binding["routing_key"])
if binding_str not in new_bindings:
self.unbind_from(Exchange(binding["source"]),
binding["routing_key"])
logger.info("Unbound from: %s", binding_str)
def get_binding_str(self, exchange, routing_key):
"""Use consistent string format for binding comparisons"""
return "{} {}".format(exchange, routing_key)
def get_bindings(self, queue_name):
"""Get list of bindings from the pulse API"""
return fetch_json("{}queue/{}/bindings".format(PULSE_GUARDIAN_URL,
queue_name))
class KombuMessageQueue(object):
"""
A queue based on kombu
"""
class_name = None
def __init__(self, dc, exchange, exchange_type, queue, binding_key,
**kwargs):
if not isinstance(dc, list):
dc = [dc]
self.dc = dc
self.exchange = exchange
self.exchange_type = exchange_type
self.exchange_durable = kwargs.get("exchange_durable", True)
self.exchange_opts = dict(durable=self.exchange_durable,
auto_delete=not self.exchange_durable,
type=self.exchange_type)
self.queue = queue
self.binding_key = binding_key
self.queue_durable = kwargs.get("queue_durable", False)
self.queue_opts = dict(durable=self.queue_durable,
auto_delete=not self.queue_durable,
routing_key=self.binding_key)
self.amqp_url = kwargs.get("amqp_url")
if not self.amqp_url:
self.amqp_url = RabbitmqCtx.get_instance().amqp_url
self.self_delete = kwargs.get("self_delete", False)
self.channel = None
self.queue_obj = None
self.closed = True
if not self.exchange:
raise RuntimeError("exchange is required")
self.Exchange_objs = set()
self.Exchange_dict = dict()
self.connect()
def _reconnect(self):
"""Reconnect to rabbitmq server"""
if self.channel and self.channel.connection and not self.channel.connection.connected: # and not self.channel.closed:
return
global connection
if connection.connected:
connection.close()
self._connect()
# get_connection(self.amqp_url)
def connect(self):
self._connect()
def _connect(self):
connection = get_connection(self.amqp_url)
self.channel = connection.channel()
# nowait, useful?
# arguments ..
logger.debug(
"{class_name}[{id:#x}] is build Communicate framework..".format(
class_name=self.class_name, id=id(self)))
for dc in self.dc:
full_ex = "{}.{}".format(dc, self.exchange)
ex = Exchange(full_ex, channel=self.channel, **self.exchange_opts)
# abstract.py line 67, __call__ will do bind
ex(self.channel).declare()
if not ex.is_bound:
logger.error(
"Exchange[{id:#x}] {exchange} is not bound to chan {chan_id}"
.format(id=id(ex),
exchange=ex,
chan_id=self.channel.channel_id))
logger.debug(
"{class_name}[{id:#x}] create Exchange[{eid:#x}]".format(
class_name=self.class_name, id=id(self), eid=id(ex)))
logger.debug("Exchange[{id:#x}] build opts: {e_opts}".format(
id=id(ex), e_opts=self.exchange_opts))
logger.debug("Exchange[{id:#x}] opts: {e_opts}".format(
id=id(ex),
e_opts=dict(durable=ex.durable,
type=ex.type,
auto_delete=ex.auto_delete)))
self.Exchange_objs.add(ex)
self.Exchange_dict[full_ex] = ex
if self.queue:
self.Queue_obj = Queue(self.queue,
self.Exchange_objs,
channel=self.channel,
**self.queue_opts)
self.Queue_obj.declare()
for ex in self.Exchange_objs:
self.Queue_obj.bind_to(ex, routing_key=self.binding_key)
logger.debug(
"{class_name}[{id:#x}] create Queue obj[{qid:#x}]: {q}".format(
class_name=self.class_name,
id=id(self),
qid=id(self.Queue_obj),
q=self.Queue_obj))
logger.debug("Queue obj[{id:#x}] build opts: {q_opts}".format(
id=id(self.Queue_obj), q_opts=self.queue_opts))
logger.debug("Queue obj[{id:#x}] opts: {q_opts}".format(
id=id(self.Queue_obj),
q_opts=dict(durable=self.Queue_obj.durable,
routing_key=self.Queue_obj.routing_key,
auto_delete=self.Queue_obj.auto_delete)))
self.closed = False
def reconnect(self):
try:
self._reconnect()
except Exception:
# try once more
logger.warn("----- Babel fail to reconnect : \n%s", sys.exc_info())
self._reconnect()
def __del__(self):
logger.debug("{cn} default del method".format(cn=self.class_name))
if not self.closed:
logger.debug(
"=============== {class_name}[{id:#x}] Enter closing... ".
format(class_name=self.class_name, id=id(self)))
self.close()
def close(self):
logger.debug(
"{cn}[{id:#x}] is calling KombuMessageQueue close method".format(
cn=self.class_name, id=id(self)))
try:
for ex in self.Exchange_objs:
if not ex.durable:
r = ex.delete()
logger.debug(
"{cn}[{id:#x}] delete {exchange}[{eid:#x}], return {r}"
.format(cn=self.class_name,
id=id(self),
exchange=ex,
eid=id(ex),
r=r))
if self.self_delete and self.queue:
r = self.channel.queue_delete(queue=self.queue, if_unused=True)
logger.debug("delete queue {q} return {r}".format(q=self.queue,
r=r))
global connection
if self.channel and connection and connection.connected:
self.channel.close()
self.channel = None
except Exception:
logger.error(
"!!! Babel Error when close {class_name}[{id:#x}]:\n{exc_info}"
.format(class_name=self.class_name,
id=id(self),
exc_info=traceback.format_exc()))
finally:
self.closed = True
class PulseConsumer(ConsumerMixin):
"""
Consume jobs from Pulse exchanges
"""
def __init__(self, connection):
self.connection = connection
self.consumers = []
self.queue = None
self.queue_name = "queue/{}/{}".format(connection.userid,
self.queue_suffix)
def get_consumers(self, Consumer, channel):
return [Consumer(**c) for c in self.consumers]
def bind_to(self, exchange, routing_key):
if not self.queue:
self.queue = Queue(
name=self.queue_name,
channel=self.connection.channel(),
exchange=exchange,
routing_key=routing_key,
durable=True,
auto_delete=False,
)
self.consumers.append(
dict(queues=self.queue, callbacks=[self.on_message]))
# just in case the queue does not already exist on Pulse
self.queue.declare()
else:
self.queue.bind_to(exchange=exchange, routing_key=routing_key)
def unbind_from(self, exchange, routing_key):
self.queue.unbind_from(exchange, routing_key)
def close(self):
self.connection.release()
def prune_bindings(self, new_bindings):
# get the existing bindings for the queue
bindings = []
try:
bindings = self.get_bindings(self.queue_name)["bindings"]
except Exception:
logger.error(
"Unable to fetch existing bindings for %s. Data ingestion may proceed, "
"but no bindings will be pruned", self.queue_name)
# Now prune any bindings from the queue that were not
# established above.
# This indicates that they are no longer in the config, and should
# therefore be removed from the durable queue bindings list.
for binding in bindings:
if binding["source"]:
binding_str = self.get_binding_str(binding["source"],
binding["routing_key"])
if binding_str not in new_bindings:
self.unbind_from(Exchange(binding["source"]),
binding["routing_key"])
logger.info("Unbound from: %s", binding_str)
def get_binding_str(self, exchange, routing_key):
"""Use consistent string format for binding comparisons"""
return "{} {}".format(exchange, routing_key)
def get_bindings(self, queue_name):
"""Get list of bindings from the pulse API"""
return fetch_json("{}queue/{}/bindings".format(PULSE_GUARDIAN_URL,
queue_name))
try:
queue = Queue(name=queue, channel=channel, **self._queue_kwargs)
except Exception, e:
logging.exception(e)
if type(queue) is not Queue:
raise ValueError('No valid queue available')
queue.declare()
queue.purge()
logging.info("Binding queue '%s' to exchange '%s' with:" % (queue.name, exchange))
routing_keys = routing_keys or ['#']
for rk in routing_keys:
try:
logging.debug("rk: %s" % rk)
queue.bind_to(exchange=exchange, routing_key=rk)
except Exception, e:
logging.exception(str(e))
logging.info('Done: binding')
consumer_tag = '%s::%s::consuming_exchange' % (self.name, queue.name)
queue.consume(consumer_tag, callback=self._consumer_producer_callback, no_ack=True)
self._queues.append(queue)
return queue
def _consumer_producer_callback(self, message):
routing_key = message.delivery_info.get('routing_key')
self.publish(message, routing_key)
logging.info("Published a message with rk '%s' to '%s'" % (routing_key, self.name))
