class JobConsumer(ConsumerMixin):
"""
Consume jobs from Pulse exchanges
"""
def __init__(self, connection):
self.connection = connection
self.consumers = []
self.queue = None
config = settings.PULSE_DATA_INGESTION_CONFIG
if not config:
raise ValueError("PULSE_DATA_INGESTION_CONFIG is required for the "
"JobConsumer class.")
self.queue_name = "queue/{}/jobs".format(config.username)
def get_consumers(self, Consumer, channel):
return [
Consumer(**c) for c in self.consumers
]
def bind_to(self, exchange, routing_key):
if not self.queue:
self.queue = Queue(
name=self.queue_name,
channel=self.connection.channel(),
exchange=exchange,
routing_key=routing_key,
durable=settings.PULSE_DATA_INGESTION_QUEUES_DURABLE,
auto_delete=settings.PULSE_DATA_INGESTION_QUEUES_AUTO_DELETE
)
self.consumers.append(dict(queues=self.queue,
callbacks=[self.on_message]))
# just in case the queue does not already exist on Pulse
self.queue.declare()
else:
self.queue.bind_to(exchange=exchange, routing_key=routing_key)
def unbind_from(self, exchange, routing_key):
self.queue.unbind_from(exchange, routing_key)
def on_message(self, body, message):
store_pulse_jobs.apply_async(
args=[body,
message.delivery_info["exchange"],
message.delivery_info["routing_key"]],
routing_key='store_pulse_jobs'
)
message.ack()
def close(self):
self.connection.release()
class JobConsumer(ConsumerMixin):
"""
Consume jobs from Pulse exchanges
"""
def __init__(self, connection):
self.connection = connection
self.consumers = []
self.queue = None
config = settings.PULSE_DATA_INGESTION_CONFIG
if not config:
raise ValueError("PULSE_DATA_INGESTION_CONFIG is required for the "
"JobConsumer class.")
self.queue_name = "queue/{}/jobs".format(config.username)
def get_consumers(self, Consumer, channel):
return [Consumer(**c) for c in self.consumers]
def bind_to(self, exchange, routing_key):
if not self.queue:
self.queue = Queue(
name=self.queue_name,
channel=self.connection.channel(),
exchange=exchange,
routing_key=routing_key,
durable=settings.PULSE_DATA_INGESTION_QUEUES_DURABLE,
auto_delete=settings.PULSE_DATA_INGESTION_QUEUES_AUTO_DELETE)
self.consumers.append(
dict(queues=self.queue, callbacks=[self.on_message]))
# just in case the queue does not already exist on Pulse
self.queue.declare()
else:
self.queue.bind_to(exchange=exchange, routing_key=routing_key)
def unbind_from(self, exchange, routing_key):
self.queue.unbind_from(exchange, routing_key)
def on_message(self, body, message):
store_pulse_jobs.apply_async(args=[
body, message.delivery_info["exchange"],
message.delivery_info["routing_key"]
],
routing_key='store_pulse_jobs')
message.ack()
def close(self):
self.connection.release()
class PulseConsumer(ConsumerMixin):
"""
Consume jobs from Pulse exchanges
"""
def __init__(self, connection):
self.connection = connection
self.consumers = []
self.queue = None
self.queue_name = "queue/{}/{}".format(connection.userid,
self.queue_suffix)
def get_consumers(self, Consumer, channel):
return [Consumer(**c) for c in self.consumers]
def bind_to(self, exchange, routing_key):
if not self.queue:
self.queue = Queue(
name=self.queue_name,
channel=self.connection.channel(),
exchange=exchange,
routing_key=routing_key,
durable=True,
auto_delete=False,
)
self.consumers.append(
dict(queues=self.queue, callbacks=[self.on_message]))
# just in case the queue does not already exist on Pulse
self.queue.declare()
else:
self.queue.bind_to(exchange=exchange, routing_key=routing_key)
def unbind_from(self, exchange, routing_key):
self.queue.unbind_from(exchange, routing_key)
def close(self):
self.connection.release()
def prune_bindings(self, new_bindings):
# get the existing bindings for the queue
bindings = []
try:
bindings = self.get_bindings(self.queue_name)["bindings"]
except Exception:
logger.error(
"Unable to fetch existing bindings for %s. Data ingestion may proceed, "
"but no bindings will be pruned", self.queue_name)
# Now prune any bindings from the queue that were not
# established above.
# This indicates that they are no longer in the config, and should
# therefore be removed from the durable queue bindings list.
for binding in bindings:
if binding["source"]:
binding_str = self.get_binding_str(binding["source"],
binding["routing_key"])
if binding_str not in new_bindings:
self.unbind_from(Exchange(binding["source"]),
binding["routing_key"])
logger.info("Unbound from: %s", binding_str)
def get_binding_str(self, exchange, routing_key):
"""Use consistent string format for binding comparisons"""
return "{} {}".format(exchange, routing_key)
def get_bindings(self, queue_name):
"""Get list of bindings from the pulse API"""
return fetch_json("{}queue/{}/bindings".format(PULSE_GUARDIAN_URL,
queue_name))
class PulseConsumer(ConsumerMixin):
"""
Consume jobs from Pulse exchanges
"""
def __init__(self, connection, queue_suffix):
self.connection = connection
self.consumers = []
self.queue = None
config = settings.PULSE_DATA_INGESTION_CONFIG
if not config:
raise ValueError("PULSE_DATA_INGESTION_CONFIG is required for the "
"JobConsumer class.")
self.queue_name = "queue/{}/{}".format(config.username, queue_suffix)
def get_consumers(self, Consumer, channel):
return [
Consumer(**c) for c in self.consumers
]
def bind_to(self, exchange, routing_key):
if not self.queue:
self.queue = Queue(
name=self.queue_name,
channel=self.connection.channel(),
exchange=exchange,
routing_key=routing_key,
durable=settings.PULSE_DATA_INGESTION_QUEUES_DURABLE,
auto_delete=settings.PULSE_DATA_INGESTION_QUEUES_AUTO_DELETE
)
self.consumers.append(dict(queues=self.queue,
callbacks=[self.on_message]))
# just in case the queue does not already exist on Pulse
self.queue.declare()
else:
self.queue.bind_to(exchange=exchange, routing_key=routing_key)
def unbind_from(self, exchange, routing_key):
self.queue.unbind_from(exchange, routing_key)
def close(self):
self.connection.release()
def prune_bindings(self, new_bindings):
# get the existing bindings for the queue
bindings = []
try:
bindings = self.get_bindings(self.queue_name)["bindings"]
except Exception:
logger.error("Unable to fetch existing bindings for {}".format(
self.queue_name))
logger.error("Data ingestion may proceed, "
"but no bindings will be pruned")
# Now prune any bindings from the queue that were not
# established above.
# This indicates that they are no longer in the config, and should
# therefore be removed from the durable queue bindings list.
for binding in bindings:
if binding["source"]:
binding_str = self.get_binding_str(binding["source"],
binding["routing_key"])
if binding_str not in new_bindings:
self.unbind_from(Exchange(binding["source"]),
binding["routing_key"])
logger.info("Unbound from: {}".format(binding_str))
def get_binding_str(self, exchange, routing_key):
"""Use consistent string format for binding comparisons"""
return "{} {}".format(exchange, routing_key)
def get_bindings(self, queue_name):
"""Get list of bindings from the pulse API"""
return fetch_json("{}queue/{}/bindings".format(
settings.PULSE_GUARDIAN_URL, queue_name))
class PulseConsumer(ConsumerMixin):
"""
Consume jobs from Pulse exchanges
"""
def __init__(self, source, build_routing_key):
self.connection = Connection(source['pulse_url'])
self.consumers = []
self.queue = None
self.queue_name = "queue/{}/{}".format(self.connection.userid,
self.queue_suffix)
self.root_url = source['root_url']
self.source = source
self.build_routing_key = build_routing_key
def get_consumers(self, Consumer, channel):
return [Consumer(**c) for c in self.consumers]
def bindings(self):
"""Get the bindings for this consumer, each of the form `<exchange>.<routing_keys>`,
with `<routing_keys>` being `:`-separated."""
return []
def prepare(self):
bindings = []
for binding in self.bindings():
# split source string into exchange and routing key sections
exchange, _, routing_keys = binding.partition('.')
# built an exchange object with our connection and exchange name
exchange = get_exchange(self.connection, exchange)
# split the routing keys up using the delimiter
for routing_key in routing_keys.split(':'):
if self.build_routing_key is not None: # build routing key
routing_key = self.build_routing_key(routing_key)
binding = self.bind_to(exchange, routing_key)
bindings.append(binding)
# prune stale queues using the binding strings
self.prune_bindings(bindings)
def bind_to(self, exchange, routing_key):
if not self.queue:
self.queue = Queue(
name=self.queue_name,
channel=self.connection.channel(),
exchange=exchange,
routing_key=routing_key,
durable=True,
auto_delete=False,
)
self.consumers.append(
dict(queues=self.queue, callbacks=[self.on_message]))
# just in case the queue does not already exist on Pulse
self.queue.declare()
else:
self.queue.bind_to(exchange=exchange, routing_key=routing_key)
# get the binding key for this consumer
binding = self.get_binding_str(exchange.name, routing_key)
logger.info("Pulse queue {} bound to: {}".format(
self.queue_name, binding))
return binding
def unbind_from(self, exchange, routing_key):
self.queue.unbind_from(exchange, routing_key)
def close(self):
self.connection.release()
def prune_bindings(self, new_bindings):
# get the existing bindings for the queue
bindings = []
try:
bindings = self.get_bindings(self.queue_name)["bindings"]
except Exception:
logger.error(
"Unable to fetch existing bindings for %s. Data ingestion may proceed, "
"but no bindings will be pruned", self.queue_name)
# Now prune any bindings from the queue that were not
# established above.
# This indicates that they are no longer in the config, and should
# therefore be removed from the durable queue bindings list.
for binding in bindings:
if binding["source"]:
binding_str = self.get_binding_str(binding["source"],
binding["routing_key"])
if binding_str not in new_bindings:
self.unbind_from(Exchange(binding["source"]),
binding["routing_key"])
logger.info("Unbound from: %s", binding_str)
def get_binding_str(self, exchange, routing_key):
"""Use consistent string format for binding comparisons"""
return "{} {}".format(exchange, routing_key)
def get_bindings(self, queue_name):
"""Get list of bindings from the pulse API"""
return fetch_json("{}queue/{}/bindings".format(PULSE_GUARDIAN_URL,
queue_name))
