def handle(self, *args, **options):
config = settings.PULSE_DATA_INGESTION_CONFIG
userid = urlparse(config).username
durable = settings.PULSE_DATA_INGESTION_QUEUES_DURABLE
auto_delete = settings.PULSE_DATA_INGESTION_QUEUES_AUTO_DELETE
connection = Connection(config)
consumer = JobConsumer(connection)
try:
for exchange_obj in settings.PULSE_DATA_INGESTION_EXCHANGES:
exchange = Exchange(exchange_obj["name"], type="topic")
exchange(connection).declare(passive=True)
self.stdout.write("Connected to Pulse Exchange: {}".format(
exchange_obj["name"]))
for project in exchange_obj["projects"]:
queue_name = "queue/{}/".format(userid)
for destination in exchange_obj['destinations']:
routing_key = "{}.{}".format(project, destination)
consumer.listen_to(exchange, routing_key, queue_name,
durable, auto_delete)
self.stdout.write(
"Pulse message consumer listening to : {} {}".
format(exchange.name, routing_key))
consumer.run()
finally:
consumer.close()
def handle(self, *args, **options):
config = settings.PULSE_DATA_INGESTION_CONFIG
userid = urlparse(config).username
durable = settings.PULSE_DATA_INGESTION_QUEUES_DURABLE
auto_delete = settings.PULSE_DATA_INGESTION_QUEUES_AUTO_DELETE
connection = Connection(config)
consumer = JobConsumer(connection)
try:
for exchange_obj in settings.PULSE_DATA_INGESTION_EXCHANGES:
exchange = Exchange(exchange_obj["name"], type="topic")
exchange(connection).declare(passive=True)
self.stdout.write("Connected to Pulse Exchange: {}".format(
exchange_obj["name"]))
for project in exchange_obj["projects"]:
queue_name = "queue/{}/".format(userid)
for destination in exchange_obj['destinations']:
routing_key = "{}.{}".format(project, destination)
consumer.listen_to(
exchange,
routing_key,
queue_name,
durable,
auto_delete)
self.stdout.write(
"Pulse message consumer listening to : {} {}".format(
exchange.name,
routing_key
))
consumer.run()
finally:
consumer.close()
def handle(self, *args, **options):
config = settings.PULSE_DATA_INGESTION_CONFIG
if not config:
raise ImproperlyConfigured(
"PULSE_DATA_INGESTION_CONFIG must be set")
sources = settings.PULSE_DATA_INGESTION_SOURCES
if not sources:
raise ImproperlyConfigured(
"PULSE_DATA_INGESTION_SOURCES must be set")
new_bindings = []
with Connection(config.geturl()) as connection:
consumer = JobConsumer(connection, "jobs")
for source in sources:
# When creating this exchange object, it is important that it
# be set to ``passive=True``. This will prevent any attempt by
# Kombu to actually create the exchange.
exchange = Exchange(source["exchange"],
type="topic",
passive=True)
# ensure the exchange exists. Throw an error if it doesn't
exchange(connection).declare()
for project in source["projects"]:
for destination in source['destinations']:
routing_key = "{}.{}".format(destination, project)
consumer.bind_to(exchange, routing_key)
new_binding_str = consumer.get_binding_str(
exchange.name, routing_key)
new_bindings.append(new_binding_str)
self.stdout.write("Pulse queue {} bound to: {}".format(
consumer.queue_name, new_binding_str))
consumer.prune_bindings(new_bindings)
try:
consumer.run()
except KeyboardInterrupt:
self.stdout.write("Pulse Job listening stopped...")
def handle(self, *args, **options):
config = settings.PULSE_DATA_INGESTION_CONFIG
if not config:
raise ImproperlyConfigured("PULSE_DATA_INGESTION_CONFIG must be set")
sources = settings.PULSE_DATA_INGESTION_SOURCES
if not sources:
raise ImproperlyConfigured("PULSE_DATA_INGESTION_SOURCES must be set")
new_bindings = []
with Connection(config.geturl()) as connection:
consumer = JobConsumer(connection, "jobs")
for source in sources:
# When creating this exchange object, it is important that it
# be set to ``passive=True``. This will prevent any attempt by
# Kombu to actually create the exchange.
exchange = Exchange(source["exchange"], type="topic",
passive=True)
# ensure the exchange exists. Throw an error if it doesn't
exchange(connection).declare()
for project in source["projects"]:
for destination in source['destinations']:
routing_key = "{}.{}".format(destination, project)
consumer.bind_to(exchange, routing_key)
new_binding_str = consumer.get_binding_str(
exchange.name,
routing_key)
new_bindings.append(new_binding_str)
self.stdout.write(
"Pulse queue {} bound to: {}".format(
consumer.queue_name,
new_binding_str
))
consumer.prune_bindings(new_bindings)
try:
consumer.run()
except KeyboardInterrupt:
self.stdout.write("Pulse Job listening stopped...")
def handle(self, *args, **options):
config = settings.PULSE_DATA_INGESTION_CONFIG
assert config, "PULSE_DATA_INGESTION_CONFIG must be set"
sources = settings.PULSE_DATA_INGESTION_SOURCES
assert sources, "PULSE_DATA_INGESTION_SOURCES must be set"
# get the existing bindings for the queue
bindings = []
new_bindings = []
with Connection(config.geturl()) as connection:
consumer = JobConsumer(connection)
try:
bindings = self.get_bindings(consumer.queue_name)["bindings"]
except Exception:
self.stderr.write(
"ERROR: Unable to fetch existing bindings for {}".format(
consumer.queue_name))
self.stderr.write("ERROR: Data ingestion may proceed, "
"but no bindings will be pruned")
for source in sources:
# When creating this exchange object, it is important that it
# be set to ``passive=True``. This will prevent any attempt by
# Kombu to actually create the exchange.
exchange = Exchange(source["exchange"],
type="topic",
passive=True)
# ensure the exchange exists. Throw an error if it doesn't
exchange(connection).declare()
for project in source["projects"]:
for destination in source['destinations']:
routing_key = "{}.{}".format(destination, project)
consumer.bind_to(exchange, routing_key)
new_binding_str = self.get_binding_str(
exchange.name, routing_key)
new_bindings.append(new_binding_str)
self.stdout.write("Pulse queue {} bound to: {}".format(
consumer.queue_name, new_binding_str))
# Now prune any bindings from the our queue that were not
# established above.
# This indicates that they are no longer in the config, and should
# therefore be removed from the durable queue bindings list.
for binding in bindings:
if binding["source"]:
binding_str = self.get_binding_str(binding["source"],
binding["routing_key"])
if binding_str not in new_bindings:
consumer.unbind_from(Exchange(binding["source"]),
binding["routing_key"])
self.stdout.write(
"Unbound from: {}".format(binding_str))
try:
consumer.run()
except KeyboardInterrupt:
self.stdout.write("Pulse listening stopped...")
def handle(self, *args, **options):
config = settings.PULSE_DATA_INGESTION_CONFIG
assert config, "PULSE_DATA_INGESTION_CONFIG must be set"
sources = settings.PULSE_DATA_INGESTION_SOURCES
assert sources, "PULSE_DATA_INGESTION_SOURCES must be set"
# get the existing bindings for the queue
bindings = []
new_bindings = []
with Connection(config.geturl()) as connection:
consumer = JobConsumer(connection)
try:
bindings = self.get_bindings(consumer.queue_name)["bindings"]
except Exception:
self.stderr.write(
"ERROR: Unable to fetch existing bindings for {}".format(
consumer.queue_name))
self.stderr.write("ERROR: Data ingestion may proceed, "
"but no bindings will be pruned")
for source in sources:
# When creating this exchange object, it is important that it
# be set to ``passive=True``. This will prevent any attempt by
# Kombu to actually create the exchange.
exchange = Exchange(source["exchange"], type="topic",
passive=True)
# ensure the exchange exists. Throw an error if it doesn't
exchange(connection).declare()
for project in source["projects"]:
for destination in source['destinations']:
routing_key = "{}.{}".format(destination, project)
consumer.bind_to(exchange, routing_key)
new_binding_str = self.get_binding_str(exchange.name,
routing_key)
new_bindings.append(new_binding_str)
self.stdout.write(
"Pulse queue {} bound to: {}".format(
consumer.queue_name,
new_binding_str
))
# Now prune any bindings from the our queue that were not
# established above.
# This indicates that they are no longer in the config, and should
# therefore be removed from the durable queue bindings list.
for binding in bindings:
if binding["source"]:
binding_str = self.get_binding_str(binding["source"],
binding["routing_key"])
if binding_str not in new_bindings:
consumer.unbind_from(Exchange(binding["source"]),
binding["routing_key"])
self.stdout.write("Unbound from: {}".format(binding_str))
try:
consumer.run()
except KeyboardInterrupt:
self.stdout.write("Pulse listening stopped...")
