def test_manual_declare(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, auto_declare=False)
self.assertTrue(p.exchange.is_bound)
self.assertNotIn("exchange_declare", channel, "auto_declare=False does not declare exchange")
p.declare()
self.assertIn("exchange_declare", channel, "p.declare() declares exchange")
def test_manual_declare(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, auto_declare=False)
self.assertTrue(p.exchange.is_bound)
self.assertNotIn('exchange_declare', channel,
'auto_declare=False does not declare exchange')
p.declare()
self.assertIn('exchange_declare', channel,
'p.declare() declares exchange')
def test_manual_declare(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, auto_declare=False)
assert p.exchange.is_bound
# auto_declare=False does not declare exchange
assert 'exchange_declare' not in channel
# p.declare() declares exchange')
p.declare()
assert 'exchange_declare' in channel
def test_manual_declare(self):
channel = self.connection.channel()
p = Producer(channel, self.exchange, auto_declare=False)
assert p.exchange.is_bound
# auto_declare=False does not declare exchange
assert 'exchange_declare' not in channel
# p.declare() declares exchange')
p.declare()
assert 'exchange_declare' in channel
def send(self, message):
print("Send called", message)
with self._create_connection() as connection:
producer = Producer(connection,
exchange=self._exchange,
routing_key=Config.rabbit_mq_routing_key)
payload = message.payload()
producer.declare()
producer.publish(payload,
exchange=self._exchange,
routing_key=Config.rabbit_mq_routing_key)
connection = Connection(auth_url,
heartbeat=60,
transport_options=transport_options)
# noqa http://docs.celeryproject.org/projects/kombu/en/latest/userguide/producers.html
print("getting channel")
channel = connection.channel()
exchange_type = "topic"
exchange = Exchange("east-coast", type=exchange_type, durable=True)
print("creating producer")
producer = Producer(channel=channel, auto_declare=True, serializer="json")
print("declaring producer")
producer.declare()
print(("declaring exchange={}").format(exchange.name))
# noqa http://docs.celeryproject.org/projects/kombu/en/latest/reference/kombu.html#queue
queues = [
Queue("us.east.charlotte",
exchange,
routing_key="us.east.charlotte",
durable=True),
Queue("us.east.newyork",
exchange,
routing_key="us.east.newyork",
durable=False)
]
def send(self): with self.make_connection() as connection: producer = Producer(connection, exchange = self.exchange, routing_key = self.routing_key) producer.declare() producer.publish(self.payload)
def task_publish_to_core(self=None, publish_node=None):
"""task_publish_to_core
:param self: parent task object for bind=True
:param publish_node: dictionary to send to the AntiNex Core Worker
"""
if settings.ANTINEX_WORKER_ENABLED:
conn = None
dataset = publish_node["body"].get("dataset", None)
predict_rows = publish_node["body"].get("predict_rows", None)
if not dataset and not predict_rows:
log.info(
("skipping antinex core publish body={} - "
"is missing dataset and predict_rows").format(publish_node))
return None
# end of checking for supported requests to the core
log.info(("task_publish_to_core - start req={}").format(
str(publish_node)[0:32]))
if not predict_rows:
log.info(("building predict_rows from dataset={}").format(dataset))
predict_rows = []
predict_rows_df = pd.read_csv(dataset)
for idx, org_row in predict_rows_df.iterrows():
new_row = json.loads(org_row.to_json())
new_row["idx"] = len(predict_rows) + 1
predict_rows.append(new_row)
# end of building predict rows
publish_node["body"]["apply_scaler"] = True
publish_node["body"]["predict_rows"] = pd.DataFrame(
predict_rows).to_json()
# end of validating
publish_node["body"]["ml_type"] = \
publish_node["body"]["manifest"]["ml_type"]
log.debug(("NEXCORE - ssl={} exchange={} routing_key={}").format(
settings.ANTINEX_SSL_OPTIONS, settings.ANTINEX_EXCHANGE_NAME,
settings.ANTINEX_ROUTING_KEY))
try:
if settings.ANTINEX_WORKER_SSL_ENABLED:
log.debug("connecting with ssl")
conn = Connection(settings.ANTINEX_AUTH_URL,
login_method="EXTERNAL",
ssl=settings.ANTINEX_SSL_OPTIONS)
else:
log.debug("connecting without ssl")
conn = Connection(settings.ANTINEX_AUTH_URL)
# end of connecting
conn.connect()
log.debug("getting channel")
channel = conn.channel()
core_exchange = Exchange(settings.ANTINEX_EXCHANGE_NAME,
type=settings.ANTINEX_EXCHANGE_TYPE,
durable=True)
log.debug("creating producer")
producer = Producer(channel=channel,
auto_declare=True,
serializer="json")
try:
log.debug("declaring exchange")
producer.declare()
except Exception as k:
log.error(("declare exchange failed with ex={}").format(k))
# end of try to declare exchange which can fail if it exists
core_queue = Queue(settings.ANTINEX_QUEUE_NAME,
core_exchange,
routing_key=settings.ANTINEX_ROUTING_KEY,
durable=True)
try:
log.debug("declaring queue")
core_queue.maybe_bind(conn)
core_queue.declare()
except Exception as k:
log.error(("declare queue={} routing_key={} failed with ex={}"
).format(settings.ANTINEX_QUEUE_NAME,
settings.ANTINEX_ROUTING_KEY, k))
# end of try to declare queue which can fail if it exists
log.info(
("publishing exchange={} routing_key={} persist={}").format(
core_exchange.name, settings.ANTINEX_ROUTING_KEY,
settings.ANTINEX_DELIVERY_MODE))
producer.publish(body=publish_node["body"],
exchange=core_exchange.name,
routing_key=settings.ANTINEX_ROUTING_KEY,
auto_declare=True,
serializer="json",
delivery_mode=settings.ANTINEX_DELIVERY_MODE)
except Exception as e:
log.info(("Failed to publish to core req={} with ex={}").format(
publish_node, e))
# try/ex
if conn:
conn.release()
log.info(("task_publish_to_core - done"))
else:
log.debug("core - disabled")
# publish to the core if enabled
return None
class Publisher:
def __init__(self,
name=ev("PUBLISHER_NAME", "kombu-publisher"),
auth_url=ev("BROKER_URL", "redis://localhost:6379/0"),
ssl_options={},
max_general_failures=-1): # infinite retries
"""
Available Transports:
https://github.com/celery/kombu#transport-comparison
"""
self.state = "not_ready"
self.name = name
self.auth_url = auth_url
self.ssl_options = ssl_options
self.exchange = None
self.queue = None
self.declare_entities = []
self.conn = None
self.channel = None
self.producer = None
self.num_setup_failures = 0
self.num_publish_failures = 0
self.max_general_failures = max_general_failures
self.exchange_name = ""
self.exchange_type = "direct"
self.queue_name = ""
self.routing_key = ""
self.serializer = "json"
# end of __init__
def setup_routing(self,
exchange_name,
queue_name,
routing_key,
serializer="json",
on_return=None,
transport_options={},
*args,
**kwargs):
self.exchange_name = exchange_name
self.exchange = None
self.routing_key = routing_key
self.queue_name = queue_name
self.serializer = serializer
if self.routing_key:
log.debug(("creating Exchange={} topic rk={}").format(
self.exchange_name, self.routing_key))
self.exchange_type = "topic"
else:
log.debug(
("creating Exchange={} direct").format(self.exchange_name,
self.routing_key))
self.exchange_type = "direct"
# end of if/else
self.exchange = Exchange(self.exchange_name, type=self.exchange_type)
self.queue = Queue(self.queue_name,
exchange=self.exchange,
routing_key=self.routing_key)
self.declare_entities = [self.exchange, self.queue]
# https://redis.io/topics/security
#
# Redis does not support encryption, but I would like to try out ssl-termination
# using an haproxy/nginx container running as an ssl-proxy to see if this works.
# import ssl
# Connection("amqp://", login_method='EXTERNAL', ssl={
# "ca_certs": '/etc/pki/tls/certs/something.crt',
# "keyfile": '/etc/something/system.key',
# "certfile": '/etc/something/system.cert',
# "cert_reqs": ssl.CERT_REQUIRED,
# })
#
self.conn = Connection(self.auth_url,
transport_options=transport_options)
self.channel = self.conn.channel()
log.debug(("creating kombu.Producer broker={} "
"ssl={} ex={} rk={} serializer={}").format(
self.auth_url, self.ssl_options, self.exchange_name,
self.routing_key, self.serializer))
self.producer = Producer(channel=self.channel,
exchange=self.exchange,
routing_key=self.routing_key,
auto_declare=True,
serializer=self.serializer,
on_return=None,
*args,
**kwargs)
log.debug("creating kombu.Exchange={}".format(self.exchange))
self.producer.declare()
log.debug("creating kombu.Queue={}".format(self.queue_name))
self.queue.maybe_bind(self.conn)
self.queue.declare()
self.state = "ready"
# end of setup_routing
def publish(
self,
body,
exchange,
routing_key,
queue,
priority=0,
ttl=None,
delivery_mode=2, # 1 - transient, 2 - persistent
serializer="json",
retry=True,
silent=False,
transport_options={},
*args,
**kwargs):
"""
Redis does not have an Exchange or Routing Keys, but RabbitMQ does.
Redis producers uses only the queue name to both publish and consume messages:
http://docs.celeryproject.org/en/latest/getting-started/brokers/redis.html#configuration
"""
msg_sent = False
if self.state != "ready":
try:
self.setup_routing(exchange_name=exchange,
queue_name=queue,
routing_key=routing_key,
serializer=serializer,
on_return=None,
transport_options=transport_options)
self.num_setup_failures = 0
self.num_publish_failures = 0
except Exception as c:
sleep_duration = calc_backoff_timer(self.num_setup_failures)
log.info(("SEND - Failed setup_routing with"
"exchange={} rk={} ex={} sleep seconds={}").format(
self.exchange.name, self.routing_key, c,
sleep_duration))
self.num_setup_failures += 1
self.state = "not_ready"
time.sleep(sleep_duration)
# end try/ex to setup the broker
if self.state != "ready":
log.info(("not in a ready state after "
"setup_routing - {} - stopping").format(
self.state.upper()))
return msg_sent
# end of initializing for the first time
if not silent:
log.info(("SEND - "
"exch={} rk={}").format(self.exchange.name,
self.routing_key))
# http://docs.celeryproject.org/projects/kombu/en/latest/_modules/kombu/messaging.html#Producer.publish
# http://docs.celeryproject.org/projects/kombu/en/latest/reference/kombu.html#kombu.Exchange.delivery_mode
try:
self.producer.publish(body=body,
exchange=self.exchange.name,
routing_key=self.routing_key,
auto_declare=True,
serializer=self.serializer,
priority=priority,
delivery_mode=delivery_mode,
expiration=ttl,
retry=False,
*args,
**kwargs)
msg_sent = True
self.num_publish_failures = 0
except Exception as e:
msg_sent = False
sleep_duration = calc_backoff_timer(self.num_publish_failures)
log.info(("SEND - Failed publish with"
"exchange={} rk={} ex={} sleep seconds={}").format(
self.exchange.name, self.routing_key, e,
sleep_duration))
self.num_publish_failures += 1
self.state = "not_ready"
time.sleep(sleep_duration)
# end of try/ex publish
if not silent:
log.debug(
("DONE - "
"exch={} queues={} sent={}").format(self.state.upper(),
self.exchange.name,
self.queue, msg_sent))
return msg_sent
