def get_publisher(self, broker_url=None, ssl_options=None):
"""get_publisher
:param broker_url: broker url
:param ssl_options: sll options
"""
if self.pub:
return self.pub
self.pub = Publisher(name=self.name,
auth_url=self.broker_url,
ssl_options=self.ssl_options)
return self.pub
def get_publisher():
"""get_publisher"""
log.info("initializing publisher")
pub = None
auth_url = ""
if FORWARD_ENDPOINT_TYPE == "redis":
auth_url = FORWARD_BROKER_URL
else:
auth_url = FORWARD_BROKER_URL
pub = Publisher(name="{}_{}".format(SOURCE, "-redis"),
auth_url=auth_url,
ssl_options=FORWARD_SSL_OPTIONS)
log.info("publisher={}".format(pub))
return pub
def connect_pub(self,
auth_url=None,
ssl_options={},
attrs={}):
use_auth_url = self.pub_auth_url
use_ssl_options = self.pub_ssl_options
use_pub_attrs = self.pub_attrs
if auth_url:
use_auth_url = auth_url
if len(ssl_options) > 0:
use_ssl_options = ssl_options
if len(ssl_options) > 0:
use_pub_attrs = use_pub_attrs
self.pub = Publisher("test-pub",
use_auth_url,
use_ssl_options)
def publish_regression_prediction_to_broker():
"""publish_regression_prediction_to_broker
Publish a Regression Prediction message to the
Celery Worker's broker queue. This message
is a JSON Dictionary.
Default Broker: ``redis://localhost:6379/6``
Default Exchange: ``webapp.predict.requests``
Default Routing Key: ``webapp.predict.requests``
Default Queue: ``webapp.predict.requests``
"""
parser = argparse.ArgumentParser(description=("Launch a Predict Dataset "
"Request into the AntiNex "
"core"))
parser.add_argument("-f",
help=("request json file to use default: "
"./training/scaler-regression.json"),
required=False,
dest="request_file")
parser.add_argument("-d",
help="debug",
required=False,
dest="debug",
action="store_true")
args = parser.parse_args()
name = "predict-publisher"
log.info("{} - start".format(name))
request_file = "./training/scaler-regression.json"
if args.request_file:
request_file = args.request_file
exchange_name = "webapp.predict.requests"
routing_key = "webapp.predict.requests"
queue_name = "webapp.predict.requests"
auth_url = "redis://localhost:6379/6"
serializer = "json"
if not os.path.exists(request_file):
log.error(("Missing request file={}").format(request_file))
sys.exit(1)
req_data = None
with open(request_file, "r") as cur_file:
req_data = json.loads(cur_file.read())
if not os.path.exists(request_file):
log.error(
("Did not find request data in file={}").format(request_file))
sys.exit(1)
# import ssl
# Connection("amqp://", login_method='EXTERNAL', ssl={
# "ca_certs": '/etc/pki/tls/certs/something.crt',
# "keyfile": '/etc/something/system.key',
# "certfile": '/etc/something/system.cert',
# "cert_reqs": ssl.CERT_REQUIRED,
# })
#
ssl_options = {}
app = Publisher(name, auth_url, ssl_options)
if not app:
log.error(("Failed to connect to broker={}").format(auth_url))
else:
# Now send:
now = datetime.datetime.now().isoformat()
body = req_data
body["created"] = now
log.info("loading predict_rows")
predict_rows_df = pd.read_csv(req_data["dataset"])
predict_rows = []
for idx, org_row in predict_rows_df.iterrows():
new_row = json.loads(org_row.to_json())
new_row["idx"] = len(predict_rows) + 1
predict_rows.append(new_row)
body["predict_rows"] = pd.DataFrame(predict_rows).to_json()
log.info(("using predict_rows={}").format(len(predict_rows)))
log.info(("Sending msg={} "
"ex={} rk={}").format(
str(body)[0:10], exchange_name, routing_key))
# Publish the message:
msg_sent = app.publish(body=body,
exchange=exchange_name,
routing_key=routing_key,
queue=queue_name,
serializer=serializer,
retry=True)
log.info(("End - {} sent={}").format(name, msg_sent))
def get_pub(self):
if not self.pub:
self.pub = Publisher("msg-pub", self.pub_auth_url,
self.pub_ssl_options)
return self.pub
name = "kombu-sqs-publisher"
log = build_colorized_logger(name=name)
log.info("Start - {}".format(name))
# Initialize Publisher
# http://docs.celeryproject.org/en/latest/getting-started/brokers/sqs.html
# https://github.com/celery/kombu/blob/master/kombu/transport/SQS.py
aws_key = ev("SQS_AWS_ACCESS_KEY", "not_a_key")
aws_secret = ev("SQS_AWS_SECRET_KEY", "not_a_secret")
sqs_auth_url = ev("SUB_BROKER_URL", "sqs://{}:{}@".format(aws_key, aws_secret))
ssl_options = {}
pub = Publisher("kombu-sqs-publisher", sqs_auth_url, ssl_options)
# sample: "sqs://*****:*****@"
# ^ from the doc: 'you must remember to include the "@" at the end.'
# Now consume:
seconds_to_consume = 10.0
serializer = "json"
exchange = ev("CONSUME_EXCHANGE", "test1")
routing_key = ev("CONSUME_ROUTING_KEY", "test1")
queue = ev("CONSUME_QUEUE", "test1")
max_timeout = 43200
transport_options = {}
if not pub:
log.error(("Failed to connect to " "broker={}").format(sqs_auth_url))
else:
exchange_name = ev("PUBLISH_EXCHANGE", "reporting")
routing_key = ev("PUBLISH_ROUTING_KEY", "reporting.accounts")
queue_name = ev("PUBLISH_QUEUE", "reporting.accounts")
auth_url = ev("PUB_BROKER_URL", "pyamqp://*****:*****@localhost:5672//")
serializer = "json"
# import ssl
# Connection("amqp://", login_method='EXTERNAL', ssl={
# "ca_certs": '/etc/pki/tls/certs/something.crt',
# "keyfile": '/etc/something/system.key',
# "certfile": '/etc/something/system.cert',
# "cert_reqs": ssl.CERT_REQUIRED,
# })
#
ssl_options = {}
app = Publisher("rabbitmq-publisher", auth_url, ssl_options)
if not app:
log.error("Failed to connect to broker={}".format(auth_url))
else:
# Create the message:
now = datetime.datetime.now().isoformat()
body = {"account_id": 456, "created": now}
log.info(("Sending msg={} "
"ex={} rk={}").format(body, exchange_name, routing_key))
# Publish the message:
msg_sent = app.publish(body=body,
exchange=exchange_name,
# https://redis.io/topics/security
#
# Redis does not support encryption, but I would like to try out ssl-termination
# using an haproxy/nginx container running as an ssl-proxy to see if this works.
# import ssl
# Connection("amqp://", login_method='EXTERNAL', ssl={
# "ca_certs": '/etc/pki/tls/certs/something.crt',
# "keyfile": '/etc/something/system.key',
# "certfile": '/etc/something/system.cert',
# "cert_reqs": ssl.CERT_REQUIRED,
# })
#
ssl_options = {}
app = Publisher("redis-publisher", auth_url, ssl_options)
if not app:
log.error(("Failed to connect to broker={}").format(auth_url))
else:
# Now send:
now = datetime.datetime.now().isoformat()
body = {"account_id": 123, "created": now}
log.info(("Sending msg={} "
"ex={} rk={}").format(body, exchange_name, routing_key))
# Publish the message:
msg_sent = app.publish(body=body,
exchange=exchange_name,
exchange_name = ev("PUBLISH_EXCHANGE", "user.events")
routing_key = ev("PUBLISH_ROUTING_KEY", "user.events.conversions")
queue_name = ev("PUBLISH_QUEUE", "user.events.conversions")
auth_url = ev("PUB_BROKER_URL", "redis://localhost:6379/0")
serializer = "json"
# import ssl
# Connection("amqp://", login_method='EXTERNAL', ssl={
# "ca_certs": '/etc/pki/tls/certs/something.crt',
# "keyfile": '/etc/something/system.key',
# "certfile": '/etc/something/system.cert',
# "cert_reqs": ssl.CERT_REQUIRED,
# })
#
ssl_options = {}
app = Publisher("publish-uce-redis", auth_url, ssl_options)
if not app:
log.error("Failed to connect to broker={}".format(auth_url))
else:
# Create the message:
now = datetime.datetime.now().isoformat()
body = {
"account_id": 123,
"subscription_id": 456,
"stripe_id": 789,
"product_id": "ABC",
"created": now
}