def put(self, msg, msg_id=None):
""" Put a message. """
delivery_mode = 1
if msg.header.get("persistent", "false") == "true":
delivery_mode = 2
header = dict()
for key, val in msg.header.items():
if type(key) == six.text_type:
key = key.encode("utf-8")
if type(val) == six.text_type:
val = val.encode("utf-8")
header[key] = val
msg.header = header
properties = self._pika.BasicProperties(timestamp=time.time(),
headers=msg.header,
delivery_mode=delivery_mode)
if "content-type" in msg.header:
content_type = msg.header["content-type"]
if content_type.startswith("text/") or \
"charset=" in content_type:
if not msg.is_text():
raise AmqpcltError("unexpected text content-type "
"for binary message: %s" % content_type)
else:
if msg.is_text():
raise AmqpcltError("unexpected binary content-type for "
"text message: %s" % content_type)
properties.content_type = content_type
elif msg.is_text():
properties.content_type = "text/unknown"
# Send the message
if "destination" not in msg.header:
raise AmqpcltError("message doesn't have a destination: %s" % msg)
destination = common.parse_sender_destination(
msg.header["destination"])
if "queue" in destination:
queue_name = self._maybe_declare_queue(destination["queue"])
exch_name = self._maybe_declare_exchange(
destination.get("exchange", dict()))
if exch_name and "queue" in destination:
self._maybe_bind(queue_name, exch_name,
destination.get("routing_key", ""))
if type(msg.body) == six.text_type:
body = msg.body.encode("utf-8")
else:
body = msg.body
self._channel.basic_publish(exchange=exch_name,
routing_key=destination.get(
"routing_key", ""),
body=body,
properties=properties)
if msg_id is None:
return list()
else:
return [
msg_id,
]
def _maybe_subscribe(self, subscription):
""" May be subscribe to queue. """
if "queue" in subscription:
queue_name = self._maybe_declare_queue(subscription["queue"])
else:
raise AmqpcltError("subscription must contain a queue")
exchange_name = None
if "exchange" in subscription:
exchange_name = self._maybe_declare_exchange(
subscription["exchange"])
if exchange_name:
self._maybe_bind(queue_name, exchange_name,
subscription.get("routing_key", ""))
if queue_name not in self._consume:
LOGGER.debug("incoming consume from queue: %s" % (queue_name, ))
tag = get_uuid()
params = {
"consumer_callback": self._handle_delivery,
"queue": queue_name,
"consumer_tag": tag
}
if not self._config["reliable"]:
params["no_ack"] = True
self._channel.basic_consume(**params)
self._consume[queue_name] = tag
def put(self, msg, msg_id=None):
""" Put a message. """
delivery_mode = 1
if msg.header.get("persistent", "false") == "true":
delivery_mode = 2
# Send the message
if "destination" not in msg.header:
raise AmqpcltError("message doesn't have a destination: %s" % msg)
destination = common.parse_sender_destination(
msg.header["destination"])
if "queue" in destination:
queue_name = self._maybe_declare_queue(destination["queue"])
exch_name = self._maybe_declare_exchange(
destination.get("exchange", dict()))
if exch_name and "queue" in destination:
self._maybe_bind(queue_name, exch_name,
destination.get("routing_key", ""))
extra = dict()
if "content-type" in msg.header:
content_type = msg.header["content-type"]
if content_type.startswith("text/") or \
"charset=" in content_type:
if not msg.is_text():
raise AmqpcltError("unexpected text content-type "
"for binary message: %s" % content_type)
else:
if msg.is_text():
raise AmqpcltError("unexpected binary content-type for "
"text message: %s" % content_type)
extra["content_type"] = content_type
elif msg.is_text():
extra["content_type"] = "text/unknown"
self._producer.publish(exchange=exch_name,
routing_key=destination.get("routing_key", ""),
delivery_mode=delivery_mode,
serializer=None,
compression=None,
headers=msg.header,
body=msg.body,
**extra)
if msg_id is None:
return list()
else:
return [
msg_id,
]
def _initialize_callback(self):
""" Initialize callback. """
if self._config.get("callback", None) is None:
return
callback_code = self._config.get("callback-code")
callback_path = self._config.get("callback-path")
if callback_code is not None:
self.logger.debug("callback inline")
if re.search("^\s*def ", callback_code, re.MULTILINE):
code = callback_code
else:
code = "\n "
code += "\n ".join(callback_code.splitlines())
code = ("""
def check(self, msg):
hdr = msg.header
""" + code + """
return msg""")
elif callback_path is not None:
self.logger.debug("callback file %s" % (callback_path, ))
try:
call_file = open(callback_path, "r")
code = call_file.read()
except IOError:
error = sys.exc_info()[1]
raise AmqpcltError("invalid callback file: %s" % error)
else:
call_file.close()
else:
raise AmqpcltError("callback parameters not complete")
try:
self._callback = _callback_module(code)()
except SyntaxError:
error = sys.exc_info()[1]
raise AmqpcltError("syntax error in the callback: %s" % error)
if not hasattr(self._callback, "check"):
raise AmqpcltError("check(message) missing in callback: %s")
callback_data = self._config.get("callback-data")
if callback_data is None:
self._config["callback-data"] = list()
else:
callback_data = callback_data.split(",")
self._config["callback-data"] = callback_data
def connect(self):
""" Create a pika AMQP connection and channel. """
direction = self.direction
config = self._config[direction]["broker"]
params = common.parse_amqp_uri(config["uri"])
cred = config.get("auth")
if cred is None:
cred = credential.new(scheme="none")
if cred.scheme == "x509":
if self._pika.__version__ < "0.9.6":
raise AmqpcltError(
"x509 authentication not supported in pika %s" %
self._pika.__version__)
ssl_options = dict()
for key, keyval in {
"cert": "certfile",
"key": "keyfile",
"ca": "ca_certs"
}.items():
if key in cred:
ssl_options[keyval] = cred[key]
ssl_options["cert_reqs"] = ssl.CERT_REQUIRED
ssl_options["ssl_version"] = ssl.PROTOCOL_SSLv3
extra = {
"ssl": True,
"ssl_options": ssl_options,
"credentials": self._pika.credentials.ExternalCredentials()
}
elif cred.scheme == "plain":
extra = {
"credentials":
self._pika.credentials.PlainCredentials(
cred['name'], cred['pass']),
}
else:
# none
extra = dict()
# if self._config.get("heartbeat") is not None:
# extra["heartbeat"] = self._config["heartbeat"]
parameters = self._pika.connection.ConnectionParameters(
params['host'].encode(), int(params['port']),
params.get('virtual_host', "rabbitmq").encode(), **extra)
self._connection = self._pika.BlockingConnection(parameters)
self._channel = self._connection.channel()
self._server_properties = self._connection._impl.server_properties
LOGGER.debug("%s broker %s:%s: %s %s" %
(direction, params['host'], params['port'],
self.server_type(), self.server_version()))
if self._config.get("%s-broker-type" % direction) is None:
self._config["%s-broker-type" % direction] = self.server_type()
return True
def work(self):
""" Do it! """
pending = dict()
put_list = list()
timek = dict()
timek["start"] = time.time()
self.logger.debug("starting")
signal.signal(signal.SIGINT, self._handle_sig)
signal.signal(signal.SIGTERM, self._handle_sig)
signal.signal(signal.SIGHUP, self._handle_sig)
if self._config.get("incoming-broker") is not None:
mtype = self._config.get("incoming-broker-module") or "pika"
if mtype == "kombu":
incoming = amqpclt.kombu.KombuIncomingBroker(self._config)
elif mtype == "pika":
incoming = amqpclt.pika.PikaIncomingBroker(self._config)
else:
raise AmqpcltError("invalid incoming broker module: %s" %
(mtype, ))
else:
incoming = amqpclt.queue.IncomingQueue(self._config)
if self._config.get("outgoing-broker") is not None:
mtype = self._config.get("outgoing-broker-module") or "pika"
if mtype == "kombu":
outgoing = amqpclt.kombu.KombuOutgoingBroker(self._config)
elif mtype == "pika":
outgoing = amqpclt.pika.PikaOutgoingBroker(self._config)
else:
raise AmqpcltError("invalid outgoing broker module: %s" %
(mtype, ))
else:
outgoing = amqpclt.queue.OutgoingQueue(self._config)
incoming.start()
if not self._config["lazy"]:
outgoing.start()
if self._config.get("callback", None) is not None:
self._callback.start(*self._config["callback"]["data"])
self.logger.debug("running")
count = size = 0
if self._config.get("duration") is not None:
timek["max"] = time.time() + self._config["duration"]
else:
timek["max"] = 0
tina = self._config.get("timeout-inactivity")
if tina is not None:
timek["ina"] = time.time() + tina
else:
timek["ina"] = 0
self._running = True
while self._running:
# are we done
if self._config.get("count") is not None and \
count >= self._config["count"]:
break
if timek["max"] and time.time() > timek["max"]:
break
# get message
if self._config["reliable"]:
if self._config.get("window") >= 0 and \
len(pending) > self._config("window"):
incoming.idle()
(msg, msg_id) = ("too many pending acks", None)
else:
(msg, msg_id) = incoming.get()
if type(msg) != str:
if msg_id in pending:
self.logger.debug("duplicate ack id: %s" %
(msg_id, ))
sys.exit(1)
else:
pending[msg_id] = True
else:
(msg, msg_id) = incoming.get()
# check inactivity
if timek.get("ina"):
if isinstance(msg, Message):
timek["ina"] = time.time() + tina
elif time.time() >= timek["ina"]:
break
# count and statistics
if isinstance(msg, Message):
count += 1
if self._config["statistics"]:
size += msg.size()
if count == 1:
timek["first"] = time.time()
# callback
if self._config.get("callback") is not None:
if type(msg) != str:
msg = self._callback.check(msg)
if not isinstance(msg, Message):
self.logger.debug("message discarded by callback: %s" %
(msg, ))
# message discarded by callback
if self._config["reliable"]:
if msg_id not in pending:
raise AmqpcltError("unexpected ack id: %s" %
(msg_id, ))
del (pending[msg_id])
incoming.ack(msg_id)
if self._config["statistics"]:
timek["last"] = time.time()
else:
self._callback.idle()
# put | idle
if isinstance(msg, Message):
self.logger.debug("loop got new message")
if self._config["lazy"]:
outgoing.start()
self._config["lazy"] = False
put_list = outgoing.put(msg, msg_id)
if self._config["statistics"]:
timek["last"] = time.time()
else:
if msg:
self.logger.debug("loop %s" % (msg, ))
else:
self.logger.debug("loop end")
self._running = False
if self._config["lazy"]:
put_list = list()
else:
put_list = outgoing.idle()
# ack
for msg_id in put_list:
if msg_id not in pending:
raise AmqpcltError("unexpected ack id: %s" % (msg_id, ))
del (pending[msg_id])
incoming.ack(msg_id)
# check --quit and show that we are alive
if self._config.get("pidfile"):
action = pid_check(self._config["pidfile"])
if action == "quit":
self.logger.debug("asked to quit")
break
pid_touch(self._config["pidfile"])
# linger
self.logger.debug("linger")
timeout_linger = self._config.get("timeout-linger")
if timeout_linger:
timek["max"] = time.time() + timeout_linger
else:
timek["max"] = 0
self._running = True
while self._running:
if not pending:
break
if "max" in timek and time.time() >= timek["max"]:
break
put_list = outgoing.idle()
if put_list:
for msg_id in put_list:
if msg_id not in pending:
raise AmqpcltError("unexpected ack id: %s" %
(msg_id, ))
del (pending[msg_id])
incoming.ack(msg_id)
else:
incoming.idle()
if pending:
raise AmqpcltError("%d pending messages" % len(pending))
# report statistics
if self._config.get("statistics"):
if count == 0:
print("no messages processed")
elif count == 1:
print("only 1 message processed")
else:
timek["elapsed"] = timek["last"] - timek["first"]
print(
("processed %d messages in %.3f seconds"
" (%.3f k messages/second)") %
(count, timek["elapsed"], count / timek["elapsed"] / 1000))
print("troughput is around %.3f MB/second" %
(size / timek["elapsed"] / 1024 / 1024))
print("average message size is around %d bytes" %
(int(size / count + 0.5)))
# stop
self.logger.debug("stopping")
if self._config.get("callback", None) is not None:
self.logger.debug("stopping callback")
self._callback.stop()
if not self._config.get("lazy"):
self.logger.debug("stopping outgoing")
outgoing.stop()
self.logger.debug("stopping incoming")
incoming.stop()
self.logger.debug("incoming stopped")
timek["stop"] = time.time()
timek["elapsed"] = timek["stop"] - timek["start"]
self.logger.debug("work processed %d messages in %.3f seconds" %
(count, timek["elapsed"]))