def rmr_rcvall_msgs_raw(mrc, pass_filter=[]):
"""
Same as rmr_rcvall_msgs, but the raw sbuf is also returned.
Useful, for example, if rts is to be used.
Parameters
----------
mrc: ctypes c_void_p
Pointer to the RMR context
pass_filter: list (optional)
The message type(s) to capture.
Returns
-------
list of tuple:$
List of tuples [(S, sbuf),...] where S is a message summary and sbuf is the raw message$
the caller is responsible for calling rmr.rmr_free_msg(sbuf) for each sbuf afterwards to prevent memory leaks.
"""
new_messages = []
while True:
mbuf = rmr.rmr_alloc_msg(mrc, 4096) # allocate buffer to have something for a return status
mbuf = rmr.rmr_torcv_msg(mrc, mbuf, 0) # set the timeout to 0 so this doesn't block!!
summary = rmr.message_summary(mbuf)
if summary["message status"] != "RMR_OK":
break
if len(pass_filter) == 0 or mbuf.contents.mtype in pass_filter: # no filter, or passes; capture it
new_messages.append((summary, mbuf))
else:
rmr.rmr_free_msg(mbuf)
return new_messages
def _send_msg(self, pay, mtype, subid):
"""
sends a msg
"""
for _ in range(0, RETRY_TIMES):
sbuf = rmr.rmr_alloc_msg(self.mrc,
len(pay),
payload=pay,
gen_transaction_id=True,
mtype=mtype,
sub_id=subid)
sbuf.contents.sub_id = subid
pre_send_summary = rmr.message_summary(sbuf)
sbuf = rmr.rmr_send_msg(self.mrc, sbuf) # send
if self._assert_good_send(sbuf, pre_send_summary):
rmr.rmr_free_msg(sbuf) # free
break
def test_rcv_all():
"""
test the ability to receive a batch of queued messages.
"""
pay_fmt = "send to ring msg: %d" # dynamic message format with counter
send_burst(MRC_SEND,
pay_fmt) # send a bunch of 13 messages that should queue
time.sleep(1) # ensure underlying transport gets cycles to send/receive
bundle = helpers.rmr_rcvall_msgs(
MRC_BUF_RCV
) # use the buffered receiver to read all with a single call
assert len(bundle) == 13
for i, ms in enumerate(bundle):
ms = bundle[
i] # validate each summary returned, and ordering preserved
assert ms["message state"] == 0
expected_pay = bytes(pay_fmt % i, "UTF-8")
assert ms["payload"] == expected_pay
send_burst(
MRC_SEND, pay_fmt, mtype=1,
num=10) # send a second round with msg types 1 and 2 to test filter
send_burst(MRC_SEND, pay_fmt, mtype=2, num=8)
send_burst(MRC_SEND, pay_fmt, mtype=1, num=5)
send_burst(MRC_SEND, pay_fmt, mtype=2, num=4,
counter=8) # total of 12 messages with type 2 should be queued
time.sleep(1) # ensure underlying transport gets cycles to send/receive
bundle = helpers.rmr_rcvall_msgs_raw(
MRC_BUF_RCV, [2]) # receive only message type 2 messages
assert len(
bundle) == 12 # we should only get the second batch of 12 messages
for i, (ms, sbuf) in enumerate(bundle): # test the raw version
test_summary = rmr.message_summary(sbuf)
assert test_summary == ms
assert ms["message state"] == 0 # all should be OK
assert ms["message type"] == 2 # only mtype 2 should have been received
expected_pay = bytes(
pay_fmt % i,
"UTF-8") # ordering should still jive with the counter
assert ms["payload"] == expected_pay
rmr.rmr_free_msg(sbuf)
def rmr_rcvall_msgs(mrc, pass_filter=[]):
"""
Assemble an array of all messages which can be received without
blocking. Effectively draining the message queue if RMR is started
in mt-call mode, or draining any waiting TCP buffers. If the
pass_filter parameter is supplied it is treated as one or more message
types to accept (pass through). Using the default, an empty list, results
in messages with any type being captured.
Parameters
----------
mrc: ctypes c_void_p
Pointer to the RMR context
pass_filter: list (optional)
The message type(s) to capture.
Returns
-------
list of dict
List of message summaries, one for each message captured.
"""
new_messages = []
mbuf = rmr.rmr_alloc_msg(mrc, 4096) # allocate buffer to have something for a return status
while True:
mbuf = rmr.rmr_torcv_msg(mrc, mbuf, 0) # set the timeout to 0 so this doesn't block!!
summary = rmr.message_summary(mbuf)
if summary["message status"] != "RMR_OK": # ok indicates msg received, stop on all other states
break
if len(pass_filter) == 0 or summary["message type"] in pass_filter: # no filter, or passes; capture it
new_messages.append(summary)
rmr.rmr_free_msg(mbuf) # must free message to avoid leak
return new_messages
def loop(self):
"""
This loop runs forever, and has 3 jobs:
- send out any messages that have to go out (create instance, delete instance)
- read a1s mailbox and update the status of all instances based on acks from downstream policy handlers
- clean up the database (eg delete the instance) under certain conditions based on those statuses (NOT DONE YET)
"""
# loop forever
mdc_logger.debug("Work loop starting")
while self.keep_going:
# send out all messages waiting for us
while not self.instance_send_queue.empty():
work_item = self.instance_send_queue.get(block=False,
timeout=None)
payload = json.dumps(
messages.a1_to_handler(*work_item)).encode("utf-8")
self._send_msg(payload, A1_POLICY_REQUEST, work_item[1])
# read our mailbox
for (msg, sbuf) in self.rcv_func():
# TODO: in the future we may also have to catch SDL errors
try:
mtype = msg["message type"]
except (KeyError, TypeError, json.decoder.JSONDecodeError):
mdc_logger.debug(
"Dropping malformed policy ack/query message: {0}".
format(msg))
if mtype == A1_POLICY_RESPONSE:
try:
# got a policy response, update status
pay = json.loads(msg["payload"])
data.set_policy_instance_status(
pay["policy_type_id"], pay["policy_instance_id"],
pay["handler_id"], pay["status"])
mdc_logger.debug(
"Successfully received status update: {0}".format(
pay))
except (PolicyTypeNotFound, PolicyInstanceNotFound):
mdc_logger.debug(
"Received a response for a non-existent instance")
except (KeyError, TypeError, json.decoder.JSONDecodeError):
mdc_logger.debug(
"Dropping malformed policy ack message: {0}".
format(msg))
elif mtype == A1_POLICY_QUERY:
try:
# got a query, do a lookup and send out all instances
pti = json.loads(msg["payload"])["policy_type_id"]
mdc_logger.debug("Received query for: {0}".format(pti))
for pii in data.get_instance_list(pti):
instance = data.get_policy_instance(pti, pii)
payload = json.dumps(
messages.a1_to_handler(
"CREATE", pti, pii,
instance)).encode("utf-8")
sbuf = self._rts_msg(payload, sbuf,
A1_POLICY_REQUEST)
except (PolicyTypeNotFound, PolicyInstanceNotFound):
mdc_logger.debug(
"Received a query for a non-existent type: {0}".
format(msg))
except (KeyError, TypeError, json.decoder.JSONDecodeError):
mdc_logger.debug(
"Dropping malformed policy query message: {0}".
format(msg))
else:
mdc_logger.debug(
"Received message type {0} but A1 does not handle this"
.format(mtype))
# we must free each sbuf
rmr.rmr_free_msg(sbuf)
self.last_ran = time.time()
time.sleep(1)