def flow_depos(ctx, pipe, nsend, name, address):
'''
An actor with a flow client sending depo messages.
'''
log.debug(f'actor: flow_depos({nsend}, "{name}", "{address}"')
pipe.signal()
port = Port(name, zmq.CLIENT, '')
port.connect(address)
port.online(None) # peer not needed if port only direct connects
flow = Flow(port)
fobj = dict(flow='BOT', direction='extract', credit=3, stream=name)
msg = Message(seqno=0, form='FLOW', label=json.dumps(fobj))
log.debug(f'flow_depos {name} send BOT:\n{msg}')
flow.send_bot(msg)
msg = flow.recv_bot(1000)
log.debug(f'flow_depos {name} got BOT:\n{msg}')
assert (msg)
for count in range(nsend):
depo = pb.Depo(ident=count,
pos=pb.Point(x=1, y=2, z=3),
time=100.0,
charge=1000.0,
trackid=12345,
pdg=11,
extent_long=9.6,
extent_tran=6.9)
a = Any()
a.Pack(depo)
msg = Message(form='FLOW',
seqno=count + 1,
label=json.dumps({'flow': 'DAT'}),
payload=[a.SerializeToString()])
log.debug(
f'flow_depos {name} put: {count}/{nsend}[{flow.credit}]:\n{msg}')
flow.put(msg)
log.debug(f'flow_depos {name} again [{flow.credit}]')
log.debug(f'flow_depos {name} send EOT')
flow.send_eot(Message(seqno=nsend + 1))
log.debug(f'flow_depos {name} recv EOT')
flow.recv_eot()
log.debug(f'flow_depos {name} wait for quit signal')
pipe.recv() # wait for signal to quit
log.debug(f'flow_depos {name} exiting')
return
def send_tens(number, connect, shape, verbosity, attrs):
'''
Generate and flow some TENS messages.
'''
import zmq
from zio import Port, Message, Node
from zio.flow import Flow
log.level = getattr(logging, verbosity.upper(), "INFO")
msg_attr = attrify(attrs)
cnode = Node("flow-send-tens")
cport = cnode.port("output", zmq.CLIENT)
cport.connect(connect)
cnode.online()
cflow = Flow(cport)
shape = list(map(int, shape.split(',')))
size = 1
for s in shape:
size *= s
attr = dict(credit=2, direction="extract", **msg_attr)
bot = Message(label=json.dumps(attr))
cflow.send_bot(bot)
bot = cflow.recv_bot(5000)
log.debug('flow-send-tens: BOT handshake done')
assert (bot)
tens_attr = dict(shape=shape, word=1, dtype='u') # unsigned char
attr["TENS"] = dict(tensors=[tens_attr], metadata=dict(source="gen-tens"))
label = json.dumps(attr)
payload = [b'X' * size]
for count in range(number):
msg = Message(label=label, payload=payload)
cflow.put(msg)
log.debug(f'flow-send-tens: {count}: {msg}')
log.debug(f'flow-send-tens: send EOT')
cflow.send_eot(Message())
log.debug(f'flow-send-tens: recv EOT (waiting)')
cflow.recv_eot()
log.debug(f'flow-send-tens: going offline')
cnode.offline()
log.debug(f'flow-send-tens: end')
def handler(ctx, pipe, bot, rule_object, filename, broker_addr, *rargs):
log.debug(f'actor: reader "{filename}"')
fp = h5py.File(filename, 'r')
mattr = message_to_dict(bot)
rattr = dict(rule_object["attr"], **mattr)
base_path = rule_object["grouppat"].format(**rattr)
log.debug(f'reader(msg, "{base_path}", "{broker_addr}")')
log.debug(bot)
pipe.signal()
sock = ctx.socket(CLIENT)
port = Port("read-handler", sock)
port.connect(broker_addr)
port.online(None)
flow = Flow(port)
log.debug(f'reader({base_path}) send BOT to {broker_addr}')
sg = fp.get(base_path)
if not sg:
log.error(f'reader failed to get {base_path} from {filename}')
return
fr = Reader(sg, *rargs)
obot = fr.read()
# fixme: something should be done to compare old and new and
# assert on any important differences. For now, we effectively
# drop the old one and send back the new.
# log.debug(f'new BOT: {bot}')
# log.debug(f'old BOT: {obot}')
flow.send_bot(bot) # this introduces us to the server
bot = flow.recv_bot()
log.debug(f'reader({base_path}) got response:\n{bot}')
flow.slurp_pay()
while True:
msg = fr.read()
log.debug(f'reader: {msg}')
if not msg:
break
ok = flow.put(msg)
if not ok:
break
flow.send_eot()
flow.recv_eot()
def recv_tens(number, connect, verbosity, attrs):
'''
Client to recv flow of TENS messages.
'''
import zmq
from zio import Port, Message, Node
from zio.flow import Flow
log.level = getattr(logging, verbosity.upper(), "INFO")
msg_attr = attrify(attrs)
cnode = Node("flow-recv-tens")
cport = cnode.port("input", zmq.CLIENT)
cport.connect(connect)
cnode.online()
cflow = Flow(cport)
attr = dict(credit=2, direction="inject", **msg_attr)
bot = Message(label=json.dumps(attr))
cflow.send_bot(bot)
bot = cflow.recv_bot(5000)
log.debug('flow-recv-tens: BOT handshake done')
assert (bot)
count = 0
while True:
if number > 0 and count == number:
break
++count
msg = cflow.get()
log.info(f'flow-recv-tens: {count}: {msg}')
if msg is None:
cflow.send_eot()
cnode.offline()
log.debug('flow-recv-tens: EOT whille receiving')
return
log.debug(f'flow-recv-tens: send EOT')
cflow.send_eot(Message())
log.debug(f'flow-recv-tens: recv EOT (waiting)')
cflow.recv_eot()
log.debug(f'flow-recv-tens: going offline')
cnode.offline()
log.debug(f'flow-recv-tens: end')
def handler(ctx, pipe, bot, rule_object, filename, broker_addr, *rargs):
log.debug(f'actor: reader "{filename}"')
fp = h5py.File(filename, 'r')
mattr = message_to_dict(bot)
rattr = dict(rule_object.get("attr", {}), **mattr)
base_path = rule_object.get("grouppat", "/").format(**rattr)
log.debug(f'reader(msg, "{base_path}", "{broker_addr}")')
log.debug(bot)
pipe.signal()
sock = ctx.socket(CLIENT)
port = Port("read-handler", sock)
port.connect(broker_addr)
port.online(None)
flow = Flow(port)
log.debug(f'reader({base_path}) send BOT to {broker_addr}')
sg = fp.get(base_path)
if not sg:
log.error(f'reader failed to get {base_path} from {filename}')
return
fr = TensReader(sg, *rargs)
flow.send_bot(bot) # this introduces us to the server
bot = flow.recv_bot()
log.debug(f'reader({base_path}) got response:\n{bot}')
flow.slurp_pay()
while True:
msg = fr.read()
log.debug(f'reader: {msg}')
if not msg:
break
ok = flow.put(msg)
if not ok:
break
flow.send_eot()
flow.recv_eot()
def recv_server(number, bind, shape, verbosity, attrs):
'''
A simple server to catch some TENS messages from flow and dump them.
'''
import zmq
from zio import Port, Message, Node
from zio.flow import Flow
log.level = getattr(logging, verbosity.upper(), "INFO")
msg_attr = attrify(attrs)
snode = Node("server")
sport = snode.port("sport", zmq.SERVER)
sport.bind(bind)
snode.online()
sflow = Flow(sport)
bot = sflow.recv_bot()
assert (bot)
lobj = bot.label_object
lobj["direction"] = "inject"
bot.label_object = lobj
sflow.send_bot(bot)
log.debug('flow-recv-server: BOT handshake done')
sflow.flush_pay()
log.debug('flow-recv-server: looping')
while True:
msg = sflow.get(1000)
log.info(f'flow-recv-server: {msg}')
if not msg or 'EOT' == msg.label_object['flow']:
log.debug('flow-recv-server: got EOT')
sflow.send_eot() # answer
break
snode.offline()
log.debug(f'flow-recv-server: end')
class TestFlow(unittest.TestCase):
origin = 42
def setUp(self):
self.snode = Node("server", self.origin)
sport = self.snode.port("sport", zmq.SERVER)
sport.bind()
self.snode.online()
self.sflow = Flow(sport)
self.cnode = Node("client")
cport = self.cnode.port("cport", zmq.CLIENT)
cport.connect("server", "sport")
self.cnode.online()
self.cflow = Flow(cport)
def test_conversation(self):
# cflow is recver
bot = Message(label='{"credit":2,"direction":"inject"}')
self.cflow.send_bot(bot)
bot = self.sflow.recv_bot(1000)
assert (bot)
assert (self.sflow.credit == 0)
assert (self.sflow.total_credit == 2)
# sflow is sender
bot = Message(label='{"credit":2,"direction":"extract"}')
self.sflow.send_bot(bot)
bot = self.cflow.recv_bot(1000)
assert (bot)
assert (self.cflow.credit == 2)
assert (self.cflow.total_credit == 2)
self.cflow.flush_pay()
assert (self.cflow.credit == 0)
c = self.sflow.slurp_pay()
assert (c == 2)
assert (self.sflow.credit == 2)
for count in range(10):
# note, seqno normally should sequential
self.sflow.put(Message(coord=CoordHeader(seqno=100 + count)))
self.sflow.put(Message(coord=CoordHeader(seqno=200 + count)))
dat = self.cflow.get()
assert (dat.seqno == 100 + count)
dat = self.cflow.get()
assert (dat.seqno == 200 + count)
# normally, when a flow explicitly sends EOT the other end
# will recv the EOT when its trying to recv another message
# (PAY or DAT). In this test things are synchronous and so we
# explicitly recv_eot().
self.cflow.send_eot(Message())
surprise = self.sflow.recv_eot(1000)
assert (surprise)
self.sflow.send_eot(Message())
expected = self.cflow.recv_eot(1000)
assert (expected)
def test_flow_string(self):
msg = Message(label='{"extra":42}')
msg.label = stringify('DAT', **objectify(msg))
fobj = objectify(msg)
assert (fobj["extra"] == 42)
assert (fobj["flow"] == "DAT")
def tearDown(self):
self.cnode.offline()
self.snode.offline()
pass
class TestFlow(unittest.TestCase):
origin = 42
credit = 2
def setUp(self):
self.snode = Node("server", self.origin)
sport = self.snode.port("sport", zmq.SERVER)
sport.bind()
self.snode.online()
self.sflow = Flow(sport, "extract", TestFlow.credit)
assert (self.sflow.sm.is_giver())
self.cnode = Node("client")
cport = self.cnode.port("cport", zmq.CLIENT)
cport.connect("server", "sport")
self.cnode.online()
self.cflow = Flow(cport, "inject", TestFlow.credit)
assert (self.cflow.sm.is_taker())
def test_conversation(self):
# normally, we use .bot() but here we are synchronous with
# both endpoints so have to break up the steps of at least one
# endpoint.
self.cflow.send_bot()
# this can pretend to be async
sbot = self.sflow.bot()
assert (sbot)
assert (sbot.form == 'FLOW')
cbot = self.cflow.recv()
assert (cbot)
assert (cbot.form == 'FLOW')
# here, server is giver, should start with no credit
assert (self.sflow.credit == 0)
assert (self.sflow.total_credit == TestFlow.credit)
# here, client is taker, should start with all credit
assert (self.cflow.credit == TestFlow.credit)
assert (self.cflow.total_credit == TestFlow.credit)
log.debug("flow BOT handshake done")
assert (self.cflow.sm.state == "READY")
assert (self.sflow.sm.state == "READY")
# this also imitates PAY
self.cflow.begin()
log.debug("client flow began")
assert (self.cflow.sm.state == "taking_HANDSOUT")
self.sflow.begin()
log.debug("server flow began")
assert (self.sflow.sm.state == "giving_GENEROUS")
for count in range(10):
log.debug(f"test_flow: server put in {self.sflow.sm.state}")
dat = Message(form='FLOW')
self.sflow.put(dat)
log.debug(f"test_flow: client get in {self.cflow.sm.state}")
dat = self.cflow.get()
# flow protocol: BOT=0, DAT=1+
assert (dat.seqno == 1 + count)
# normally, when a flow explicitly sends EOT the other end
# will recv the EOT when its trying to recv another message
# (PAY or DAT).
self.cflow.eotsend()
should_be_eot = self.sflow.recv()
assert (should_be_eot)
self.sflow.eotsend()
expected = self.cflow.eotrecv()
assert (expected)
# def test_flow_string(self):
# msg = Message(label='{"extra":42}')
# msg.label = stringify('DAT', **objectify(msg))
# fobj = objectify(msg)
# assert(fobj["extra"] == 42)
# assert(fobj["flow"] == "DAT")
def tearDown(self):
self.cnode.offline()
self.snode.offline()
pass
def client_handler(ctx, pipe, bot, rule_object, writer_addr, broker_addr):
'''Connect to and marshall messages between broker and writer sockets.
Parameters
----------
bot : zio.Message
The BOT message
rule_object: dicionary
A ruleset rule object.
writer_addr :: string
The address of the writer's PULL socket to connect.
broker_addr : string
The address of the broker's SERVER socket to connect.
'''
# An HDF path to be added to every message we send to writer.
mattr = message_to_dict(bot)
rattr = dict(rule_object["attr"], **mattr)
base_path = rule_object["grouppat"].format(**rattr)
log.debug(f'client_handler(msg, "{base_path}", "{broker_addr}", "{writer_addr}")')
log.debug(bot)
pipe.signal()
push = ctx.socket(PUSH)
push.connect(writer_addr)
sock = ctx.socket(CLIENT)
port = Port("write-handler", sock)
port.connect(broker_addr)
port.online(None)
flow = Flow(port)
log.debug (f'writer({base_path}) send BOT to {broker_addr}')
flow.send_bot(bot) # this introduces us to the server
bot = flow.recv_bot()
log.debug (f'writer({base_path}) got response:\n{bot}')
flow.flush_pay()
def push_message(m):
log.debug (f'write_handler({base_path}) push {m}')
attr = message_to_dict(m)
attr['hdfgroup'] = base_path
m.label = json.dumps(attr)
push.send(m.encode())
push_message(bot)
poller = Poller()
poller.register(pipe, POLLIN)
poller.register(sock, POLLIN)
while True:
for which,_ in poller.poll():
if not which:
return
if which == pipe: # signal exit
log.debug ('write_handler pipe hit')
return
# o.w. we have flow
try:
msg = flow.get()
except Exception as err:
log.warning('flow.get error: %s %s' % (type(err),err))
continue
if not msg:
log.debug("write_handler: got EOT")
flow.send_eot()
# fixme: send an EOT also to push socket?.
break
push_message(msg)
continue
log.debug ('write_handler exiting')
pipe.signal()