def client_actor(ctx, pipe, address):
'An actor function talking to a broker on given address'
pipe.signal()
port = zio.Port("client", zmq.CLIENT,'')
port.connect(address)
port.online(None) # peer not needed if port only direct connects
log.debug ("made flow")
direction='extract'
credit=2
cflow = Flow(port, direction, credit)
bot = cflow.bot()
log.debug (f'client did BOT: {bot}')
assert(bot)
cflow.begin()
msg = zio.Message(form='FLOW', label_object={'flow':'DAT'})
log.debug (f'client put DAT with {cflow.credit}/{cflow.total_credit} {msg}')
cflow.put(msg)
log.debug (f'client did DAT')
cflow.eot()
log.debug (f'client did EOT')
pipe.recv() # wait for signal to exit
def handler(ctx, pipe, bot, rule_object, filename, broker_addr, *rargs):
log.debug(f'actor: reader "{filename}"')
fp = h5py.File(filename, 'r')
mattr = message_to_dict(bot)
rattr = dict(rule_object.get("attr", {}), **mattr)
base_path = rule_object.get("grouppat", "/").format(**rattr)
log.debug(f'reader(msg, "{base_path}", "{broker_addr}")')
log.debug(bot)
pipe.signal()
sock = ctx.socket(CLIENT)
port = Port("read-handler", sock)
port.connect(broker_addr)
port.online(None)
direction = mattr["direction"]
if direction != "extract":
raise RuntimeError(f'zio.flow.hdf.reader bad direction: "{direction}"')
credit = mattr["credit"]
flow = Flow(port, direction, credit)
log.debug(f'reader({base_path}) send BOT to {broker_addr}')
sg = fp.get(base_path)
if not sg:
log.error(f'reader failed to get {base_path} from {filename}')
return
fr = TensReader(sg, *rargs)
bot = flow.bot(bot) # this introduces us to the server
log.debug(f'reader({base_path}) got response: {bot}')
flow.begin()
while True:
msg = fr.read()
log.debug(f'reader: {msg}')
if not msg:
break
flow.put(msg)
flow.eot()
def client_handler(ctx, pipe, bot, rule_object, writer_addr, broker_addr):
'''Connect to and marshall messages between broker and writer sockets.
Parameters
----------
bot : zio.Message
The BOT message
rule_object: dicionary
A ruleset rule object.
writer_addr :: string
The address of the writer's PULL socket to connect.
broker_addr : string
The address of the broker's SERVER socket to connect.
'''
# An HDF path to be added to every message we send to writer.
mattr = message_to_dict(bot)
rattr = dict(rule_object.get("attr",{}), **mattr)
log.info(f'writer: attrs: {rattr}')
try:
base_path = rule_object.get("grouppat","/").format(**rattr)
except KeyError as e:
log.error(f'writer: missing attribute: {e}')
raise
log.debug(f'client_handler(msg, "{base_path}", "{broker_addr}", "{writer_addr}")')
log.debug(bot)
pipe.signal()
push = ctx.socket(PUSH)
push.connect(writer_addr)
sock = ctx.socket(CLIENT)
port = Port("write-handler", sock)
port.connect(broker_addr)
port.online(None)
direction = mattr["direction"]
if direction != "inject":
raise RuntimeError(f'zio.flow.hdf.writer bad direction: "{direction}"')
credit = mattr["credit"]
flow = Flow(port, direction, credit)
log.debug (f'writer({base_path}) send BOT to {broker_addr}')
bot = flow.bot(bot) # this introduces us to the server
log.debug (f'writer({base_path}) got response:\n{bot}')
flow.begin()
def push_message(m):
log.debug (f'write_handler({base_path}) push {m}')
attr = message_to_dict(m)
attr['hdfgroup'] = base_path
m.label = json.dumps(attr)
push.send(m.encode())
#push_message(bot)
poller = Poller()
poller.register(pipe, POLLIN)
poller.register(sock, POLLIN)
while True:
for which,_ in poller.poll():
if not which:
return
if which == pipe: # signal exit
log.debug ('write_handler pipe hit')
return
# o.w. we have flow
try:
msg = flow.get()
except TransmissionEnd as te:
flow.eotsend()
break
push_message(msg)
continue
log.debug ('write_handler exiting')
pipe.signal()
def dumper(ctx, pipe, bot, address):
'''
A dump handler which may be used as an actor talking to a broker's botport.
Parameters
----------
bot : zio.Message
Our initiating BOT message
address : string
A ZeroMQ address string for a bound broker SERVER socket
'''
poller = zmq.Poller()
poller.register(pipe, zmq.POLLIN)
pipe.signal() # ready
port = zio.Port("dumper", zmq.CLIENT,'')
port.connect(address)
port.online(None) # peer not needed if port only direct connects
fobj = bot.label_object
direction=fobj["direction"]
credit=fobj["credit"]
flow = Flow(port, direction, credit)
poller.register(flow.port.sock, zmq.POLLIN)
log.debug (f'dumper: send {bot}')
bot = flow.bot(bot)
assert(bot)
flow.begin()
interupted = False
keep_going = True
while keep_going:
for sock,_ in poller.poll():
if sock == pipe:
log.debug ("dumper: pipe hit")
data = pipe.recv()
if data == b'STOP':
log.debug ("dumper: got STOP")
if len(data) == 0:
log.debug ("dumper: got signal")
interupted = True
return
# got flow messages
try:
msg = flow.get()
except TransmissionEnd:
log.debug ("dumper: get gives EOT")
flow.eotsend()
poller.unregister(sock)
keep_going = False
break
log.debug (f'dumper: sock hit: {msg}')
log.debug("dumper: taking port offline")
port.offline()
if not interupted:
log.debug("dumper: waiting for quit")
pipe.recv()
log.debug("dumper: done")
return
class TestFlow(unittest.TestCase):
origin = 42
credit = 2
def setUp(self):
self.snode = Node("server", self.origin)
sport = self.snode.port("sport", zmq.SERVER)
sport.bind()
self.snode.online()
self.sflow = Flow(sport, "extract", TestFlow.credit)
assert (self.sflow.sm.is_giver())
self.cnode = Node("client")
cport = self.cnode.port("cport", zmq.CLIENT)
cport.connect("server", "sport")
self.cnode.online()
self.cflow = Flow(cport, "inject", TestFlow.credit)
assert (self.cflow.sm.is_taker())
def test_conversation(self):
# normally, we use .bot() but here we are synchronous with
# both endpoints so have to break up the steps of at least one
# endpoint.
self.cflow.send_bot()
# this can pretend to be async
sbot = self.sflow.bot()
assert (sbot)
assert (sbot.form == 'FLOW')
cbot = self.cflow.recv()
assert (cbot)
assert (cbot.form == 'FLOW')
# here, server is giver, should start with no credit
assert (self.sflow.credit == 0)
assert (self.sflow.total_credit == TestFlow.credit)
# here, client is taker, should start with all credit
assert (self.cflow.credit == TestFlow.credit)
assert (self.cflow.total_credit == TestFlow.credit)
log.debug("flow BOT handshake done")
assert (self.cflow.sm.state == "READY")
assert (self.sflow.sm.state == "READY")
# this also imitates PAY
self.cflow.begin()
log.debug("client flow began")
assert (self.cflow.sm.state == "taking_HANDSOUT")
self.sflow.begin()
log.debug("server flow began")
assert (self.sflow.sm.state == "giving_GENEROUS")
for count in range(10):
log.debug(f"test_flow: server put in {self.sflow.sm.state}")
dat = Message(form='FLOW')
self.sflow.put(dat)
log.debug(f"test_flow: client get in {self.cflow.sm.state}")
dat = self.cflow.get()
# flow protocol: BOT=0, DAT=1+
assert (dat.seqno == 1 + count)
# normally, when a flow explicitly sends EOT the other end
# will recv the EOT when its trying to recv another message
# (PAY or DAT).
self.cflow.eotsend()
should_be_eot = self.sflow.recv()
assert (should_be_eot)
self.sflow.eotsend()
expected = self.cflow.eotrecv()
assert (expected)
# def test_flow_string(self):
# msg = Message(label='{"extra":42}')
# msg.label = stringify('DAT', **objectify(msg))
# fobj = objectify(msg)
# assert(fobj["extra"] == 42)
# assert(fobj["flow"] == "DAT")
def tearDown(self):
self.cnode.offline()
self.snode.offline()
pass