def file_server(bind, format, name, port, verbosity, ruleset):
'''Source and sink ZIO flow protocol to file
This brings back-end reader and writer handlers to external
clients via the flow broker. A ruleset factory dynamically spawns
handlers based on incoming BOT flow messages. The file format
maps to different flavors of handlers.
'''
import zmq
from zio import Port, Message, Node
from zio.flow.broker import Broker
from zio.flow.factories import Ruleset as Factory
from zio.jsonnet import load as jsonnet_load
# For now we only support HDF. In future this may be replaced by
# a mapping from supported format to module providing handlers
from zio.flow.hdf import writer, reader
assert (format == "hdf")
getLogger("transitions").level = logging.WARNING
log.level = getattr(logging, verbosity.upper(), "INFO")
ruleset = jsonnet_load(ruleset)
# fixme: is it really needed to create a common ctx?
ctx = zmq.Context()
factory = Factory(ctx,
ruleset,
wactors=((writer.file_handler,
("inproc://" + format + "writer{port}")),
(writer.client_handler, (bind, ))),
ractor=(reader.handler, (bind, )))
node = Node(name)
sport = node.port(port, zmq.SERVER)
sport.bind(bind)
node.online()
log.info(f'flow-broker {name}:{port} online at {bind}')
# this may throw
broker = Broker(sport, factory)
log.info(f'flow-broker {name} entering loop')
while True:
try:
broker.poll(10000)
except TimeoutError:
node.peer.drain()
log.debug(f'flow-broker {name} is lonely')
log.debug(node.peer.peers)
except Exception as e:
log.error(e)
continue
broker.stop()
def setUp(self):
self.snode = Node("server", self.origin)
sport = self.snode.port("sport", zmq.SERVER)
sport.bind()
self.snode.online()
self.cnode = Node("client")
cport = self.cnode.port("cport", zmq.CLIENT)
cport.connect("server", "sport")
self.cnode.online()
class TestPort(unittest.TestCase):
origin = 42
def setUp(self):
self.snode = Node("server", self.origin)
sport = self.snode.port("sport", zmq.SERVER)
sport.bind()
self.snode.online()
self.cnode = Node("client")
cport = self.cnode.port("cport", zmq.CLIENT)
cport.connect("server", "sport")
self.cnode.online()
def tearDown(self):
pass
def test_sendrecv(self):
sport = self.snode.port("sport")
cport = self.cnode.port("cport")
lobj = dict(foo='bar')
msg = Message(form='TEST', label_object=lobj)
log.debug(f'{msg}')
cport.send(msg)
log.debug('now recv')
msg2 = sport.recv()
assert (msg2)
assert (msg2.form == 'TEST')
lobj2 = msg2.label_object
assert (lobj2)
assert (lobj2 == lobj)
def file_server(ruleset, bind, format, name, port, verbosity):
'''
Serve files over ZPB/ZIO.
'''
import zmq
from zio import Port, Message, Node
from zio.flow.broker import Broker
from .factory import Ruleset as Factory
from . import pb as pbmod
from .jsonnet import load as jsonnet_load
# for now we only support HDF
from .hdf import writer, reader, frompb, topb
assert (format == "hdf")
log.level = getattr(logging, verbosity.upper(), "INFO")
ruleset = jsonnet_load(ruleset)
# fixme: is it really needed to create a common ctx?
ctx = zmq.Context()
factory = Factory(ctx,
ruleset,
wactors=((writer.file_handler,
("inproc://hdfwriter{port}", (pbmod, frompb))),
(writer.client_handler, (bind, ))),
ractor=(reader.handler, (bind, pbmod, topb)))
node = Node(name)
sport = node.port(port, zmq.SERVER)
sport.bind(bind)
node.online()
log.info(f'broker {name}:{port} online at {bind}')
broker = Broker(sport, factory)
log.info(f'broker {name} entering loop')
while True:
try:
broker.poll(10000)
except TimeoutError:
node.peer.drain()
log.debug(f'broker {name} is lonely')
log.debug(node.peer.peers)
except Exception as e:
log.critical(e)
raise
break
broker.stop()
def setUp(self):
self.snode = Node("server", self.origin)
sport = self.snode.port("sport", zmq.SERVER)
sport.bind()
self.snode.online()
self.sflow = Flow(sport, "extract", TestFlow.credit)
assert (self.sflow.sm.is_giver())
self.cnode = Node("client")
cport = self.cnode.port("cport", zmq.CLIENT)
cport.connect("server", "sport")
self.cnode.online()
self.cflow = Flow(cport, "inject", TestFlow.credit)
assert (self.cflow.sm.is_taker())
def recv_tens(number, connect, verbosity, attrs):
'''
Client to recv flow of TENS messages.
'''
import zmq
from zio import Port, Message, Node
from zio.flow import Flow
log.level = getattr(logging, verbosity.upper(), "INFO")
msg_attr = attrify(attrs)
cnode = Node("flow-recv-tens")
cport = cnode.port("input", zmq.CLIENT)
cport.connect(connect)
cnode.online()
cflow = Flow(cport)
attr = dict(credit=2, direction="inject", **msg_attr)
bot = Message(label=json.dumps(attr))
cflow.send_bot(bot)
bot = cflow.recv_bot(5000)
log.debug('flow-recv-tens: BOT handshake done')
assert (bot)
count = 0
while True:
if number > 0 and count == number:
break
++count
msg = cflow.get()
log.info(f'flow-recv-tens: {count}: {msg}')
if msg is None:
cflow.send_eot()
cnode.offline()
log.debug('flow-recv-tens: EOT whille receiving')
return
log.debug(f'flow-recv-tens: send EOT')
cflow.send_eot(Message())
log.debug(f'flow-recv-tens: recv EOT (waiting)')
cflow.recv_eot()
log.debug(f'flow-recv-tens: going offline')
cnode.offline()
log.debug(f'flow-recv-tens: end')
def send_tens(number, connect, shape, verbosity, attrs):
'''
Generate and flow some TENS messages.
'''
import zmq
from zio import Port, Message, Node
from zio.flow import Flow
log.level = getattr(logging, verbosity.upper(), "INFO")
msg_attr = attrify(attrs)
cnode = Node("flow-send-tens")
cport = cnode.port("output", zmq.CLIENT)
cport.connect(connect)
cnode.online()
cflow = Flow(cport)
shape = list(map(int, shape.split(',')))
size = 1
for s in shape:
size *= s
attr = dict(credit=2, direction="extract", **msg_attr)
bot = Message(label=json.dumps(attr))
cflow.send_bot(bot)
bot = cflow.recv_bot(5000)
log.debug('flow-send-tens: BOT handshake done')
assert (bot)
tens_attr = dict(shape=shape, word=1, dtype='u') # unsigned char
attr["TENS"] = dict(tensors=[tens_attr], metadata=dict(source="gen-tens"))
label = json.dumps(attr)
payload = [b'X' * size]
for count in range(number):
msg = Message(label=label, payload=payload)
cflow.put(msg)
log.debug(f'flow-send-tens: {count}: {msg}')
log.debug(f'flow-send-tens: send EOT')
cflow.send_eot(Message())
log.debug(f'flow-send-tens: recv EOT (waiting)')
cflow.recv_eot()
log.debug(f'flow-send-tens: going offline')
cnode.offline()
log.debug(f'flow-send-tens: end')
def recv_server(number, bind, shape, verbosity, attrs):
'''
A simple server to catch some TENS messages from flow and dump them.
'''
import zmq
from zio import Port, Message, Node
from zio.flow import Flow
log.level = getattr(logging, verbosity.upper(), "INFO")
msg_attr = attrify(attrs)
snode = Node("server")
sport = snode.port("sport", zmq.SERVER)
sport.bind(bind)
snode.online()
sflow = Flow(sport)
bot = sflow.recv_bot()
assert (bot)
lobj = bot.label_object
lobj["direction"] = "inject"
bot.label_object = lobj
sflow.send_bot(bot)
log.debug('flow-recv-server: BOT handshake done')
sflow.flush_pay()
log.debug('flow-recv-server: looping')
while True:
msg = sflow.get(1000)
log.info(f'flow-recv-server: {msg}')
if not msg or 'EOT' == msg.label_object['flow']:
log.debug('flow-recv-server: got EOT')
sflow.send_eot() # answer
break
snode.offline()
log.debug(f'flow-recv-server: end')
class TestFlow(unittest.TestCase):
origin = 42
def setUp(self):
self.snode = Node("server", self.origin)
sport = self.snode.port("sport", zmq.SERVER)
sport.bind()
self.snode.online()
self.sflow = Flow(sport)
self.cnode = Node("client")
cport = self.cnode.port("cport", zmq.CLIENT)
cport.connect("server", "sport")
self.cnode.online()
self.cflow = Flow(cport)
def test_conversation(self):
# cflow is recver
bot = Message(label='{"credit":2,"direction":"inject"}')
self.cflow.send_bot(bot)
bot = self.sflow.recv_bot(1000)
assert (bot)
assert (self.sflow.credit == 0)
assert (self.sflow.total_credit == 2)
# sflow is sender
bot = Message(label='{"credit":2,"direction":"extract"}')
self.sflow.send_bot(bot)
bot = self.cflow.recv_bot(1000)
assert (bot)
assert (self.cflow.credit == 2)
assert (self.cflow.total_credit == 2)
self.cflow.flush_pay()
assert (self.cflow.credit == 0)
c = self.sflow.slurp_pay()
assert (c == 2)
assert (self.sflow.credit == 2)
for count in range(10):
# note, seqno normally should sequential
self.sflow.put(Message(coord=CoordHeader(seqno=100 + count)))
self.sflow.put(Message(coord=CoordHeader(seqno=200 + count)))
dat = self.cflow.get()
assert (dat.seqno == 100 + count)
dat = self.cflow.get()
assert (dat.seqno == 200 + count)
# normally, when a flow explicitly sends EOT the other end
# will recv the EOT when its trying to recv another message
# (PAY or DAT). In this test things are synchronous and so we
# explicitly recv_eot().
self.cflow.send_eot(Message())
surprise = self.sflow.recv_eot(1000)
assert (surprise)
self.sflow.send_eot(Message())
expected = self.cflow.recv_eot(1000)
assert (expected)
def test_flow_string(self):
msg = Message(label='{"extra":42}')
msg.label = stringify('DAT', **objectify(msg))
fobj = objectify(msg)
assert (fobj["extra"] == 42)
assert (fobj["flow"] == "DAT")
def tearDown(self):
self.cnode.offline()
self.snode.offline()
pass
class TestFlow(unittest.TestCase):
origin = 42
credit = 2
def setUp(self):
self.snode = Node("server", self.origin)
sport = self.snode.port("sport", zmq.SERVER)
sport.bind()
self.snode.online()
self.sflow = Flow(sport, "extract", TestFlow.credit)
assert (self.sflow.sm.is_giver())
self.cnode = Node("client")
cport = self.cnode.port("cport", zmq.CLIENT)
cport.connect("server", "sport")
self.cnode.online()
self.cflow = Flow(cport, "inject", TestFlow.credit)
assert (self.cflow.sm.is_taker())
def test_conversation(self):
# normally, we use .bot() but here we are synchronous with
# both endpoints so have to break up the steps of at least one
# endpoint.
self.cflow.send_bot()
# this can pretend to be async
sbot = self.sflow.bot()
assert (sbot)
assert (sbot.form == 'FLOW')
cbot = self.cflow.recv()
assert (cbot)
assert (cbot.form == 'FLOW')
# here, server is giver, should start with no credit
assert (self.sflow.credit == 0)
assert (self.sflow.total_credit == TestFlow.credit)
# here, client is taker, should start with all credit
assert (self.cflow.credit == TestFlow.credit)
assert (self.cflow.total_credit == TestFlow.credit)
log.debug("flow BOT handshake done")
assert (self.cflow.sm.state == "READY")
assert (self.sflow.sm.state == "READY")
# this also imitates PAY
self.cflow.begin()
log.debug("client flow began")
assert (self.cflow.sm.state == "taking_HANDSOUT")
self.sflow.begin()
log.debug("server flow began")
assert (self.sflow.sm.state == "giving_GENEROUS")
for count in range(10):
log.debug(f"test_flow: server put in {self.sflow.sm.state}")
dat = Message(form='FLOW')
self.sflow.put(dat)
log.debug(f"test_flow: client get in {self.cflow.sm.state}")
dat = self.cflow.get()
# flow protocol: BOT=0, DAT=1+
assert (dat.seqno == 1 + count)
# normally, when a flow explicitly sends EOT the other end
# will recv the EOT when its trying to recv another message
# (PAY or DAT).
self.cflow.eotsend()
should_be_eot = self.sflow.recv()
assert (should_be_eot)
self.sflow.eotsend()
expected = self.cflow.eotrecv()
assert (expected)
# def test_flow_string(self):
# msg = Message(label='{"extra":42}')
# msg.label = stringify('DAT', **objectify(msg))
# fobj = objectify(msg)
# assert(fobj["extra"] == 42)
# assert(fobj["flow"] == "DAT")
def tearDown(self):
self.cnode.offline()
self.snode.offline()
pass