class TestPort(unittest.TestCase): origin = 42 def setUp(self): self.snode = Node("server", self.origin) sport = self.snode.port("sport", zmq.SERVER) sport.bind() self.snode.online() self.cnode = Node("client") cport = self.cnode.port("cport", zmq.CLIENT) cport.connect("server", "sport") self.cnode.online() def tearDown(self): pass def test_sendrecv(self): sport = self.snode.port("sport") cport = self.cnode.port("cport") lobj = dict(foo='bar') msg = Message(form='TEST', label_object=lobj) log.debug(f'{msg}') cport.send(msg) log.debug('now recv') msg2 = sport.recv() assert (msg2) assert (msg2.form == 'TEST') lobj2 = msg2.label_object assert (lobj2) assert (lobj2 == lobj)
def file_server(bind, format, name, port, verbosity, ruleset): '''Source and sink ZIO flow protocol to file This brings back-end reader and writer handlers to external clients via the flow broker. A ruleset factory dynamically spawns handlers based on incoming BOT flow messages. The file format maps to different flavors of handlers. ''' import zmq from zio import Port, Message, Node from zio.flow.broker import Broker from zio.flow.factories import Ruleset as Factory from zio.jsonnet import load as jsonnet_load # For now we only support HDF. In future this may be replaced by # a mapping from supported format to module providing handlers from zio.flow.hdf import writer, reader assert (format == "hdf") getLogger("transitions").level = logging.WARNING log.level = getattr(logging, verbosity.upper(), "INFO") ruleset = jsonnet_load(ruleset) # fixme: is it really needed to create a common ctx? ctx = zmq.Context() factory = Factory(ctx, ruleset, wactors=((writer.file_handler, ("inproc://" + format + "writer{port}")), (writer.client_handler, (bind, ))), ractor=(reader.handler, (bind, ))) node = Node(name) sport = node.port(port, zmq.SERVER) sport.bind(bind) node.online() log.info(f'flow-broker {name}:{port} online at {bind}') # this may throw broker = Broker(sport, factory) log.info(f'flow-broker {name} entering loop') while True: try: broker.poll(10000) except TimeoutError: node.peer.drain() log.debug(f'flow-broker {name} is lonely') log.debug(node.peer.peers) except Exception as e: log.error(e) continue broker.stop()
def file_server(ruleset, bind, format, name, port, verbosity): ''' Serve files over ZPB/ZIO. ''' import zmq from zio import Port, Message, Node from zio.flow.broker import Broker from .factory import Ruleset as Factory from . import pb as pbmod from .jsonnet import load as jsonnet_load # for now we only support HDF from .hdf import writer, reader, frompb, topb assert (format == "hdf") log.level = getattr(logging, verbosity.upper(), "INFO") ruleset = jsonnet_load(ruleset) # fixme: is it really needed to create a common ctx? ctx = zmq.Context() factory = Factory(ctx, ruleset, wactors=((writer.file_handler, ("inproc://hdfwriter{port}", (pbmod, frompb))), (writer.client_handler, (bind, ))), ractor=(reader.handler, (bind, pbmod, topb))) node = Node(name) sport = node.port(port, zmq.SERVER) sport.bind(bind) node.online() log.info(f'broker {name}:{port} online at {bind}') broker = Broker(sport, factory) log.info(f'broker {name} entering loop') while True: try: broker.poll(10000) except TimeoutError: node.peer.drain() log.debug(f'broker {name} is lonely') log.debug(node.peer.peers) except Exception as e: log.critical(e) raise break broker.stop()
def send_tens(number, connect, shape, verbosity, attrs): ''' Generate and flow some TENS messages. ''' import zmq from zio import Port, Message, Node from zio.flow import Flow log.level = getattr(logging, verbosity.upper(), "INFO") msg_attr = attrify(attrs) cnode = Node("flow-send-tens") cport = cnode.port("output", zmq.CLIENT) cport.connect(connect) cnode.online() cflow = Flow(cport) shape = list(map(int, shape.split(','))) size = 1 for s in shape: size *= s attr = dict(credit=2, direction="extract", **msg_attr) bot = Message(label=json.dumps(attr)) cflow.send_bot(bot) bot = cflow.recv_bot(5000) log.debug('flow-send-tens: BOT handshake done') assert (bot) tens_attr = dict(shape=shape, word=1, dtype='u') # unsigned char attr["TENS"] = dict(tensors=[tens_attr], metadata=dict(source="gen-tens")) label = json.dumps(attr) payload = [b'X' * size] for count in range(number): msg = Message(label=label, payload=payload) cflow.put(msg) log.debug(f'flow-send-tens: {count}: {msg}') log.debug(f'flow-send-tens: send EOT') cflow.send_eot(Message()) log.debug(f'flow-send-tens: recv EOT (waiting)') cflow.recv_eot() log.debug(f'flow-send-tens: going offline') cnode.offline() log.debug(f'flow-send-tens: end')
def recv_tens(number, connect, verbosity, attrs): ''' Client to recv flow of TENS messages. ''' import zmq from zio import Port, Message, Node from zio.flow import Flow log.level = getattr(logging, verbosity.upper(), "INFO") msg_attr = attrify(attrs) cnode = Node("flow-recv-tens") cport = cnode.port("input", zmq.CLIENT) cport.connect(connect) cnode.online() cflow = Flow(cport) attr = dict(credit=2, direction="inject", **msg_attr) bot = Message(label=json.dumps(attr)) cflow.send_bot(bot) bot = cflow.recv_bot(5000) log.debug('flow-recv-tens: BOT handshake done') assert (bot) count = 0 while True: if number > 0 and count == number: break ++count msg = cflow.get() log.info(f'flow-recv-tens: {count}: {msg}') if msg is None: cflow.send_eot() cnode.offline() log.debug('flow-recv-tens: EOT whille receiving') return log.debug(f'flow-recv-tens: send EOT') cflow.send_eot(Message()) log.debug(f'flow-recv-tens: recv EOT (waiting)') cflow.recv_eot() log.debug(f'flow-recv-tens: going offline') cnode.offline() log.debug(f'flow-recv-tens: end')
def recv_server(number, bind, shape, verbosity, attrs): ''' A simple server to catch some TENS messages from flow and dump them. ''' import zmq from zio import Port, Message, Node from zio.flow import Flow log.level = getattr(logging, verbosity.upper(), "INFO") msg_attr = attrify(attrs) snode = Node("server") sport = snode.port("sport", zmq.SERVER) sport.bind(bind) snode.online() sflow = Flow(sport) bot = sflow.recv_bot() assert (bot) lobj = bot.label_object lobj["direction"] = "inject" bot.label_object = lobj sflow.send_bot(bot) log.debug('flow-recv-server: BOT handshake done') sflow.flush_pay() log.debug('flow-recv-server: looping') while True: msg = sflow.get(1000) log.info(f'flow-recv-server: {msg}') if not msg or 'EOT' == msg.label_object['flow']: log.debug('flow-recv-server: got EOT') sflow.send_eot() # answer break snode.offline() log.debug(f'flow-recv-server: end')
class TestFlow(unittest.TestCase): origin = 42 def setUp(self): self.snode = Node("server", self.origin) sport = self.snode.port("sport", zmq.SERVER) sport.bind() self.snode.online() self.sflow = Flow(sport) self.cnode = Node("client") cport = self.cnode.port("cport", zmq.CLIENT) cport.connect("server", "sport") self.cnode.online() self.cflow = Flow(cport) def test_conversation(self): # cflow is recver bot = Message(label='{"credit":2,"direction":"inject"}') self.cflow.send_bot(bot) bot = self.sflow.recv_bot(1000) assert (bot) assert (self.sflow.credit == 0) assert (self.sflow.total_credit == 2) # sflow is sender bot = Message(label='{"credit":2,"direction":"extract"}') self.sflow.send_bot(bot) bot = self.cflow.recv_bot(1000) assert (bot) assert (self.cflow.credit == 2) assert (self.cflow.total_credit == 2) self.cflow.flush_pay() assert (self.cflow.credit == 0) c = self.sflow.slurp_pay() assert (c == 2) assert (self.sflow.credit == 2) for count in range(10): # note, seqno normally should sequential self.sflow.put(Message(coord=CoordHeader(seqno=100 + count))) self.sflow.put(Message(coord=CoordHeader(seqno=200 + count))) dat = self.cflow.get() assert (dat.seqno == 100 + count) dat = self.cflow.get() assert (dat.seqno == 200 + count) # normally, when a flow explicitly sends EOT the other end # will recv the EOT when its trying to recv another message # (PAY or DAT). In this test things are synchronous and so we # explicitly recv_eot(). self.cflow.send_eot(Message()) surprise = self.sflow.recv_eot(1000) assert (surprise) self.sflow.send_eot(Message()) expected = self.cflow.recv_eot(1000) assert (expected) def test_flow_string(self): msg = Message(label='{"extra":42}') msg.label = stringify('DAT', **objectify(msg)) fobj = objectify(msg) assert (fobj["extra"] == 42) assert (fobj["flow"] == "DAT") def tearDown(self): self.cnode.offline() self.snode.offline() pass
class TestFlow(unittest.TestCase): origin = 42 credit = 2 def setUp(self): self.snode = Node("server", self.origin) sport = self.snode.port("sport", zmq.SERVER) sport.bind() self.snode.online() self.sflow = Flow(sport, "extract", TestFlow.credit) assert (self.sflow.sm.is_giver()) self.cnode = Node("client") cport = self.cnode.port("cport", zmq.CLIENT) cport.connect("server", "sport") self.cnode.online() self.cflow = Flow(cport, "inject", TestFlow.credit) assert (self.cflow.sm.is_taker()) def test_conversation(self): # normally, we use .bot() but here we are synchronous with # both endpoints so have to break up the steps of at least one # endpoint. self.cflow.send_bot() # this can pretend to be async sbot = self.sflow.bot() assert (sbot) assert (sbot.form == 'FLOW') cbot = self.cflow.recv() assert (cbot) assert (cbot.form == 'FLOW') # here, server is giver, should start with no credit assert (self.sflow.credit == 0) assert (self.sflow.total_credit == TestFlow.credit) # here, client is taker, should start with all credit assert (self.cflow.credit == TestFlow.credit) assert (self.cflow.total_credit == TestFlow.credit) log.debug("flow BOT handshake done") assert (self.cflow.sm.state == "READY") assert (self.sflow.sm.state == "READY") # this also imitates PAY self.cflow.begin() log.debug("client flow began") assert (self.cflow.sm.state == "taking_HANDSOUT") self.sflow.begin() log.debug("server flow began") assert (self.sflow.sm.state == "giving_GENEROUS") for count in range(10): log.debug(f"test_flow: server put in {self.sflow.sm.state}") dat = Message(form='FLOW') self.sflow.put(dat) log.debug(f"test_flow: client get in {self.cflow.sm.state}") dat = self.cflow.get() # flow protocol: BOT=0, DAT=1+ assert (dat.seqno == 1 + count) # normally, when a flow explicitly sends EOT the other end # will recv the EOT when its trying to recv another message # (PAY or DAT). self.cflow.eotsend() should_be_eot = self.sflow.recv() assert (should_be_eot) self.sflow.eotsend() expected = self.cflow.eotrecv() assert (expected) # def test_flow_string(self): # msg = Message(label='{"extra":42}') # msg.label = stringify('DAT', **objectify(msg)) # fobj = objectify(msg) # assert(fobj["extra"] == 42) # assert(fobj["flow"] == "DAT") def tearDown(self): self.cnode.offline() self.snode.offline() pass