Exemplo n.º 1
0
def client_actor(ctx, pipe, address):
    'An actor function talking to a broker on given address'
    pipe.signal()
    
    port = zio.Port("client", zmq.CLIENT,'')
    port.connect(address)
    port.online(None)       # peer not needed if port only direct connects
    log.debug ("made flow")

    direction='extract'
    credit=2
    cflow = Flow(port, direction, credit)

    bot = cflow.bot()
    log.debug (f'client did BOT: {bot}')
    assert(bot)
    cflow.begin()

    msg = zio.Message(form='FLOW', label_object={'flow':'DAT'})
    log.debug (f'client put DAT with {cflow.credit}/{cflow.total_credit} {msg}')
    cflow.put(msg)
    log.debug (f'client did DAT')
    cflow.eot()
    log.debug (f'client did EOT')

    pipe.recv()                 # wait for signal to exit
Exemplo n.º 2
0
def handler(ctx, pipe, bot, rule_object, filename, broker_addr, *rargs):

    log.debug(f'actor: reader "{filename}"')
    fp = h5py.File(filename, 'r')

    mattr = message_to_dict(bot)
    rattr = dict(rule_object.get("attr", {}), **mattr)
    base_path = rule_object.get("grouppat", "/").format(**rattr)
    log.debug(f'reader(msg, "{base_path}", "{broker_addr}")')
    log.debug(bot)
    pipe.signal()

    sock = ctx.socket(CLIENT)
    port = Port("read-handler", sock)
    port.connect(broker_addr)
    port.online(None)

    direction = mattr["direction"]
    if direction != "extract":
        raise RuntimeError(f'zio.flow.hdf.reader bad direction: "{direction}"')
    credit = mattr["credit"]
    flow = Flow(port, direction, credit)
    log.debug(f'reader({base_path}) send BOT to {broker_addr}')

    sg = fp.get(base_path)
    if not sg:
        log.error(f'reader failed to get {base_path} from {filename}')
        return
    fr = TensReader(sg, *rargs)

    bot = flow.bot(bot)  # this introduces us to the server
    log.debug(f'reader({base_path}) got response: {bot}')
    flow.begin()

    while True:
        msg = fr.read()
        log.debug(f'reader: {msg}')
        if not msg:
            break
        flow.put(msg)
    flow.eot()
Exemplo n.º 3
0
def client_handler(ctx, pipe, bot, rule_object, writer_addr, broker_addr):
    '''Connect to and marshall messages between broker and writer sockets.

    Parameters
    ----------

    bot : zio.Message

        The BOT message

    rule_object: dicionary 

        A ruleset rule object.

    writer_addr :: string

        The address of the writer's PULL socket to connect.

    broker_addr : string

        The address of the broker's SERVER socket to connect.

    '''
    # An HDF path to be added to every message we send to writer.
    mattr = message_to_dict(bot)
    rattr = dict(rule_object.get("attr",{}), **mattr)
    log.info(f'writer: attrs: {rattr}')
    try:
        base_path =  rule_object.get("grouppat","/").format(**rattr)
    except KeyError as e:
        log.error(f'writer: missing attribute: {e}')
        raise
    log.debug(f'client_handler(msg, "{base_path}", "{broker_addr}", "{writer_addr}")')
    log.debug(bot)
    pipe.signal()

    push = ctx.socket(PUSH)
    push.connect(writer_addr)

    sock = ctx.socket(CLIENT)
    port = Port("write-handler", sock)
    port.connect(broker_addr)
    port.online(None)

    direction = mattr["direction"]
    if direction != "inject":
        raise RuntimeError(f'zio.flow.hdf.writer bad direction: "{direction}"')
    credit = mattr["credit"]
    flow = Flow(port, direction, credit)
    log.debug (f'writer({base_path}) send BOT to {broker_addr}')

    bot = flow.bot(bot)         # this introduces us to the server
    log.debug (f'writer({base_path}) got response:\n{bot}')
    flow.begin()

    def push_message(m):
        log.debug (f'write_handler({base_path}) push {m}')
        attr = message_to_dict(m)
        attr['hdfgroup'] = base_path
        m.label = json.dumps(attr)
        push.send(m.encode())

    #push_message(bot)

    poller = Poller()
    poller.register(pipe, POLLIN)
    poller.register(sock, POLLIN)
    while True:

        for which,_ in poller.poll():
            if not which:
                return

            if which == pipe: # signal exit
                log.debug ('write_handler pipe hit')
                return          

            # o.w. we have flow

            try:
                msg = flow.get()
            except TransmissionEnd as te:
                flow.eotsend()
                break
            push_message(msg)

            continue

    log.debug ('write_handler exiting')
    pipe.signal()
Exemplo n.º 4
0
def dumper(ctx, pipe, bot, address):
    '''
    A dump handler which may be used as an actor talking to a broker's botport.

    Parameters
    ----------
    bot : zio.Message
        Our initiating BOT message
    address : string
        A ZeroMQ address string for a bound broker SERVER socket
    '''
    poller = zmq.Poller()
    poller.register(pipe, zmq.POLLIN)
    pipe.signal()               # ready

    port = zio.Port("dumper", zmq.CLIENT,'')
    port.connect(address)
    port.online(None)       # peer not needed if port only direct connects

    fobj = bot.label_object
    direction=fobj["direction"]
    credit=fobj["credit"]
    flow = Flow(port, direction, credit)
    poller.register(flow.port.sock, zmq.POLLIN)

    log.debug (f'dumper: send {bot}')

    bot = flow.bot(bot)
    assert(bot)

    flow.begin()

    interupted = False
    keep_going = True
    while keep_going:

        for sock,_ in poller.poll():

            if sock == pipe:
                log.debug ("dumper: pipe hit")
                data = pipe.recv()
                if data == b'STOP':
                    log.debug ("dumper: got STOP")
                if len(data) == 0:
                    log.debug ("dumper: got signal")
                interupted = True
                return

            # got flow messages
            try:
                msg = flow.get()
            except TransmissionEnd:
                log.debug ("dumper: get gives EOT")
                flow.eotsend()
                poller.unregister(sock)
                keep_going = False
                break

            log.debug (f'dumper: sock hit: {msg}')

    log.debug("dumper: taking port offline")
    port.offline()
    if not interupted:
        log.debug("dumper: waiting for quit")
        pipe.recv()
    log.debug("dumper: done")
    return
Exemplo n.º 5
0
class TestFlow(unittest.TestCase):

    origin = 42
    credit = 2

    def setUp(self):
        self.snode = Node("server", self.origin)
        sport = self.snode.port("sport", zmq.SERVER)
        sport.bind()
        self.snode.online()
        self.sflow = Flow(sport, "extract", TestFlow.credit)
        assert (self.sflow.sm.is_giver())

        self.cnode = Node("client")
        cport = self.cnode.port("cport", zmq.CLIENT)
        cport.connect("server", "sport")
        self.cnode.online()
        self.cflow = Flow(cport, "inject", TestFlow.credit)
        assert (self.cflow.sm.is_taker())

    def test_conversation(self):

        # normally, we use .bot() but here we are synchronous with
        # both endpoints so have to break up the steps of at least one
        # endpoint.
        self.cflow.send_bot()

        # this can pretend to be async
        sbot = self.sflow.bot()
        assert (sbot)
        assert (sbot.form == 'FLOW')

        cbot = self.cflow.recv()
        assert (cbot)
        assert (cbot.form == 'FLOW')

        # here, server is giver, should start with no credit
        assert (self.sflow.credit == 0)
        assert (self.sflow.total_credit == TestFlow.credit)
        # here, client is taker, should start with all credit
        assert (self.cflow.credit == TestFlow.credit)
        assert (self.cflow.total_credit == TestFlow.credit)

        log.debug("flow BOT handshake done")
        assert (self.cflow.sm.state == "READY")
        assert (self.sflow.sm.state == "READY")

        # this also imitates PAY
        self.cflow.begin()
        log.debug("client flow began")
        assert (self.cflow.sm.state == "taking_HANDSOUT")

        self.sflow.begin()
        log.debug("server flow began")
        assert (self.sflow.sm.state == "giving_GENEROUS")

        for count in range(10):
            log.debug(f"test_flow: server put in {self.sflow.sm.state}")
            dat = Message(form='FLOW')
            self.sflow.put(dat)
            log.debug(f"test_flow: client get in {self.cflow.sm.state}")
            dat = self.cflow.get()
            # flow protocol: BOT=0, DAT=1+
            assert (dat.seqno == 1 + count)

        # normally, when a flow explicitly sends EOT the other end
        # will recv the EOT when its trying to recv another message
        # (PAY or DAT).
        self.cflow.eotsend()

        should_be_eot = self.sflow.recv()

        assert (should_be_eot)
        self.sflow.eotsend()
        expected = self.cflow.eotrecv()
        assert (expected)

    # def test_flow_string(self):
    #     msg = Message(label='{"extra":42}')
    #     msg.label = stringify('DAT', **objectify(msg))
    #     fobj = objectify(msg)
    #     assert(fobj["extra"] == 42)
    #     assert(fobj["flow"] == "DAT")

    def tearDown(self):
        self.cnode.offline()
        self.snode.offline()

        pass