예제 #1
0
def flow_depos(ctx, pipe, nsend, name, address):
    '''
    An actor with a flow client sending depo messages.
    '''
    log.debug(f'actor: flow_depos({nsend}, "{name}", "{address}"')

    pipe.signal()

    port = Port(name, zmq.CLIENT, '')
    port.connect(address)
    port.online(None)  # peer not needed if port only direct connects
    flow = Flow(port)

    fobj = dict(flow='BOT', direction='extract', credit=3, stream=name)
    msg = Message(seqno=0, form='FLOW', label=json.dumps(fobj))
    log.debug(f'flow_depos {name} send BOT:\n{msg}')
    flow.send_bot(msg)
    msg = flow.recv_bot(1000)
    log.debug(f'flow_depos {name} got BOT:\n{msg}')
    assert (msg)

    for count in range(nsend):
        depo = pb.Depo(ident=count,
                       pos=pb.Point(x=1, y=2, z=3),
                       time=100.0,
                       charge=1000.0,
                       trackid=12345,
                       pdg=11,
                       extent_long=9.6,
                       extent_tran=6.9)
        a = Any()
        a.Pack(depo)
        msg = Message(form='FLOW',
                      seqno=count + 1,
                      label=json.dumps({'flow': 'DAT'}),
                      payload=[a.SerializeToString()])
        log.debug(
            f'flow_depos {name} put: {count}/{nsend}[{flow.credit}]:\n{msg}')
        flow.put(msg)
        log.debug(f'flow_depos {name} again [{flow.credit}]')

    log.debug(f'flow_depos {name} send EOT')
    flow.send_eot(Message(seqno=nsend + 1))
    log.debug(f'flow_depos {name} recv EOT')
    flow.recv_eot()
    log.debug(f'flow_depos {name} wait for quit signal')
    pipe.recv()  # wait for signal to quit
    log.debug(f'flow_depos {name} exiting')

    return
예제 #2
0
def send_tens(number, connect, shape, verbosity, attrs):
    '''
    Generate and flow some TENS messages.
    '''
    import zmq
    from zio import Port, Message, Node
    from zio.flow import Flow

    log.level = getattr(logging, verbosity.upper(), "INFO")

    msg_attr = attrify(attrs)

    cnode = Node("flow-send-tens")
    cport = cnode.port("output", zmq.CLIENT)
    cport.connect(connect)
    cnode.online()
    cflow = Flow(cport)

    shape = list(map(int, shape.split(',')))
    size = 1
    for s in shape:
        size *= s

    attr = dict(credit=2, direction="extract", **msg_attr)
    bot = Message(label=json.dumps(attr))
    cflow.send_bot(bot)
    bot = cflow.recv_bot(5000)
    log.debug('flow-send-tens: BOT handshake done')
    assert (bot)

    tens_attr = dict(shape=shape, word=1, dtype='u')  # unsigned char
    attr["TENS"] = dict(tensors=[tens_attr], metadata=dict(source="gen-tens"))
    label = json.dumps(attr)
    payload = [b'X' * size]

    for count in range(number):
        msg = Message(label=label, payload=payload)
        cflow.put(msg)
        log.debug(f'flow-send-tens: {count}: {msg}')

    log.debug(f'flow-send-tens: send EOT')
    cflow.send_eot(Message())
    log.debug(f'flow-send-tens: recv EOT (waiting)')
    cflow.recv_eot()
    log.debug(f'flow-send-tens: going offline')
    cnode.offline()
    log.debug(f'flow-send-tens: end')
예제 #3
0
def handler(ctx, pipe, bot, rule_object, filename, broker_addr, *rargs):
    log.debug(f'actor: reader "{filename}"')
    fp = h5py.File(filename, 'r')

    mattr = message_to_dict(bot)
    rattr = dict(rule_object["attr"], **mattr)
    base_path = rule_object["grouppat"].format(**rattr)
    log.debug(f'reader(msg, "{base_path}", "{broker_addr}")')
    log.debug(bot)
    pipe.signal()

    sock = ctx.socket(CLIENT)
    port = Port("read-handler", sock)
    port.connect(broker_addr)
    port.online(None)
    flow = Flow(port)
    log.debug(f'reader({base_path}) send BOT to {broker_addr}')

    sg = fp.get(base_path)
    if not sg:
        log.error(f'reader failed to get {base_path} from {filename}')
        return
    fr = Reader(sg, *rargs)
    obot = fr.read()

    # fixme: something should be done to compare old and new and
    # assert on any important differences.  For now, we effectively
    # drop the old one and send back the new.
    # log.debug(f'new BOT: {bot}')
    # log.debug(f'old BOT: {obot}')

    flow.send_bot(bot)  # this introduces us to the server
    bot = flow.recv_bot()
    log.debug(f'reader({base_path}) got response:\n{bot}')
    flow.slurp_pay()

    while True:
        msg = fr.read()
        log.debug(f'reader: {msg}')
        if not msg:
            break
        ok = flow.put(msg)
        if not ok:
            break
    flow.send_eot()
    flow.recv_eot()
예제 #4
0
def recv_tens(number, connect, verbosity, attrs):
    '''
    Client to recv flow of TENS messages.
    '''
    import zmq
    from zio import Port, Message, Node
    from zio.flow import Flow

    log.level = getattr(logging, verbosity.upper(), "INFO")

    msg_attr = attrify(attrs)

    cnode = Node("flow-recv-tens")
    cport = cnode.port("input", zmq.CLIENT)
    cport.connect(connect)
    cnode.online()
    cflow = Flow(cport)

    attr = dict(credit=2, direction="inject", **msg_attr)
    bot = Message(label=json.dumps(attr))
    cflow.send_bot(bot)
    bot = cflow.recv_bot(5000)
    log.debug('flow-recv-tens: BOT handshake done')
    assert (bot)

    count = 0
    while True:
        if number > 0 and count == number:
            break
        ++count
        msg = cflow.get()
        log.info(f'flow-recv-tens: {count}: {msg}')
        if msg is None:
            cflow.send_eot()
            cnode.offline()
            log.debug('flow-recv-tens: EOT whille receiving')
            return

    log.debug(f'flow-recv-tens: send EOT')
    cflow.send_eot(Message())
    log.debug(f'flow-recv-tens: recv EOT (waiting)')
    cflow.recv_eot()
    log.debug(f'flow-recv-tens: going offline')
    cnode.offline()
    log.debug(f'flow-recv-tens: end')
예제 #5
0
파일: reader.py 프로젝트: HaiwangYu/zio
def handler(ctx, pipe, bot, rule_object, filename, broker_addr, *rargs):

    log.debug(f'actor: reader "{filename}"')
    fp = h5py.File(filename, 'r')

    mattr = message_to_dict(bot)
    rattr = dict(rule_object.get("attr", {}), **mattr)
    base_path = rule_object.get("grouppat", "/").format(**rattr)
    log.debug(f'reader(msg, "{base_path}", "{broker_addr}")')
    log.debug(bot)
    pipe.signal()

    sock = ctx.socket(CLIENT)
    port = Port("read-handler", sock)
    port.connect(broker_addr)
    port.online(None)
    flow = Flow(port)
    log.debug(f'reader({base_path}) send BOT to {broker_addr}')

    sg = fp.get(base_path)
    if not sg:
        log.error(f'reader failed to get {base_path} from {filename}')
        return
    fr = TensReader(sg, *rargs)

    flow.send_bot(bot)  # this introduces us to the server
    bot = flow.recv_bot()
    log.debug(f'reader({base_path}) got response:\n{bot}')
    flow.slurp_pay()

    while True:
        msg = fr.read()
        log.debug(f'reader: {msg}')
        if not msg:
            break
        ok = flow.put(msg)
        if not ok:
            break
    flow.send_eot()
    flow.recv_eot()
예제 #6
0
def recv_server(number, bind, shape, verbosity, attrs):
    '''
    A simple server to catch some TENS messages from flow and dump them.
    '''
    import zmq
    from zio import Port, Message, Node
    from zio.flow import Flow

    log.level = getattr(logging, verbosity.upper(), "INFO")

    msg_attr = attrify(attrs)

    snode = Node("server")
    sport = snode.port("sport", zmq.SERVER)
    sport.bind(bind)
    snode.online()
    sflow = Flow(sport)

    bot = sflow.recv_bot()
    assert (bot)
    lobj = bot.label_object
    lobj["direction"] = "inject"
    bot.label_object = lobj
    sflow.send_bot(bot)
    log.debug('flow-recv-server: BOT handshake done')
    sflow.flush_pay()
    log.debug('flow-recv-server: looping')
    while True:
        msg = sflow.get(1000)
        log.info(f'flow-recv-server: {msg}')
        if not msg or 'EOT' == msg.label_object['flow']:
            log.debug('flow-recv-server: got EOT')
            sflow.send_eot()  # answer
            break

    snode.offline()
    log.debug(f'flow-recv-server: end')
예제 #7
0
class TestFlow(unittest.TestCase):

    origin = 42

    def setUp(self):
        self.snode = Node("server", self.origin)
        sport = self.snode.port("sport", zmq.SERVER)
        sport.bind()
        self.snode.online()
        self.sflow = Flow(sport)

        self.cnode = Node("client")
        cport = self.cnode.port("cport", zmq.CLIENT)
        cport.connect("server", "sport")
        self.cnode.online()
        self.cflow = Flow(cport)

    def test_conversation(self):

        # cflow is recver
        bot = Message(label='{"credit":2,"direction":"inject"}')
        self.cflow.send_bot(bot)
        bot = self.sflow.recv_bot(1000)
        assert (bot)
        assert (self.sflow.credit == 0)
        assert (self.sflow.total_credit == 2)

        # sflow is sender
        bot = Message(label='{"credit":2,"direction":"extract"}')
        self.sflow.send_bot(bot)
        bot = self.cflow.recv_bot(1000)
        assert (bot)
        assert (self.cflow.credit == 2)
        assert (self.cflow.total_credit == 2)

        self.cflow.flush_pay()
        assert (self.cflow.credit == 0)
        c = self.sflow.slurp_pay()
        assert (c == 2)
        assert (self.sflow.credit == 2)

        for count in range(10):
            # note, seqno normally should sequential
            self.sflow.put(Message(coord=CoordHeader(seqno=100 + count)))
            self.sflow.put(Message(coord=CoordHeader(seqno=200 + count)))
            dat = self.cflow.get()
            assert (dat.seqno == 100 + count)
            dat = self.cflow.get()
            assert (dat.seqno == 200 + count)

        # normally, when a flow explicitly sends EOT the other end
        # will recv the EOT when its trying to recv another message
        # (PAY or DAT).  In this test things are synchronous and so we
        # explicitly recv_eot().
        self.cflow.send_eot(Message())

        surprise = self.sflow.recv_eot(1000)
        assert (surprise)
        self.sflow.send_eot(Message())

        expected = self.cflow.recv_eot(1000)
        assert (expected)

    def test_flow_string(self):
        msg = Message(label='{"extra":42}')
        msg.label = stringify('DAT', **objectify(msg))
        fobj = objectify(msg)
        assert (fobj["extra"] == 42)
        assert (fobj["flow"] == "DAT")

    def tearDown(self):
        self.cnode.offline()
        self.snode.offline()

        pass
예제 #8
0
def client_handler(ctx, pipe, bot, rule_object, writer_addr, broker_addr):
    '''Connect to and marshall messages between broker and writer sockets.

    Parameters
    ----------

    bot : zio.Message

        The BOT message

    rule_object: dicionary 

        A ruleset rule object.

    writer_addr :: string

        The address of the writer's PULL socket to connect.

    broker_addr : string

        The address of the broker's SERVER socket to connect.

    '''
    # An HDF path to be added to every message we send to writer.
    mattr = message_to_dict(bot)
    rattr = dict(rule_object["attr"], **mattr)
    base_path =  rule_object["grouppat"].format(**rattr)
    log.debug(f'client_handler(msg, "{base_path}", "{broker_addr}", "{writer_addr}")')
    log.debug(bot)
    pipe.signal()

    push = ctx.socket(PUSH)
    push.connect(writer_addr)

    sock = ctx.socket(CLIENT)
    port = Port("write-handler", sock)
    port.connect(broker_addr)
    port.online(None)
    flow = Flow(port)
    log.debug (f'writer({base_path}) send BOT to {broker_addr}')
    flow.send_bot(bot)          # this introduces us to the server
    bot = flow.recv_bot()
    log.debug (f'writer({base_path}) got response:\n{bot}')
    flow.flush_pay()

    def push_message(m):
        log.debug (f'write_handler({base_path}) push {m}')
        attr = message_to_dict(m)
        attr['hdfgroup'] = base_path
        m.label = json.dumps(attr)
        push.send(m.encode())

    push_message(bot)

    poller = Poller()
    poller.register(pipe, POLLIN)
    poller.register(sock, POLLIN)
    while True:

        for which,_ in poller.poll():
            if not which:
                return

            if which == pipe: # signal exit
                log.debug ('write_handler pipe hit')
                return          

            # o.w. we have flow

            try:
                msg = flow.get()
            except Exception as err:
                log.warning('flow.get error: %s %s' % (type(err),err))
                continue

            if not msg:
                log.debug("write_handler: got EOT")
                flow.send_eot()
                # fixme: send an EOT also to push socket?.
                break

            push_message(msg)

            continue

    log.debug ('write_handler exiting')
    pipe.signal()