def make_broker(ctx, ruleset, address):
sock = ctx.socket(zmq.SERVER)
sport = Port("broker", sock)
sport.bind(address)
sport.do_binds()
sport.online()
log.debug("make factory")
factory = hdf.Factory(ctx, ruleset, address, wargs=(pb, hdf.frompb))
return Broker(sport, factory)
def flow_depos(ctx, pipe, nsend, name, address):
'''
An actor with a flow client sending depo messages.
'''
log.debug(f'actor: flow_depos({nsend}, "{name}", "{address}"')
pipe.signal()
port = Port(name, zmq.CLIENT, '')
port.connect(address)
port.online(None) # peer not needed if port only direct connects
flow = Flow(port)
fobj = dict(flow='BOT', direction='extract', credit=3, stream=name)
msg = Message(seqno=0, form='FLOW', label=json.dumps(fobj))
log.debug(f'flow_depos {name} send BOT:\n{msg}')
flow.send_bot(msg)
msg = flow.recv_bot(1000)
log.debug(f'flow_depos {name} got BOT:\n{msg}')
assert (msg)
for count in range(nsend):
depo = pb.Depo(ident=count,
pos=pb.Point(x=1, y=2, z=3),
time=100.0,
charge=1000.0,
trackid=12345,
pdg=11,
extent_long=9.6,
extent_tran=6.9)
a = Any()
a.Pack(depo)
msg = Message(form='FLOW',
seqno=count + 1,
label=json.dumps({'flow': 'DAT'}),
payload=[a.SerializeToString()])
log.debug(
f'flow_depos {name} put: {count}/{nsend}[{flow.credit}]:\n{msg}')
flow.put(msg)
log.debug(f'flow_depos {name} again [{flow.credit}]')
log.debug(f'flow_depos {name} send EOT')
flow.send_eot(Message(seqno=nsend + 1))
log.debug(f'flow_depos {name} recv EOT')
flow.recv_eot()
log.debug(f'flow_depos {name} wait for quit signal')
pipe.recv() # wait for signal to quit
log.debug(f'flow_depos {name} exiting')
return
def handler(ctx, pipe, bot, rule_object, filename, broker_addr, *rargs):
log.debug(f'actor: reader "{filename}"')
fp = h5py.File(filename, 'r')
mattr = message_to_dict(bot)
rattr = dict(rule_object["attr"], **mattr)
base_path = rule_object["grouppat"].format(**rattr)
log.debug(f'reader(msg, "{base_path}", "{broker_addr}")')
log.debug(bot)
pipe.signal()
sock = ctx.socket(CLIENT)
port = Port("read-handler", sock)
port.connect(broker_addr)
port.online(None)
flow = Flow(port)
log.debug(f'reader({base_path}) send BOT to {broker_addr}')
sg = fp.get(base_path)
if not sg:
log.error(f'reader failed to get {base_path} from {filename}')
return
fr = Reader(sg, *rargs)
obot = fr.read()
# fixme: something should be done to compare old and new and
# assert on any important differences. For now, we effectively
# drop the old one and send back the new.
# log.debug(f'new BOT: {bot}')
# log.debug(f'old BOT: {obot}')
flow.send_bot(bot) # this introduces us to the server
bot = flow.recv_bot()
log.debug(f'reader({base_path}) got response:\n{bot}')
flow.slurp_pay()
while True:
msg = fr.read()
log.debug(f'reader: {msg}')
if not msg:
break
ok = flow.put(msg)
if not ok:
break
flow.send_eot()
flow.recv_eot()
def handler(ctx, pipe, bot, rule_object, filename, broker_addr, *rargs):
log.debug(f'actor: reader "{filename}"')
fp = h5py.File(filename, 'r')
mattr = message_to_dict(bot)
rattr = dict(rule_object.get("attr", {}), **mattr)
base_path = rule_object.get("grouppat", "/").format(**rattr)
log.debug(f'reader(msg, "{base_path}", "{broker_addr}")')
log.debug(bot)
pipe.signal()
sock = ctx.socket(CLIENT)
port = Port("read-handler", sock)
port.connect(broker_addr)
port.online(None)
direction = mattr["direction"]
if direction != "extract":
raise RuntimeError(f'zio.flow.hdf.reader bad direction: "{direction}"')
credit = mattr["credit"]
flow = Flow(port, direction, credit)
log.debug(f'reader({base_path}) send BOT to {broker_addr}')
sg = fp.get(base_path)
if not sg:
log.error(f'reader failed to get {base_path} from {filename}')
return
fr = TensReader(sg, *rargs)
bot = flow.bot(bot) # this introduces us to the server
log.debug(f'reader({base_path}) got response: {bot}')
flow.begin()
while True:
msg = fr.read()
log.debug(f'reader: {msg}')
if not msg:
break
flow.put(msg)
flow.eot()
def handler(ctx, pipe, bot, rule_object, filename, broker_addr, *rargs):
log.debug(f'actor: reader "{filename}"')
fp = h5py.File(filename, 'r')
mattr = message_to_dict(bot)
rattr = dict(rule_object.get("attr", {}), **mattr)
base_path = rule_object.get("grouppat", "/").format(**rattr)
log.debug(f'reader(msg, "{base_path}", "{broker_addr}")')
log.debug(bot)
pipe.signal()
sock = ctx.socket(CLIENT)
port = Port("read-handler", sock)
port.connect(broker_addr)
port.online(None)
flow = Flow(port)
log.debug(f'reader({base_path}) send BOT to {broker_addr}')
sg = fp.get(base_path)
if not sg:
log.error(f'reader failed to get {base_path} from {filename}')
return
fr = TensReader(sg, *rargs)
flow.send_bot(bot) # this introduces us to the server
bot = flow.recv_bot()
log.debug(f'reader({base_path}) got response:\n{bot}')
flow.slurp_pay()
while True:
msg = fr.read()
log.debug(f'reader: {msg}')
if not msg:
break
ok = flow.put(msg)
if not ok:
break
flow.send_eot()
flow.recv_eot()
def client_handler(ctx, pipe, bot, rule_object, writer_addr, broker_addr):
'''Connect to and marshall messages between broker and writer sockets.
Parameters
----------
bot : zio.Message
The BOT message
rule_object: dicionary
A ruleset rule object.
writer_addr :: string
The address of the writer's PULL socket to connect.
broker_addr : string
The address of the broker's SERVER socket to connect.
'''
# An HDF path to be added to every message we send to writer.
mattr = message_to_dict(bot)
rattr = dict(rule_object.get("attr",{}), **mattr)
log.info(f'writer: attrs: {rattr}')
try:
base_path = rule_object.get("grouppat","/").format(**rattr)
except KeyError as e:
log.error(f'writer: missing attribute: {e}')
raise
log.debug(f'client_handler(msg, "{base_path}", "{broker_addr}", "{writer_addr}")')
log.debug(bot)
pipe.signal()
push = ctx.socket(PUSH)
push.connect(writer_addr)
sock = ctx.socket(CLIENT)
port = Port("write-handler", sock)
port.connect(broker_addr)
port.online(None)
direction = mattr["direction"]
if direction != "inject":
raise RuntimeError(f'zio.flow.hdf.writer bad direction: "{direction}"')
credit = mattr["credit"]
flow = Flow(port, direction, credit)
log.debug (f'writer({base_path}) send BOT to {broker_addr}')
bot = flow.bot(bot) # this introduces us to the server
log.debug (f'writer({base_path}) got response:\n{bot}')
flow.begin()
def push_message(m):
log.debug (f'write_handler({base_path}) push {m}')
attr = message_to_dict(m)
attr['hdfgroup'] = base_path
m.label = json.dumps(attr)
push.send(m.encode())
#push_message(bot)
poller = Poller()
poller.register(pipe, POLLIN)
poller.register(sock, POLLIN)
while True:
for which,_ in poller.poll():
if not which:
return
if which == pipe: # signal exit
log.debug ('write_handler pipe hit')
return
# o.w. we have flow
try:
msg = flow.get()
except TransmissionEnd as te:
flow.eotsend()
break
push_message(msg)
continue
log.debug ('write_handler exiting')
pipe.signal()
def client_handler(ctx, pipe, bot, rule_object, writer_addr, broker_addr):
'''Connect to and marshall messages between broker and writer sockets.
Parameters
----------
bot : zio.Message
The BOT message
rule_object: dicionary
A ruleset rule object.
writer_addr :: string
The address of the writer's PULL socket to connect.
broker_addr : string
The address of the broker's SERVER socket to connect.
'''
# An HDF path to be added to every message we send to writer.
mattr = message_to_dict(bot)
rattr = dict(rule_object["attr"], **mattr)
base_path = rule_object["grouppat"].format(**rattr)
log.debug(f'client_handler(msg, "{base_path}", "{broker_addr}", "{writer_addr}")')
log.debug(bot)
pipe.signal()
push = ctx.socket(PUSH)
push.connect(writer_addr)
sock = ctx.socket(CLIENT)
port = Port("write-handler", sock)
port.connect(broker_addr)
port.online(None)
flow = Flow(port)
log.debug (f'writer({base_path}) send BOT to {broker_addr}')
flow.send_bot(bot) # this introduces us to the server
bot = flow.recv_bot()
log.debug (f'writer({base_path}) got response:\n{bot}')
flow.flush_pay()
def push_message(m):
log.debug (f'write_handler({base_path}) push {m}')
attr = message_to_dict(m)
attr['hdfgroup'] = base_path
m.label = json.dumps(attr)
push.send(m.encode())
push_message(bot)
poller = Poller()
poller.register(pipe, POLLIN)
poller.register(sock, POLLIN)
while True:
for which,_ in poller.poll():
if not which:
return
if which == pipe: # signal exit
log.debug ('write_handler pipe hit')
return
# o.w. we have flow
try:
msg = flow.get()
except Exception as err:
log.warning('flow.get error: %s %s' % (type(err),err))
continue
if not msg:
log.debug("write_handler: got EOT")
flow.send_eot()
# fixme: send an EOT also to push socket?.
break
push_message(msg)
continue
log.debug ('write_handler exiting')
pipe.signal()