def run_mpi(self):
rankreq = np.empty(1, dtype='i')
for (smd_chunk, update_chunk) in self.smdr_man.chunks():
# Creates a chunk from smd and epics data to send to SmdNode
# Anatomy of a chunk (pf=packet_footer):
# [ [smd0][smd1][smd2][pf] ][ [epics0][epics1][epics2][pf] ][ pf ]
# ----- smd_chunk ------ ---------epics_chunk-------
# -------------------------- chunk ------------------------------
# Read new epics data as available in the queue
# then send only unseen portion of data to the evtbuilder rank.
update_pf = PacketFooter(view=update_chunk)
self.epics_man.extend_buffers(update_pf.split_packets())
smd_comm.Recv(rankreq, source=MPI.ANY_SOURCE)
epics_chunk = self.epics_man.get_buffer(rankreq[0])
pf = PacketFooter(2)
pf.set_size(0, memoryview(smd_chunk).shape[0])
pf.set_size(1, memoryview(epics_chunk).shape[0])
chunk = smd_chunk + epics_chunk + pf.footer
smd_comm.Send(chunk, dest=rankreq[0])
for i in range(PS_SMD_NODES):
smd_comm.Recv(rankreq, source=MPI.ANY_SOURCE)
smd_comm.Send(bytearray(), dest=rankreq[0])
def chunks(self):
""" Generates a tuple of smd and update dgrams """
got_events = -1
while got_events != 0:
self.smdr.get(self.n_events)
got_events = self.smdr.got_events
self.processed_events += got_events
smd_view = bytearray()
smd_pf = PacketFooter(n_packets=self.n_files)
update_view = bytearray()
update_pf = PacketFooter(n_packets=self.n_files)
for i in range(self.n_files):
_smd_view = self.smdr.view(i)
if _smd_view != 0:
smd_view.extend(_smd_view)
smd_pf.set_size(i, memoryview(_smd_view).shape[0])
_update_view = self.smdr.view(i, update=True)
if _update_view != 0:
update_view.extend(_update_view)
update_pf.set_size(i, memoryview(_update_view).shape[0])
if smd_view or update_view:
if smd_view:
smd_view.extend(smd_pf.footer)
if update_view:
update_view.extend(update_pf.footer)
yield (smd_view, update_view)
if self.max_events:
if self.processed_events >= self.max_events:
break
def chunks(self):
""" Generates a tuple of smd and step dgrams """
self._read()
while self.got_events > 0:
smd_view = bytearray()
smd_pf = PacketFooter(n_packets=self.n_files)
step_view = bytearray()
step_pf = PacketFooter(n_packets=self.n_files)
for i in range(self.n_files):
_smd_view = self.smdr.view(i)
if _smd_view != 0:
smd_view.extend(_smd_view)
smd_pf.set_size(i, memoryview(_smd_view).shape[0])
_step_view = self.smdr.view(i, step=True)
if _step_view != 0:
step_view.extend(_step_view)
step_pf.set_size(i, memoryview(_step_view).shape[0])
if smd_view or step_view:
if smd_view:
smd_view.extend(smd_pf.footer)
if step_view:
step_view.extend(step_pf.footer)
yield (smd_view, step_view)
if self.run.max_events:
if self.processed_events >= self.run.max_events:
break
self._read()
def pack(self, *args):
pf = PacketFooter(len(args))
batch = bytearray()
for i, arg in enumerate(args):
pf.set_size(i, memoryview(arg).shape[0])
batch += arg
batch += pf.footer
return batch
def _to_bytes(self):
event_bytes = bytearray()
pf = PacketFooter(self._size)
for i, d in enumerate(self._dgrams):
event_bytes.extend(bytearray(d))
pf.set_size(i, memoryview(bytearray(d)).shape[0])
if event_bytes:
event_bytes.extend(pf.footer)
return event_bytes
def step_chunk(self):
""" Returns list of steps in all smd files."""
step_view = bytearray()
step_pf = PacketFooter(n_packets=self.n_files)
for i in range(self.n_files):
_step_view = self.eb.step_view(i)
if _step_view != 0:
step_view.extend(_step_view)
step_pf.set_size(i, memoryview(_step_view).shape[0])
if step_view:
step_view.extend(step_pf.footer)
return step_view
def test_contents(self):
view = bytearray()
pf = PacketFooter(2)
for i, msg in enumerate([b'packet0', b'packet1']):
view.extend(msg)
pf.set_size(i, memoryview(msg).shape[0])
view.extend(pf.footer)
pf2 = PacketFooter(view=view)
assert pf2.n_packets == 2
views = pf2.split_packets()
assert memoryview(views[0]).shape[0] == 7
assert memoryview(views[1]).shape[0] == 7
def chunks(self):
""" Generates a tuple of smd and step dgrams """
is_done = False
while not is_done:
if self.smdr.is_complete():
mmrv_bufs, mmrv_step_bufs = self.smdr.view(batch_size=self.batch_size)
self.got_events = self.smdr.view_size
self.processed_events += self.got_events
# sending data to prometheus
if self.run.prom_man:
logging.debug('Smd0 got %d events'%(self.got_events))
self.c_read.labels('evts', 'None').inc(self.got_events)
self.c_read.labels('batches', 'None').inc()
if self.run.max_events and self.processed_events >= self.run.max_events:
is_done = True
smd_view = bytearray()
smd_pf = PacketFooter(n_packets=self.n_files)
step_view = bytearray()
step_pf = PacketFooter(n_packets=self.n_files)
for i, (mmrv_buf, mmrv_step_buf) in enumerate(zip(mmrv_bufs, mmrv_step_bufs)):
if mmrv_buf != 0:
smd_view.extend(mmrv_buf)
smd_pf.set_size(i, memoryview(mmrv_buf).nbytes)
if mmrv_step_buf != 0:
step_view.extend(mmrv_step_buf)
step_pf.set_size(i, memoryview(mmrv_step_buf).nbytes)
if smd_view or step_view:
if smd_view:
smd_view.extend(smd_pf.footer)
if step_view:
step_view.extend(step_pf.footer)
yield (smd_view, step_view)
else:
self.smdr.get()
if self.run.prom_man:
logging.debug('Smd0 read %.2f MB'%(self.smdr.got/1e6))
self.c_read.labels('MB', 'None').inc(self.smdr.got/1e6)
if not self.smdr.is_complete():
is_done = True
break
def get_buffer(self, client_id):
""" Returns new epics data (if any) for this client
then updates the sent record."""
update_chunk = bytearray()
if self.n_smds: # do nothing if no epics data found
indexed_id = client_id - 1 # rank 0 has no send history.
pf = PacketFooter(self.n_smds)
for i, buf in enumerate(self.bufs):
current_buf = self.bufs[i]
current_offset = self.send_history[indexed_id][i]
current_buf_size = memoryview(current_buf).shape[0]
pf.set_size(i, current_buf_size - current_offset)
update_chunk.extend(current_buf[current_offset:])
self.send_history[indexed_id][i] = current_buf_size
update_chunk.extend(pf.footer)
return update_chunk
def repack_for_eb(smd_chunk, step_views, configs):
""" Smd0 uses this to prepend missing step views
to the smd_chunk (just data with the same limit timestamp from all
smd files - not event-built yet).
"""
if step_views:
smd_chunk_pf = PacketFooter(view=smd_chunk)
new_chunk_pf = PacketFooter(n_packets=smd_chunk_pf.n_packets)
new_chunk = bytearray()
for i, (smd_view, step_view) in enumerate(
zip(smd_chunk_pf.split_packets(), step_views)):
new_chunk.extend(step_view + bytearray(smd_view))
new_chunk_pf.set_size(
i,
memoryview(step_view).nbytes + smd_view.nbytes)
new_chunk.extend(new_chunk_pf.footer)
return new_chunk
else:
return smd_chunk
def repack_for_bd(smd_batch, step_views, configs):
""" EventBuilder Node uses this to prepend missing step views
to the smd_batch. Unlike repack_for_eb (used by Smd0), this output
chunk contains list of pre-built events."""
if step_views:
batch_pf = PacketFooter(view=smd_batch)
# Create bytearray containing a list of events from step_views
steps = bytearray()
n_smds = len(step_views)
offsets = [0] * n_smds
n_steps = 0
step_sizes = []
while offsets[0] < memoryview(step_views[0]).nbytes:
step_pf = PacketFooter(n_packets=n_smds)
step_size = 0
for i, (config, view) in enumerate(zip(configs, step_views)):
d = Dgram(config=config, view=view, offset=offsets[i])
steps.extend(d)
offsets[i] += d._size
step_size += d._size
step_pf.set_size(i, d._size)
steps.extend(step_pf.footer)
step_sizes.append(step_size + memoryview(step_pf.footer).nbytes)
n_steps += 1
# Create new batch with total_events = smd_batch_events + step_events
new_batch_pf = PacketFooter(n_packets=batch_pf.n_packets + n_steps)
for i in range(n_steps):
new_batch_pf.set_size(i, step_sizes[i])
for i in range(n_steps, new_batch_pf.n_packets):
new_batch_pf.set_size(i, batch_pf.get_size(i - n_steps))
new_batch = bytearray()
new_batch.extend(steps)
new_batch.extend(smd_batch[:memoryview(smd_batch).nbytes -
memoryview(batch_pf.footer).nbytes])
new_batch.extend(new_batch_pf.footer)
return new_batch
else:
return smd_batch