def __next__(self):
if self.flag_empty_smd_batch:
raise StopIteration
if not self.get_smd:
# RunSerial - get smd chuk from SmdReader iterator
try:
return self._get_evt_and_update_store()
except StopIteration:
try:
batch_dict = next(self._batch_iter)
except StopIteration:
self._batch_iter = next(self._smdr_man)
batch_dict = next(self._batch_iter)
self._evt_man = EventManager(batch_dict[0][0], self.run.configs, \
self.run.dm, filter_fn=self.run.filter_callback)
return self._get_evt_and_update_store()
else:
# RunParallel - get smd chunk from a callback (MPI receive)
try:
return self._get_evt_and_update_store()
except StopIteration:
smd_batch = self.get_smd()
if smd_batch == bytearray():
self.flag_empty_smd_batch = True
raise StopIteration
else:
self._evt_man = EventManager(smd_batch, self.run.configs, \
self.run.dm, filter_fn=self.run.filter_callback)
return self._get_evt_and_update_store()
def __next__(self):
if self.flag_empty_smd_batch:
raise StopIteration
if self.get_smd:
# RunParallel - get smd chunk from a callback (MPI receive)
try:
return self._get_evt_and_update_store()
except StopIteration:
smd_batch = self.get_smd()
if smd_batch == bytearray():
self.flag_empty_smd_batch = True
raise StopIteration
else:
if self.run.prom_man:
self.c_read.labels('batches', 'None').inc()
self._evt_man = EventManager(
smd_batch,
self.run.configs,
self.run.dm,
filter_fn=self.run.filter_callback,
prometheus_counter=self.c_read)
return self._get_evt_and_update_store()
else:
if self.dm:
# RunSingleFile or RunShmem - get event from DgramManager
evt = next(self.dm)
if evt.service() != TransitionId.L1Accept:
self.run.esm.update_by_event(evt)
return evt
else:
# RunSerial - get smd chunk from SmdReader iterator
try:
return self._get_evt_and_update_store()
except StopIteration:
try:
batch_dict, _ = next(self._batch_iter)
except StopIteration:
self._batch_iter = next(self._smdr_man)
batch_dict, _ = next(self._batch_iter)
self._evt_man = EventManager(
batch_dict[0][0],
self.run.configs,
self.run.dm,
filter_fn=self.run.filter_callback,
prometheus_counter=self.c_read)
return self._get_evt_and_update_store()
class BigDataNode(object):
def __init__(self, run):
self.evt_man = EventManager(run.configs, run.dm, \
filter_fn=run.filter_callback)
self.run = run
def run_mpi(self):
while True:
bd_comm.Send(np.array([bd_rank], dtype='i'), dest=0)
info = MPI.Status()
bd_comm.Probe(source=0, tag=MPI.ANY_TAG, status=info)
count = info.Get_elements(MPI.BYTE)
chunk = bytearray(count)
bd_comm.Recv(chunk, source=0)
if count == 0:
break
if chunk == bytearray(b'wait'):
continue
pf = PacketFooter(view=chunk)
smd_chunk, epics_chunk = pf.split_packets()
pfe = PacketFooter(view=epics_chunk)
epics_views = pfe.split_packets()
self.run.epics_store.update(epics_views)
if self.run.scan:
yield Step(self.run, smd_batch=smd_chunk)
else:
for event in self.evt_man.events(smd_chunk):
yield event
def events(self):
# Event generator that yields list of events from
# an smd_chunk: data from smd0 that needs to be event built (serial mode) or
# an smd_batch: prebuilt smd data (parallel mode)
ev_man = EventManager(self.run.configs, self.run.dm, \
filter_fn=self.run.filter_callback)
if self.eb_man:
for batch_dict in self.eb_man.batches(limit_ts=self.limit_ts):
batch, _ = batch_dict[0]
for evt in ev_man.events(batch):
if evt._dgrams[0].seq.service() != 12: continue
yield evt
else:
for evt in ev_man.events(self.smd_batch):
if evt._dgrams[0].seq.service() != 12: continue
yield evt
def events(self):
ev_man = EventManager(self.configs, self.dm, \
filter_fn=self.filter_callback)
#get smd chunks
smdr_man = SmdReaderManager(self.smd_dm.fds, self.max_events)
for (smd_chunk, step_chunk) in smdr_man.chunks():
# Update epics_store for each chunk
step_pf = PacketFooter(view=step_chunk)
step_views = step_pf.split_packets()
self.epics_store.update(step_views)
eb_man = EventBuilderManager(smd_chunk,
self.configs,
batch_size=self.batch_size,
filter_fn=self.filter_callback)
for batch_dict in eb_man.batches():
batch, _ = batch_dict[
0] # there's only 1 dest_rank for serial run
for evt in ev_man.events(batch):
if evt._dgrams[0].seq.service() != 12: continue
yield evt
def run_bigdata_task(batch, run):
evt_man = EventManager(run.configs, run.dm, run.filter_callback)
for evt in evt_man.events(batch):
if evt._dgrams[0].seq.service() != 12: continue
run.event_fn(evt, run.det)
def __init__(self, run):
self.evt_man = EventManager(run.configs, run.dm, \
filter_fn=run.filter_callback)
self.run = run