def __init__(self,
view,
smd_configs,
dm,
filter_fn=0,
prometheus_counter=None,
max_retries=0):
if view:
pf = PacketFooter(view=view)
self.smd_events = pf.split_packets()
self.n_events = pf.n_packets
else:
self.smd_events = None
self.n_events = 0
self.smd_configs = smd_configs
self.dm = dm
self.n_smd_files = len(self.smd_configs)
self.filter_fn = filter_fn
self.cn_events = 0
self.prometheus_counter = prometheus_counter
self.max_retries = max_retries
if not self.filter_fn and len(self.dm.xtc_files) > 0:
self._read_bigdata_in_chunk()
def __init__(self, view, configs, dsparms):
self.configs = configs
self.batch_size = dsparms.batch_size
self.filter_fn = dsparms.filter
self.destination = dsparms.destination
self.n_files = len(self.configs)
pf = PacketFooter(view=view)
views = pf.split_packets()
self.eb = EventBuilder(views, self.configs)
self.c_filter = PrometheusManager.get_metric('psana_eb_filter')
def _to_bytes(self):
event_bytes = bytearray()
pf = PacketFooter(self._size)
for i, d in enumerate(self._dgrams):
event_bytes.extend(bytearray(d))
pf.set_size(i, memoryview(bytearray(d)).shape[0])
if event_bytes:
event_bytes.extend(pf.footer)
return event_bytes
def __init__(self, view, smd_configs, dm, esm,
filter_fn=0, prometheus_counter=None,
max_retries=0, use_smds=[]):
if view:
pf = PacketFooter(view=view)
self.n_events = pf.n_packets
else:
self.n_events = 0
self.smd_configs = smd_configs
self.dm = dm
self.esm = esm
self.n_smd_files = len(self.smd_configs)
self.filter_fn = filter_fn
self.prometheus_counter = prometheus_counter
self.max_retries = max_retries
self.use_smds = use_smds
self.smd_view = view
self.i_evt = 0
# Each chunk must fit in BD_CHUNKSIZE and we only fill bd buffers
# when bd_offset reaches the size of buffer.
self.BD_CHUNKSIZE = int(os.environ.get('PS_BD_CHUNKSIZE', 0x1000000))
self._get_offset_and_size()
if self.dm.n_files > 0:
self._init_bd_chunks()
def _from_bytes(cls, configs, event_bytes, run=None):
dgrams = []
if event_bytes:
pf = PacketFooter(view=event_bytes)
views = pf.split_packets()
assert len(configs) == pf.n_packets
dgrams = [None
] * pf.n_packets # make sure that dgrams are arranged
# according to the smd files.
for i in range(pf.n_packets):
if views[i].shape[0] > 0: # do not include any missing dgram
dgrams[i] = dgram.Dgram(config=configs[i], view=views[i])
evt = cls(dgrams, run=run)
return evt
def _get_offset_and_size(self):
"""
Use fast step-through to read off offset and size from smd_view.
Format of smd_view
[
[[d_bytes][d_bytes]....[evt_footer]] <-- 1 event
[[d_bytes][d_bytes]....[evt_footer]]
[chunk_footer]]
"""
offset = 0
i_smd = 0
smd_chunk_pf = PacketFooter(view=self.smd_view)
dtype = np.int64
# Row - events, col = smd files
self.bd_offset_array = np.zeros((smd_chunk_pf.n_packets, self.n_smd_files), dtype=dtype)
self.bd_size_array = np.zeros((smd_chunk_pf.n_packets, self.n_smd_files), dtype=dtype)
self.smd_offset_array = np.zeros((smd_chunk_pf.n_packets, self.n_smd_files), dtype=dtype)
self.smd_size_array = np.zeros((smd_chunk_pf.n_packets, self.n_smd_files), dtype=dtype)
self.new_chunk_id_array = np.zeros((smd_chunk_pf.n_packets, self.n_smd_files), dtype=dtype)
self.cutoff_flag_array = np.ones((smd_chunk_pf.n_packets, self.n_smd_files), dtype=dtype)
self.services = np.zeros(smd_chunk_pf.n_packets, dtype=dtype)
smd_aux_sizes = np.zeros(self.n_smd_files, dtype=dtype)
self.i_first_L1s = np.zeros(self.n_smd_files, dtype=dtype) + 0xffffff
# For comparing if the next dgram should be in the same read
current_bd_offsets = np.zeros(self.n_smd_files, dtype=dtype)
# Current chunk size (gets reset at boundary)
current_bd_chunk_sizes = np.zeros(self.n_smd_files, dtype=dtype)
i_evt = 0
while offset < memoryview(self.smd_view).nbytes - memoryview(smd_chunk_pf.footer).nbytes:
if i_smd == 0:
smd_evt_size = smd_chunk_pf.get_size(i_evt)
smd_evt_pf = PacketFooter(view=self.smd_view[offset: offset+smd_evt_size])
smd_aux_sizes[:] = [smd_evt_pf.get_size(i) for i in range(smd_evt_pf.n_packets)]
# Only get offset and size of non-missing dgram
# TODO: further optimization by looking for the first L1 and read in a big chunk
# anything that comes after. Right now, all transitions mark the cutoff points.
if smd_aux_sizes[i_smd] == 0:
self.cutoff_flag_array[i_evt, i_smd] = 0
else:
d = dgram.Dgram(config=self.smd_configs[i_smd], view=self.smd_view, offset=offset)
self.smd_offset_array[i_evt, i_smd] = offset
self.smd_size_array[i_evt, i_smd] = d._size
self.services[i_evt] = d.service()
# Any dgrams after the first L1 (as long as they fit in the chunk size)
# will be read in together.
if self.dm.n_files > 0:
if d.service() == TransitionId.L1Accept:
if self.i_first_L1s[i_smd] == 0xffffff:
self.i_first_L1s[i_smd] = i_evt
print(f'i_smd={i_smd} i_first_L1={self.i_first_L1s[i_smd]}')
# For SlowUpdate, we need to check if the next dgram gets cutoff
elif d.service() == TransitionId.SlowUpdate and hasattr(d, 'chunkinfo'):
stream_id = self.dm.get_stream_id(i_smd)
_chunk_ids = [getattr(d.chunkinfo[seg_id].chunkinfo, 'chunkid') for seg_id in d.chunkinfo]
# There must be only one unique epics var
if _chunk_ids: self.new_chunk_id_array[i_evt, i_smd] = _chunk_ids[0]
self._get_bd_offset_and_size(d, current_bd_offsets, current_bd_chunk_sizes,
i_evt, i_smd, self.i_first_L1s[i_smd])
"""
# For L1 with bigdata files, store offset and size found in smd dgrams.
# For SlowUpdate, store new chunk id (if found). TODO: check if
# we need to always check for epics for SlowUpdate.
if d.service() == TransitionId.L1Accept and self.dm.n_files > 0:
if i_first_L1 == -1:
i_first_L1 = i_evt
print(f'i_smd={i_smd} i_first_L1={i_first_L1}')
self._get_bd_offset_and_size(d, current_bd_offsets, current_bd_chunk_sizes, i_evt, i_smd, i_first_L1)
elif d.service() == TransitionId.SlowUpdate and hasattr(d, 'chunkinfo'):
# We only support chunking on bigdata
if self.dm.n_files > 0:
stream_id = self.dm.get_stream_id(i_smd)
_chunk_ids = [getattr(d.chunkinfo[seg_id].chunkinfo, 'chunkid') for seg_id in d.chunkinfo]
# There must be only one unique epics var
if _chunk_ids: self.new_chunk_id_array[i_evt, i_smd] = _chunk_ids[0]
"""
offset += smd_aux_sizes[i_smd]
i_smd += 1
if i_smd == self.n_smd_files:
offset += PacketFooter.n_bytes * (self.n_smd_files + 1) # skip the footer
i_smd = 0 # reset to the first smd file
i_evt += 1 # done with this smd event
# end while offset
# Precalculate cutoff indices
self.cutoff_indices = []
self.chunk_indices = np.zeros(self.n_smd_files, dtype=dtype)
for i_smd in range(self.n_smd_files):
self.cutoff_indices.append(np.where(self.cutoff_flag_array[:, i_smd] == 1)[0])
print(f'i_smd={i_smd} cutoff_index={self.cutoff_indices[i_smd]} services={self.services[self.cutoff_indices[i_smd]]}')
def _get_offset_and_size(self):
"""
Use fast step-through to read off offset and size from smd_view.
Format of smd_view
[
[[d_bytes][d_bytes]....[evt_footer]] <-- 1 event
[[d_bytes][d_bytes]....[evt_footer]]
[chunk_footer]]
"""
offset = 0
i_smd = 0
smd_chunk_pf = PacketFooter(view=self.smd_view)
dtype = np.int64
# Row - events, col = smd files
self.bd_offset_array = np.zeros((smd_chunk_pf.n_packets, self.n_smd_files), dtype=dtype)
self.bd_size_array = np.zeros((smd_chunk_pf.n_packets, self.n_smd_files), dtype=dtype)
self.smd_offset_array = np.zeros((smd_chunk_pf.n_packets, self.n_smd_files), dtype=dtype)
self.smd_size_array = np.zeros((smd_chunk_pf.n_packets, self.n_smd_files), dtype=dtype)
self.new_chunk_id_array = np.zeros((smd_chunk_pf.n_packets, self.n_smd_files), dtype=dtype)
self.missing_flag_array = np.zeros((smd_chunk_pf.n_packets, self.n_smd_files), dtype=dtype)
self.cutoff_flag_array = np.ones((smd_chunk_pf.n_packets, self.n_smd_files), dtype=dtype)
self.services = np.zeros(smd_chunk_pf.n_packets, dtype=dtype)
smd_aux_sizes = np.zeros(self.n_smd_files, dtype=dtype)
# For comparing if the next dgram should be in the same read
current_bd_offsets = np.zeros(self.n_smd_files, dtype=dtype)
# Current chunk size (gets reset at boundary)
current_bd_chunk_sizes = np.zeros(self.n_smd_files, dtype=dtype)
i_evt = 0
i_first_L1 = -1
ts = None
while offset < memoryview(self.smd_view).nbytes - memoryview(smd_chunk_pf.footer).nbytes:
if i_smd == 0:
smd_evt_size = smd_chunk_pf.get_size(i_evt)
smd_evt_pf = PacketFooter(view=self.smd_view[offset: offset+smd_evt_size])
smd_aux_sizes[:] = [smd_evt_pf.get_size(i) for i in range(smd_evt_pf.n_packets)]
# Only get offset and size of non-missing dgram
# TODO: further optimization by looking for the first L1 and read in a big chunk
# anything that comes after. Right now, all transitions mark the cutoff points.
if smd_aux_sizes[i_smd] == 0:
self.cutoff_flag_array[i_evt, i_smd] = 0
self.missing_flag_array[i_evt, i_smd] = 1
else:
d = dgram.Dgram(config=self.smd_configs[i_smd], view=self.smd_view, offset=offset)
self.smd_offset_array[i_evt, i_smd] = offset
self.smd_size_array[i_evt, i_smd] = d._size
self.services[i_evt] = d.service()
# For L1 with bigdata files, store offset and size found in smd dgrams.
# For Enable, store new chunk id (if found).
if d.service() == TransitionId.L1Accept and self.dm.n_files > 0:
if i_first_L1 == -1:
i_first_L1 = i_evt
self._get_bd_offset_and_size(d, current_bd_offsets, current_bd_chunk_sizes, i_evt, i_smd, i_first_L1)
elif d.service() == TransitionId.Enable and hasattr(d, 'chunkinfo'):
# We only support chunking on bigdata
if self.dm.n_files > 0:
_chunk_ids = [getattr(d.chunkinfo[seg_id].chunkinfo, 'chunkid') for seg_id in d.chunkinfo]
# Only flag new chunk when there's chunkinfo and that chunkid is new
if _chunk_ids:
# There must be only one unique chunkid name
new_chunk_id = _chunk_ids[0]
current_chunk_id = self.dm.get_chunk_id(i_smd)
if new_chunk_id > current_chunk_id:
self.new_chunk_id_array[i_evt, i_smd] = new_chunk_id
offset += smd_aux_sizes[i_smd]
i_smd += 1
if i_smd == self.n_smd_files:
offset += PacketFooter.n_bytes * (self.n_smd_files + 1) # skip the footer
i_smd = 0 # reset to the first smd file
i_evt += 1 # done with this smd event
ts = None
# end while offset
# Precalculate cutoff indices
self.cutoff_indices = []
self.chunk_indices = np.zeros(self.n_smd_files, dtype=dtype)
for i_smd in range(self.n_smd_files):
self.cutoff_indices.append(np.where(self.cutoff_flag_array[:, i_smd] == 1)[0])