def _setup_beginruns(self):
""" Determines if there is a next run as
1) New run found in the same smalldata files
2) New run found in the new smalldata files
"""
while True:
if nodetype == 'smd0':
dgrams = self.smdr_man.get_next_dgrams()
nbytes = np.zeros(len(self.smd_files), dtype='i')
if dgrams is not None:
nbytes = np.array([memoryview(d).shape[0] for d in dgrams],
dtype='i')
else:
dgrams = None
nbytes = np.empty(len(self.smd_files), dtype='i')
self.comms.psana_comm.Bcast(nbytes, root=0)
if np.sum(nbytes) == 0: return False
if nodetype != 'smd0':
dgrams = [np.empty(nbyte, dtype='b') for nbyte in nbytes]
for i in range(len(dgrams)):
self.comms.psana_comm.Bcast([dgrams[i], nbytes[i], MPI.BYTE],
root=0)
if nodetype != 'smd0':
dgrams = [dgram.Dgram(view=d, config=config, offset=0) \
for d, config in zip(dgrams,self._configs)]
if dgrams[0].service() == TransitionId.BeginRun:
self.beginruns = dgrams
return True
def events(self, view):
pf = PacketFooter(view=view)
views = pf.split_packets()
# Keeps offset, size, & timestamp for all events in the batch
# for batch reading (if filter_fn is not given).
ofsz_batch = np.zeros((pf.n_packets, self.n_smd_files, 2),
dtype=np.intp)
has_offset = True
for i, event_bytes in enumerate(views):
if event_bytes:
evt = Event._from_bytes(self.smd_configs, event_bytes)
if not evt._has_offset:
has_offset = False
yield evt # no offset info in the smalldata event
else:
segment = 0 # "d.info" "detectors" have only one segment
ofsz = np.asarray([[d.info[segment].offsetAlg.intOffset, d.info[segment].offsetAlg.intDgramSize] \
for d in evt])
ofsz_batch[i, :, :] = ofsz
# Only get big data one event at a time when filter is off
if self.filter_fn:
bd_evt = self.dm.jump(ofsz[:, 0], ofsz[:, 1])
yield bd_evt
if self.filter_fn == 0 and has_offset:
# Read chunks of 'size' bytes and store them in views
views = [None] * self.n_smd_files
view_sizes = np.zeros(self.n_smd_files)
for i in range(self.n_smd_files):
# If no data were filtered, we can assume that all bigdata
# dgrams starting from the first offset are stored consecutively
# in the file. We read a chunk of sum(all dgram sizes) and
# store in a view.
offset = ofsz_batch[0, i, 0]
size = np.sum(ofsz_batch[:, i, 1])
view_sizes[i] = size
os.lseek(self.dm.fds[i], offset, 0)
views[i] = os.read(self.dm.fds[i], size)
# Build each event from these views
dgrams = [None] * self.n_smd_files
offsets = [0] * self.n_smd_files
for i in range(pf.n_packets):
for j in range(self.n_smd_files):
if offsets[j] >= view_sizes[j]:
continue
size = ofsz_batch[i, j, 1]
if size:
dgrams[j] = dgram.Dgram(view=views[j],
config=self.dm.configs[j],
offset=offsets[j])
offsets[j] += size
bd_evt = Event(dgrams)
yield bd_evt
def __next__(self):
if self.cn_events == self.n_events:
raise StopIteration
if len(self.dm.xtc_files) == 0:
smd_evt = Event._from_bytes(self.smd_configs,
self.smd_events[self.cn_events],
run=self.dm.run())
self.cn_events += 1
return smd_evt
if self.filter_fn:
smd_evt = Event._from_bytes(self.smd_configs,
self.smd_events[self.cn_events],
run=self.dm.run())
self.cn_events += 1
ofsz = np.asarray([[d.smdinfo[0].offsetAlg.intOffset, \
d.smdinfo[0].offsetAlg.intDgramSize] for d in smd_evt])
bd_evt = self.dm.jump(ofsz[:, 0], ofsz[:, 1])
return bd_evt
dgrams = [None] * self.n_smd_files
ofsz = self.ofsz_batch[self.cn_events, :, :]
for j in range(self.n_smd_files):
if ofsz[j, 1]:
dgrams[j] = dgram.Dgram(view=self.bigdata[j],
config=self.dm.configs[j],
offset=ofsz[j, 0])
bd_evt = Event(dgrams, run=self.dm.run())
self.cn_events += 1
return bd_evt
def __init__(self, xtc_files, configs=[], tag=None, run=None):
""" Opens xtc_files and stores configs."""
self.xtc_files = []
self.shmem_cli = None
self.shmem_kwargs = {'index':-1,'size':0,'cli_cptr':None}
self.configs = []
self.fds = []
self._timestamps = [] # built when iterating
self._run = run
if isinstance(xtc_files, (str)):
self.xtc_files = np.array([xtc_files], dtype='U%s'%FN_L)
assert len(self.xtc_files) > 0
elif isinstance(xtc_files, (list, np.ndarray)):
if len(xtc_files) > 0: # handles smalldata-only case
if xtc_files[0] == 'shmem':
self.shmem_cli = PyShmemClient()
#establish connection to available server - blocking
status = int(self.shmem_cli.connect(tag,0))
assert not status,'shmem connect failure %d' % status
#wait for first configure datagram - blocking
view = self.shmem_cli.get(self.shmem_kwargs)
assert view
d = dgram.Dgram(view=view, \
shmem_index=self.shmem_kwargs['index'], \
shmem_size=self.shmem_kwargs['size'], \
shmem_cli_cptr=self.shmem_kwargs['cli_cptr'], \
shmem_cli_pyobj=self.shmem_cli)
self.configs += [d]
else:
self.xtc_files = np.asarray(xtc_files, dtype='U%s'%FN_L)
assert len(self.xtc_files) > 0
given_configs = True if len(configs) > 0 else False
if given_configs:
self.configs = configs
for i, xtcdata_filename in enumerate(self.xtc_files):
self.fds.append(os.open(xtcdata_filename,
os.O_RDONLY))
if not given_configs:
d = dgram.Dgram(file_descriptor=self.fds[-1])
self.configs += [d]
self.det_class_table, self.xtc_info = self.get_det_class_table()
self.calibs = {} # initialize to empty dict - will be populated by run class
def get_next_dgrams(self, configs=None):
dgrams = None
if not self.smdr.is_complete():
self.smdr.get()
if self.smdr.is_complete():
mmrv_bufs, _ = self.smdr.view(batch_size=1)
# For configs, we need to copy data from smdreader's buffers
# This prevents it from getting overwritten by other dgrams.
bytearray_bufs = [bytearray(mmrv_buf) for mmrv_buf in mmrv_bufs]
if configs is None:
dgrams = [dgram.Dgram(view=ba_buf, offset=0) for ba_buf in bytearray_bufs]
else:
dgrams = [dgram.Dgram(view=ba_buf, config=config, offset=0) for ba_buf, config in zip(bytearray_bufs, configs)]
return dgrams
def _check_missing_endrun(self, beginruns=None):
fake_endruns = None
if not self.found_endrun: # there's no previous EndRun
sec = (self._timestamps[-1] >> 32) & 0xffffffff
usec = int((self._timestamps[-1] & 0xffffffff) * 1e3 + 1)
if beginruns:
self.buffered_beginruns = [
dgram.Dgram(config=config, view=d, offset=0, size=d._size)
for d, config in zip(beginruns, self.configs)
]
fake_endruns = [dgram.Dgram(config=config, fake_endrun=1, \
fake_endrun_sec=sec, fake_endrun_usec=usec) \
for config in self.configs]
self.found_endrun = True
else:
self.found_endrun = False
return fake_endruns
def from_bytes(configs, event_bytes):
dgrams = []
if event_bytes:
dgrams_bytes = event_bytes.split(b'eod')
assert len(configs) == len(dgrams_bytes)
dgrams = [dgram.Dgram(config=configs[i], view=dgrams_bytes[i]) \
for i in range(len(configs))]
evt = Event(dgrams=dgrams)
return evt
def __init__(self, comms, exp, run_no, run_src, **kwargs):
""" Parallel read requires that rank 0 does the file system works.
Configs and calib constants are sent to other ranks by MPI.
Note that destination callback only works with RunParallel.
"""
super(RunParallel, self).__init__(exp, run_no, max_events=kwargs['max_events'], \
batch_size=kwargs['batch_size'], filter_callback=kwargs['filter_callback'], \
destination=kwargs['destination'])
xtc_files, smd_files, other_files = run_src
self.comms = comms
psana_comm = comms.psana_comm # TODO tjl and cpo to review
rank = psana_comm.Get_rank()
size = psana_comm.Get_size()
if rank == 0:
self.smd_dm = DgramManager(smd_files, run=self)
self.dm = DgramManager(xtc_files,
configs=self.smd_dm.configs,
run=self)
self.configs = self.dm.configs
nbytes = np.array([memoryview(config).shape[0] for config in self.configs], \
dtype='i')
self.calibs = {}
for det_name in self.detnames:
self.calibs[det_name] = super(RunParallel,
self)._get_calib(det_name)
else:
self.smd_dm = None
self.dm = None
self.configs = None
self.calibs = None
nbytes = np.empty(len(smd_files), dtype='i')
psana_comm.Bcast(nbytes, root=0) # no. of bytes is required for mpich
# create empty views of known size
if rank > 0:
self.configs = [np.empty(nbyte, dtype='b') for nbyte in nbytes]
for i in range(len(self.configs)):
psana_comm.Bcast([self.configs[i], nbytes[i], MPI.BYTE], root=0)
self.calibs = psana_comm.bcast(self.calibs, root=0)
if rank > 0:
# Create dgram objects using views from rank 0 (no disk operation).
self.configs = [
dgram.Dgram(view=config, offset=0) for config in self.configs
]
self.dm = DgramManager(xtc_files, configs=self.configs, run=self)
self.esm = EnvStoreManager(self.configs, 'epics', 'scan')
def _get_next_evt(self):
""" Generate bd evt for different cases:
1) No bigdata or is a Transition Event prior to i_first_L1
create dgrams from smd_view
2) L1Accept event
create dgrams from bd_bufs
3) L1Accept with some smd files replaced by bigdata files
create dgram from smd_view if use_smds[i_smd] is set
otherwise create dgram from bd_bufs
"""
dgrams = [None] * self.n_smd_files
for i_smd in range(self.n_smd_files):
# Check in case we need to switch to the next bigdata chunk file
if self.services[self.i_evt] != TransitionId.L1Accept:
if self.new_chunk_id_array[self.i_evt, i_smd] != 0:
print(f'open_new_bd_file i_smd={i_smd} chunk_id={self.new_chunk_id_array[self.i_evt, i_smd]}')
self._open_new_bd_file(i_smd,
self.new_chunk_id_array[self.i_evt, i_smd])
view, offset, size = (bytearray(),0,0)
# Try to create dgram from smd view
if self.dm.n_files == 0 or self.use_smds[i_smd] \
or self.i_evt < self.i_first_L1s[i_smd]:
view = self.smd_view
offset = self.smd_offset_array[self.i_evt, i_smd]
size = self.smd_size_array[self.i_evt, i_smd]
# Non L1 dgram prior to i_first_L1 are counted as a new "chunk"
# because their cutoff flag is set (data coming from smd view
# instead of bd chunk). We'll need to update chunk index for
# this smd when we see non L1.
self.chunk_indices[i_smd] += 1
else:
# Check if we need to fill bd buf if this dgram doesn't fit in the current view
if self.bd_buf_offsets[i_smd] + self.bd_size_array[self.i_evt, i_smd] \
> memoryview(self.bd_bufs[i_smd]).nbytes:
self._fill_bd_chunk(i_smd)
self.chunk_indices[i_smd] += 1
# This is the offset of bd buffer! and not what stored in smd dgram,
# which in contrast points to the location of disk.
offset = self.bd_buf_offsets[i_smd]
size = self.bd_size_array[self.i_evt, i_smd]
view = self.bd_bufs[i_smd]
self.bd_buf_offsets[i_smd] += size
if size: # handles missing dgram
dgrams[i_smd] = dgram.Dgram(config=self.dm.configs[i_smd], view=view, offset=offset)
self.i_evt += 1
self._inc_prometheus_counter('evts')
evt = Event(dgrams=dgrams, run=self.dm.get_run())
print(f'YIELD ts={evt.timestamp} service={evt.service()}')
return evt
def next(self):
""" only support sequential read - no event building"""
if self.shmem:
view = self.shmem.get(self.shmem_kwargs)
if view:
# use the most recent configure datagram
config = self.configs[len(self.configs) - 1]
d = dgram.Dgram(config=config,view=view, \
shmem_index=self.shmem_kwargs['index'], \
shmem_size=self.shmem_kwargs['size'], \
shmem_cli=self.shmem_kwargs['cli'])
dgrams = [d]
else:
raise StopIteration
else:
dgrams = [dgram.Dgram(config=config) for config in self.configs]
evt = Event(dgrams)
self._timestamps += [evt.timestamp]
return evt
def _get_next_evt(self):
""" Generate bd evt for different cases:
1) No bigdata or Transition Event
create dgrams from smd_view
2) L1Accept event
create dgrams from bd_bufs
3) L1Accept with some smd files replaced by bigdata files
create dgram from smd_view if use_smds[i_smd] is set
otherwise create dgram from bd_bufs
"""
dgrams = [None] * self.n_smd_files
for i_smd in range(self.n_smd_files):
if self.dm.n_files == 0 or \
self.services[self.i_evt] != TransitionId.L1Accept or \
self.use_smds[i_smd]:
view = self.smd_view
offset = self.smd_offset_array[self.i_evt, i_smd]
size = self.smd_size_array[self.i_evt, i_smd]
# Non L1 always are counted as a new "chunk" since they
# ther cutoff flag is set (data coming from smd view
# instead of bd chunk. We'll need to update chunk index for
# this smd when we see non L1.
self.chunk_indices[i_smd] += 1
# Check in case we need to switch to the next bigdata chunk file
if self.services[self.i_evt] != TransitionId.L1Accept:
if self.new_chunk_id_array[self.i_evt, i_smd] != 0:
self._open_new_bd_file(
i_smd, self.new_chunk_id_array[self.i_evt, i_smd])
else:
# Fill up bd buf if this dgram doesn't fit in the current view
if self.bd_buf_offsets[i_smd] + self.bd_size_array[self.i_evt, i_smd] \
> memoryview(self.bd_bufs[i_smd]).nbytes:
self._fill_bd_chunk(i_smd)
self.chunk_indices[i_smd] += 1
# This is the offset of bd buffer! and not what stored in smd dgram,
# which in contrast points to the location of disk.
offset = self.bd_buf_offsets[i_smd]
size = self.bd_size_array[self.i_evt, i_smd]
view = self.bd_bufs[i_smd]
self.bd_buf_offsets[i_smd] += size
if size > 0: # handles missing dgram
dgrams[i_smd] = dgram.Dgram(config=self.dm.configs[i_smd],
view=view,
offset=offset)
self.i_evt += 1
self._inc_prometheus_counter('evts')
evt = Event(dgrams=dgrams, run=self.dm.get_run())
return evt
def jump(self, offsets, sizes):
""" Jumps to the offset and reads out dgram on each xtc file.
This is used in normal mode (multiple detectors with MPI).
"""
assert len(offsets) > 0 and len(sizes) > 0
dgrams = []
for fd, config, offset, size in zip(self.fds, self.configs, offsets, sizes):
d = dgram.Dgram(file_descriptor=fd, config=config, offset=offset, size=size)
dgrams += [d]
evt = Event(dgrams, run=self.run())
return evt
def get_next_dgrams(self):
""" Returns list of dgrams as appeared in the current offset of the smd chunks.
Currently used to retrieve Configure and BeginRun. This allows read with wait
for these two types of dgram.
"""
if self.dsparms.max_events > 0 and \
self.processed_events >= self.dsparms.max_events:
logger.debug(f'max_events={self.dsparms.max_events} reached')
return None
dgrams = None
if not self.smdr.is_complete():
self._get()
if self.smdr.is_complete():
# Get chunks with only one dgram each. There's no need to set
# integrating stream id here since Configure and BeginRun
# must exist in this stream too.
self.smdr.view(batch_size=1)
# For configs, we need to copy data from smdreader's buffers
# This prevents it from getting overwritten by other dgrams.
bytearray_bufs = [
bytearray(self.smdr.show(i)) for i in range(self.n_files)
]
if self.configs is None:
dgrams = [
dgram.Dgram(view=ba_buf, offset=0)
for ba_buf in bytearray_bufs
]
self.configs = dgrams
self.smdr.set_configs(self.configs)
else:
dgrams = [
dgram.Dgram(view=ba_buf, config=config, offset=0)
for ba_buf, config in zip(bytearray_bufs, self.configs)
]
return dgrams
def jumps(self, dgram_i, offset, size):
if offset == 0 and size == 0:
d = None
else:
try:
d = dgram.Dgram(file_descriptor=self.fds[dgram_i],
config=self.configs[dgram_i],
offset=offset,
size=size,
max_retries=self.max_retries)
except StopIteration:
d = None
return d
def next(self, offsets=[], sizes=[], read_chunk=True):
assert len(self.offsets) > 0 or len(offsets) > 0
if len(offsets) == 0: offsets = self.offsets
if len(sizes) == 0: sizes = [0] * len(offsets)
dgrams = []
for fd, config, offset, size in zip(self.fds, self.configs, offsets,
sizes):
if (read_chunk):
d = dgram.Dgram(config=config, offset=offset)
else:
assert size > 0
d = dgram.Dgram(file_descriptor=fd,
config=config,
offset=offset,
size=size)
dgrams += [d]
evt = Event(dgrams=dgrams)
self.offsets = evt.offsets
return evt
def main():
fd = os.open('/reg/d/psdm/xpp/xpptut15/scratch/mona/smd.xtc', os.O_RDONLY)
config = dgram.Dgram(file_descriptor=fd)
print(config)
# Iterate chunks of n_events
n_events = 10000
dchunk = dgramchunk.DgramChunk(fd)
displacement = memoryview(config).shape[0]
view = dchunk.get(displacement, n_events)
while view != 0:
do_chunk(bytes(view)) # FIXME: Find a way to pickle without copying
displacement += view.nbytes
view = dchunk.get(displacement, n_events)
def _from_bytes(cls, configs, event_bytes, run=None):
dgrams = []
if event_bytes:
pf = PacketFooter(view=event_bytes)
views = pf.split_packets()
assert len(configs) == pf.n_packets
dgrams = [None]*pf.n_packets # make sure that dgrams are arranged
# according to the smd files.
for i in range(pf.n_packets):
if views[i].shape[0] > 0: # do not include any missing dgram
dgrams[i] = dgram.Dgram(config=configs[i], view=views[i])
evt = cls(dgrams, run=run)
return evt
def smd_client(config):
while True:
comm.Send(np.array([rank], dtype='i'), dest=0)
info = MPI.Status()
comm.Probe(MPI.ANY_SOURCE, MPI.ANY_TAG, info)
count = info.Get_elements(MPI.BYTE)
view = bytearray(count)
comm.Recv(view, source=0)
if view.startswith(b'end'):
break
offset = 0
while offset < count:
d = dgram.Dgram(config=config, view=view, offset=offset)
offset += memoryview(d).shape[0]
def _set_configinfo(self):
""" From configs, we generate a dictionary lookup with det_name as a key.
The information stored the value field contains:
- configs specific to that detector
- sorted_segment_ids
used by Detector cls for checking if an event has correct no. of segments
- detid_dict
has segment_id as a key
- dettype
- uniqueid
"""
self.configinfo_dict = {}
for _, det_class in self.dm.det_classes.items(
): # det_class is either normal or envstore
for (det_name, _), _ in det_class.items():
# Create a copy of list of configs for this detector
det_configs = [dgram.Dgram(view=config) for config in self.dm.configs \
if hasattr(config.software, det_name)]
sorted_segment_ids = []
# a dictionary of the ids (a.k.a. serial-number) of each segment
detid_dict = {}
dettype = ""
uniqueid = ""
for config in det_configs:
seg_dict = getattr(config.software, det_name)
sorted_segment_ids += list(seg_dict.keys())
for segment, det in seg_dict.items():
detid_dict[segment] = det.detid
dettype = det.dettype
sorted_segment_ids.sort()
uniqueid = dettype
for segid in sorted_segment_ids:
uniqueid += '_' + detid_dict[segid]
self.configinfo_dict[det_name] = type("ConfigInfo", (), {\
"configs": det_configs, \
"sorted_segment_ids": sorted_segment_ids, \
"detid_dict": detid_dict, \
"dettype": dettype, \
"uniqueid": uniqueid})
def __next__(self):
if self.cn_events == self.n_events:
raise StopIteration
smd_evt = Event._from_bytes(self.smd_configs,
self.smd_events[self.cn_events],
run=self.dm.get_run())
if len(self.dm.xtc_files
) == 0 or smd_evt.service() != TransitionId.L1Accept:
self.cn_events += 1
self._inc_prometheus_counter('evts')
return smd_evt
if self.filter_fn:
bd_dgrams = []
read_size = 0
for smd_i, smd_dgram in enumerate(smd_evt._dgrams):
if self.use_smds[smd_i]:
bd_dgrams.append(smd_dgram)
else:
offset_and_size = smd_evt.get_offset_and_size(smd_i)
read_size += offset_and_size[0, 1]
bd_dgrams.append(
self._read_dgram_from_disk(smd_i, offset_and_size))
bd_evt = Event(dgrams=bd_dgrams, run=self.dm.get_run())
self.cn_events += 1
self._inc_prometheus_counter('evts')
return bd_evt
dgrams = [None] * self.n_smd_files
ofsz = self.ofsz_batch[self.cn_events, :, :]
for i_smd in range(self.n_smd_files):
d_offset, d_size = ofsz[i_smd]
if d_size and d_offset + d_size <= \
memoryview(self.bigdata[i_smd]).nbytes:
dgrams[i_smd] = dgram.Dgram(view=self.bigdata[i_smd],
config=self.dm.configs[i_smd],
offset=d_offset)
bd_evt = Event(dgrams, run=self.dm.get_run())
self.cn_events += 1
self._inc_prometheus_counter('evts')
return bd_evt
def __next__(self):
if self.cn_events == self.n_events:
raise StopIteration
if len(self.dm.xtc_files) == 0:
smd_evt = Event._from_bytes(self.smd_configs,
self.smd_events[self.cn_events],
run=self.dm.run())
self.cn_events += 1
self._inc_prometheus_counter('evts')
return smd_evt
if self.filter_fn:
smd_evt = Event._from_bytes(self.smd_configs,
self.smd_events[self.cn_events],
run=self.dm.run())
self.cn_events += 1
if smd_evt.service() == TransitionId.L1Accept:
offset_and_size_array = smd_evt.get_offsets_and_sizes()
bd_evt = self._read_event_from_disk(
offset_and_size_array[:, 0], offset_and_size_array[:, 1])
self._inc_prometheus_counter(
'MB',
np.sum(offset_and_size_array[:, 1]) / 1e6)
else:
bd_evt = smd_evt
self._inc_prometheus_counter('evts')
return bd_evt
dgrams = [None] * self.n_smd_files
ofsz = self.ofsz_batch[self.cn_events, :, :]
for j in range(self.n_smd_files):
d_offset, d_size = ofsz[j]
if d_size and d_offset + d_size <= \
memoryview(self.bigdata[j]).nbytes:
dgrams[j] = dgram.Dgram(view=self.bigdata[j],
config=self.dm.configs[j],
offset=d_offset)
bd_evt = Event(dgrams, run=self.dm.run())
self.cn_events += 1
self._inc_prometheus_counter('evts')
return bd_evt
def _setup_configs(self):
""" Creates and broadcasts configs
only called by _setup_run()
"""
g_ts = self.prom_man.get_metric("psana_timestamp")
if nodetype == 'smd0':
super()._close_opened_smd_files()
self.smd_fds = np.array([
os.open(smd_file, os.O_RDONLY) for smd_file in self.smd_files
],
dtype=np.int32)
logging.info(f'mpi_ds: smd0 opened smd_fds: {self.smd_fds}')
self.smdr_man = SmdReaderManager(self.smd_fds, self.dsparms)
self._configs = self.smdr_man.get_next_dgrams()
super()._apply_detector_selection()
super()._setup_det_class_table()
super()._set_configinfo()
g_ts.labels("first_event").set(time.time())
nbytes = np.array([memoryview(config).shape[0] for config in self._configs], \
dtype='i')
else:
self._configs = None
nbytes = np.empty(len(self.smd_files), dtype='i')
self.comms.psana_comm.Bcast(
nbytes, root=0) # no. of bytes is required for mpich
if nodetype != 'smd0':
self._configs = [np.empty(nbyte, dtype='b') for nbyte in nbytes]
for i in range(len(self._configs)):
self.comms.psana_comm.Bcast(
[self._configs[i], nbytes[i], MPI.BYTE], root=0)
if nodetype != 'smd0':
self._configs = [
dgram.Dgram(view=config, offset=0) for config in self._configs
]
g_ts.labels("first_event").set(time.time())
self._setup_det_class_table()
self._set_configinfo()
def run_smd0():
filenames = glob.glob(
'/reg/d/psdm/xpp/xpptut15/scratch/mona/test/smalldata/*.smd.xtc')
#filenames = glob.glob('/u1/mona/smalldata/*.smd.xtc')
fds = [os.open(filename, os.O_RDONLY) for filename in filenames]
# read configs so that fileptr is in the right place
configs = [dgram.Dgram(file_descriptor=fd) for fd in fds]
limit = int(sys.argv[1])
st = time.time()
smdr = SmdReader(fds[:limit])
got_events = -1
n_events = 10000
processed_events = 0
while got_events != 0:
smdr.get(n_events)
got_events = smdr.got_events
processed_events += got_events
#if processed_events >= 100: break
#print("processed_events: %d"%processed_events)
en = time.time()
print("Elapsed Time (s): %f #Events: %d Rate: %f" %
((en - st), processed_events, processed_events / ((en - st) * 1e6)))
def jump(self, offsets, sizes):
""" Jumps to the offset and reads out dgram on each xtc file.
This is used in normal mode (multiple detectors with MPI).
"""
assert len(offsets) > 0 and len(sizes) > 0
dgrams = []
for fd, config, offset, size in zip(self.fds, self.configs, offsets,
sizes):
if offset == 0 and size == 0:
d = None
else:
try:
d = dgram.Dgram(file_descriptor=fd,
config=config,
offset=offset,
size=size,
max_retries=self.max_retries)
except StopIteration:
d = None
dgrams += [d]
evt = Event(dgrams, run=self.run())
return evt
def __init__(self, xtc_files, configs=[]):
if isinstance(xtc_files, (str)):
self.xtc_files = np.array([xtc_files], dtype='U%s' % FN_L)
elif isinstance(xtc_files, (list, np.ndarray)):
self.xtc_files = np.asarray(xtc_files, dtype='U%s' % FN_L)
assert len(self.xtc_files) > 0
given_configs = True if len(configs) > 0 else False
self.configs = []
if given_configs:
self.configs = configs
for i in range(len(self.configs)):
self.configs[i]._assign_dict()
self.fds = []
for i, xtcdata_filename in enumerate(self.xtc_files):
self.fds.append(
os.open(xtcdata_filename, os.O_RDONLY | os.O_LARGEFILE))
if not given_configs:
d = dgram.Dgram(file_descriptor=self.fds[-1])
self.configs += [d]
self.offsets = [_config._offset for _config in self.configs]
if __name__ == "__main__":
comm.Barrier()
ts0 = MPI.Wtime()
nfiles = 2
# broadcast smd files
if rank == 0:
smd_files = np.array(['/reg/d/psdm/xpp/xpptut15/scratch/mona/smd-00.xtc', '/reg/d/psdm/xpp/xpptut15/scratch/mona/smd-01.xtc'], dtype='U%s'%FN_L)
else:
smd_files = np.empty(nfiles, dtype='U%s'%FN_L)
comm.Bcast([smd_files, MPI.CHAR], root=0)
# broadcast configs
if rank == 0:
fds = [os.open(smd_file, os.O_RDONLY) for smd_file in smd_files]
configs = [dgram.Dgram(file_descriptor=fd) for fd in fds]
nbytes = np.array([memoryview(config).shape[0] for config in configs], dtype='i')
else:
configs = [dgram.Dgram()] * nfiles
nbytes = np.empty(nfiles, dtype='i')
comm.Bcast(nbytes, root=0)
for i in range(nfiles):
comm.Bcast([configs[i], nbytes[i], MPI.BYTE], root=0)
comm.Barrier()
ts1 = MPI.Wtime()
# start smd-bd nodes
n_smd_nodes = int(round((size - 1) * .25))
n_bd_nodes = size - 1 - n_smd_nodes
if rank == 0:
import dgram
from psana import dgram as psana_dgram
from psana.smdreader import SmdReader
import os
fd = os.open('data-r0001-s00.smd.xtc2', os.O_RDONLY)
config = psana_dgram.Dgram(file_descriptor=fd)
smdr = SmdReader([fd])
smdr.get(1)
print(smdr.got_events)
dgram.get(smdr.view(0))
def __init__(self, expstr, filter=filter, batch_size=1):
"""Initializes datasource.
Keyword arguments:
expstr -- experiment string (eg. exp=xpptut13:run=1) or
a file or list of files (eg. 'data.xtc' or ['data0.xtc','dataN.xtc'])
batch_size -- length of batched offsets
Supports:
reading file(s) -- DgramManager handles those files (single core)
reading an exp -- Single core owns both smd and xtc DgramManagers
Multiple cores: rank 0 owns smd DgramManager
rank 1 owns xtc DgramManager
"""
assert batch_size > 0
self.batch_size = batch_size
self.filter = filter
self.nodetype = 'bd'
# Check if we are reading file(s) or an experiment
self.read_files = False
if isinstance(expstr, (str)):
if expstr.find("exp") == -1:
self.xtc_files = np.array([expstr], dtype='U%s' % FN_L)
self.smd_files = None
else:
self.read_files = True
elif isinstance(expstr, (list, np.ndarray)):
self.xtc_files = np.asarray(expstr, dtype='U%s' % FN_L)
self.smd_files = None
if not self.read_files:
self.dm = DgramManager(self.xtc_files)
self.configs = self.dm.configs
else:
# Fetch xtc files
if rank == 0:
opts = expstr.split(':')
exp = {}
for opt in opts:
items = opt.split('=')
assert len(items) == 2
exp[items[0]] = items[1]
run = ''
if 'dir' in exp:
xtc_path = exp['dir']
else:
xtc_dir = os.environ.get('SIT_PSDM_DATA', '/reg/d/psdm')
xtc_path = os.path.join(xtc_dir, exp['exp'][:3],
exp['exp'], 'xtc')
if 'run' in exp:
run = exp['run']
if run:
self.xtc_files = np.array(glob.glob(
os.path.join(xtc_path, '*r%s*.xtc' % (run.zfill(4)))),
dtype='U%s' % FN_L)
else:
self.xtc_files = np.array(glob.glob(
os.path.join(xtc_path, '*.xtc')),
dtype='U%s' % FN_L)
self.xtc_files.sort()
self.smd_files = np.empty(len(self.xtc_files),
dtype='U%s' % FN_L)
smd_dir = os.path.join(xtc_path, 'smalldata')
for i, xtc_file in enumerate(self.xtc_files):
smd_file = os.path.join(
smd_dir,
os.path.splitext(os.path.basename(xtc_file))[0] +
'.smd.xtc')
if os.path.isfile(smd_file):
self.smd_files[i] = smd_file
else:
raise InputError(smd_file, "File not found.")
self.nfiles = np.array([len(self.xtc_files)], dtype='i')
assert self.nfiles[0] > 0
else:
self.nfiles = np.zeros(1, dtype='i')
if size == 1:
self.smd_dm = DgramManager(self.smd_files)
self.dm = DgramManager(self.xtc_files,
configs=self.smd_dm.configs)
self.configs = self.smd_dm.configs
else:
# Send filenames
comm.Bcast(self.nfiles, root=0)
if rank > 0:
self.xtc_files = np.empty(self.nfiles[0],
dtype='U%s' % FN_L)
self.smd_files = np.empty(self.nfiles[0],
dtype='U%s' % FN_L)
comm.Bcast([self.xtc_files, MPI.CHAR], root=0)
comm.Bcast([self.smd_files, MPI.CHAR], root=0)
# Send configs
if rank == 0:
self.dm = DgramManager(self.smd_files)
self.configs = self.dm.configs
nbytes = np.array([memoryview(config).shape[0] for config in self.configs], \
dtype='i')
else:
self.dm = None
self.configs = [
dgram.Dgram() for i in range(self.nfiles[0])
]
nbytes = np.empty(self.nfiles[0], dtype='i')
comm.Bcast(nbytes,
root=0) # no. of bytes is required for mpich
for i in range(self.nfiles[0]):
comm.Bcast([self.configs[i], nbytes[i], MPI.BYTE], root=0)
# Assign node types
self.nsmds = int(
os.environ.get('PS_SMD_NODES',
np.ceil((size - 1) * PERCENT_SMD)))
if rank == 0:
self.nodetype = 'smd0'
elif rank < self.nsmds + 1:
self.nodetype = 'smd'
# Big-data nodes own big data
if self.nodetype == 'bd':
self.dm = DgramManager(self.xtc_files,
configs=self.configs)
self.Detector = Detector(self.configs)
def main():
# Process arguments
parser = argparse.ArgumentParser()
parser.add_argument('pvbase', help='EPICS PV base (e.g. DAQ:LAB2:PART:2)')
parser.add_argument('-p',
type=int,
choices=range(0, 8),
default=0,
help='platform (default 0)')
parser.add_argument('-v', action='store_true', help='be verbose')
args = parser.parse_args()
if args.v:
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s - %(levelname)s - %(message)s')
else:
logging.basicConfig(level=logging.WARNING,
format='%(asctime)s - %(levelname)s - %(message)s')
logging.info('control server starting')
# CM state
yy = ControlStateMachine(args.pvbase)
logging.debug("ControlStateMachine state: %s" % yy.state())
# context and sockets
ctx = zmq.Context()
cmd = ctx.socket(zmq.ROUTER)
pull = ctx.socket(zmq.PULL)
cmd.bind("tcp://*:%d" % ControlMsg.router_port(args.p))
pull.bind("tcp://*:%d" % ControlMsg.pull_port(args.p))
sequence = 0
poller = zmq.Poller()
poller.register(cmd, zmq.POLLIN)
poller.register(pull, zmq.POLLIN)
try:
while True:
items = dict(poller.poll(1000))
# Handle pull socket
if pull in items:
msg = pull.recv()
config = dgram.Dgram(view=msg)
# now it's in dgram.Dgram object
ttt = config.seq.timestamp()
print('Timestamp:', ttt) # FIXME
# Execute state cmd request
if cmd in items:
msg = cmd.recv_multipart()
identity = msg[0]
request = msg[1]
logging.debug('Received <%s> from cmd' % request.decode())
if request == ControlMsg.PING:
# Send reply to client
logging.debug("Sending <PONG> reply")
cmd.send(identity, zmq.SNDMORE)
cmmsg = ControlMsg(sequence, key=ControlMsg.PONG)
cmmsg.send(cmd)
continue
if request == ControlMsg.PONG:
continue
if request in [
Transition.configure, Transition.beginrun,
Transition.enable, Transition.disable,
Transition.endrun, Transition.unconfigure,
ControlMsg.GETSTATE
]:
if request != ControlMsg.GETSTATE:
oldstate = yy.state()
# Do transition
yy.on_transition(request)
newstate = yy.state()
if newstate != oldstate:
logging.debug("ControlStateMachine state: %s" %
newstate)
# Send reply to client
cmd.send(identity, zmq.SNDMORE)
cmmsg = ControlMsg(sequence, key=yy.state().key())
cmmsg.send(cmd)
continue
else:
logging.warning("Unknown msg <%s>" % request.decode())
# Send reply to client
logging.debug("Sending <HUH?> reply")
cmd.send(identity, zmq.SNDMORE)
cmmsg = ControlMsg(sequence, key=ControlMsg.HUH)
cmmsg.send(cmd)
continue
except KeyboardInterrupt:
logging.debug("Interrupt received")
# Clean up
logging.debug("Clean up")
time.sleep(.25)
# close zmq sockets
cmd.close()
pull.close()
# terminate zmq context
ctx.term()
logging.info('control server exiting')
def _get_offset_and_size(self):
"""
Use fast step-through to read off offset and size from smd_view.
Format of smd_view
[
[[d_bytes][d_bytes]....[evt_footer]] <-- 1 event
[[d_bytes][d_bytes]....[evt_footer]]
[chunk_footer]]
"""
offset = 0
i_smd = 0
smd_chunk_pf = PacketFooter(view=self.smd_view)
dtype = np.int64
# Row - events, col = smd files
self.bd_offset_array = np.zeros((smd_chunk_pf.n_packets, self.n_smd_files), dtype=dtype)
self.bd_size_array = np.zeros((smd_chunk_pf.n_packets, self.n_smd_files), dtype=dtype)
self.smd_offset_array = np.zeros((smd_chunk_pf.n_packets, self.n_smd_files), dtype=dtype)
self.smd_size_array = np.zeros((smd_chunk_pf.n_packets, self.n_smd_files), dtype=dtype)
self.new_chunk_id_array = np.zeros((smd_chunk_pf.n_packets, self.n_smd_files), dtype=dtype)
self.cutoff_flag_array = np.ones((smd_chunk_pf.n_packets, self.n_smd_files), dtype=dtype)
self.services = np.zeros(smd_chunk_pf.n_packets, dtype=dtype)
smd_aux_sizes = np.zeros(self.n_smd_files, dtype=dtype)
self.i_first_L1s = np.zeros(self.n_smd_files, dtype=dtype) + 0xffffff
# For comparing if the next dgram should be in the same read
current_bd_offsets = np.zeros(self.n_smd_files, dtype=dtype)
# Current chunk size (gets reset at boundary)
current_bd_chunk_sizes = np.zeros(self.n_smd_files, dtype=dtype)
i_evt = 0
while offset < memoryview(self.smd_view).nbytes - memoryview(smd_chunk_pf.footer).nbytes:
if i_smd == 0:
smd_evt_size = smd_chunk_pf.get_size(i_evt)
smd_evt_pf = PacketFooter(view=self.smd_view[offset: offset+smd_evt_size])
smd_aux_sizes[:] = [smd_evt_pf.get_size(i) for i in range(smd_evt_pf.n_packets)]
# Only get offset and size of non-missing dgram
# TODO: further optimization by looking for the first L1 and read in a big chunk
# anything that comes after. Right now, all transitions mark the cutoff points.
if smd_aux_sizes[i_smd] == 0:
self.cutoff_flag_array[i_evt, i_smd] = 0
else:
d = dgram.Dgram(config=self.smd_configs[i_smd], view=self.smd_view, offset=offset)
self.smd_offset_array[i_evt, i_smd] = offset
self.smd_size_array[i_evt, i_smd] = d._size
self.services[i_evt] = d.service()
# Any dgrams after the first L1 (as long as they fit in the chunk size)
# will be read in together.
if self.dm.n_files > 0:
if d.service() == TransitionId.L1Accept:
if self.i_first_L1s[i_smd] == 0xffffff:
self.i_first_L1s[i_smd] = i_evt
print(f'i_smd={i_smd} i_first_L1={self.i_first_L1s[i_smd]}')
# For SlowUpdate, we need to check if the next dgram gets cutoff
elif d.service() == TransitionId.SlowUpdate and hasattr(d, 'chunkinfo'):
stream_id = self.dm.get_stream_id(i_smd)
_chunk_ids = [getattr(d.chunkinfo[seg_id].chunkinfo, 'chunkid') for seg_id in d.chunkinfo]
# There must be only one unique epics var
if _chunk_ids: self.new_chunk_id_array[i_evt, i_smd] = _chunk_ids[0]
self._get_bd_offset_and_size(d, current_bd_offsets, current_bd_chunk_sizes,
i_evt, i_smd, self.i_first_L1s[i_smd])
"""
# For L1 with bigdata files, store offset and size found in smd dgrams.
# For SlowUpdate, store new chunk id (if found). TODO: check if
# we need to always check for epics for SlowUpdate.
if d.service() == TransitionId.L1Accept and self.dm.n_files > 0:
if i_first_L1 == -1:
i_first_L1 = i_evt
print(f'i_smd={i_smd} i_first_L1={i_first_L1}')
self._get_bd_offset_and_size(d, current_bd_offsets, current_bd_chunk_sizes, i_evt, i_smd, i_first_L1)
elif d.service() == TransitionId.SlowUpdate and hasattr(d, 'chunkinfo'):
# We only support chunking on bigdata
if self.dm.n_files > 0:
stream_id = self.dm.get_stream_id(i_smd)
_chunk_ids = [getattr(d.chunkinfo[seg_id].chunkinfo, 'chunkid') for seg_id in d.chunkinfo]
# There must be only one unique epics var
if _chunk_ids: self.new_chunk_id_array[i_evt, i_smd] = _chunk_ids[0]
"""
offset += smd_aux_sizes[i_smd]
i_smd += 1
if i_smd == self.n_smd_files:
offset += PacketFooter.n_bytes * (self.n_smd_files + 1) # skip the footer
i_smd = 0 # reset to the first smd file
i_evt += 1 # done with this smd event
# end while offset
# Precalculate cutoff indices
self.cutoff_indices = []
self.chunk_indices = np.zeros(self.n_smd_files, dtype=dtype)
for i_smd in range(self.n_smd_files):
self.cutoff_indices.append(np.where(self.cutoff_flag_array[:, i_smd] == 1)[0])
print(f'i_smd={i_smd} cutoff_index={self.cutoff_indices[i_smd]} services={self.services[self.cutoff_indices[i_smd]]}')
