def run_smd0(fds, max_events=0):
n_events = int(os.environ.get('PS_SMD_N_EVENTS', 100))
if max_events:
if max_events < n_events:
n_events = max_events
smdr = SmdReader(fds)
got_events = -1
processed_events = 0
while got_events != 0:
smdr.get(n_events)
got_events = smdr.got_events
processed_events += got_events
print("processed_events", processed_events, "got_events", got_events)
views = bytearray()
for i in range(len(fds)):
view = smdr.view(i)
if view != 0:
views.extend(view)
if i < len(fds) - 1:
views.extend(b'endofstream')
if views:
#run_smd_task(views, run)
print("got views")
if max_events:
if processed_events >= max_events:
break
"""
def smd_0(fds, n_smd_nodes):
""" Sends blocks of smds to smd_node
Identifies limit timestamp of the slowest detector then
sends all smds within that timestamp to an smd_node.
"""
assert len(fds) > 0
smdr = SmdReader(fds)
got_events = -1
rankreq = np.empty(1, dtype='i')
while got_events != 0:
smdr.get(n_events)
got_events = smdr.got_events
views = bytearray()
for i in range(len(fds)):
view = smdr.view(i)
if view != 0:
views.extend(view)
if i < len(fds) - 1:
views.extend(b'endofstream')
if views:
comm.Recv(rankreq, source=MPI.ANY_SOURCE)
comm.Send(views, dest=rankreq[0], tag=12)
for i in range(n_smd_nodes):
comm.Recv(rankreq, source=MPI.ANY_SOURCE)
comm.Send(bytearray(b'eof'), dest=rankreq[0], tag=12)
def __init__(self, fds, max_events):
self.n_files = len(fds)
assert self.n_files > 0
self.smdr = SmdReader(fds)
self.n_events = int(os.environ.get('PS_SMD_N_EVENTS', 1000))
self.max_events = max_events
self.processed_events = 0
if self.max_events:
if self.max_events < self.n_events:
self.n_events = self.max_events
class SmdReaderManager(object):
def __init__(self, fds, max_events):
self.n_files = len(fds)
assert self.n_files > 0
self.smdr = SmdReader(fds)
self.n_events = int(os.environ.get('PS_SMD_N_EVENTS', 1000))
self.max_events = max_events
self.processed_events = 0
if self.max_events:
if self.max_events < self.n_events:
self.n_events = self.max_events
def chunks(self):
""" Generates a tuple of smd and update dgrams """
got_events = -1
while got_events != 0:
self.smdr.get(self.n_events)
got_events = self.smdr.got_events
self.processed_events += got_events
smd_view = bytearray()
smd_pf = PacketFooter(n_packets=self.n_files)
update_view = bytearray()
update_pf = PacketFooter(n_packets=self.n_files)
for i in range(self.n_files):
_smd_view = self.smdr.view(i)
if _smd_view != 0:
smd_view.extend(_smd_view)
smd_pf.set_size(i, memoryview(_smd_view).shape[0])
_update_view = self.smdr.view(i, update=True)
if _update_view != 0:
update_view.extend(_update_view)
update_pf.set_size(i, memoryview(_update_view).shape[0])
if smd_view or update_view:
if smd_view:
smd_view.extend(smd_pf.footer)
if update_view:
update_view.extend(update_pf.footer)
yield (smd_view, update_view)
if self.max_events:
if self.processed_events >= self.max_events:
break
@property
def min_ts(self):
return self.smdr.min_ts
@property
def max_ts(self):
return self.smdr.max_ts
def __init__(self, run):
self.n_files = len(run.smd_dm.fds)
assert self.n_files > 0
self.run = run
self.smdr = SmdReader(run.smd_dm.fds)
self.n_events = int(os.environ.get('PS_SMD_N_EVENTS', 1000))
self.processed_events = 0
if self.run.max_events:
if self.run.max_events < self.n_events:
self.n_events = self.run.max_events
self.got_events = -1
def __init__(self, run):
self.n_files = len(run.smd_dm.fds)
assert self.n_files > 0
self.run = run
self.batch_size = int(os.environ.get('PS_SMD_N_EVENTS', 13500 * 16))
if self.run.max_events:
if self.run.max_events < self.batch_size:
self.batch_size = self.run.max_events
self.chunksize = int(os.environ.get('PS_SMD_CHUNKSIZE', 0x1000000))
self.smdr = SmdReader(run.smd_dm.fds, self.chunksize)
self.processed_events = 0
self.got_events = -1
def __init__(self, ds):
Node.__init__(self, ds.mpi)
self.fds = ds.smd_dm.fds
assert len(self.fds) > 0
self.smdr = SmdReader(self.fds)
self.n_smd_nodes = ds.nsmds
self.n_events = int(os.environ.get('PS_SMD_N_EVENTS', 100))
self.max_events = ds.max_events
if self.max_events:
if self.max_events < self.n_events:
self.n_events = self.max_events
if mode == 'mpi':
self.run_mpi()
elif mode == 'legion':
self.run_legion()
def __init__(self, run):
self.n_files = len(run.smd_fds)
assert self.n_files > 0
self.run = run
self.batch_size = int(os.environ.get('PS_SMD_N_EVENTS', 1000))
if self.run.max_events:
if self.run.max_events < self.batch_size:
self.batch_size = self.run.max_events
self.chunksize = int(os.environ.get('PS_SMD_CHUNKSIZE', 0x100000))
self.smdr = SmdReader(run.smd_fds, self.chunksize)
self.processed_events = 0
self.got_events = -1
# Collecting Smd0 performance using prometheus
if self.run.prom_man:
self.c_read = self.run.prom_man.get_counter('psana_smd0_read')
def __init__(self, smd_fds, dsparms, configs=None):
self.n_files = len(smd_fds)
self.dsparms = dsparms
self.configs = configs
assert self.n_files > 0
self.smd0_n_events = int(os.environ.get('PS_SMD_N_EVENTS', 1000))
if self.dsparms.max_events:
if self.dsparms.max_events < self.smd0_n_events:
self.smd0_n_events = self.dsparms.max_events
self.chunksize = int(os.environ.get('PS_SMD_CHUNKSIZE', 0x1000000))
self.smdr = SmdReader(smd_fds, self.chunksize,
self.dsparms.max_retries)
self.processed_events = 0
self.got_events = -1
self._run = None
# Collecting Smd0 performance using prometheus
self.c_read = self.dsparms.prom_man.get_metric('psana_smd0_read')
def smd_0(fds, n_smd_nodes, max_events=0):
""" Sends blocks of smds to smd_node
Identifies limit timestamp of the slowest detector then
sends all smds within that timestamp to an smd_node.
"""
assert len(fds) > 0
smdr = SmdReader(fds)
got_events = -1
processed_events = 0
rankreq = np.empty(1, dtype='i')
n_events = int(os.environ.get('PS_SMD_N_EVENTS', 100))
if max_events:
if max_events < n_events:
n_events = max_events
while got_events != 0:
smdr.get(n_events)
got_events = smdr.got_events
processed_events += got_events
views = bytearray()
for i in range(len(fds)):
view = smdr.view(i)
if view != 0:
views.extend(view)
if i < len(fds) - 1:
views.extend(b'endofstream')
if views:
comm.Recv(rankreq, source=MPI.ANY_SOURCE)
comm.Send(views, dest=rankreq[0], tag=12)
if max_events:
if processed_events == max_events:
break
for i in range(n_smd_nodes):
comm.Recv(rankreq, source=MPI.ANY_SOURCE)
comm.Send(bytearray(b'eof'), dest=rankreq[0], tag=12)
def __init__(self, smd_fds, dsparms, configs=None):
self.n_files = len(smd_fds)
self.dsparms = dsparms
self.configs = configs
assert self.n_files > 0
# Sets no. of events Smd0 sends to each EventBuilder core. This gets
# overridden by max_events set by DataSource if max_events is smaller.
self.smd0_n_events = int(os.environ.get('PS_SMD_N_EVENTS', 1000))
if self.dsparms.max_events:
if self.dsparms.max_events < self.smd0_n_events:
self.smd0_n_events = self.dsparms.max_events
# Sets the memory size for smalldata buffer for each stream file.
self.chunksize = int(os.environ.get('PS_SMD_CHUNKSIZE', 0x1000000))
self.smdr = SmdReader(smd_fds, self.chunksize,
self.dsparms.max_retries)
self.processed_events = 0
self.got_events = -1
self._run = None
# Collecting Smd0 performance using prometheus
self.c_read = self.dsparms.prom_man.get_metric('psana_smd0_read')
def run_smd0(n_events):
filenames = glob.glob(os.path.join(xtc_dir, '.tmp', 'smalldata', '*.xtc2'))
fds = [os.open(filename, os.O_RDONLY) for filename in filenames]
# Move file ptrs to datagram part
configs = [Dgram(file_descriptor=fd) for fd in fds]
limit = len(filenames)
if len(sys.argv) > 1:
limit = int(sys.argv[1])
st = time.time()
smdr = SmdReader(fds[:limit])
got_events = -1
processed_events = 0
smdr.get(n_events)
got_events = smdr.got_events
result = {'each_read': [], 'total_n_events': 0}
cn_i = 0
while got_events != 0:
step_chunk_nbytes = 0
smd_chunk_nbytes = 0
for i in range(limit):
smd_view = smdr.view(i)
if smd_view:
smd_chunk_nbytes += smd_view.nbytes
step_view = smdr.view(i, update=True)
if step_view:
step_chunk_nbytes += step_view.nbytes
result['each_read'].append(
[got_events, smd_chunk_nbytes, step_chunk_nbytes])
processed_events += got_events
# Read more events
smdr.get(n_events)
got_events = smdr.got_events
cn_i += 1
en = time.time()
result['total_n_events'] = processed_events
for fd in fds:
os.close(fd)
return result
class SmdReaderManager(object):
def __init__(self, run):
self.n_files = len(run.smd_dm.fds)
assert self.n_files > 0
self.run = run
self.batch_size = int(os.environ.get('PS_SMD_N_EVENTS', 13500 * 16))
if self.run.max_events:
if self.run.max_events < self.batch_size:
self.batch_size = self.run.max_events
self.chunksize = int(os.environ.get('PS_SMD_CHUNKSIZE', 0x1000000))
self.smdr = SmdReader(run.smd_dm.fds, self.chunksize)
self.processed_events = 0
self.got_events = -1
def __iter__(self):
return self
def _read(self):
"""
Reads 'batch_size' no. of events. If they don't fit in the 'chunksize',
returns only events that fit.
If user specifies max_events when creating DataSource, only asks SmdReader
to read this amount of events (see how_many is being set below).
"""
max_retries = int(os.environ.get('PS_SMD_MAX_RETRIES', '5'))
sleep_secs = int(os.environ.get('PS_SMD_SLEEP_SECS', '1'))
how_many = self.batch_size
if self.run.max_events:
to_be_read = self.run.max_events - self.processed_events
if to_be_read < how_many:
how_many = to_be_read
self.smdr.get(how_many)
cn_retries = 0
while self.smdr.got_events == 0:
self.smdr.retry()
cn_retries += 1
if cn_retries == max_retries:
break
time.sleep(sleep_secs)
self.got_events = self.smdr.got_events
self.processed_events += self.got_events
def __next__(self):
"""
Returns a batch of events as an iterator object.
This is used by non-parallel run. Parallel run uses chunks
generator that yields chunks of raw smd data and steps (no
event building).
The iterator stops reading under two conditions. Either there's
nothing to read (_read() comes back with got_events=0) or no.
of processed_events = max_events (specified at DataSource).
"""
if self.run.max_events:
if self.processed_events >= self.run.max_events:
raise StopIteration
self._read()
if self.got_events == 0: raise StopIteration
views = []
for i in range(self.n_files):
view = self.smdr.view(i)
if view:
views.append(view)
else:
views.append(memoryview(bytearray()))
batch_iter = BatchIterator(views, batch_size=self.run.batch_size, \
filter_fn=self.run.filter_callback, destination=self.run.destination)
return batch_iter
def chunks(self):
""" Generates a tuple of smd and step dgrams """
self._read()
while self.got_events > 0:
smd_view = bytearray()
smd_pf = PacketFooter(n_packets=self.n_files)
step_view = bytearray()
step_pf = PacketFooter(n_packets=self.n_files)
for i in range(self.n_files):
_smd_view = self.smdr.view(i)
if _smd_view != 0:
smd_view.extend(_smd_view)
smd_pf.set_size(i, memoryview(_smd_view).shape[0])
_step_view = self.smdr.view(i, step=True)
if _step_view != 0:
step_view.extend(_step_view)
step_pf.set_size(i, memoryview(_step_view).shape[0])
if smd_view or step_view:
if smd_view:
smd_view.extend(smd_pf.footer)
if step_view:
step_view.extend(step_pf.footer)
yield (smd_view, step_view)
if self.run.max_events:
if self.processed_events >= self.run.max_events:
break
self._read()
@property
def min_ts(self):
return self.smdr.min_ts
@property
def max_ts(self):
return self.smdr.max_ts
def run_smd0():
#filenames = glob.glob('/reg/neh/home/monarin/psana-nersc/psana2/.tmp/smalldata/*.xtc2')
filenames = glob.glob('/ffb01/mona/.tmp/smalldata/*.xtc2')
fds = np.array([os.open(filename, os.O_RDONLY) for filename in filenames],
dtype=np.int32)
# Move file ptrs to datagram part
configs = [Dgram(file_descriptor=fd) for fd in fds]
beginRun = [Dgram(config=config) for config in configs]
limit = len(filenames)
if len(sys.argv) > 1:
limit = int(sys.argv[1])
st = time.time()
smdr = SmdReader(fds[:limit], chunksize)
got_events = -1
processed_events = 0
offsets = np.zeros(limit, dtype=np.uint64)
how_many = smd0_batch_size
to_be_read = max_events - processed_events
if to_be_read < how_many:
how_many = to_be_read
smdr.get(how_many)
while smdr.got_events > 0:
for i in range(limit):
view = smdr.view(i)
"""
if view:
cn_dgrams = 0
while offsets[i] < view.shape[0]:
d = Dgram(config=configs[i], view=view, offset=offsets[i])
print(f' buf{i} d_id: {cn_dgrams} d_ts {d.timestamp() & 0xffffffff}')
offsets[i] += d._size
cn_dgrams += 1
#print(f'smdr_man got {memoryview(view).nbytes}')
else:
#print(f' buf[{i} empty')
pass
"""
processed_events += smdr.got_events
if processed_events >= max_events:
break
how_many = smd0_batch_size
to_be_read = max_events - processed_events
if to_be_read < how_many:
how_many = to_be_read
smdr.get(how_many)
offsets[:] = 0
"""
while smdr.got_events != 0:
if smdr.got_events > 0:
processed_events += smdr.got_events
if processed_events >= max_events:
break
for i in range(limit):
smdr.view(i)
smdr.get(n_events)
print(f'smdr.got_events={smdr.got_events}')
"""
en = time.time()
print("#Events: %d Elapsed Time (s): %f Rate (MHz): %f" %
(processed_events, (en - st), processed_events / ((en - st) * 1e6)))
import dgram
from psana import dgram as psana_dgram
from psana.smdreader import SmdReader
import os
fd = os.open('data-r0001-s00.smd.xtc2', os.O_RDONLY)
config = psana_dgram.Dgram(file_descriptor=fd)
smdr = SmdReader([fd])
smdr.get(1)
print(smdr.got_events)
dgram.get(smdr.view(0))
"/var/opt/cray/dws/mounts/batch/psana2_hsd_16718487_striped_scratch/hsd/smalldata/data-*.smd.xtc"
)
if not xtc_files:
print("smd files not found.")
exit()
n_files = int(sys.argv[1])
fds = [os.open(xtc_file, os.O_RDONLY) for xtc_file in xtc_files]
# do this to move to dgram part of the xtc
# since config doesn't have correct timestamp so the
# code won't work - won't happen in real situation
configs = [dgram.Dgram(file_descriptor=fd) for fd in fds]
st = time.time()
smdr = DummyReader(fds[:n_files])
got_events = -1
processed_events = 0
while got_events != 0:
smdr.get(1000)
got_events = smdr.got_events
processed_events += got_events
for i in range(n_files):
view = smdr.view(i)
#if processed_events >= 1000: break
en = time.time()
print("processed_events %d total elapsed %f s rate %f MHz" %
(processed_events, en - st, processed_events /
((en - st) * 1000000)))
#print("DeltaT get_init(s): %f"%(smdr.dt_get_init))
class SmdReaderManager(object):
def __init__(self, run):
self.n_files = len(run.smd_fds)
assert self.n_files > 0
self.run = run
self.batch_size = int(os.environ.get('PS_SMD_N_EVENTS', 1000))
if self.run.max_events:
if self.run.max_events < self.batch_size:
self.batch_size = self.run.max_events
self.chunksize = int(os.environ.get('PS_SMD_CHUNKSIZE', 0x100000))
self.smdr = SmdReader(run.smd_fds, self.chunksize)
self.processed_events = 0
self.got_events = -1
# Collecting Smd0 performance using prometheus
if self.run.prom_man:
self.c_read = self.run.prom_man.get_counter('psana_smd0_read')
def get_next_dgrams(self, configs=None):
dgrams = None
if not self.smdr.is_complete():
self.smdr.get()
if self.smdr.is_complete():
mmrv_bufs, _ = self.smdr.view(batch_size=1)
# For configs, we need to copy data from smdreader's buffers
# This prevents it from getting overwritten by other dgrams.
bytearray_bufs = [bytearray(mmrv_buf) for mmrv_buf in mmrv_bufs]
if configs is None:
dgrams = [dgram.Dgram(view=ba_buf, offset=0) for ba_buf in bytearray_bufs]
else:
dgrams = [dgram.Dgram(view=ba_buf, config=config, offset=0) for ba_buf, config in zip(bytearray_bufs, configs)]
return dgrams
def __iter__(self):
return self
def __next__(self):
"""
Returns a batch of events as an iterator object.
This is used by non-parallel run. Parallel run uses chunks
generator that yields chunks of raw smd data and steps (no
event building).
The iterator stops reading under two conditions. Either there's
no more data or max_events reached.
"""
if self.run.max_events and self.processed_events >= self.run.max_events:
raise StopIteration
if not self.smdr.is_complete():
self.smdr.get()
if self.run.prom_man:
self.c_read.labels('MB', 'None').inc(self.smdr.got/1e6)
if not self.smdr.is_complete():
raise StopIteration
mmrv_bufs, _ = self.smdr.view(batch_size=self.batch_size)
batch_iter = BatchIterator(mmrv_bufs, batch_size=self.run.batch_size, \
filter_fn=self.run.filter_callback, destination=self.run.destination)
self.got_events = self.smdr.view_size
self.processed_events += self.got_events
# sending data to prometheus
if self.run.prom_man:
self.c_read.labels('evts', 'None').inc(self.got_events)
self.c_read.labels('batches', 'None').inc()
return batch_iter
def chunks(self):
""" Generates a tuple of smd and step dgrams """
is_done = False
while not is_done:
if self.smdr.is_complete():
mmrv_bufs, mmrv_step_bufs = self.smdr.view(batch_size=self.batch_size)
self.got_events = self.smdr.view_size
self.processed_events += self.got_events
# sending data to prometheus
if self.run.prom_man:
logging.debug('Smd0 got %d events'%(self.got_events))
self.c_read.labels('evts', 'None').inc(self.got_events)
self.c_read.labels('batches', 'None').inc()
if self.run.max_events and self.processed_events >= self.run.max_events:
is_done = True
smd_view = bytearray()
smd_pf = PacketFooter(n_packets=self.n_files)
step_view = bytearray()
step_pf = PacketFooter(n_packets=self.n_files)
for i, (mmrv_buf, mmrv_step_buf) in enumerate(zip(mmrv_bufs, mmrv_step_bufs)):
if mmrv_buf != 0:
smd_view.extend(mmrv_buf)
smd_pf.set_size(i, memoryview(mmrv_buf).nbytes)
if mmrv_step_buf != 0:
step_view.extend(mmrv_step_buf)
step_pf.set_size(i, memoryview(mmrv_step_buf).nbytes)
if smd_view or step_view:
if smd_view:
smd_view.extend(smd_pf.footer)
if step_view:
step_view.extend(step_pf.footer)
yield (smd_view, step_view)
else:
self.smdr.get()
if self.run.prom_man:
logging.debug('Smd0 read %.2f MB'%(self.smdr.got/1e6))
self.c_read.labels('MB', 'None').inc(self.smdr.got/1e6)
if not self.smdr.is_complete():
is_done = True
break
@property
def min_ts(self):
return self.smdr.min_ts
@property
def max_ts(self):
return self.smdr.max_ts
class SmdReaderManager(object):
def __init__(self, smd_fds, dsparms, configs=None):
self.n_files = len(smd_fds)
self.dsparms = dsparms
self.configs = configs
assert self.n_files > 0
self.smd0_n_events = int(os.environ.get('PS_SMD_N_EVENTS', 1000))
if self.dsparms.max_events:
if self.dsparms.max_events < self.smd0_n_events:
self.smd0_n_events = self.dsparms.max_events
self.chunksize = int(os.environ.get('PS_SMD_CHUNKSIZE', 0x1000000))
self.smdr = SmdReader(smd_fds, self.chunksize,
self.dsparms.max_retries)
self.processed_events = 0
self.got_events = -1
self._run = None
# Collecting Smd0 performance using prometheus
self.c_read = self.dsparms.prom_man.get_metric('psana_smd0_read')
def _get(self):
st = time.time()
self.smdr.get()
en = time.time()
logger.debug(
f'smdreader_manager: read {self.smdr.got/1e6:.5f} MB took {en-st}s. rate: {self.smdr.got/(1e6*(en-st))} MB/s'
)
self.c_read.labels('MB', 'None').inc(self.smdr.got / 1e6)
self.c_read.labels('seconds', 'None').inc(en - st)
if self.smdr.chunk_overflown > 0:
msg = f"SmdReader found dgram ({self.smdr.chunk_overflown} MB) larger than chunksize ({self.chunksize/1e6} MB)"
raise ValueError(msg)
def get_next_dgrams(self):
if self.dsparms.max_events > 0 and \
self.processed_events >= self.dsparms.max_events:
logger.debug(
f'smdreader_manager: get_next_dgrams max_events={self.dsparms.max_events} reached'
)
return None
dgrams = None
if not self.smdr.is_complete():
self._get()
if self.smdr.is_complete():
mmrv_bufs, _ = self.smdr.view(batch_size=1)
# For configs, we need to copy data from smdreader's buffers
# This prevents it from getting overwritten by other dgrams.
bytearray_bufs = [bytearray(mmrv_buf) for mmrv_buf in mmrv_bufs]
if self.configs is None:
dgrams = [
dgram.Dgram(view=ba_buf, offset=0)
for ba_buf in bytearray_bufs
]
self.configs = dgrams
else:
dgrams = [
dgram.Dgram(view=ba_buf, config=config, offset=0)
for ba_buf, config in zip(bytearray_bufs, self.configs)
]
return dgrams
def __iter__(self):
return self
def __next__(self):
"""
Returns a batch of events as an iterator object.
This is used by non-parallel run. Parallel run uses chunks
generator that yields chunks of raw smd data and steps (no
event building).
The iterator stops reading under two conditions. Either there's
no more data or max_events reached.
"""
if self.dsparms.max_events and self.processed_events >= self.dsparms.max_events:
raise StopIteration
if not self.smdr.is_complete():
self._get()
if not self.smdr.is_complete():
raise StopIteration
mmrv_bufs, _ = self.smdr.view(batch_size=self.smd0_n_events)
batch_iter = BatchIterator(mmrv_bufs,
self.configs,
self._run,
batch_size=self.dsparms.batch_size,
filter_fn=self.dsparms.filter,
destination=self.dsparms.destination)
self.got_events = self.smdr.view_size
self.processed_events += self.got_events
return batch_iter
def chunks(self):
""" Generates a tuple of smd and step dgrams """
is_done = False
while not is_done:
if self.smdr.is_complete():
mmrv_bufs, mmrv_step_bufs = self.smdr.view(
batch_size=self.smd0_n_events)
self.got_events = self.smdr.view_size
self.processed_events += self.got_events
# sending data to prometheus
logger.debug('smdreader_manager: smd0 got %d events' %
(self.got_events))
if self.dsparms.max_events and self.processed_events >= self.dsparms.max_events:
logger.debug(
f'smdreader_manager: max_events={self.dsparms.max_events} reached'
)
is_done = True
smd_view = bytearray()
smd_pf = PacketFooter(n_packets=self.n_files)
step_view = bytearray()
step_pf = PacketFooter(n_packets=self.n_files)
for i, (mmrv_buf, mmrv_step_buf) in enumerate(
zip(mmrv_bufs, mmrv_step_bufs)):
if mmrv_buf != 0:
smd_view.extend(mmrv_buf)
smd_pf.set_size(i, memoryview(mmrv_buf).nbytes)
if mmrv_step_buf != 0:
step_view.extend(mmrv_step_buf)
step_pf.set_size(i, memoryview(mmrv_step_buf).nbytes)
if smd_view or step_view:
if smd_view:
smd_view.extend(smd_pf.footer)
if step_view:
step_view.extend(step_pf.footer)
yield (smd_view, step_view)
else: # if self.smdr.is_complete()
self._get()
if not self.smdr.is_complete():
is_done = True
break
@property
def min_ts(self):
return self.smdr.min_ts
@property
def max_ts(self):
return self.smdr.max_ts
def set_run(self, run):
self._run = run
def get_run(self):
return self._run
class Smd0(Node):
""" Sends blocks of smds to smd_node
Identifies limit timestamp of the slowest detector then
sends all smds within that timestamp to an smd_node.
"""
def __init__(self, ds):
Node.__init__(self, ds.mpi)
self.fds = ds.smd_dm.fds
assert len(self.fds) > 0
self.smdr = SmdReader(self.fds)
self.n_smd_nodes = ds.nsmds
self.n_events = int(os.environ.get('PS_SMD_N_EVENTS', 100))
self.max_events = ds.max_events
if self.max_events:
if self.max_events < self.n_events:
self.n_events = self.max_events
if mode == 'mpi':
self.run_mpi()
elif mode == 'legion':
self.run_legion()
def run_mpi(self):
rank = self.mpi.rank
comm = self.mpi.comm
rankreq = np.empty(1, dtype='i')
for chunk in self.chunks():
comm.Recv(rankreq, source=MPI.ANY_SOURCE)
comm.Send(chunk, dest=rankreq[0], tag=12)
for i in range(self.n_smd_nodes):
comm.Recv(rankreq, source=MPI.ANY_SOURCE)
comm.Send(bytearray(b'eof'), dest=rankreq[0], tag=12)
def run_legion(self, configs, batch_size=1, filter=0):
for chunk in self.chunks():
SmdNode.run_legion_task(chunk, configs, batch_size, filter)
def chunks(self):
processed_events = 0
got_events = -1
while got_events != 0:
self.smdr.get(self.n_events)
got_events = self.smdr.got_events
processed_events += got_events
views = bytearray()
for i in range(len(self.fds)):
view = self.smdr.view(i)
if view != 0:
views.extend(view)
if i < len(self.fds) - 1:
views.extend(b'endofstream')
if views:
yield views
if self.max_events:
if processed_events == self.max_events:
break
class SmdReaderManager(object):
def __init__(self, smd_fds, dsparms, configs=None):
self.n_files = len(smd_fds)
self.dsparms = dsparms
self.configs = configs
assert self.n_files > 0
# Sets no. of events Smd0 sends to each EventBuilder core. This gets
# overridden by max_events set by DataSource if max_events is smaller.
self.smd0_n_events = int(os.environ.get('PS_SMD_N_EVENTS', 1000))
if self.dsparms.max_events:
if self.dsparms.max_events < self.smd0_n_events:
self.smd0_n_events = self.dsparms.max_events
# Sets the memory size for smalldata buffer for each stream file.
self.chunksize = int(os.environ.get('PS_SMD_CHUNKSIZE', 0x1000000))
self.smdr = SmdReader(smd_fds, self.chunksize,
self.dsparms.max_retries)
self.processed_events = 0
self.got_events = -1
self._run = None
# Collecting Smd0 performance using prometheus
self.c_read = self.dsparms.prom_man.get_metric('psana_smd0_read')
def _get(self):
st = time.monotonic()
self.smdr.get(self.dsparms.smd_inprogress_converted)
en = time.monotonic()
logger.debug(
f'read {self.smdr.got/1e6:.3f} MB took {en-st}s. rate: {self.smdr.got/(1e6*(en-st))} MB/s'
)
self.c_read.labels('MB', 'None').inc(self.smdr.got / 1e6)
self.c_read.labels('seconds', 'None').inc(en - st)
if self.smdr.chunk_overflown > 0:
msg = f"SmdReader found dgram ({self.smdr.chunk_overflown} MB) larger than chunksize ({self.chunksize/1e6} MB)"
raise ValueError(msg)
def get_next_dgrams(self):
""" Returns list of dgrams as appeared in the current offset of the smd chunks.
Currently used to retrieve Configure and BeginRun. This allows read with wait
for these two types of dgram.
"""
if self.dsparms.max_events > 0 and \
self.processed_events >= self.dsparms.max_events:
logger.debug(f'max_events={self.dsparms.max_events} reached')
return None
dgrams = None
if not self.smdr.is_complete():
self._get()
if self.smdr.is_complete():
# Get chunks with only one dgram each. There's no need to set
# integrating stream id here since Configure and BeginRun
# must exist in this stream too.
self.smdr.view(batch_size=1)
# For configs, we need to copy data from smdreader's buffers
# This prevents it from getting overwritten by other dgrams.
bytearray_bufs = [
bytearray(self.smdr.show(i)) for i in range(self.n_files)
]
if self.configs is None:
dgrams = [
dgram.Dgram(view=ba_buf, offset=0)
for ba_buf in bytearray_bufs
]
self.configs = dgrams
self.smdr.set_configs(self.configs)
else:
dgrams = [
dgram.Dgram(view=ba_buf, config=config, offset=0)
for ba_buf, config in zip(bytearray_bufs, self.configs)
]
return dgrams
def __iter__(self):
return self
def __next__(self):
"""
Returns a batch of events as an iterator object.
This is used by non-parallel run. Parallel run uses chunks
generator that yields chunks of raw smd data and steps (no
event building).
The iterator stops reading under two conditions. Either there's
no more data or max_events reached.
"""
intg_stream_id = self.dsparms.intg_stream_id
if self.dsparms.max_events and self.processed_events >= self.dsparms.max_events:
raise StopIteration
if not self.smdr.is_complete():
self._get()
if not self.smdr.is_complete():
raise StopIteration
self.smdr.view(batch_size=self.smd0_n_events,
intg_stream_id=intg_stream_id)
mmrv_bufs = [self.smdr.show(i) for i in range(self.n_files)]
batch_iter = BatchIterator(mmrv_bufs,
self.configs,
self._run,
batch_size=self.dsparms.batch_size,
filter_fn=self.dsparms.filter,
destination=self.dsparms.destination,
timestamps=self.dsparms.timestamps,
intg_stream_id=intg_stream_id)
self.got_events = self.smdr.view_size
self.processed_events += self.got_events
return batch_iter
def chunks(self):
""" Generates a tuple of smd and step dgrams """
is_done = False
d_view, d_read = 0, 0
cn_chunks = 0
while not is_done:
logger.debug(f'SMD0 1. STARTCHUNK {time.monotonic()}')
st_view, en_view, st_read, en_read = 0, 0, 0, 0
l1_size = 0
tr_size = 0
got_events = 0
if self.smdr.is_complete():
st_view = time.monotonic()
# Gets the next batch of already read-in data.
self.smdr.view(batch_size=self.smd0_n_events,
intg_stream_id=self.dsparms.intg_stream_id)
self.got_events = self.smdr.view_size
got_events = self.got_events
self.processed_events += self.got_events
# sending data to prometheus
logger.debug('got %d events' % (self.got_events))
if self.dsparms.max_events and self.processed_events >= self.dsparms.max_events:
logger.debug(
f'max_events={self.dsparms.max_events} reached')
is_done = True
en_view = time.monotonic()
d_view += en_view - st_view
logger.debug(f'SMD0 2. DONECREATEVIEW {time.monotonic()}')
if self.got_events:
cn_chunks += 1
yield cn_chunks
else: # if self.smdr.is_complete()
st_read = time.monotonic()
self._get()
en_read = time.monotonic()
logger.debug(f'SMD0 3. DONEREAD {time.monotonic()}')
d_read += en_read - st_read
if not self.smdr.is_complete():
is_done = True
break
@property
def min_ts(self):
return self.smdr.min_ts
@property
def max_ts(self):
return self.smdr.max_ts
def set_run(self, run):
self._run = run
def get_run(self):
return self._run
class SmdReaderManager(object):
def __init__(self, run):
self.n_files = len(run.smd_dm.fds)
assert self.n_files > 0
self.run = run
self.smdr = SmdReader(run.smd_dm.fds)
self.n_events = int(os.environ.get('PS_SMD_N_EVENTS', 1000))
self.processed_events = 0
if self.run.max_events:
if self.run.max_events < self.n_events:
self.n_events = self.run.max_events
self.got_events = -1
def __iter__(self):
return self
def __next__(self):
if self.got_events == 0: raise StopIteration
self.smdr.get(self.n_events)
self.got_events = self.smdr.got_events
self.processed_events += self.got_events
views = [self.smdr.view(i) for i in range(self.n_files)]
batch_iter = BatchIterator(views, batch_size=self.run.batch_size, \
filter_fn=self.run.filter_callback, destination=self.run.destination)
if self.run.max_events:
if self.processed_events >= self.run.max_events:
self.got_events = 0
return batch_iter
def chunks(self):
""" Generates a tuple of smd and step dgrams """
got_events = -1
while got_events != 0:
self.smdr.get(self.n_events)
got_events = self.smdr.got_events
self.processed_events += got_events
smd_view = bytearray()
smd_pf = PacketFooter(n_packets=self.n_files)
step_view = bytearray()
step_pf = PacketFooter(n_packets=self.n_files)
for i in range(self.n_files):
_smd_view = self.smdr.view(i)
if _smd_view != 0:
smd_view.extend(_smd_view)
smd_pf.set_size(i, memoryview(_smd_view).shape[0])
_step_view = self.smdr.view(i, step=True)
if _step_view != 0:
step_view.extend(_step_view)
step_pf.set_size(i, memoryview(_step_view).shape[0])
if smd_view or step_view:
if smd_view:
smd_view.extend(smd_pf.footer)
if step_view:
step_view.extend(step_pf.footer)
yield (smd_view, step_view)
if self.run.max_events:
if self.processed_events >= self.run.max_events:
break
@property
def min_ts(self):
return self.smdr.min_ts
@property
def max_ts(self):
return self.smdr.max_ts
