def run_smd0(fds, max_events=0):
n_events = int(os.environ.get('PS_SMD_N_EVENTS', 100))
if max_events:
if max_events < n_events:
n_events = max_events
smdr = SmdReader(fds)
got_events = -1
processed_events = 0
while got_events != 0:
smdr.get(n_events)
got_events = smdr.got_events
processed_events += got_events
print("processed_events", processed_events, "got_events", got_events)
views = bytearray()
for i in range(len(fds)):
view = smdr.view(i)
if view != 0:
views.extend(view)
if i < len(fds) - 1:
views.extend(b'endofstream')
if views:
#run_smd_task(views, run)
print("got views")
if max_events:
if processed_events >= max_events:
break
"""
def smd_0(fds, n_smd_nodes):
""" Sends blocks of smds to smd_node
Identifies limit timestamp of the slowest detector then
sends all smds within that timestamp to an smd_node.
"""
assert len(fds) > 0
smdr = SmdReader(fds)
got_events = -1
rankreq = np.empty(1, dtype='i')
while got_events != 0:
smdr.get(n_events)
got_events = smdr.got_events
views = bytearray()
for i in range(len(fds)):
view = smdr.view(i)
if view != 0:
views.extend(view)
if i < len(fds) - 1:
views.extend(b'endofstream')
if views:
comm.Recv(rankreq, source=MPI.ANY_SOURCE)
comm.Send(views, dest=rankreq[0], tag=12)
for i in range(n_smd_nodes):
comm.Recv(rankreq, source=MPI.ANY_SOURCE)
comm.Send(bytearray(b'eof'), dest=rankreq[0], tag=12)
def __init__(self, fds, max_events):
self.n_files = len(fds)
assert self.n_files > 0
self.smdr = SmdReader(fds)
self.n_events = int(os.environ.get('PS_SMD_N_EVENTS', 1000))
self.max_events = max_events
self.processed_events = 0
if self.max_events:
if self.max_events < self.n_events:
self.n_events = self.max_events
def __init__(self, run):
self.n_files = len(run.smd_dm.fds)
assert self.n_files > 0
self.run = run
self.smdr = SmdReader(run.smd_dm.fds)
self.n_events = int(os.environ.get('PS_SMD_N_EVENTS', 1000))
self.processed_events = 0
if self.run.max_events:
if self.run.max_events < self.n_events:
self.n_events = self.run.max_events
self.got_events = -1
def __init__(self, run):
self.n_files = len(run.smd_dm.fds)
assert self.n_files > 0
self.run = run
self.batch_size = int(os.environ.get('PS_SMD_N_EVENTS', 13500 * 16))
if self.run.max_events:
if self.run.max_events < self.batch_size:
self.batch_size = self.run.max_events
self.chunksize = int(os.environ.get('PS_SMD_CHUNKSIZE', 0x1000000))
self.smdr = SmdReader(run.smd_dm.fds, self.chunksize)
self.processed_events = 0
self.got_events = -1
def run_smd0(n_events):
filenames = glob.glob(os.path.join(xtc_dir, '.tmp', 'smalldata', '*.xtc2'))
fds = [os.open(filename, os.O_RDONLY) for filename in filenames]
# Move file ptrs to datagram part
configs = [Dgram(file_descriptor=fd) for fd in fds]
limit = len(filenames)
if len(sys.argv) > 1:
limit = int(sys.argv[1])
st = time.time()
smdr = SmdReader(fds[:limit])
got_events = -1
processed_events = 0
smdr.get(n_events)
got_events = smdr.got_events
result = {'each_read': [], 'total_n_events': 0}
cn_i = 0
while got_events != 0:
step_chunk_nbytes = 0
smd_chunk_nbytes = 0
for i in range(limit):
smd_view = smdr.view(i)
if smd_view:
smd_chunk_nbytes += smd_view.nbytes
step_view = smdr.view(i, update=True)
if step_view:
step_chunk_nbytes += step_view.nbytes
result['each_read'].append(
[got_events, smd_chunk_nbytes, step_chunk_nbytes])
processed_events += got_events
# Read more events
smdr.get(n_events)
got_events = smdr.got_events
cn_i += 1
en = time.time()
result['total_n_events'] = processed_events
for fd in fds:
os.close(fd)
return result
def __init__(self, ds):
Node.__init__(self, ds.mpi)
self.fds = ds.smd_dm.fds
assert len(self.fds) > 0
self.smdr = SmdReader(self.fds)
self.n_smd_nodes = ds.nsmds
self.n_events = int(os.environ.get('PS_SMD_N_EVENTS', 100))
self.max_events = ds.max_events
if self.max_events:
if self.max_events < self.n_events:
self.n_events = self.max_events
if mode == 'mpi':
self.run_mpi()
elif mode == 'legion':
self.run_legion()
def __init__(self, run):
self.n_files = len(run.smd_fds)
assert self.n_files > 0
self.run = run
self.batch_size = int(os.environ.get('PS_SMD_N_EVENTS', 1000))
if self.run.max_events:
if self.run.max_events < self.batch_size:
self.batch_size = self.run.max_events
self.chunksize = int(os.environ.get('PS_SMD_CHUNKSIZE', 0x100000))
self.smdr = SmdReader(run.smd_fds, self.chunksize)
self.processed_events = 0
self.got_events = -1
# Collecting Smd0 performance using prometheus
if self.run.prom_man:
self.c_read = self.run.prom_man.get_counter('psana_smd0_read')
def __init__(self, smd_fds, dsparms, configs=None):
self.n_files = len(smd_fds)
self.dsparms = dsparms
self.configs = configs
assert self.n_files > 0
self.smd0_n_events = int(os.environ.get('PS_SMD_N_EVENTS', 1000))
if self.dsparms.max_events:
if self.dsparms.max_events < self.smd0_n_events:
self.smd0_n_events = self.dsparms.max_events
self.chunksize = int(os.environ.get('PS_SMD_CHUNKSIZE', 0x1000000))
self.smdr = SmdReader(smd_fds, self.chunksize,
self.dsparms.max_retries)
self.processed_events = 0
self.got_events = -1
self._run = None
# Collecting Smd0 performance using prometheus
self.c_read = self.dsparms.prom_man.get_metric('psana_smd0_read')
def smd_0(fds, n_smd_nodes, max_events=0):
""" Sends blocks of smds to smd_node
Identifies limit timestamp of the slowest detector then
sends all smds within that timestamp to an smd_node.
"""
assert len(fds) > 0
smdr = SmdReader(fds)
got_events = -1
processed_events = 0
rankreq = np.empty(1, dtype='i')
n_events = int(os.environ.get('PS_SMD_N_EVENTS', 100))
if max_events:
if max_events < n_events:
n_events = max_events
while got_events != 0:
smdr.get(n_events)
got_events = smdr.got_events
processed_events += got_events
views = bytearray()
for i in range(len(fds)):
view = smdr.view(i)
if view != 0:
views.extend(view)
if i < len(fds) - 1:
views.extend(b'endofstream')
if views:
comm.Recv(rankreq, source=MPI.ANY_SOURCE)
comm.Send(views, dest=rankreq[0], tag=12)
if max_events:
if processed_events == max_events:
break
for i in range(n_smd_nodes):
comm.Recv(rankreq, source=MPI.ANY_SOURCE)
comm.Send(bytearray(b'eof'), dest=rankreq[0], tag=12)
def __init__(self, smd_fds, dsparms, configs=None):
self.n_files = len(smd_fds)
self.dsparms = dsparms
self.configs = configs
assert self.n_files > 0
# Sets no. of events Smd0 sends to each EventBuilder core. This gets
# overridden by max_events set by DataSource if max_events is smaller.
self.smd0_n_events = int(os.environ.get('PS_SMD_N_EVENTS', 1000))
if self.dsparms.max_events:
if self.dsparms.max_events < self.smd0_n_events:
self.smd0_n_events = self.dsparms.max_events
# Sets the memory size for smalldata buffer for each stream file.
self.chunksize = int(os.environ.get('PS_SMD_CHUNKSIZE', 0x1000000))
self.smdr = SmdReader(smd_fds, self.chunksize,
self.dsparms.max_retries)
self.processed_events = 0
self.got_events = -1
self._run = None
# Collecting Smd0 performance using prometheus
self.c_read = self.dsparms.prom_man.get_metric('psana_smd0_read')
def run_smd0():
#filenames = glob.glob('/reg/neh/home/monarin/psana-nersc/psana2/.tmp/smalldata/*.xtc2')
filenames = glob.glob('/ffb01/mona/.tmp/smalldata/*.xtc2')
fds = np.array([os.open(filename, os.O_RDONLY) for filename in filenames],
dtype=np.int32)
# Move file ptrs to datagram part
configs = [Dgram(file_descriptor=fd) for fd in fds]
beginRun = [Dgram(config=config) for config in configs]
limit = len(filenames)
if len(sys.argv) > 1:
limit = int(sys.argv[1])
st = time.time()
smdr = SmdReader(fds[:limit], chunksize)
got_events = -1
processed_events = 0
offsets = np.zeros(limit, dtype=np.uint64)
how_many = smd0_batch_size
to_be_read = max_events - processed_events
if to_be_read < how_many:
how_many = to_be_read
smdr.get(how_many)
while smdr.got_events > 0:
for i in range(limit):
view = smdr.view(i)
"""
if view:
cn_dgrams = 0
while offsets[i] < view.shape[0]:
d = Dgram(config=configs[i], view=view, offset=offsets[i])
print(f' buf{i} d_id: {cn_dgrams} d_ts {d.timestamp() & 0xffffffff}')
offsets[i] += d._size
cn_dgrams += 1
#print(f'smdr_man got {memoryview(view).nbytes}')
else:
#print(f' buf[{i} empty')
pass
"""
processed_events += smdr.got_events
if processed_events >= max_events:
break
how_many = smd0_batch_size
to_be_read = max_events - processed_events
if to_be_read < how_many:
how_many = to_be_read
smdr.get(how_many)
offsets[:] = 0
"""
while smdr.got_events != 0:
if smdr.got_events > 0:
processed_events += smdr.got_events
if processed_events >= max_events:
break
for i in range(limit):
smdr.view(i)
smdr.get(n_events)
print(f'smdr.got_events={smdr.got_events}')
"""
en = time.time()
print("#Events: %d Elapsed Time (s): %f Rate (MHz): %f" %
(processed_events, (en - st), processed_events / ((en - st) * 1e6)))
import dgram
from psana import dgram as psana_dgram
from psana.smdreader import SmdReader
import os
fd = os.open('data-r0001-s00.smd.xtc2', os.O_RDONLY)
config = psana_dgram.Dgram(file_descriptor=fd)
smdr = SmdReader([fd])
smdr.get(1)
print(smdr.got_events)
dgram.get(smdr.view(0))