def __init__(self, expstr, filter=0, batch_size=1, max_events=0):
"""Initializes datasource.
Keyword arguments:
expstr -- experiment string (eg. exp=xpptut13:run=1) or
a file or list of files (eg. 'data.xtc' or ['data0.xtc','dataN.xtc'])
batch_size -- length of batched offsets
max_events -- no. of maximum events
"""
self.filter = filter
assert batch_size > 0
self.batch_size = batch_size
self.max_events = max_events
self.mpi = MpiComm()
ds_helper = DataSourceHelper(expstr, self)
ds_helper.assign_node_type()
if self.nodetype == 'smd0':
ds_helper.parse_expstr()
ds_helper.run(job="bcast_files")
if self.nodetype == 'smd0':
ds_helper.read_configs()
else:
ds_helper.init_configs()
ds_helper.run(job="bcast_configs")
if self.nodetype != 'smd0':
self.smd_dm = DgramManager(self.smd_files,
configs=ds_helper.smd_configs)
self.dm = DgramManager(self.xtc_files, configs=ds_helper.configs)
if self.nodetype in ('bd', 'smd0'):
self.Detector = Detector(self.dm.configs, calib=self.calib)
def __init__(self, exp, run_no, run_src, **kwargs):
""" Parallel read using Legion """
super(RunLegion,
self).__init__(exp,
run_no,
max_events=kwargs['max_events'],
batch_size=kwargs['batch_size'],
filter_callback=kwargs['filter_callback'],
prom_man=kwargs['prom_man'])
xtc_files, smd_files, other_files = run_src
# get Configure and BeginRun using SmdReader
self.smd_fds = np.array(
[os.open(smd_file, os.O_RDONLY) for smd_file in smd_files],
dtype=np.int32)
self.smdr_man = SmdReaderManager(self)
self.configs = self.smdr_man.get_next_dgrams()
self.beginruns = self.smdr_man.get_next_dgrams(configs=self.configs)
self._get_runinfo()
self.smd_dm = DgramManager(smd_files, configs=self.configs)
self.dm = DgramManager(xtc_files, configs=self.smd_dm.configs)
super()._set_configinfo()
super()._set_calibconst()
self.esm = EnvStoreManager(self.configs, 'epics', 'scan')
def setup_dgram_manager(self, ds, xtc_files, smd_files):
if smd_files is not None:
ds.smd_dm = DgramManager(smd_files)
ds.smd_configs = ds.smd_dm.configs # FIXME: there should only be one type of config
ds.dm = DgramManager(xtc_files)
ds.configs = ds.dm.configs
ds.calib = self.get_calib_dict(run_no=ds.run)
def __init__(self, comms, exp, run_no, run_src, **kwargs):
""" Parallel read requires that rank 0 does the file system works.
Configs and calib constants are sent to other ranks by MPI.
Note that destination callback only works with RunParallel.
"""
super(RunParallel, self).__init__(exp, run_no, max_events=kwargs['max_events'], \
batch_size=kwargs['batch_size'], filter_callback=kwargs['filter_callback'], \
destination=kwargs['destination'])
xtc_files, smd_files, other_files = run_src
self.comms = comms
psana_comm = comms.psana_comm # TODO tjl and cpo to review
rank = psana_comm.Get_rank()
size = psana_comm.Get_size()
if rank == 0:
self.smd_dm = DgramManager(smd_files, run=self)
self.dm = DgramManager(xtc_files,
configs=self.smd_dm.configs,
run=self)
self.configs = self.dm.configs
nbytes = np.array([memoryview(config).shape[0] for config in self.configs], \
dtype='i')
self.calibs = {}
for det_name in self.detnames:
self.calibs[det_name] = super(RunParallel,
self)._get_calib(det_name)
else:
self.smd_dm = None
self.dm = None
self.configs = None
self.calibs = None
nbytes = np.empty(len(smd_files), dtype='i')
psana_comm.Bcast(nbytes, root=0) # no. of bytes is required for mpich
# create empty views of known size
if rank > 0:
self.configs = [np.empty(nbyte, dtype='b') for nbyte in nbytes]
for i in range(len(self.configs)):
psana_comm.Bcast([self.configs[i], nbytes[i], MPI.BYTE], root=0)
self.calibs = psana_comm.bcast(self.calibs, root=0)
if rank > 0:
# Create dgram objects using views from rank 0 (no disk operation).
self.configs = [
dgram.Dgram(view=config, offset=0) for config in self.configs
]
self.dm = DgramManager(xtc_files, configs=self.configs, run=self)
self.esm = EnvStoreManager(self.configs, 'epics', 'scan')
def __init__(self, exp, run_no, run_src, **kwargs):
""" Parallel read using Legion """
super(RunLegion, self).__init__(exp, run_no, max_events=kwargs['max_events'], \
batch_size=kwargs['batch_size'], filter_callback=kwargs['filter_callback'])
xtc_files, smd_files, other_files = run_src
self.smd_dm = DgramManager(smd_files)
self.dm = DgramManager(xtc_files, configs=self.smd_dm.configs)
self.configs = self.dm.configs
self.epics_store = StepStore(self.configs, 'epics')
self.calibs = {}
for det_name in self.detnames:
self.calibs[det_name] = super(RunLegion, self)._get_calib(det_name)
def __init__(self, exp, run_no, run_src, **kwargs):
""" Parallel read using Legion """
super(RunLegion, self).__init__(exp, run_no, max_events=kwargs['max_events'], \
batch_size=kwargs['batch_size'], filter_callback=kwargs['filter_callback'])
xtc_files, smd_files, other_files = run_src
self.smd_dm = DgramManager(smd_files)
self.dm = DgramManager(xtc_files, configs=self.smd_dm.configs)
self.configs = self.dm.configs
super()._get_runinfo()
super()._set_configinfo()
super()._set_calibconst()
self.esm = EnvStoreManager(self.configs, 'epics', 'scan')
def __init__(self, exp, run_no, run_src, **kwargs):
super(RunSerial, self).__init__(exp, run_no, \
max_events=kwargs['max_events'], batch_size=kwargs['batch_size'], \
filter_callback=kwargs['filter_callback'])
xtc_files, smd_files, other_files = run_src
self.smd_dm = DgramManager(smd_files)
self.dm = DgramManager(xtc_files, configs=self.smd_dm.configs)
self.configs = self.dm.configs
self.esm = EnvStoreManager(self.smd_dm.configs, 'epics', 'scan')
self.calibs = {}
for det_name in self.detnames:
self.calibs[det_name] = self._get_calib(det_name)
def read_event(ds, event_type=None):
"""Reads an event.
Keywords:
ds -- DataSource with access to the smd or xtc files
event_type -- used to determined config updates or other
types of event.
"""
rank = ds.mpi.rank
size = ds.mpi.size
if size == 1:
dm = DgramManager(ds.xtc_files, configs=ds.dm.configs)
for evt in dm:
yield evt # safe for python2
else:
if ds.nodetype == 'smd0':
Smd0(ds)
elif ds.nodetype == 'smd':
bd_node_ids = (np.arange(size)[ds.nsmds + 1:] % ds.nsmds) + 1
smd_node = SmdNode(ds, len(bd_node_ids[bd_node_ids == rank]))
smd_node.run_mpi()
elif ds.nodetype == 'bd':
smd_node_id = (rank % ds.nsmds) + 1
bd_node = BigDataNode(ds, smd_node_id)
for evt in bd_node.run_mpi():
yield evt
def _setup_run(self):
if self.runnum_list_index == len(self.runnum_list):
return False
runnum = self.runnum_list[self.runnum_list_index]
self.runnum_list_index += 1
if nodetype == 'smd0':
super()._setup_run_files(runnum)
super()._apply_detector_selection()
else:
self.xtc_files = None
self.smd_files = None
self.dsparms.use_smds = None
self.xtc_files = self.comms.psana_comm.bcast(self.xtc_files, root=0)
self.smd_files = self.comms.psana_comm.bcast(self.smd_files, root=0)
self.dsparms.use_smds = self.comms.psana_comm.bcast(self.dsparms.use_smds, root=0)
configs = self._get_configs()
self.dm = DgramManager(self.xtc_files, configs=configs, config_consumers=[self.dsparms])
if nodetype == 'smd0':
self.smd0 = Smd0(self.comms, configs, self.smdr_man, self.dsparms)
elif nodetype == 'eb':
self.eb_node = EventBuilderNode(self.comms, configs, self.dsparms, self.dm)
elif nodetype == 'bd':
self.bd_node = BigDataNode(self.comms, configs, self.dsparms, self.dm)
return True
def read_configs(self):
""" Reads configs."""
ds = self.ds
ds.dm = DgramManager(self.ds.xtc_files)
self.configs = ds.dm.configs
self.xtc_nbytes = np.array([memoryview(config).shape[0] \
for config in self.configs], dtype='i')
self.smd_configs = []
if self.ds.smd_files is not None:
ds.smd_dm = DgramManager(self.ds.smd_files)
self.smd_configs = ds.smd_dm.configs # FIXME: there should only be one type of config
self.smd_nbytes = np.array([memoryview(config).shape[0] \
for config in self.smd_configs], dtype='i')
ds.calib = self.get_calib_dict(run_no=ds.run)
def __init__(self, expstr, ds):
ds.nodetype = 'bd'
ds.run = -1
rank = ds.mpi.rank
size = ds.mpi.size
comm = ds.mpi.comm
if rank == 0:
xtc_files, smd_files, run = self.parse_expstr(expstr)
if run is not None:
ds.run = run
else:
xtc_files, smd_files = None, None
# Setup DgramManager, Configs, and Calib
if size == 1:
# This is one-core read.
self.setup_dgram_manager(ds, xtc_files, smd_files)
else:
# This is parallel read.
xtc_files = comm.bcast(xtc_files, root=0)
smd_files = comm.bcast(smd_files, root=0)
# Send configs
if rank == 0:
self.setup_dgram_manager(ds, xtc_files, smd_files)
smd_nbytes = np.array([memoryview(config).shape[0] for config in ds.smd_configs], \
dtype='i')
nbytes = np.array([memoryview(config).shape[0] for config in ds.configs], \
dtype='i')
else:
ds.smd_dm = None
ds.smd_configs = [dgram.Dgram() for i in range(len(smd_files))]
smd_nbytes = np.empty(len(smd_files), dtype='i')
ds.dm = None
ds.configs = [dgram.Dgram() for i in range(len(xtc_files))]
nbytes = np.empty(len(xtc_files), dtype='i')
ds.calib = None
comm.Bcast(smd_nbytes, root=0) # no. of bytes is required for mpich
for i in range(len(smd_files)):
comm.Bcast([ds.smd_configs[i], smd_nbytes[i], MPI.BYTE], root=0)
comm.Bcast(nbytes, root=0) # no. of bytes is required for mpich
for i in range(len(xtc_files)):
comm.Bcast([ds.configs[i], nbytes[i], MPI.BYTE], root=0)
# Send calib
ds.calib = comm.bcast(ds.calib, root=0)
# Assign node types
ds.nsmds = int(os.environ.get('PS_SMD_NODES', np.ceil((size-1)*PERCENT_SMD)))
if rank == 0:
ds.nodetype = 'smd0'
elif rank < ds.nsmds + 1:
ds.nodetype = 'smd'
if ds.nodetype == 'bd':
ds.dm = DgramManager(xtc_files, configs=ds.configs)
def _setup_run(self):
if self.runnum_list_index == len(self.runnum_list):
return False
runnum = self.runnum_list[self.runnum_list_index]
self.dm = DgramManager(self.files[self.runnum_list_index],
config_consumers=[self.dsparms])
self.runnum_list_index += 1
return True
def __init__(self, exp, run_no, xtc_files, max_events, batch_size, filter_callback, tag, prom_man):
super(RunShmem, self).__init__(exp, run_no, max_events=max_events, batch_size=batch_size, filter_callback=filter_callback, prom_man=prom_man)
self.dm = DgramManager(xtc_files,tag=tag)
self.configs = self.dm.configs
super()._get_runinfo()
super()._set_configinfo()
super()._set_calibconst()
self.dm.calibconst = self.calibconst
self.esm = EnvStoreManager(self.dm.configs, 'epics', 'scan')
def __init__(self, exp, run_no, run_src, **kwargs):
super(RunSingleFile, self).__init__(exp, run_no, \
max_events=kwargs['max_events'], batch_size=kwargs['batch_size'], \
filter_callback=kwargs['filter_callback'])
xtc_files, smd_files, epics_file = run_src
self.dm = DgramManager(xtc_files)
self.configs = self.dm.configs
self.calibs = {}
for det_name in self.detnames:
self.calibs[det_name] = self._get_calib(det_name)
def _setup_run(self):
if self.runnum_list_index == len(self.runnum_list):
return False
runnum = self.runnum_list[self.runnum_list_index]
self.runnum_list_index += 1
super()._setup_run_files(runnum)
self._setup_configs()
self.dm = DgramManager(self.xtc_files, configs=self._configs)
return True
def _setup_run(self):
if self.runnum_list_index == len(self.runnum_list):
return False
runnum = self.runnum_list[self.runnum_list_index]
self.dm = DgramManager(self.files[self.runnum_list_index])
self._configs = self.dm.configs
super()._setup_det_class_table()
super()._set_configinfo()
self.runnum_list_index += 1
return True
def _setup_run(self):
if self.runnum_list_index == len(self.runnum_list):
return False
runnum = self.runnum_list[self.runnum_list_index]
self.runnum_list_index += 1
super()._setup_run_files(runnum)
super()._apply_detector_selection()
configs = self._get_configs()
self.dm = DgramManager(self.xtc_files, configs=configs, config_consumers=[self.dsparms])
return True
def __init__(self, exp, run_no, run_src, **kwargs):
super(RunSingleFile, self).__init__(exp, run_no, \
max_events=kwargs['max_events'], batch_size=kwargs['batch_size'], \
filter_callback=kwargs['filter_callback'])
xtc_files, smd_files, epics_file = run_src
self.dm = DgramManager(xtc_files)
self.configs = self.dm.configs
super()._get_runinfo()
super()._set_configinfo()
super()._set_calibconst()
self.esm = EnvStoreManager(self.dm.configs, 'epics', 'scan')
def translate_xtc_demo(job_type, offset=1):
configure_file = '%s_configure.xtc' % job_type
event_file = '%s_evts.xtc' % job_type
try:
os.remove(FILE_NAME)
except:
pass
lcls1_xtc = load_json("%s.json" % job_type)
alg = dc.alg('raw', [1, 2, 3])
ninfo = dc.nameinfo('DscCsPad', 'cspad', 'detnum1234', 0)
# This is needed for some reason.
# Perhaps a collision of lcls1 xtc "version" with lcls2 counterpart
try:
lcls1_xtc[0]['version.'] = lcls1_xtc[0]['version']
del lcls1_xtc[0]['version']
except KeyError:
pass
cydgram = dc.CyDgram()
with open(configure_file, 'wb') as f:
cydgram.addDet(ninfo, alg, lcls1_xtc[0])
df = cydgram.get()
f.write(df)
del cydgram
cydgram = dc.CyDgram()
with open(event_file, 'wb') as f:
for event_dgram in lcls1_xtc[offset:]:
cydgram.addDet(ninfo, alg, event_dgram)
df = cydgram.get()
f.write(df)
ds_cfg = DgramManager(configure_file)
ds_evt = DgramManager(event_file)
def __init__(self, exp, run_no, xtc_files, max_events, batch_size,
filter_callback, tag):
super(RunShmem, self).__init__(exp,
run_no,
max_events=max_events,
batch_size=batch_size,
filter_callback=filter_callback)
self.dm = DgramManager(xtc_files, tag=tag)
self.configs = self.dm.configs
self.calibs = {}
for det_name in self.detnames:
self.calibs[det_name] = self._get_calib(det_name)
self.dm.calibs = self.calibs
def test_cydgram(self):
fname = 'data_cydgram.xtc'
try:
os.remove(fname)
except:
pass
# read in an old xtc file
ds = DgramManager('data.xtc')
pyxtc = dc.parse_xtc(ds)
for evt in ds:
pyxtc.parse_event(evt)
# put the dictionaries in a new xtc file
cydgram = dc.CyDgram()
pyxtc.write_events(fname, cydgram)
# test that the values in the new file are correct
xtc(fname)
def test_copy(self):
try:
os.remove('data_copy.xtc')
except:
pass
input = 'data.xtc'
input2 = 'data_copy.xtc'
ds = DgramManager(input)
pyxtc = dc.parse_xtc(ds)
for evt in ds:
pyxtc.parse_event(evt)
cydgram = dc.CyDgram()
pyxtc.write_events(input2, cydgram)
h1 = self.hash_xtc(input)
h2 = self.hash_xtc(input2)
assert (h1 == h2)
for ct, dgrm in enumerate(lq):
if ct == config_ind:
continue
dgr = dgram.Dgram(view=dgrm[1:], config=config)
dgrams.append(dgr)
timestamps = [dgr.seq.timestamp() for dgr in dgrams]
_, dgrams = zip(*sorted(zip(timestamps, dgrams)))
return dgrams
if __name__ == "__main__":
input = 'data-00.xtc'
ds = DgramManager(input)
dgram_queue = queue.Queue()
recv_queue = queue.Queue()
for ct, evt in enumerate(ds):
if ct == 0:
dgrm = ds.configs[0]
serv = 2
elif ct > 10:
break
else:
dgrm = evt.dgrams[0]
serv = 0
evt_bytes = memoryview(dgrm).tobytes()
msg = chr(serv).encode() + evt_bytes
dgram_queue.put(msg)
def __init__(self, expstr, filter=filter, batch_size=1):
"""Initializes datasource.
Keyword arguments:
expstr -- experiment string (eg. exp=xpptut13:run=1) or
a file or list of files (eg. 'data.xtc' or ['data0.xtc','dataN.xtc'])
batch_size -- length of batched offsets
Supports:
reading file(s) -- DgramManager handles those files (single core)
reading an exp -- Single core owns both smd and xtc DgramManagers
Multiple cores: rank 0 owns smd DgramManager
rank 1 owns xtc DgramManager
"""
assert batch_size > 0
self.batch_size = batch_size
self.filter = filter
self.nodetype = 'bd'
# Check if we are reading file(s) or an experiment
self.read_files = False
if isinstance(expstr, (str)):
if expstr.find("exp") == -1:
self.xtc_files = np.array([expstr], dtype='U%s' % FN_L)
self.smd_files = None
else:
self.read_files = True
elif isinstance(expstr, (list, np.ndarray)):
self.xtc_files = np.asarray(expstr, dtype='U%s' % FN_L)
self.smd_files = None
if not self.read_files:
self.dm = DgramManager(self.xtc_files)
self.configs = self.dm.configs
else:
# Fetch xtc files
if rank == 0:
opts = expstr.split(':')
exp = {}
for opt in opts:
items = opt.split('=')
assert len(items) == 2
exp[items[0]] = items[1]
run = ''
if 'dir' in exp:
xtc_path = exp['dir']
else:
xtc_dir = os.environ.get('SIT_PSDM_DATA', '/reg/d/psdm')
xtc_path = os.path.join(xtc_dir, exp['exp'][:3],
exp['exp'], 'xtc')
if 'run' in exp:
run = exp['run']
if run:
self.xtc_files = np.array(glob.glob(
os.path.join(xtc_path, '*r%s*.xtc' % (run.zfill(4)))),
dtype='U%s' % FN_L)
else:
self.xtc_files = np.array(glob.glob(
os.path.join(xtc_path, '*.xtc')),
dtype='U%s' % FN_L)
self.xtc_files.sort()
self.smd_files = np.empty(len(self.xtc_files),
dtype='U%s' % FN_L)
smd_dir = os.path.join(xtc_path, 'smalldata')
for i, xtc_file in enumerate(self.xtc_files):
smd_file = os.path.join(
smd_dir,
os.path.splitext(os.path.basename(xtc_file))[0] +
'.smd.xtc')
if os.path.isfile(smd_file):
self.smd_files[i] = smd_file
else:
raise InputError(smd_file, "File not found.")
self.nfiles = np.array([len(self.xtc_files)], dtype='i')
assert self.nfiles[0] > 0
else:
self.nfiles = np.zeros(1, dtype='i')
if size == 1:
self.smd_dm = DgramManager(self.smd_files)
self.dm = DgramManager(self.xtc_files,
configs=self.smd_dm.configs)
self.configs = self.smd_dm.configs
else:
# Send filenames
comm.Bcast(self.nfiles, root=0)
if rank > 0:
self.xtc_files = np.empty(self.nfiles[0],
dtype='U%s' % FN_L)
self.smd_files = np.empty(self.nfiles[0],
dtype='U%s' % FN_L)
comm.Bcast([self.xtc_files, MPI.CHAR], root=0)
comm.Bcast([self.smd_files, MPI.CHAR], root=0)
# Send configs
if rank == 0:
self.dm = DgramManager(self.smd_files)
self.configs = self.dm.configs
nbytes = np.array([memoryview(config).shape[0] for config in self.configs], \
dtype='i')
else:
self.dm = None
self.configs = [
dgram.Dgram() for i in range(self.nfiles[0])
]
nbytes = np.empty(self.nfiles[0], dtype='i')
comm.Bcast(nbytes,
root=0) # no. of bytes is required for mpich
for i in range(self.nfiles[0]):
comm.Bcast([self.configs[i], nbytes[i], MPI.BYTE], root=0)
# Assign node types
self.nsmds = int(
os.environ.get('PS_SMD_NODES',
np.ceil((size - 1) * PERCENT_SMD)))
if rank == 0:
self.nodetype = 'smd0'
elif rank < self.nsmds + 1:
self.nodetype = 'smd'
# Big-data nodes own big data
if self.nodetype == 'bd':
self.dm = DgramManager(self.xtc_files,
configs=self.configs)
self.Detector = Detector(self.configs)
# FIXME: discover how many channels there are
chan0 = evt._dgrams[0].xpphsd.hsd.chan0
chan1 = evt._dgrams[0].xpphsd.hsd.chan1
chan2 = evt._dgrams[0].xpphsd.hsd.chan2
chan3 = evt._dgrams[0].xpphsd.hsd.chan3
chans = [chan0, chan1, chan2, chan3]
nonOpaqueHsd = Hsd.hsd("1.2.3", chans) # make an object per event
return nonOpaqueHsd
class _Factory:
def create(self, name, config):
return hsd(name, config)
from psana.dgrammanager import DgramManager
ds = DgramManager(os.path.join(cwd, 'hsd_061918_n3.xtc2'))
rawData = OpaqueRawData('xpphsd', ds.configs[0])
waveformStack = []
myPeaks = []
chanNum = 2
for i in range(5):
print("### Event: ", i)
evt = ds.next()
myrawhsd = rawData(evt)
print(dir(myrawhsd))
raw = myrawhsd.waveform()
listOfPeaks, sPos = myrawhsd.peaks(chanNum)
plt.plot(raw[0], 'o-')
def __init__(self, comms, exp, run_no, run_src, **kwargs):
""" Parallel read requires that rank 0 does the file system works.
Configs and calib constants are sent to other ranks by MPI.
Note that destination callback only works with RunParallel.
"""
super(RunParallel,
self).__init__(exp,
run_no,
max_events=kwargs['max_events'],
batch_size=kwargs['batch_size'],
filter_callback=kwargs['filter_callback'],
destination=kwargs['destination'],
prom_man=kwargs['prom_man'])
xtc_files, smd_files, other_files = run_src
self.comms = comms
psana_comm = comms.psana_comm # TODO tjl and cpo to review
rank = psana_comm.Get_rank()
size = psana_comm.Get_size()
g_ts = self.prom_man.get_metric("psana_timestamp")
if rank == 0:
# get Configure and BeginRun using SmdReader
self.smd_fds = np.array(
[os.open(smd_file, os.O_RDONLY) for smd_file in smd_files],
dtype=np.int32)
self.smdr_man = SmdReaderManager(self)
self.configs = self.smdr_man.get_next_dgrams()
g_ts.labels("first_event").set(time.time())
self.beginruns = self.smdr_man.get_next_dgrams(
configs=self.configs)
self._get_runinfo()
self.smd_dm = DgramManager(smd_files,
configs=self.configs,
run=self,
fds=self.smd_fds)
self.dm = DgramManager(xtc_files,
configs=self.smd_dm.configs,
run=self)
nbytes = np.array([memoryview(config).shape[0] for config in self.configs], \
dtype='i')
super()._set_configinfo()
super()._set_calibconst()
self.bcast_packets = {'calibconst': self.calibconst, \
'expt': self.expt, 'runnum': self.runnum, 'timestamp': self.timestamp}
else:
self.smd_dm = None
self.dm = None
self.configs = None
nbytes = np.empty(len(smd_files), dtype='i')
self.bcast_packets = None
# Send configs without pickling
psana_comm.Bcast(nbytes, root=0) # no. of bytes is required for mpich
if rank > 0:
self.configs = [np.empty(nbyte, dtype='b') for nbyte in nbytes]
for i in range(len(self.configs)):
psana_comm.Bcast([self.configs[i], nbytes[i], MPI.BYTE], root=0)
# Send other small things using small-case bcast
self.bcast_packets = psana_comm.bcast(self.bcast_packets, root=0)
if rank > 0:
self.configs = [
dgram.Dgram(view=config, offset=0) for config in self.configs
]
g_ts.labels("first_event").set(time.time())
self.dm = DgramManager(xtc_files, configs=self.configs, run=self)
super()._set_configinfo(
) # after creating a dgrammanger, we can setup config info
self.calibconst = self.bcast_packets['calibconst']
self.expt = self.bcast_packets['expt']
self.runnum = self.bcast_packets['runnum']
self.timestamp = self.bcast_packets['timestamp']
self.esm = EnvStoreManager(self.configs, 'epics', 'scan')
# FIXME: discover how many channels there are
chan0 = evt.dgrams[0].xpphsd.hsd.chan0
chan1 = evt.dgrams[0].xpphsd.hsd.chan1
chan2 = evt.dgrams[0].xpphsd.hsd.chan2
chan3 = evt.dgrams[0].xpphsd.hsd.chan3
chans = [chan0, chan1, chan2, chan3]
nonOpaqueHsd = Hsd.hsd("1.0.0", chans) # make an object per event
return nonOpaqueHsd
class _Factory:
def create(self, name, config):
return hsd(name, config)
from psana.dgrammanager import DgramManager
ds = DgramManager('/reg/neh/home/yoon82/Software/lcls2/hsd_052218b.xtc')
rawData = OpaqueRawData('xpphsd', ds.configs[0])
waveformStack = []
myPeaks = []
chanNum = 2
for i in range(5):
print("### Event: ", i)
evt = ds.next()
myrawhsd = rawData(evt)
raw = myrawhsd.raw()
listOfPeaks, sPos = myrawhsd.fex(chanNum)
plt.plot(raw[0], 'o-')
plt.title('Event {}, Waveform channel: {}'.format(i, chanNum))
import os, time, glob, sys
from psana.dgrammanager import DgramManager
filenames = glob.glob(
'/reg/d/psdm/xpp/xpptut15/scratch/mona/test/smalldata/*.smd.xtc')
dm = DgramManager(filenames)
cn_evt = 0
for evt in dm:
print('evt %d' % cn_evt)
for d in evt:
print(d.seq.timestamp())
cn_evt += 1
if cn_evt == 2: break
def _setup_xtcs(self):
exp = None
run_dict = {
} # stores list of runs with corresponding xtc_files, smd_files, and epic file
if self.shmem:
self.tag = self.shmem
run_dict[-1] = (['shmem'], None, None)
return exp, run_dict
# Reading xtc files in one of these two ways
assert self.exp != self.files
read_exp = False
if self.exp:
read_exp = True
elif self.files:
if isinstance(self.files, (str)):
f = pathlib.Path(self.files)
if not f.exists():
err = f"File {self.files} not found"
raise XtcFileNotFound(err)
xtc_files = [self.files]
elif isinstance(self.files, (list, np.ndarray)):
for xtc_file in self.files:
f = pathlib.Path(xtc_file)
if not f.exists():
err = f"File {xtc_file} not found"
raise XtcFileNotFound(err)
xtc_files = self.files
else:
raise InvalidFileType(
"Only accept filename string or list of files.")
# In case of reading file(s), user negative integers for the index.
# If files is a list, separate each file to an individual run.
for num, xtc_file in enumerate(xtc_files):
run_dict[-1 * (num + 1)] = ([xtc_file], None, None)
# Reads list of xtc files from experiment folder
if read_exp:
if self.dir:
xtc_path = self.dir
else:
xtc_dir = os.environ.get('SIT_PSDM_DATA', '/reg/d/psdm')
xtc_path = os.path.join(xtc_dir, self.exp[:3], self.exp, 'xtc')
# Get a list of runs (or just one run if user specifies it) then
# setup corresponding xtc_files and smd_files for each run in run_dict
run_list = []
if self.run_num > -1:
run_list = [self.run_num]
else:
run_list = [int(os.path.splitext(os.path.basename(_dummy))[0].split('-r')[1].split('-')[0]) \
for _dummy in glob.glob(os.path.join(xtc_path, '*-r*.xtc2'))]
run_list.sort()
smd_dir = os.path.join(xtc_path, 'smalldata')
for r in run_list:
all_smd_files = glob.glob(
os.path.join(smd_dir,
'*r%s-s*.smd.xtc2' % (str(r).zfill(4))))
if self.detectors:
# Create a dgrammanager to access the configs. This will be
# done on only core 0.
s1 = set(self.detectors)
smd_dm = DgramManager(all_smd_files)
smd_files = [all_smd_files[i] for i in range(len(all_smd_files)) \
if s1.intersection(set(smd_dm.configs[i].__dict__.keys()))]
else:
smd_files = all_smd_files
xtc_files = [os.path.join(xtc_path, \
os.path.basename(smd_file).split('.smd')[0] + '.xtc2') \
for smd_file in smd_files \
if os.path.isfile(os.path.join(xtc_path, \
os.path.basename(smd_file).split('.smd')[0] + '.xtc2'))]
all_files = glob.glob(
os.path.join(xtc_path, '*r%s-*.xtc2' % (str(r).zfill(4))))
other_files = [f for f in all_files if f not in xtc_files]
run_dict[r] = (xtc_files, smd_files, other_files)
return self.exp, run_dict